Repository: AccessOwl/open_owl Branch: main Commit: 2ced9f72db66 Files: 44 Total size: 117.4 KB Directory structure: gitextract_mi7z9xmv/ ├── .dockerignore ├── .formatter.exs ├── .github/ │ └── workflows/ │ └── tests.yml ├── .gitignore ├── .tool-versions ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yml ├── lib/ │ ├── api_client.ex │ ├── cli.ex │ ├── helpers/ │ │ ├── api_utils.ex │ │ ├── cli_utils.ex │ │ └── struct_utils.ex │ ├── login_flow_wrapper.ex │ ├── pagination_strategies/ │ │ ├── pagination_next_cursor_in_body_to_send_as_json.ex │ │ ├── pagination_next_page_in_body.ex │ │ ├── pagination_offset_params_in_url.ex │ │ ├── pagination_page_params_in_url.ex │ │ ├── pagination_strategy.ex │ │ └── pagination_url_in_body.ex │ ├── recipes/ │ │ ├── action.ex │ │ └── recipe.ex │ ├── recipes.ex │ ├── response_transformer.ex │ └── saas_owl.ex ├── mix.exs ├── owl.sh ├── package.json ├── recipes.yml ├── test/ │ ├── api_client_test.exs │ ├── fixtures/ │ │ └── test_recipes.yml │ ├── helpers/ │ │ ├── api_utils_test.exs │ │ ├── cli_utils_test.exs │ │ └── struct_utils_test.exs │ ├── login_flow_wrapper_test.exs │ ├── recipes_test.exs │ ├── response_transformer_test.exs │ └── test_helper.exs ├── ts_src/ │ ├── login_flow.ts │ ├── main.ts │ ├── recipe_manager.ts │ └── types.ts └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ npm-debug.log yarn.lock _build !_build/dev/lib/castore deps !deps/castore node_modules dist .gitignore ================================================ FILE: .formatter.exs ================================================ # Used by "mix format" [ inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] ] ================================================ FILE: .github/workflows/tests.yml ================================================ name: Run tests concurrency: ci_tests on: push: branches: [ main ] pull_request: branches: [ main ] jobs: build: name: Build and test runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v3 with: fetch-depth: 0 # cache the ASDF directory, using the values from .tool-versions - name: ASDF cache uses: actions/cache@v3 with: path: ~/.asdf key: ${{ runner.os }}-asdf-v2-${{ hashFiles('.tool-versions') }} id: asdf-cache # only run `asdf install` if we didn't hit the cache - uses: asdf-vm/actions/install@v1 if: steps.asdf-cache.outputs.cache-hit != 'true' # if we did hit the cache, set up the environment - name: Setup ASDF environment run: | echo "ASDF_DIR=$HOME/.asdf" >> $GITHUB_ENV echo "ASDF_DATA_DIR=$HOME/.asdf" >> $GITHUB_ENV if: steps.asdf-cache.outputs.cache-hit == 'true' - name: Reshim ASDF run: | echo "$ASDF_DIR/bin" >> $GITHUB_PATH echo "$ASDF_DIR/shims" >> $GITHUB_PATH $ASDF_DIR/bin/asdf reshim - name: Restore dependencies cache uses: actions/cache@v3 with: path: deps key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} restore-keys: ${{ runner.os }}-mix- - name: Cache compiled build id: cache-build uses: actions/cache@v3 env: cache-name: cache-compiled-build with: path: _build key: ${{ runner.os }}-mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }} restore-keys: | ${{ runner.os }}-mix-${{ env.cache-name }}- ${{ runner.os }}-mix- - name: Install dependencies run: | mix local.rebar --force mix local.hex --force mix deps.get - name: Compile run: mix compile - name: Run tests run: mix test ================================================ FILE: .gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage *.lcov # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # TypeScript cache *.tsbuildinfo # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ .rts2_cache_es/ .rts2_cache_umd/ # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env .env.test # parcel-bundler cache (https://parceljs.org/) .cache # Next.js build output .next # Nuxt.js build / generate output .nuxt dist # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and *not* Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output .vuepress/dist # Serverless directories .serverless/ # FuseBox cache .fusebox/ # DynamoDB Local files .dynamodb/ # TernJS port file .tern-port /test-results/ /playwright-report/ /playwright/.cache/ # The directory Mix will write compiled artifacts to. /_build/ # If you run "mix test --cover", coverage assets end up here. /cover/ # The directory Mix downloads your dependencies sources to. /deps/ # Where 3rd-party dependencies like ExDoc output generated docs. /doc/ # Ignore .fetch files in case you like to edit your project deps locally. /.fetch # If the VM crashes, it generates a dump, let's ignore it too. erl_crash.dump # Also ignore archive artifacts (built via "mix archive.build"). *.ez .DS_Store .elixir_ls/ .owl_env.tmp auth_cache/ results/ ================================================ FILE: .tool-versions ================================================ elixir 1.14.3-otp-24 erlang 24.2.1 nodejs 19.6.0 ================================================ FILE: Dockerfile ================================================ ### ### First Stage - Building the Elixir app as escript ### FROM hexpm/elixir:1.14.3-erlang-23.2.6-alpine-3.16.0 AS build # install build dependencies RUN apk add --no-cache build-base git # prepare build dir WORKDIR /app # extend hex timeout ENV HEX_HTTP_TIMEOUT=20 # install hex + rebar RUN mix local.hex --force && \ mix local.rebar --force # Copy over the mix.exs and mix.lock files to load the dependencies. If those # files don't change, then we don't keep re-fetching and rebuilding the deps. COPY mix.exs mix.lock ./ RUN mix deps.get && \ mix deps.compile && \ mkdir priv && \ # because of https://github.com/elixir-mint/castore/issues/35 cp _build/dev/lib/castore/priv/cacerts.pem priv/cacerts.pem COPY lib lib RUN mix compile && \ mix escript.build ### ### Second Stage - Setup the Runtime Environment ### FROM node:16-bullseye-slim AS app ENV LANG=C.UTF-8 # Install Firefox dependencies + tools RUN sh -c 'echo "deb http://ftp.us.debian.org/debian bullseye main non-free" >> /etc/apt/sources.list.d/fonts.list' && \ DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends erlang && \ # clean apt cache rm -rf /var/lib/apt/lists/* WORKDIR /app RUN npm config --global set update-notifier false ENV PLAYWRIGHT_BROWSERS_PATH=/playwright ENV REQ_MINT_CACERTFILE=/app/bin/cacerts.pem COPY package*.json ./ RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --include=dev --omit=optional --audit=false --progress=false --loglevel=error RUN npm_config_ignore_scripts=1 npx playwright install-deps firefox && \ npx playwright install firefox COPY ts_src ts_src COPY tsconfig.json ./ RUN npm run build COPY --from=build /app/owl ./bin/owl COPY --from=build /app/priv/cacerts.pem ./bin/cacerts.pem RUN mkdir -p /app/results ENV HOME=/app ENV MIX_ENV=dev ENTRYPOINT ["./bin/owl"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2023 Omni Owl GmbH Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # OpenOwl 🦉 Release Build OpenOwl lets you download user lists including user permissions (and other additional data) from various SaaS applications without the need for a public API. This tool is commonly used to check for orphaned user accounts or as preparation for an access review. This project is made with IT Ops, InfoSec and Compliance teams in mind - no developer experience needed. The [`recipes.yml`](recipes.yml) lists all supported applications. You are welcome to contribute to this project by setting up additional vendor integrations. ## How to run it Since OpenOwl uses various technologies running it with Docker is recommended. There are additional instructions if you want to run it directly. ### Option 1: Shell Wrapper with [`Docker`](https://docs.docker.com/get-docker/) (Recommended) The Docker image is built automatically with the first launch. #### Show all available applications a.k.a recipes ```bash ./owl.sh recipes list ``` #### Step 1. Run `login` action Following is an example on how to use it with [`Mezmo`](https://www.mezmo.com/) Parameters like `OWL_USERNAME` and `OWL_PASSWORD` are passed via the environment variable. ```bash OWL_USERNAME=someone@acme.com OWL_PASSWORD=abc123 ./owl.sh mezmo login ``` #### Step 2. Run `download_users` action ```bash ./owl.sh mezmo download_users ``` Depending on the application there might be additonal parameters required which will be listed after running the command. If that is the case, add the parameters and re-run the command. Example for additional parameters: In [recipes.yml](recipes.yml) you can see that there is a placeholder defined for Mezmo. Placeholders start with a `:` and look like this: `:account_id`. Depending on the recipe they are either passed as parameter or populated from other data (e.g. see Adobe). To pass the `:account_id` parameter, you prefix it with `OWL_` and upcase it. So `:account_id` becomes `OWL_ACCOUNT_ID`. ```bash OWL_ACCOUNT_ID=YOUR_ACCOUNT_ID ./owl.sh mezmo download_users ``` ### Option 2: Docker Compose Examples: ```bash docker compose run owl recipes list docker compose run -e 'OWL_USERNAME=YOUR_USERNAME' -e 'OWL_PASSWORD=YOUR_PASSWORD' owl mezmo login docker compose run -e 'OWL_ACCOUNT_ID=YOUR_ACCOUNT_ID' owl mezmo download_users ``` ### Option 3: Directly You can leverage the available [`.tool-versions`](.tool-versions) file to install requirements with [`asdf`](https://asdf-vm.com). ```bash asdf install ``` Alternatively you can install the required Node, Elixir and Erlang version manually based on the version of the [`.tool-versions`](.tool-versions) file. Run it with: ```bash mix run lib/cli.exs ``` ## How does it work? OpenOwl signs in like a regular user by entering username and password (RPA via Playwright). It then uses the SaaS' internal APIs to request the list of users on your behalf. This approach only works for SaaS applications with internal APIs. With the rise of single page apps (SPA) (think about React.js, Vue etc.), most applications can be supported. Besides SaaS applications this tool can also be used for internal apps. ## Quick Demo Watch the video ## Known limitations 1. User Account: The user account needs administrator rights (or a similar permissions that grants access to user lists). 2. Login: Currently only direct logins via username and password are supported. Logins via SSO (Google, Okta, Onelogin,...) will be added in the future. 3. Captchas: Login flows that include captchas are currently not supported. ## How to contribute [Here](recipes.yml) is the list of available integrations. Open a PR to add further [recipes](recipes.yml), adjust existing ones or extend missing capabilities (like further pagination strategies) to support even more applications. When all the required capabilities exist, a further integration can take just 30m. ### How to add a new vendor recipe 1. Check that your intended SaaS application has an internal API. 1. Open the Developer Tools/Inspector of your favourite browser. 1. Navigate to the page that shows all users. 1. In your inspector filter by XHR requests and reload the page. When you find some that belong to the same domain like your tool, the application is probably supported. 1. Use the Network tab in the inspector, find the request that includes your list of users with their permissions. 1. Copy the request as `curl`-request and paste it into a tool like [Postman](https://www.postman.com/downloads/). You can import the request by clicking *Import*, selecting *Raw text* and pasting the copied `curl`-request. 1. Execute the request in Postman and remove as many headers and parameters as possible to keep the request clean. Check that the request still works. Add the request to the [`recipes.yml`](recipes.yml). 1. When you have many users in your SaaS account, you will not see all users at once but 10, 50 or 100. Adjust the pagination parameters and try to get all data by traversing through it. Check which [pagination strategy](lib/pagination_strategies/) applies and add it to your added recipe. 1. Open the direct login page of the new vendor and find the right [selector](https://www.cuketest.com/playwright/docs/selectors) for the username and password field. Adjust the [`recipes.yml`](recipes.yml) accordingly. 1. Test that the login flow works properly and the configured action. 1. Open a PR. ## Troubleshooting ### Build Docker image In case the Shell Wrapper cannot build the Docker image as expected, try the following command to build the image manually: ```bash docker build -t open_owl-owl:latest . ``` ## The history of OpenOwl Knowing who has access to your SaaS tools is often guesswork at best and manually adding and deleting user accounts is tedious and error prone. The fact that SCIM/SAML are often locked up behind an enterprise-subscription is adding to the frustration. That's why Mathias and Philip decided to build [AccessOwl](https://www.accessowl.io/), a tool to automate SaaS account provisioning for employees for any tool. OpenOwl is essentially an open-source version of the underlying technology of AccessOwl. It was created out of a discussion that having access to your own teams user data should be a basic right for anybody. No matter whether it's used for audits, to discover orphaned user accounts or run access reviews. That's why we decided to open source a core part of AccessOwl to let anybody read out crucial information out of their SaaS stack. ================================================ FILE: docker-compose.yml ================================================ services: owl: build: ./ volumes: - ${PWD}/auth_cache:/app/auth_cache - ${PWD}/results:/app/results - ${PWD}/recipes.yml:/app/recipes.yml ================================================ FILE: lib/api_client.ex ================================================ defmodule OpenOwl.ApiClient do import OpenOwl.Helpers.ApiUtils @base_header [ {"accept-encoding", "gzip"}, {"accept", "*/*"} ] @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" def do_request( cookies, session_storage, http_method, body, url, headers, response_path, pagination, %{} = placeholder_params, placeholders_from_session_storage ) do cookie_string = filter_relevant_cookies(cookies, url) |> build_cookie_params_string() placeholder_params = get_params_from_session_storage(session_storage, placeholders_from_session_storage) |> Map.merge(placeholder_params) # required for local mix run Application.ensure_all_started(:telemetry) Application.ensure_all_started(:req) connect_options = if cacertfile = System.get_env("REQ_MINT_CACERTFILE"), do: [transport_opts: [cacertfile: cacertfile]], else: [] req = Req.new( method: http_method, body: body, url: url, headers: build_headers(headers, placeholder_params, cookie_string), path_params: placeholder_params, connect_options: connect_options, user_agent: @user_agent ) case Req.request(req) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> if pagination != nil do pagination.__struct__.handle_paginated_response( req, body, pagination, response_path, &extract_data_from_response_body/2 ) else {:ok, extract_data_from_response_body(body, response_path)} end {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end end defp extract_data_from_response_body(body, response_path) do get_field_via_response_path(body, response_path) end defp build_headers(headers, placeholders, cookie_string) do req_headers = [{"cookie", cookie_string} | @base_header] [headers | req_headers] |> List.flatten() |> Enum.map(fn {key, value} -> {key, apply_placeholder_params(value, placeholders)} end) end end ================================================ FILE: lib/cli.ex ================================================ defmodule OpenOwl.CLI do alias OpenOwl.Recipes alias OpenOwl.Recipes.Recipe alias OpenOwl.Recipes.Action alias OpenOwl.ResponseTransformer alias OpenOwl.LoginFlowWrapper import OpenOwl.Helpers.ApiUtils, only: [apply_placeholder_params: 2] import OpenOwl.Helpers.CliUtils @owl_cli_name "./owl.sh" @env_prefix "OWL_" @auth_cache_folder "auth_cache" # escript def main(args) do args |> parse_args() |> process_params() end # direct call via mix run def main() do System.argv() |> parse_args() |> process_params() end defp parse_args(args) do {flags, commands, _} = OptionParser.parse(args, strict: [debug: :boolean, pwdebug: :boolean, output: :string, help: :boolean] ) {commands, flags} end defp process_params({["recipes", "list"], _}) do IO.puts("Listing recipes with actions and parameters...\n") recipes = load_recipes() trailing_padding = get_trailing_padding(recipes) recipes |> Enum.each(fn {title, %Recipe{actions: actions} = recipe} -> title = title |> Atom.to_string() |> String.capitalize() |> then(&"#{&1}: ") |> String.pad_trailing(trailing_padding) login_parameters = recipe |> Recipes.get_required_parameters_for_recipe("login") |> parameter_list_as_env() |> maybe_in_brackets() IO.puts(title <> "login #{login_parameters}") Enum.each(actions, fn %Action{} = action -> parameters = recipe |> Recipes.get_required_parameters_for_recipe(action.name) |> parameter_list_as_env() |> maybe_in_brackets() IO.puts(String.pad_trailing("", trailing_padding) <> "#{action.name} #{parameters}") end) end) end defp process_params({[vendor, "login"], flags}) do recipes = load_recipes() # TODO: until we pull that out from a config or sth app_env_params = get_app_env_params() with %Recipe{} = recipe <- Recipes.get_recipe(recipes, vendor), {:ok, %{username: username, password: password}} <- Recipes.validate_recipe_parameters(recipe, "login", app_env_params) do IO.puts("Starting #{vendor}: login...") if Keyword.get(flags, :debug, false) do IO.puts("DEBUG=true\n") LoginFlowWrapper.get_local_cmd_command( vendor, apply_placeholder_params(recipe.login_url, app_env_params), apply_placeholder_params(recipe.destination_url_pattern, app_env_params), recipe.username_selector, recipe.password_selector, username, password ) |> IO.puts() else case LoginFlowWrapper.call_local_cmd( vendor, apply_placeholder_params(recipe.login_url, app_env_params), apply_placeholder_params(recipe.destination_url_pattern, app_env_params), recipe.username_selector, recipe.password_selector, username, password, Keyword.get(flags, :pwdebug, false) ) do {:ok, _} -> IO.puts("... authentication info saved") IO.puts("DONE") {:error, %{exit_code: exit_code, result: result}} -> write_error("#{exit_code}: #{inspect(result)}") end end else {:error, missing_parameters} -> write_error("missing parameters: #{parameter_list_as_env(missing_parameters)}") nil -> write_error("vendor #{vendor} not found") end end defp process_params({[vendor, action_name], flags}) when vendor != "recipes" do recipes = load_recipes() app_env_params = get_app_env_params() IO.puts("Starting #{vendor}: #{action_name}...") cookies_path = Path.join([File.cwd!(), @auth_cache_folder, "#{vendor}_cookies.json"]) session_storage_path = Path.join([File.cwd!(), @auth_cache_folder, "#{vendor}_session_storage.json"]) session_storage = case File.read(session_storage_path) do {:ok, data} -> Jason.decode!(data) {:error, _} -> %{} end with {:ok, cookie_binary_json} <- File.read(cookies_path), {:ok, cookies} <- Jason.decode(cookie_binary_json), %Recipe{actions: actions} = recipe <- Recipes.get_recipe(recipes, vendor), %Action{} = action <- Recipes.get_action_for_name(actions, action_name), {:ok, _} <- Recipes.validate_recipe_parameters(recipe, action_name, app_env_params) do IO.puts( "... #{action.http_method}-request #{apply_placeholder_params(action.url, app_env_params)}..." ) case OpenOwl.ApiClient.do_request( cookies, session_storage, action.http_method, action.body, action.url, action.headers, action.response_path, action.pagination, app_env_params, action.populate_placeholders_from_session_storage ) do {:ok, response} -> output_flag = Keyword.get(flags, :output) filename = if output_flag == nil, do: timebased_filename("#{vendor}.csv"), else: output_flag path = Path.join([File.cwd!(), "results", filename]) content = ResponseTransformer.records_to_csv(response) File.write!(path, content) IO.puts("... wrote results to #{filename}") IO.puts("DONE") {:http_error, {status, body}} -> write_error("HTTP (#{status}): #{inspect(body)}") {:error, reason} -> write_error("#{inspect(reason)}") end else nil -> write_error("Vendor #{vendor} or action #{action_name} not found") {:error, missing_parameters} when is_list(missing_parameters) -> write_error("missing parameters: #{parameter_list_as_env(missing_parameters)}") {:error, %Jason.DecodeError{} = reason} -> write_error("#{inspect(reason)}") {:error, reason} -> write_error( "Authentication cookies file #{cookies_path} could not be loaded: #{inspect(reason)}" ) end end defp process_params(_), do: print_help() defp print_help() do version = OpenOwl.version() IO.puts("OpenOwl v#{version}\n") IO.puts("Usage:") IO.puts( " #{@owl_cli_name} recipes list - Show list of recipes with their actions" ) IO.puts( "[env_vars] #{@owl_cli_name} login - Authenticate and get required authentication" ) IO.puts( "[env_vars] #{@owl_cli_name} [flags] - Triggers defined action of recipes.yml\n" ) IO.puts("Env vars:") IO.puts(" Some commands need set environment variables. You can prepend commands with them.") IO.puts(" For instance OWL_USERNAME=user #{@owl_cli_name} login\n") IO.puts("Flags:") IO.puts(" --output - Set a custom output filename. Otherwise a timebased filename is used.") IO.puts(" --help - Print this help message") end defp load_recipes() do case Recipes.load_recipes() do {:ok, recipes} -> recipes {:error, reason} -> raise "File could not be loaded: #{inspect(reason)}" end end defp get_trailing_padding(%{} = recipes) do Enum.map(recipes, fn {title, _} -> "#{title}: " end) |> get_trailing_padding() end defp get_trailing_padding(title_list) when is_list(title_list) do title_list |> Enum.max(fn left, right -> String.length(left) >= String.length(right) end) |> String.length() end defp parameter_list_as_env(list) do list |> Enum.map(&to_string/1) |> Enum.map(&"#{@env_prefix}#{String.upcase(&1)}") |> Enum.join(", ") end defp maybe_in_brackets(""), do: "" defp maybe_in_brackets(string) do "(#{string})" end defp write_error(msg) do IO.puts("ERROR: #{msg}") System.halt(1) end end OpenOwl.CLI.main() ================================================ FILE: lib/helpers/api_utils.ex ================================================ defmodule OpenOwl.Helpers.ApiUtils do @path_separator "." alias OpenOwl.Recipes @doc """ Resolves a response path and get the described string field out of a map. Some APIs have a one-element list as root. We handle it by removing this layer. If the first argument is a list and the second is nil, the list is returned directly. iex> get_field_via_response_path(%{}, nil) nil iex> get_field_via_response_path(%{}, "a.b") nil iex> get_field_via_response_path(%{"a" => "hit"}, nil) nil iex> get_field_via_response_path(%{"a" => "hit", "b" => "nob"}, "a") "hit" iex> get_field_via_response_path([%{"a" => "hit", "b" => "nob"}], "a") "hit" iex> get_field_via_response_path(%{"a" => "hit", "b" => "nob"}, "a.a1") nil iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => "noa1"}, "b" => "nob"}, "a.a1") "hit" iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => "hit2"}, "b" => "nob"}, "a.a2") "hit2" iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => %{"a2i" => "hit2", "a2ii" => "noa2ii"}}, "b" => "nob"}, "a.a2.a2i") "hit2" iex> get_field_via_response_path([%{"a" => "a1"}, %{"a" => "a2"}], nil) [%{"a" => "a1"}, %{"a" => "a2"}] iex> get_field_via_response_path([%{"a" => "a1"}, %{"a" => "a2"}], "a") ** (FunctionClauseError) no function clause matching in OpenOwl.Helpers.ApiUtils.get_field_via_response_path/2 """ def get_field_via_response_path(list, nil) when is_list(list), do: list def get_field_via_response_path([one], response_path), do: get_field_via_response_path(one, response_path) def get_field_via_response_path(_map, nil), do: nil def get_field_via_response_path(%{} = map, response_path) do path = String.split(response_path, @path_separator) get_in(map, path) rescue _ -> nil end @doc """ Sets a specific field in a map via the passed path. The (nested) structure must exist, otherwise it will raise. Some APIs pass a one-element list as root. Such a structure would be kept. iex> flat_map = %{"a" => "a1", "b" => "b1"} iex> set_field_via_path(flat_map, nil, "v") flat_map iex> set_field_via_path(flat_map, "", "v") flat_map iex> set_field_via_path(flat_map, "c", "v") %{"a" => "a1", "b" => "b1", "c" => "v"} iex> set_field_via_path(flat_map, "b", "v") %{"a" => "a1", "b" => "v"} iex> set_field_via_path([flat_map], "b", "v") [%{"a" => "a1", "b" => "v"}] iex> deep_map = %{"a" => "a1", "b" => "b1", "d" => %{"d1" => "d1v"}} iex> set_field_via_path(deep_map, "d.d1", "v") %{"a" => "a1", "b" => "b1", "d" => %{"d1" => "v"}} iex> set_field_via_path([deep_map], "d.d1", "v") [%{"a" => "a1", "b" => "b1", "d" => %{"d1" => "v"}}] iex> set_field_via_path(deep_map, "d.d2", "v") %{"a" => "a1", "b" => "b1", "d" => %{"d1" => "d1v", "d2" => "v"}} iex> set_field_via_path(deep_map, "d.d1.deep", "v") ** (FunctionClauseError) no function clause matching in Access.get_and_update/3 """ def set_field_via_path(%{} = map, nil, _value), do: map def set_field_via_path(%{} = map, "", _value), do: map def set_field_via_path([one], field_path, value), do: set_field_via_path(one, field_path, value, true) def set_field_via_path(%{} = map, field_path, value, list? \\ false) do path = String.split(field_path, @path_separator) result = put_in(map, path, value) if list?, do: [result], else: result end @doc """ Filters for cookies of the passed url (domain + subdomain). iex> cookie_domain = %{"name" => "session","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"} iex> cookie_subdomain = %{"name" => "session","value" => "123","domain" => ".example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"} iex> cookies = [cookie_domain, cookie_subdomain] iex> filter_relevant_cookies(cookies, "http://example.com/a?b=2") cookies iex> filter_relevant_cookies(cookies, "http://s.bla.example.com/a?b=2") cookies """ def filter_relevant_cookies(cookies, url) do %URI{host: host} = URI.parse(url) host = host |> String.split(@path_separator) |> Enum.slice(-2..-1) |> Enum.join(@path_separator) Enum.filter(cookies, fn cookie -> String.contains?(cookie["domain"], host) end) end @doc ~S""" Gets the passed params (support paths separated by ".") from session storage. Searches in values of all the session storage and merges with maps one level deeper. Also decodes JSON on first level if necessary. Current implementation is not flexible enough yet to support many use cases. Depending on future requirements we will adjust it (including the interface towards recipes). iex> get_params_from_session_storage(%{}, []) %{} iex> payload = %{"adobeid_ims_access_token/ONESIE1/false/AdobeID,ab.manage" => "{\"REAUTH_SCOPE\":\"reauthenticated\",\"client_id\":\"ONESIE1\",\"scope\":\"openid,AdobeID,additional_info.projectedProductContext,read_organizations,read_members,read_countries_regions,additional_info.roles,adobeio_api,read_auth_src_domains,authSources.rwd,bis.read.pi,app_policies.read,app_policies.write,client.read,publisher.read,client.scopes.read,creative_cloud,service_principals.write,aps.read.app_merchandising,aps.eval_licensesforapps,ab.manage,aps.device_activation_mgmt\",\"expire\":\"2023-01-01T13:37:00.342Z\",\"user_id\":\"thisiauser_id\",\"tokenValue\":\"longtoken\",\"sid\":\"thisissid\",\"state\":{},\"fromFragment\":false,\"impersonatorId\":\"\",\"isImpersonatedSession\":false,\"other\":\"{}\",\"pbaSatisfiedPolicies\":[\"MedSecNoEV\",\"LowSec\"]}", "adobeid_ims_profile/ONESIE1/false/AdobeID,ab.manage,additional_info" => "{\"account_type\":\"type2e\",\"utcOffset\":\"null\",\"preferred_languages\":[\"en-us\"],\"displayName\":\"Integration Account\",\"roles\":[{\"principal\":\"LONGID@AdobeOrg:123456\",\"organization\":\"LONGID@AdobeOrg\",\"named_role\":\"org_admin\",\"target\":\"LONGID@AdobeOrg\",\"target_type\":\"TRG_ORG\",\"target_data\":{}}],\"last_name\":\"Account\",\"userId\":\"someuserid.e\",\"authId\":\"someauthid@AdobeID\",\"tags\":[\"agegroup_18plus\"],\"projectedProductContext\":[],\"emailVerified\":\"true\",\"toua\":[{\"touName\":\"creative_cloud\",\"current\":true}],\"phoneNumber\":null,\"countryCode\":\"US\",\"name\":\"Integration Account\",\"mrktPerm\":\"\",\"mrktPermEmail\":null,\"first_name\":\"Integration\",\"email\":\"bruce@wayne.org\"}", "sherlockId" => "sherlockUUID"} iex> get_params_from_session_storage(payload, []) %{} iex> get_params_from_session_storage(payload, ~w(client_id tokenValue notexistent)) %{client_id: "ONESIE1", tokenValue: "longtoken", notexistent: nil} iex> get_params_from_session_storage(payload, ~w(client_id roles.organization)) %{client_id: "ONESIE1", organization: "LONGID@AdobeOrg"} """ def get_params_from_session_storage(session_storage, params) when is_map(session_storage) and is_list(params) do result = session_storage |> Map.values() |> Enum.map(fn item -> case Jason.decode(item) do {:ok, result} -> result {:error, _} -> item end end) |> Enum.reduce(%{}, fn item, acc when is_map(item) -> Map.merge(acc, item) # we ignore flat items here for the moment. We might need it later which means refactoring # of this approach _item, acc -> acc end) |> Enum.map(fn {key, []} -> {key, []} {key, value} when is_list(value) -> {key, hd(value)} {key, value} -> {key, value} end) |> Map.new() Enum.reduce(params, %{}, fn param, acc -> Map.put( acc, get_last_response_path_part(param) |> String.to_atom(), get_field_via_response_path(result, param) ) end) end @doc """ Returns the last part of a response path that is splitted by #{@path_separator}. iex> get_last_response_path_part("a.b.c") "c" iex> get_last_response_path_part("abc") "abc" iex> get_last_response_path_part("") "" """ def get_last_response_path_part(path) do path |> String.split(@path_separator) |> Enum.reverse() |> hd() end @doc """ Builds a string properly formatted to send it as cookies header. iex> cookie_a = %{"name" => "session","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"} iex> cookie_b = %{"name" => "bla","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"} iex> cookies = [cookie_a, cookie_b] iex> build_cookie_params_string([cookie_a]) "session=123" iex> build_cookie_params_string(cookies) "session=123; bla=123" """ def build_cookie_params_string(cookies) do Enum.map_join(cookies, "; ", &(&1["name"] <> "=" <> &1["value"])) end @doc """ Replaces placeholder in subject using passed params. iex> apply_placeholder_params("cool", %{org: "o1", team_id: 123}) "cool" iex> apply_placeholder_params("cool:org", %{org: "o1", team_id: 123}) "coolo1" iex> apply_placeholder_params("cool_:team_id_:org_", %{org: "o1", team_id: 123}) "cool_123_o1_" iex> apply_placeholder_params("cool_:team_bla_:org_", %{_org: "o1", team_bla_id: 123}) "cool_:team_bla_:org_" """ def apply_placeholder_params(subject, params) do Regex.replace(Recipes.placeholder_regex(), subject, fn whole_match, key -> to_string(params[String.to_atom(key)] || whole_match) end) end end ================================================ FILE: lib/helpers/cli_utils.ex ================================================ defmodule OpenOwl.Helpers.CliUtils do @prefix "OWL_" @doc """ Returns all app environment params downcased and without app prefix. iex> get_app_env_params(%{"SHELL" => "/bin/zsh", "DOWL_BLA" => "no", "OWL_USERNAME" => "Oo", "OWL_PASSWORD" => "secret", "OWL_TEAM_ID" => "123"}) %{username: "Oo", password: "secret", team_id: "123"} iex> get_app_env_params(%{"SHELL" => "/bin/zsh", "DOWL_BLA" => "no"}) %{} iex> get_app_env_params(%{}) %{} """ def get_app_env_params(params \\ System.get_env()) do params |> Enum.filter(fn {@prefix <> _, _value} -> true _ -> false end) |> Enum.map(fn {key, value} -> {normalize_var(key), value} end) |> Map.new() end defp normalize_var(var) do var |> String.replace(@prefix, "") |> String.downcase() |> String.to_atom() end @doc """ Generates a timebased filename. iex> datetime = ~N[2017-11-06 00:23:51.123456] iex> timebased_filename("miro.csv", datetime) "20171106T002351_miro.csv" iex> timebased_filename(nil) ** (ArgumentError) filename empty iex> timebased_filename("") ** (ArgumentError) filename empty """ def timebased_filename(filename, datetime \\ NaiveDateTime.utc_now()) def timebased_filename(nil, _), do: raise(ArgumentError, "filename empty") def timebased_filename("", _), do: raise(ArgumentError, "filename empty") def timebased_filename(filename, datetime) do now = datetime |> NaiveDateTime.truncate(:second) |> NaiveDateTime.to_iso8601(:basic) "#{now}_#{filename}" end end ================================================ FILE: lib/helpers/struct_utils.ex ================================================ defmodule OpenOwl.Helpers.StructUtils do @moduledoc """ Converts a map with strings, a map with atoms or an existing struct with all of the necessary keys into a target struct. """ def to_struct(kind, attrs) do struct = struct(kind) map_list = Map.to_list(struct) Enum.reduce(map_list, struct, fn {:__struct__, _}, acc -> acc {key, _}, acc -> fetch_value(attrs, key, acc) end) end defp fetch_value(attrs, key, acc) when is_atom(key) do case Map.fetch(attrs, key) do {:ok, value} -> %{acc | key => value} :error -> fetch_value(attrs, Atom.to_string(key), acc) end end defp fetch_value(attrs, key, acc) do case Map.fetch(attrs, key) do # credo:disable-for-next-line {:ok, value} -> %{acc | String.to_atom(key) => value} :error -> acc end end end ================================================ FILE: lib/login_flow_wrapper.ex ================================================ defmodule OpenOwl.LoginFlowWrapper do def call_local_cmd( slug, url, destination_url_pattern, username_selector, password_selector, username, password, pw_debug? ) do case System.cmd("npm", ["run", "start"], env: [ {"SLUG", slug}, {"URL", url}, {"DESTINATION_URL_PATTERN", destination_url_pattern}, {"USERNAME_SELECTOR", username_selector}, {"PASSWORD_SELECTOR", password_selector}, {"USER", username}, {"PASSWORD", password}, {"PWDEBUG", if(pw_debug?, do: "1", else: "0")} ] ) do {result, 0} -> {:ok, result} {result, exit_code} -> {:error, %{exit_code: exit_code, result: result}} end end @doc """ Returns the command to execute for the login flow. ## Examples iex> get_local_cmd_command("slug", "url", "dest", "usel", "psel", "user", "password") ~s(PWDEBUG=1 SLUG=slug URL="url" DESTINATION_URL_PATTERN="dest" USERNAME_SELECTOR="usel" PASSWORD_SELECTOR="psel" USER="user" PASSWORD="password" npm run start) """ def get_local_cmd_command( slug, url, destination_url_pattern, username_selector, password_selector, username, password ) do env = [ "PWDEBUG=1", "SLUG=#{slug}", ~s(URL="#{url}"), ~s(DESTINATION_URL_PATTERN="#{destination_url_pattern}"), ~s(USERNAME_SELECTOR="#{username_selector}"), ~s(PASSWORD_SELECTOR="#{password_selector}"), ~s(USER="#{username}"), ~s(PASSWORD="#{password}") ] "#{Enum.join(env, " ")} npm run start" end end ================================================ FILE: lib/pagination_strategies/pagination_next_cursor_in_body_to_send_as_json.ex ================================================ defmodule OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson do @moduledoc """ Handles APIs that return a cursor in the response body. Puts the cursur into a field of the JSON request body using `json_body_cursor_path`. It needs a boolean flag (`has_next_page_response_path`) to indicate whether the last page was reached. ## Example response: ```json { "results": [{"id": 1}, {"id": 2}], "pagination": {"next_cursor": "abc", "has_next_page": true} } ``` You can use the `next_cursor_response_path` parameter to define the field that has the cursor. You can traverse into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it would be `pagination.next_cursor`. """ use OpenOwl.PaginationStrategy alias OpenOwl.Helpers.ApiUtils @enforce_keys [ :strategy, :next_cursor_response_path, :has_next_page_response_path, :json_body_cursor_path ] defstruct strategy: nil, next_cursor_response_path: nil, has_next_page_response_path: nil, json_body_cursor_path: nil @type t :: %__MODULE__{ strategy: :next_cursor_in_body_to_send_as_json, next_cursor_response_path: String.t(), has_next_page_response_path: String.t(), json_body_cursor_path: String.t() } @impl true def handle_paginated_response( %Req.Request{} = req, body, %__MODULE__{} = pagination, response_path, data_extraction_fn, data_acc \\ [] ) do data = data_extraction_fn.(body, response_path) data_acc = data_acc ++ data next_cursor = ApiUtils.get_field_via_response_path(body, pagination.next_cursor_response_path) new_req_body = req.body |> Jason.decode!() |> ApiUtils.set_field_via_path(pagination.json_body_cursor_path, next_cursor) has_next_page? = ApiUtils.get_field_via_response_path(body, pagination.has_next_page_response_path) if has_next_page? do case Req.request(req, json: new_req_body) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> handle_paginated_response( req, body, pagination, response_path, data_extraction_fn, data_acc ) {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end else {:ok, data_acc} end end end ================================================ FILE: lib/pagination_strategies/pagination_next_page_in_body.ex ================================================ defmodule OpenOwl.PaginationStrategy.NextPageInBody do @moduledoc """ Handles APIs that return the next page cursor in the response body. Furthermore the cursor needs to be passed as query parameter. ## Example response: ```json { "results": [{"id": 1}, {"id": 2}], "pagination": {"next_page": 42"} } ``` Use `page_query_param` to define the name of the query parameter. You can use the `next_page_response_path` parameter to define the next page field. You can traverse into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it would be `pagination.next_page`. """ use OpenOwl.PaginationStrategy alias OpenOwl.Helpers.ApiUtils @enforce_keys [:strategy, :next_page_response_path, :page_query_param] defstruct strategy: nil, next_page_response_path: nil, page_query_param: nil @type t :: %__MODULE__{ strategy: :next_page_in_body, next_page_response_path: String.t(), page_query_param: String.t() } @impl true def handle_paginated_response( %Req.Request{} = req, body, %__MODULE__{} = pagination, response_path, data_extraction_fn, data_acc \\ [] ) do data = data_extraction_fn.(body, response_path) data_acc = data_acc ++ data next_page = ApiUtils.get_field_via_response_path(body, pagination.next_page_response_path) query_param = Keyword.new([{String.to_atom(pagination.page_query_param), next_page}]) if next_page != nil and next_page != "" do case Req.request(req, params: query_param) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> handle_paginated_response( req, body, pagination, response_path, data_extraction_fn, data_acc ) {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end else {:ok, data_acc} end end end ================================================ FILE: lib/pagination_strategies/pagination_offset_params_in_url.ex ================================================ defmodule OpenOwl.PaginationStrategy.OffsetParamsInUrl do @moduledoc """ Handles APIs that have offset params in URL query parameters. By default, it assumes `offset` is used as parameter. The parameter must be part of the URL in the first request. Subsequent requests will increment this parameter by the value of the `limit` query parameter. The name of this parameter can be defined with `limit_query_param` and is `limit` by default. The name of the passed `offset` param can be defined with `offset_query_param`. Request loop stops when the page has no data anymore (empty result). """ use OpenOwl.PaginationStrategy @enforce_keys [:strategy] defstruct strategy: nil, offset_query_param: "offset", limit_query_param: "limit" @type t :: %__MODULE__{ strategy: :page_params_in_url, offset_query_param: String.t(), limit_query_param: String.t() } @impl true def handle_paginated_response( %Req.Request{} = req, body, %__MODULE__{} = pagination, response_path, data_extraction_fn, data_acc \\ [] ) do data = data_extraction_fn.(body, response_path) data_acc = data_acc ++ data %URI{query: query} = req.url limit_query_param = pagination.limit_query_param %{^limit_query_param => limit} = query_params = URI.decode_query(query) query_params_string = query_params |> Map.update!( pagination.offset_query_param, &(String.to_integer(&1) + String.to_integer(limit)) ) |> URI.encode_query() req = update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end) if data != [] do case Req.request(req) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> handle_paginated_response( req, body, pagination, response_path, data_extraction_fn, data_acc ) {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end else {:ok, data_acc} end end end ================================================ FILE: lib/pagination_strategies/pagination_page_params_in_url.ex ================================================ defmodule OpenOwl.PaginationStrategy.PageParamsInUrl do @moduledoc """ Handles APIs that have page params in URL query parameters. By default, it assumes `page` is used as parameter. The parameter must be part of the URL in the first request. Subsequent requests will increment this parameter. The name of the passed `page` param can be defined with `page_query_param`. Request loop stops when the page has no data anymore (empty result). """ use OpenOwl.PaginationStrategy @enforce_keys [:strategy] defstruct strategy: nil, page_query_param: "page" @type t :: %__MODULE__{strategy: :page_params_in_url, page_query_param: String.t()} @impl true def handle_paginated_response( %Req.Request{} = req, body, %__MODULE__{} = pagination, response_path, data_extraction_fn, data_acc \\ [] ) do data = data_extraction_fn.(body, response_path) data_acc = data_acc ++ data %URI{query: query} = req.url query_params_string = query |> URI.decode_query() |> Map.update!(pagination.page_query_param, &(String.to_integer(&1) + 1)) |> URI.encode_query() req = update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end) if data != [] do case Req.request(req) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> handle_paginated_response( req, body, pagination, response_path, data_extraction_fn, data_acc ) {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end else {:ok, data_acc} end end end ================================================ FILE: lib/pagination_strategies/pagination_strategy.ex ================================================ defmodule OpenOwl.PaginationStrategy do @type t :: module() @type status() :: pos_integer() @type body() :: map() @type pagination() :: %{atom() => atom() | String.t()} @type response_path() :: String.t() @doc """ Casts a map to a typed struct. """ @callback cast(map()) :: %{ :__struct__ => atom(), :strategy => atom(), optional(atom()) => any() } @doc """ Follows the pagination strategy to accumulate all the data and returns it at once. """ @callback handle_paginated_response( %Req.Request{}, body(), pagination(), response_path(), fun(), [map()] ) :: {:ok, map()} | {:http_error, {status(), body()}} | {:error, any()} defmacro __using__(_options) do quote do @behaviour OpenOwl.PaginationStrategy @impl true def cast(attrs) do OpenOwl.Helpers.StructUtils.to_struct(__MODULE__, attrs) end end end end ================================================ FILE: lib/pagination_strategies/pagination_url_in_body.ex ================================================ defmodule OpenOwl.PaginationStrategy.UrlInBody do @moduledoc """ Handles APIs that return a URL in the response body where the next page can be fetched. ## Example response: ```json { "results": [{"id": 1}, {"id": 2}], "nextLink": "https://api.example.com/?nextLink=456" } ``` You can use the `next_url_response_path` parameter to define the field that has the URL. You can traverse into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it would be just `nextLink`. """ use OpenOwl.PaginationStrategy alias OpenOwl.Helpers.ApiUtils @enforce_keys [:strategy, :next_url_response_path] defstruct strategy: nil, next_url_response_path: nil @type t :: %__MODULE__{ strategy: :url_in_body, next_url_response_path: String.t() } @impl true def handle_paginated_response( %Req.Request{} = req, body, %__MODULE__{} = pagination, response_path, data_extraction_fn, data_acc \\ [] ) do data = data_extraction_fn.(body, response_path) data_acc = data_acc ++ data next_url = ApiUtils.get_field_via_response_path(body, pagination.next_url_response_path) if next_url != nil and next_url != "" do case Req.request(req, url: next_url) do {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] -> handle_paginated_response( req, body, pagination, response_path, data_extraction_fn, data_acc ) {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] -> {:http_error, {status, body}} {:error, exception} -> {:error, exception} end else {:ok, data_acc} end end end ================================================ FILE: lib/recipes/action.ex ================================================ defmodule OpenOwl.Recipes.Action do alias OpenOwl.Helpers.StructUtils @enforce_keys [:name, :http_method, :url] defstruct name: nil, http_method: nil, body: nil, url: nil, headers: [], response_path: nil, pagination: nil, populate_placeholders_from_session_storage: [] @type t :: %__MODULE__{ name: String.t(), http_method: :get | :post | :put | :patch | :delete, body: String.t(), url: String.t(), headers: [{String.t(), String.t()}], response_path: String.t() | nil, pagination: OpenOwl.PaginationStrategy.t() | nil, populate_placeholders_from_session_storage: [String.t()] | nil } def cast(attrs) do http_method = case attrs do %{http_method: http_method} -> http_method %{"http_method" => http_method} -> http_method end headers = case attrs do %{headers: headers} -> headers %{"headers" => headers} -> headers _ -> [] end |> Enum.map(fn header -> Map.to_list(header) |> hd() end) {pagination_strategy, pagination_attrs} = case attrs do %{pagination: pagination} -> {pagination[:strategy], pagination} %{"pagination" => pagination} -> {pagination["strategy"], pagination} _ -> {nil, nil} end pagination_mod = modulize(pagination_strategy) action = StructUtils.to_struct(__MODULE__, attrs) |> Map.put(:http_method, http_method_to_atom(http_method)) |> Map.put(:headers, headers) if pagination_mod != nil, do: Map.put(action, :pagination, apply(pagination_mod, :cast, [pagination_attrs])), else: action end defp http_method_to_atom(http_method) do http_method |> String.downcase() |> String.to_atom() end defp modulize(nil), do: nil defp modulize(underscored_string) do underscored_string |> String.split("_") |> Enum.map_join(&String.capitalize/1) |> then(&Module.concat(OpenOwl.PaginationStrategy, &1)) end end ================================================ FILE: lib/recipes/recipe.ex ================================================ defmodule OpenOwl.Recipes.Recipe do alias OpenOwl.Helpers.StructUtils alias OpenOwl.Recipes.Action @enforce_keys [ :login_url, :destination_url_pattern, :username_selector, :password_selector ] defstruct login_url: nil, destination_url_pattern: nil, username_selector: nil, password_selector: nil, actions: [] @type t :: %__MODULE__{ login_url: String.t(), destination_url_pattern: String.t(), username_selector: String.t(), password_selector: String.t(), actions: [] | [Action.t()] } def cast(attrs, actions) do recipe = StructUtils.to_struct(__MODULE__, attrs) %{recipe | actions: Enum.map(actions, &Action.cast/1)} end end ================================================ FILE: lib/recipes.ex ================================================ defmodule OpenOwl.Recipes do @recipe_file_name "recipes.yml" @external_resource Path.join(File.cwd!(), @recipe_file_name) @placeholder_regex ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/ alias OpenOwl.Recipes.Recipe alias OpenOwl.Recipes.Action def load_recipes(rel_path \\ @recipe_file_name) do path = Path.join(File.cwd!(), rel_path) case YamlElixir.read_from_file(path) do {:ok, data} -> {:ok, parse_yaml_data(data)} {:error, reason} -> {:error, reason} end end def get_recipe(recipes, slug) when is_binary(slug) do get_recipe(recipes, String.to_atom(slug)) end def get_recipe(recipes, slug) do Map.get(recipes, slug) end def get_action_for_name(actions, name) do Enum.find(actions, &(&1.name == name)) end @doc false def get_required_parameters_for_recipe( %Recipe{login_url: login_url, destination_url_pattern: destination_url_pattern}, "login" ) do [:username, :password] ++ get_parameters([login_url, destination_url_pattern]) end @doc false def get_required_parameters_for_recipe(%Recipe{actions: actions}, action_name) do with %Action{url: url, headers: headers} <- get_action_for_name(actions, action_name) do get_parameters([url | get_header_values(headers)]) else nil -> [] end end defp get_parameters(strings) do strings |> Enum.reduce([], fn string, acc -> matches = Regex.scan(@placeholder_regex, string, capture: :first) [matches | acc] end) |> List.flatten() |> Enum.uniq() |> Enum.map(&atomize_placeholder_string/1) end defp atomize_placeholder_string(placeholder) do ":" <> name = placeholder String.to_atom(name) end defp get_header_values(headers) do Enum.map(headers, fn {_, value} -> value end) end def validate_recipe_parameters(%Recipe{} = recipe, action, %{} = parameter_map) do parameters = Map.keys(parameter_map) required_parameters = get_required_parameters_for_recipe(recipe, action) missing_parameters = required_parameters -- parameters if missing_parameters == [] do {:ok, parameter_map} else {:error, missing_parameters} end end @doc false def placeholder_regex() do @placeholder_regex end defp parse_yaml_data(%{} = data) do Enum.reduce(data, %{}, fn {key, value}, acc -> Map.put(acc, String.to_atom(key), OpenOwl.Recipes.Recipe.cast(value, value["actions"])) end) end end ================================================ FILE: lib/response_transformer.ex ================================================ defmodule OpenOwl.ResponseTransformer do alias NimbleCSV.RFC4180, as: CSV @doc """ Converts a response that consists of a list of records (maps) of the same type to csv. """ def records_to_csv(records) do records |> records_to_list_with_header() |> list_with_header_to_csv() end def list_with_header_to_csv(list_with_header) when is_list(list_with_header) do list_with_header |> CSV.dump_to_iodata() |> IO.iodata_to_binary() end @doc false def records_to_list_with_header([]) do [] end @doc false def records_to_list_with_header(map_list) when is_list(map_list) do records = map_list |> Enum.map(fn map -> flat_record(map) end) header = hd(records) |> Map.keys() content = Enum.map(records, &Map.values/1) [header | content] end @doc false def flat_record(map) do flat_record(map, nil) |> List.flatten() |> Map.new() end @doc false def flat_record(map, prefix) do map |> Enum.map(fn {key, value} when is_map(value) -> flat_record(value, [key | [prefix]]) {key, value} when is_list(value) -> if Enum.any?(value, &is_map/1) do nil else {key, value} end {key, value} -> field_name = [key | [prefix]] |> List.flatten() |> Enum.reverse() |> Enum.reject(&is_nil/1) |> Enum.join(".") {field_name, value} end) |> Enum.reject(&is_nil/1) end end ================================================ FILE: lib/saas_owl.ex ================================================ defmodule OpenOwl do @version OpenOwl.MixProject.project() |> Keyword.fetch!(:version) def version do @version end end ================================================ FILE: mix.exs ================================================ defmodule OpenOwl.MixProject do use Mix.Project def project do [ app: :open_owl, version: "0.1.0", elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, escript: escript(), deps: deps() ] end # Run "mix help compile.app" to learn about applications. def application do [ extra_applications: [:logger] ] end defp elixirc_paths(_), do: ["lib"] defp escript do [main_module: OpenOwl.CLI, name: "owl"] end # Run "mix help deps" to learn about dependencies. defp deps do [ {:yaml_elixir, "~> 2.9"}, {:jason, "~> 1.4"}, {:nimble_csv, "~> 1.2"}, {:req, "~> 0.3.5"}, {:bypass, "~> 2.1", only: [:test]}, {:mix_test_watch, "~> 1.0", only: [:dev, :test], runtime: false} ] end end ================================================ FILE: owl.sh ================================================ #!/usr/bin/env sh TMP_ENV_FILE=.owl_env.tmp IMAGE_NAME=open_owl-owl store_relevant_env_variables() { printenv|grep OWL > $TMP_ENV_FILE } exists_docker_image() { docker image ls -q $IMAGE_NAME:latest 2> /dev/null } build_docker_image() { docker build -t $IMAGE_NAME:latest . } store_relevant_env_variables if [ "$(exists_docker_image)" == "" ]; then echo "Docker image does not exist yet, creating one..." build_docker_image fi docker run --rm -it \ -v `pwd`/auth_cache:/app/auth_cache \ -v `pwd`/results:/app/results \ -v `pwd`/recipes.yml:/app/recipes.yml \ --env-file $TMP_ENV_FILE \ $IMAGE_NAME $@ result=$? rm -f $TMP_ENV_FILE exit $result ================================================ FILE: package.json ================================================ { "name": "open_owl", "version": "1.0.0", "description": "", "type": "module", "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "ts-node-esm -T ts_src/main.ts", "build": "tsc" }, "keywords": [], "dependencies": { "@types/js-yaml": "^4.0.5", "js-yaml": "^4.1.0", "playwright": "^1.31.2" }, "devDependencies": { "ts-node": "^10.9.1" } } ================================================ FILE: recipes.yml ================================================ adobe: login_url: https://adminconsole.adobe.com/team destination_url_pattern: https://adminconsole.adobe.com/**/overview username_selector: "internal:label=\"Email address\"i" password_selector: "internal:label=\"Password\"i" actions: - name: download_users http_method: GET url: https://bps-il.adobe.io/jil-api/v2/organizations/:organization/users?filter_exclude_domain=techacct.adobe.com&page=0&page_size=20&productConfigurations=true&sort=FNAME_LNAME&sort_order=ASC populate_placeholders_from_session_storage: - client_id - tokenValue - roles.organization headers: - x-api-key: :client_id - authorization: Bearer :tokenValue pagination: strategy: page_params_in_url amplitude: login_url: https://analytics.amplitude.com/login/:org_name destination_url_pattern: https://analytics.amplitude.com/:org_name username_selector: "input[placeholder=Email]" password_selector: "input[placeholder=Password]" actions: - name: download_users http_method: POST body: '{"query":"query Users {\n users {\n ...userFields\n __typename\n }\n}\n\nfragment userFields on User {\n id\n alias\n avatarVersion\n blurb\n createdAt\n defaultAllProjectRole\n defaultAppId\n email\n firstName\n fullName\n hasAvatar\n hasOutstandingInvite\n isConnectedToSlack\n lastName\n loginId\n name\n orgRole\n orgTeam\n title\n pronouns\n __typename\n}\n"}' url: https://analytics.amplitude.com/t/graphql/org/:org_id?q=Users response_path: data.users headers: - content-type: application/json - origin: https://analytics.amplitude.com calendly: login_url: https://calendly.com/app/login destination_url_pattern: https://calendly.com/event_types/user/me username_selector: "input[placeholder='email address']" password_selector: "input[placeholder=password]" actions: - name: download_users http_method: GET url: https://calendly.com/api/organization/memberships?filter_term=&sort_field=name&sort_order=asc response_path: results pagination: strategy: next_page_in_body next_page_response_path: pagination.next_page page_query_param: page figma: login_url: https://www.figma.com/login destination_url_pattern: https://www.figma.com/files/** username_selector: "input[placeholder=Email]" password_selector: "input[placeholder=Password]" actions: - name: download_users http_method: GET url: https://www.figma.com/api/teams/:team_id/members response_path: meta loom: login_url: https://www.loom.com/settings/workspace destination_url_pattern: https://www.loom.com/settings/workspace username_selector: "input[placeholder='Enter your email to continue…']" password_selector: "#password" actions: - name: download_users http_method: POST body: '[{"operationName":"FilterWorkspaceMembers","variables":{"roles":["admin","viewer","creator","creator_lite","guest"],"status":"active","query":"","first":20,"after":""},"query":"query FilterWorkspaceMembers($query: String!, $roles: [OrganizationMemberRole!]!, $status: OrganizationMemberStatus!, $first: Int!, $after: String) {\n result: searchPaginatedWorkspaceMembers {\n ... on SearchPaginatedWorkspaceMembersResult {\n accepted(\n query: $query\n roles: $roles\n status: $status\n first: $first\n after: $after\n ) {\n edges {\n node {\n member_role\n member_status\n user {\n id\n email\n display_name\n first_name\n last_name\n createdAt\n avatars {\n thumb\n __typename\n }\n __typename\n }\n pending_downgrade {\n to_role\n status\n __typename\n }\n __typename\n }\n __typename\n }\n pageInfo {\n hasPreviousPage\n hasNextPage\n startCursor\n endCursor\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n}\n"}]' url: https://www.loom.com/graphql response_path: data.result.accepted.edges headers: - content-type: application/json pagination: strategy: next_cursor_in_body_to_send_as_json next_cursor_response_path: data.result.accepted.pageInfo.endCursor has_next_page_response_path: data.result.accepted.pageInfo.hasNextPage json_body_cursor_path: variables.after mezmo: login_url: https://app.mezmo.com/account/signin destination_url_pattern: https://app.mezmo.com/**/logs/* username_selector: "data-testid=email" password_selector: "data-testid=password" actions: - name: download_users http_method: GET url: https://app.mezmo.com/manage/get-team-members response_path: users headers: - x-account-context: :account_id - x-requested-with: XMLHttpRequest miro: login_url: https://miro.com/login destination_url_pattern: https://miro.com/app/** username_selector: "data-testid=mr-form-login-email-1" password_selector: "data-testid=mr-form-login-password-1" actions: - name: download_users http_method: GET url: https://miro.com/api/v1/accounts/:team_id/users response_path: data pagination: strategy: url_in_body next_url_response_path: nextLink ================================================ FILE: test/api_client_test.exs ================================================ defmodule ApiClientTest do use ExUnit.Case, async: true alias OpenOwl.ApiClient alias OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson alias OpenOwl.PaginationStrategy.NextPageInBody alias OpenOwl.PaginationStrategy.OffsetParamsInUrl alias OpenOwl.PaginationStrategy.PageParamsInUrl alias OpenOwl.PaginationStrategy.UrlInBody # TODO: maybe add bypass to tests transparently # https://github.com/wojtekmach/req/issues/137 setup do bypass = Bypass.open() [bypass: bypass, url: "http://localhost:#{bypass.port}"] end defp get_relevant_headers(headers) do Enum.reject(headers, &(elem(&1, 0) in ["host", "content-length"])) end describe "do_request/6" do @default_headers [ {"accept", "*/*"}, {"accept-encoding", "gzip"}, {"cookie", ""}, {"user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"} ] @json_content_type {"content-type", "application/json"} @pagination %UrlInBody{strategy: :url_in_body, next_url_response_path: "nextLink"} for http_method <- [:get, :post, :put, :patch, :delete] do http_method_uppercase = Atom.to_string(http_method) |> String.upcase() test "#{http_method_uppercase} returns response body for one page when success", c do endpoint = "/do/sth" payload = [%{"k" => "v"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers {:ok, body, conn} = Plug.Conn.read_body(conn) assert body == "a=b" Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload})) end) assert {:ok, payload} == ApiClient.do_request( [], %{}, unquote(http_method), "a=b", c.url <> endpoint, [], "data", @pagination, %{}, [] ) end test "#{http_method_uppercase} returns response body for one page with deeper response path when success", c do endpoint = "/do/sth" payload = [%{"k" => "v"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => %{"deeper" => payload}})) end) assert {:ok, payload} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint, [], "data.deeper", @pagination, %{}, [] ) end test "#{http_method_uppercase} returns response body for one page without pagination strategy when success", c do endpoint = "/do/sth" payload = [%{"k" => "v"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload})) end) assert {:ok, payload} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint, [], "data", nil, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with url_in_body strategy when success", c do endpoint1 = "/do/sth/p/1" endpoint2 = "/do/sth/p/2" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint1, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(%{"data" => payload1, "nextLink" => %{"deeper" => c.url <> endpoint2}}) ) end) Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint2, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, # TODO: test nextLink = nil Jason.encode!(%{"data" => payload2, "nextLink" => %{"deeper" => ""}}) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint1, [], "data", %UrlInBody{strategy: :url_in_body, next_url_response_path: "nextLink.deeper"}, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with next_page_in_body strategy when success", c do endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers first_page? = not Map.has_key?(conn.query_params, "page") if not first_page? do assert conn.query_params == %{"page" => "2", "param" => "stays"} end response = if first_page?, do: %{"data" => payload1, "pagination" => %{"next_page" => 2}}, # TODO: test next_page = "" else: %{"data" => payload2, "pagination" => %{"next_page" => nil}} Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint <> "?param=stays", [], "data", %NextPageInBody{ strategy: :next_page_in_body, next_page_response_path: "pagination.next_page", page_query_param: "page" }, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with next_cursor_in_body_populate_as_placeholder strategy when success", c do sent_body = %{"a" => "a1", "vars" => %{"after" => "", "sth" => "keep"}} endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> req_body = with {:ok, body, _conn} <- Plug.Conn.read_body(conn), {:ok, body} <- Jason.decode(body) do body else {:error, _} -> %{} end first_page? = req_body["vars"]["after"] == "" if first_page? do assert req_body == sent_body assert get_relevant_headers(conn.req_headers) == @default_headers else assert req_body == put_in(sent_body, ["vars", "after"], "abc") assert get_relevant_headers(conn.req_headers) == List.insert_at(@default_headers, 2, @json_content_type) end response = if first_page?, do: %{ "data" => payload1, "pagination" => %{"next_cursor" => "abc", "has_next" => true} }, else: %{ "data" => payload2, "pagination" => %{"next_cursor" => "efg", "has_next" => false} } Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), Jason.encode!(sent_body), c.url <> endpoint, [], "data", %NextCursorInBodyToSendAsJson{ strategy: :next_cursor_in_body_to_send_as_json, next_cursor_response_path: "pagination.next_cursor", has_next_page_response_path: "pagination.has_next", json_body_cursor_path: "vars.after" }, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with page_params_in_url strategy when success", c do endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] payload3 = [] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers page_param = conn.query_params["page"] assert conn.query_params == %{"page" => page_param, "param" => "stays"} response = case page_param do "0" -> payload1 "1" -> payload2 "2" -> payload3 end Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint <> "?page=0¶m=stays", [], nil, %PageParamsInUrl{strategy: :page_params_in_url}, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with page_params_in_url and custom page_query_param strategy when success", c do endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] payload3 = [] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers page_param = conn.query_params["p"] assert conn.query_params == %{"p" => page_param, "param" => "stays"} response = case page_param do "0" -> payload1 "1" -> payload2 "2" -> payload3 end Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint <> "?p=0¶m=stays", [], nil, %PageParamsInUrl{strategy: :page_params_in_url, page_query_param: "p"}, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url strategy when success", c do endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] payload3 = [] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers offset_param = conn.query_params["offset"] limit_param = conn.query_params["limit"] assert conn.query_params == %{ "offset" => offset_param, "limit" => limit_param, "param" => "stays" } response = case offset_param do "0" -> payload1 "2" -> payload2 "4" -> payload3 end Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint <> "?offset=0&limit=2¶m=stays", [], nil, %OffsetParamsInUrl{strategy: :offset_params_in_url}, %{}, [] ) end test "#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url and custom params strategy when success", c do endpoint = "/do/sth" payload1 = [%{"k" => "v1"}, %{"k" => "v2"}] payload2 = [%{"k" => "v3"}, %{"k" => "v4"}] payload3 = [] Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers offset_param = conn.query_params["start_offset"] limit_param = conn.query_params["size"] assert conn.query_params == %{ "start_offset" => offset_param, "size" => limit_param, "param" => "stays" } response = case offset_param do "0" -> payload1 "2" -> payload2 "4" -> payload3 end Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp( 200, Jason.encode!(response) ) end) assert {:ok, payload1 ++ payload2} == ApiClient.do_request( [], %{}, unquote(http_method), nil, c.url <> endpoint <> "?start_offset=0&size=2¶m=stays", [], nil, %OffsetParamsInUrl{ strategy: :offset_params_in_url, offset_query_param: "start_offset", limit_query_param: "size" }, %{}, [] ) end end test "replaces placeholder in passed URL and headers properly", c do endpoint_prefix = "/do/sth:not/" payload = [%{"k" => "v"}] Bypass.expect(c.bypass, "GET", "#{endpoint_prefix}123", fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers ++ [{"x-dynamic", "pre_123-post"}, {"x-not", ":there"}, {"x-static", "Blub 1"}] Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload})) end) assert {:ok, payload} == ApiClient.do_request( [], %{}, :get, nil, c.url <> endpoint_prefix <> ":team_id", [ {"X-Dynamic", "pre_:team_id-post"}, {"X-Not", ":there"}, {"X-Static", "Blub 1"} ], "data", @pagination, %{team_id: "123"}, [] ) end test "uses session_storage to populate new placeholders and replaces it in passed URL and headers properly", c do endpoint_prefix = "/do/sth:not/" payload = [%{"k" => "v"}] Bypass.expect(c.bypass, "GET", "#{endpoint_prefix}123/v2", fn conn -> assert get_relevant_headers(conn.req_headers) == @default_headers ++ [ {"x-passed", "pre_123-post"}, {"x-populated", "pre_v3b"}, {"x-static", "Blub 1"} ] Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload})) end) assert {:ok, payload} == ApiClient.do_request( [], %{ "whatever_1" => Jason.encode!(%{"kone" => "v1", "ktwo" => "v2"}), "whatever_2" => Jason.encode!(%{"kthree" => [%{"kthree_a" => "v3a", "kthree_b" => "v3b"}]}), "none" => "nope" }, :get, nil, c.url <> endpoint_prefix <> ":team_id/:ktwo", [ {"X-passed", "pre_:team_id-post"}, {"X-populated", "pre_:kthree_b"}, {"X-Static", "Blub 1"} ], "data", @pagination, %{team_id: "123"}, ["ktwo", "kthree.kthree_b"] ) end for status <- [400, 401, 403, 404] do test "returns http_error when status is #{status}", c do endpoint = "/do/sth" payload = %{"error" => "oops"} Bypass.expect(c.bypass, "GET", endpoint, fn conn -> Plug.Conn.put_resp_header(conn, "content-type", "application/json") |> Plug.Conn.send_resp(unquote(status), Jason.encode!(payload)) end) response = ApiClient.do_request( [], %{}, :get, nil, c.url <> endpoint, [], "data", @pagination, %{}, [] ) assert {:http_error, {unquote(status), ^payload}} = response end end end end ================================================ FILE: test/fixtures/test_recipes.yml ================================================ adobe: login_url: https://adminconsole.adobe.com/team destination_url_pattern: https://adminconsole.adobe.com/**/overview username_selector: "internal:label=\"Email address\"i" password_selector: "internal:label=\"Password\"i" actions: - name: download_users http_method: GET url: https://adobe.io/api/sth populate_placeholders_from_session_storage: - client_id - tokenValue - roles.organization headers: - x-api-key: :client_id - authorization: Bearer :tokenValue pagination: strategy: page_params_in_url page_query_param: "p" badobe: login_url: https://adminconsole.adobe.com/team destination_url_pattern: https://adminconsole.adobe.com/**/overview username_selector: "internal:label=\"Email address\"i" password_selector: "internal:label=\"Password\"i" actions: - name: download_users http_method: GET url: https://adobe.io/api/sth pagination: strategy: offset_params_in_url offset_query_param: "start_offset" limit_query_param: "size" asana: login_url: https://asana.com/login destination_url_pattern: https://asana.com/app/** username_selector: "#a" password_selector: "#b" actions: - name: change_sth http_method: POST body: "{\"query\":\"query Users\"}" url: https://asana.com/api/sth headers: - X-Dynamic: :team_id - X-Static: Blub 1 response_path: meta.data calendly: login_url: https://calendly.com/app/login destination_url_pattern: https://calendly.com/event_types/user/me username_selector: "input[placeholder='email address']" password_selector: "input[placeholder=password]" actions: - name: download_users http_method: GET url: https://calendly.com/api/organization/memberships response_path: results pagination: strategy: next_page_in_body next_page_response_path: pagination.next_page page_query_param: page loom: login_url: https://loom.com/login destination_url_pattern: https://loom.com/app/** username_selector: "#a" password_selector: "#b" actions: - name: download_users http_method: GET url: https://loom.com/api/v1 response_path: data pagination: strategy: next_cursor_in_body_to_send_as_json next_cursor_response_path: pagination.next_page has_next_page_response_path: pagination.has_next_page json_body_cursor_path: variables.after miro: login_url: https://miro.com/login destination_url_pattern: https://miro.com/app/** username_selector: "data-testid=mr1" password_selector: "data-testid=mr2" actions: - name: download_users http_method: GET url: https://miro.com/api/v1/accounts/:team_id/users response_path: data pagination: strategy: url_in_body next_url_response_path: nextLink ================================================ FILE: test/helpers/api_utils_test.exs ================================================ defmodule ApiUtilsTest do use ExUnit.Case, async: true doctest OpenOwl.Helpers.ApiUtils, import: true end ================================================ FILE: test/helpers/cli_utils_test.exs ================================================ defmodule CliUtilsTest do use ExUnit.Case, async: true doctest OpenOwl.Helpers.CliUtils, import: true end ================================================ FILE: test/helpers/struct_utils_test.exs ================================================ defmodule StructUtilsTest do use ExUnit.Case, async: true alias OpenOwl.Helpers.StructUtils defmodule TestStruct do @enforce_keys [:title] defstruct [:id, :title] end defmodule AnotherTestStruct do defstruct [:id, :title] end describe "to_struct/2 creates the specified struct" do test "with passed empty Map attrs" do assert StructUtils.to_struct(TestStruct, %{}) == %TestStruct{id: nil, title: nil} assert StructUtils.to_struct(TestStruct, %{}) |> Map.to_list() == [ __struct__: StructUtilsTest.TestStruct, id: nil, title: nil ] end test "with passed string Map attrs" do assert StructUtils.to_struct(TestStruct, %{"id" => "id"}) == %TestStruct{ id: "id", title: nil } assert StructUtils.to_struct(TestStruct, %{"title" => "title"}) == %TestStruct{ id: nil, title: "title" } end test "with passed atom Map attrs" do assert StructUtils.to_struct(TestStruct, %{id: "id"}) == %TestStruct{ id: "id", title: nil } assert StructUtils.to_struct(TestStruct, %{title: "title"}) == %TestStruct{ id: nil, title: "title" } end test "with passed struct attrs" do assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{}) |> Map.to_list() == [ __struct__: StructUtilsTest.TestStruct, id: nil, title: nil ] assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{id: "id"}) == %TestStruct{ id: "id", title: nil } assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{title: "title"}) == %TestStruct{ id: nil, title: "title" } end end end ================================================ FILE: test/login_flow_wrapper_test.exs ================================================ defmodule LoginFlowWrapperTest do use ExUnit.Case, async: true doctest OpenOwl.LoginFlowWrapper, import: true end ================================================ FILE: test/recipes_test.exs ================================================ defmodule RecipesTest do use ExUnit.Case, async: true alias OpenOwl.Recipes.Recipe alias OpenOwl.Recipes.Action alias OpenOwl.Recipes @test_recipe1 %Recipe{ login_url: "https://example1.com", destination_url_pattern: "dup", username_selector: "us", password_selector: "ps" } @test_recipe2 %{@test_recipe1 | login_url: "https://example2.com"} @action1 %Action{ name: :download, http_method: :get, url: "https://example.com/:url_id", headers: [{"x-1", "cool_:header_id_one"}, {"x-2", ":url_id/:header_id_two"}] } @action2 %Action{name: :list, http_method: :get, url: "https://example.com/:url_id"} describe "load_recipes/1" do test "returns parsed recipes from a yaml file" do assert {:ok, recipes} = Recipes.load_recipes("test/fixtures/test_recipes.yml") assert recipes == %{ adobe: %OpenOwl.Recipes.Recipe{ login_url: "https://adminconsole.adobe.com/team", destination_url_pattern: "https://adminconsole.adobe.com/**/overview", username_selector: "internal:label=\"Email address\"i", password_selector: "internal:label=\"Password\"i", actions: [ %OpenOwl.Recipes.Action{ name: "download_users", http_method: :get, url: "https://adobe.io/api/sth", populate_placeholders_from_session_storage: ~w(client_id tokenValue roles.organization), response_path: nil, headers: [ {"x-api-key", ":client_id"}, {"authorization", "Bearer :tokenValue"} ], pagination: %OpenOwl.PaginationStrategy.PageParamsInUrl{ strategy: "page_params_in_url", page_query_param: "p" } } ] }, badobe: %OpenOwl.Recipes.Recipe{ login_url: "https://adminconsole.adobe.com/team", destination_url_pattern: "https://adminconsole.adobe.com/**/overview", username_selector: "internal:label=\"Email address\"i", password_selector: "internal:label=\"Password\"i", actions: [ %OpenOwl.Recipes.Action{ name: "download_users", http_method: :get, url: "https://adobe.io/api/sth", response_path: nil, pagination: %OpenOwl.PaginationStrategy.OffsetParamsInUrl{ strategy: "offset_params_in_url", offset_query_param: "start_offset", limit_query_param: "size" } } ] }, asana: %OpenOwl.Recipes.Recipe{ login_url: "https://asana.com/login", destination_url_pattern: "https://asana.com/app/**", username_selector: "#a", password_selector: "#b", actions: [ %OpenOwl.Recipes.Action{ name: "change_sth", http_method: :post, body: "{\"query\":\"query Users\"}", url: "https://asana.com/api/sth", response_path: "meta.data", headers: [{"X-Dynamic", ":team_id"}, {"X-Static", "Blub 1"}] } ] }, calendly: %OpenOwl.Recipes.Recipe{ login_url: "https://calendly.com/app/login", destination_url_pattern: "https://calendly.com/event_types/user/me", username_selector: "input[placeholder='email address']", password_selector: "input[placeholder=password]", actions: [ %OpenOwl.Recipes.Action{ name: "download_users", http_method: :get, url: "https://calendly.com/api/organization/memberships", response_path: "results", pagination: %OpenOwl.PaginationStrategy.NextPageInBody{ strategy: "next_page_in_body", next_page_response_path: "pagination.next_page", page_query_param: "page" }, headers: [] } ] }, loom: %OpenOwl.Recipes.Recipe{ login_url: "https://loom.com/login", destination_url_pattern: "https://loom.com/app/**", username_selector: "#a", password_selector: "#b", actions: [ %OpenOwl.Recipes.Action{ name: "download_users", http_method: :get, url: "https://loom.com/api/v1", response_path: "data", pagination: %OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson{ strategy: "next_cursor_in_body_to_send_as_json", next_cursor_response_path: "pagination.next_page", has_next_page_response_path: "pagination.has_next_page", json_body_cursor_path: "variables.after" }, headers: [] } ] }, miro: %OpenOwl.Recipes.Recipe{ login_url: "https://miro.com/login", destination_url_pattern: "https://miro.com/app/**", username_selector: "data-testid=mr1", password_selector: "data-testid=mr2", actions: [ %OpenOwl.Recipes.Action{ name: "download_users", http_method: :get, url: "https://miro.com/api/v1/accounts/:team_id/users", response_path: "data", pagination: %OpenOwl.PaginationStrategy.UrlInBody{ strategy: "url_in_body", next_url_response_path: "nextLink" }, headers: [] } ] } } end test "returns error if file was not found" do assert {:error, %YamlElixir.FileNotFoundError{}} = Recipes.load_recipes("test/fixtures/not_existent.yml") end end test "get_recipe/2 returns recipe for a slug" do recipes = %{test1: @test_recipe1, test2: @test_recipe2} assert Recipes.get_recipe(%{}, :test) == nil assert Recipes.get_recipe(recipes, :test) == nil assert Recipes.get_recipe(recipes, :test1) == @test_recipe1 assert Recipes.get_recipe(recipes, :test2) == @test_recipe2 assert Recipes.get_recipe(recipes, "test") == nil assert Recipes.get_recipe(recipes, "test1") == @test_recipe1 assert Recipes.get_recipe(recipes, "test2") == @test_recipe2 end test "get_action_for_name/2 returns action for name or nil" do actions = [@action1, @action2] assert Recipes.get_action_for_name(actions, :not_existent) == nil assert Recipes.get_action_for_name(actions, :download) == @action1 assert Recipes.get_action_for_name(actions, :list) == @action2 end describe "get_required_parameters_for_recipe/2" do test "returns required parameters for special recipe action login" do recipe_login_param = %{@test_recipe1 | login_url: "https://example.com/:org_id/:bla"} recipe_dest_param = %{ @test_recipe1 | destination_url_pattern: "https://example.com/*/:bla" } recipe_both_duplicated = %{ @test_recipe1 | login_url: "https://example.com/:org_id/:bla", destination_url_pattern: "https://example.com/*/:bla" } assert Recipes.get_required_parameters_for_recipe(@test_recipe1, "login") == [ :username, :password ] assert Recipes.get_required_parameters_for_recipe(recipe_login_param, "login") == [ :username, :password, :org_id, :bla ] assert Recipes.get_required_parameters_for_recipe(recipe_dest_param, "login") == [ :username, :password, :bla ] assert Recipes.get_required_parameters_for_recipe(recipe_both_duplicated, "login") == [ :username, :password, :bla, :org_id ] end test "returns required parameters for dynamic recipe action" do recipe_base = %{ @test_recipe1 | login_url: "https://example.com/:org_id", destination_url_pattern: ":dest_id" } recipe_with_actions = %{recipe_base | actions: [@action1, @action2]} # ignores login parameters assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :not_existent) == [] assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :download) == [ :url_id, :header_id_two, :header_id_one ] assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :list) == [:url_id] end end describe "validate_recipe_parameters/3" do @recipe_with_params %{ @test_recipe1 | login_url: "https://example.com/:org_id", actions: [@action1] } test "returns ok with passed parameter map if all required parameters are present" do # without params login_parameters_pure = %{username: "u", password: "p"} assert Recipes.validate_recipe_parameters(@test_recipe1, "login", login_parameters_pure) == {:ok, login_parameters_pure} login_parameters_not_existing = Map.put(login_parameters_pure, :not_existent, 1) assert Recipes.validate_recipe_parameters( @test_recipe1, "login", login_parameters_not_existing ) == {:ok, login_parameters_not_existing} login_parameters_required = Map.put(login_parameters_pure, :org_id, "1") assert Recipes.validate_recipe_parameters( @recipe_with_params, "login", login_parameters_required ) == {:ok, login_parameters_required} action_parameters = %{ url_id: "1", header_id_two: 2, header_id_one: "a3" } assert Recipes.validate_recipe_parameters( @recipe_with_params, @action1.name, action_parameters ) == {:ok, action_parameters} end test "returns error with list of missing parameters if not all required parameters are present" do assert Recipes.validate_recipe_parameters(@recipe_with_params, "login", %{wrong: 1}) == {:error, [:username, :password, :org_id]} assert Recipes.validate_recipe_parameters(@recipe_with_params, "login", %{ password: "p", wrong: 1 }) == {:error, [:username, :org_id]} assert Recipes.validate_recipe_parameters(@recipe_with_params, @action1.name, %{ wrong: 1, header_id_two: "2" }) == {:error, [:url_id, :header_id_one]} end end test "placeholder_regex/0 returns regex for placeholders" do assert Recipes.placeholder_regex() == ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/ end end ================================================ FILE: test/response_transformer_test.exs ================================================ defmodule ResponseTransformerTest do use ExUnit.Case, async: true alias OpenOwl.ResponseTransformer @record1 %{ "email" => "batman@wayne.org", "emailConfirmed" => true, "id" => 123, "lastLoginDate" => "2022-12-13T14:07:59.371Z", "name" => "Bruce Wayne", "team_role" => %{ "level" => 999, "pending" => false, "pending_email" => nil, "resource_type" => "team", "role" => %{ "name" => "Heroes", "id" => "abc" }, "user_id" => "456" } } @record2 %{@record1 | "email" => "robin@wayne.org", "name" => "Robin"} @map_list [@record1, @record2] describe "records_to_list_with_header/1" do test "returns table structure with header" do assert ResponseTransformer.records_to_list_with_header(@map_list) == [ [ "email", "emailConfirmed", "id", "lastLoginDate", "name", "team_role.level", "team_role.pending", "team_role.pending_email", "team_role.resource_type", "team_role.role.id", "team_role.role.name", "team_role.user_id" ], [ "batman@wayne.org", true, 123, "2022-12-13T14:07:59.371Z", "Bruce Wayne", 999, false, nil, "team", "abc", "Heroes", "456" ], [ "robin@wayne.org", true, 123, "2022-12-13T14:07:59.371Z", "Robin", 999, false, nil, "team", "abc", "Heroes", "456" ] ] end test "returns empty list if input was an empty list" do assert ResponseTransformer.records_to_list_with_header([]) == [] end test "raises error if it is not a list" do assert_raise FunctionClauseError, fn -> ResponseTransformer.records_to_list_with_header(@record1) end end end describe "list_with_header_to_csv/1" do test "returns a CSV string" do assert ResponseTransformer.list_with_header_to_csv([]) == "" assert ResponseTransformer.list_with_header_to_csv([~w(col_a col_b), ~w(hey you)]) == "col_a,col_b\r\nhey,you\r\n" end test "raises error if it is not a list" do assert_raise FunctionClauseError, fn -> ResponseTransformer.list_with_header_to_csv(%{}) end end end describe "flat_record/1" do test "returns zero-level map untouched" do record = %{"hey" => "you", "what" => 4, "or" => true} assert ResponseTransformer.flat_record(record) == record end test "ignores fields that hold lists with maps" do record = %{ "hey" => "you", "simple_array" => ["you", 1], "complex_array" => [%{"sub_map" => 1}] } assert ResponseTransformer.flat_record(record) == %{ "hey" => "you", "simple_array" => ["you", 1] } end test "returns one-level map with concatted field names" do record = %{"hey" => "you", "what" => 4, "deep" => %{"name" => "cracker", "good" => true}} assert ResponseTransformer.flat_record(record) == %{ "hey" => "you", "what" => 4, "deep.name" => "cracker", "deep.good" => true } end test "returns multi-level map with concatted field names" do assert ResponseTransformer.flat_record(@record1) == %{ "email" => "batman@wayne.org", "emailConfirmed" => true, "id" => 123, "lastLoginDate" => "2022-12-13T14:07:59.371Z", "name" => "Bruce Wayne", "team_role.level" => 999, "team_role.pending" => false, "team_role.pending_email" => nil, "team_role.resource_type" => "team", "team_role.role.name" => "Heroes", "team_role.role.id" => "abc", "team_role.user_id" => "456" } end end end ================================================ FILE: test/test_helper.exs ================================================ ExUnit.start(exclude: :skip) ================================================ FILE: ts_src/login_flow.ts ================================================ import { firefox, Page } from 'playwright'; import path from 'path'; import * as fs from 'fs'; const authCacheFolder = 'auth_cache'; const resultFolder = 'results'; const screenshotsFolder = 'login_screenshots'; function maybeCreateScreenshotsFolder() { fs.mkdirSync(path.join(resultFolder, screenshotsFolder), { recursive: true }); } function clearScreenshots() { fs.readdirSync(path.join(resultFolder, screenshotsFolder)).forEach(file => { fs.rmSync(path.join(resultFolder, screenshotsFolder, file)); }); } async function makeScreenshot(page: Page, slug: String) { const screenshot = await page.screenshot(); fs.writeFileSync(path.join(resultFolder, screenshotsFolder, Date.now() + '_' + slug + '.png'), screenshot); } export async function loginAndSaveCookies( slug: string, url: string, destination_url_pattern: string, username_selector: string, password_selector: string, username: string, password: string): Promise { maybeCreateScreenshotsFolder(); clearScreenshots(); const browser = await firefox.launch({ headless: true }); const context = await browser.newContext() const page = await context.newPage() await page.goto(url); await page.fill(username_selector, username); await makeScreenshot(page, slug); // if password field is not visible, press actively enter so that it appears try { await page.waitForSelector(password_selector, { timeout: 2 }); } catch (e) { await page.press(username_selector, 'Enter'); } await makeScreenshot(page, slug); await page.fill(password_selector, password); await page.press(password_selector, 'Enter'); await makeScreenshot(page, slug); await page.waitForURL(destination_url_pattern, { timeout: 10000 }); await makeScreenshot(page, slug); const cookies = await context.cookies(); const cookieJson = JSON.stringify(cookies); fs.writeFileSync(path.join(authCacheFolder, slug + '_cookies.json'), cookieJson); const sessionStorage = await page.evaluate(() => JSON.stringify(sessionStorage)); fs.writeFileSync(path.join(authCacheFolder, slug + '_session_storage.json'), sessionStorage, 'utf-8'); clearScreenshots(); await browser.close(); } ================================================ FILE: ts_src/main.ts ================================================ import { loginAndSaveCookies } from './login_flow.js'; console.log("started..."); loginAndSaveCookies(process.env.SLUG, process.env.URL, process.env.DESTINATION_URL_PATTERN, process.env.USERNAME_SELECTOR, process.env.PASSWORD_SELECTOR, process.env.USER, process.env.PASSWORD); console.log("DONE"); ================================================ FILE: ts_src/recipe_manager.ts ================================================ import type { VendorTemplate, VendorAction } from "./types"; import path from 'path'; import { fileURLToPath } from 'url'; import fs from 'fs'; import yaml from 'js-yaml'; class RecipeManager { recipes: { [key: string]: VendorTemplate }; constructor() { // https://flaviocopes.com/fix-dirname-not-defined-es-module-scope/ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); let recipesPath = path.join(__dirname, '../recipes.yml'); this.recipes = yaml.load(fs.readFileSync(recipesPath).toString()) as { string: VendorTemplate } } getRecipes(): { [key: string]: VendorTemplate } { return this.recipes; } } export default new RecipeManager(); ================================================ FILE: ts_src/types.ts ================================================ export interface VendorTemplate { login_url: string; destination_url_pattern: string; username_selector: string; password_selector: string; actions?: VendorAction[]; } export interface VendorAction { name: string; http_method: HttpMethod; url: string; response_path: string; } export enum HttpMethod { GET = 'GET', POST = 'POST', PUT = 'PUT', PATCH = 'POST', DELETE = 'DELETE' } ================================================ FILE: tsconfig.json ================================================ { "compileOnSave": true, "compilerOptions": { "module": "ES2022", "target": "ES2022", "outDir": "dist", "moduleResolution": "node", "allowSyntheticDefaultImports": true, "noUnusedLocals": false, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "lib": [ "ESNext" ] }, "baseUrl": "./", "include": [ "ts_src/**/*" ] }