Repository: AccessOwl/open_owl
Branch: main
Commit: 2ced9f72db66
Files: 44
Total size: 117.4 KB
Directory structure:
gitextract_mi7z9xmv/
├── .dockerignore
├── .formatter.exs
├── .github/
│ └── workflows/
│ └── tests.yml
├── .gitignore
├── .tool-versions
├── Dockerfile
├── LICENSE
├── README.md
├── docker-compose.yml
├── lib/
│ ├── api_client.ex
│ ├── cli.ex
│ ├── helpers/
│ │ ├── api_utils.ex
│ │ ├── cli_utils.ex
│ │ └── struct_utils.ex
│ ├── login_flow_wrapper.ex
│ ├── pagination_strategies/
│ │ ├── pagination_next_cursor_in_body_to_send_as_json.ex
│ │ ├── pagination_next_page_in_body.ex
│ │ ├── pagination_offset_params_in_url.ex
│ │ ├── pagination_page_params_in_url.ex
│ │ ├── pagination_strategy.ex
│ │ └── pagination_url_in_body.ex
│ ├── recipes/
│ │ ├── action.ex
│ │ └── recipe.ex
│ ├── recipes.ex
│ ├── response_transformer.ex
│ └── saas_owl.ex
├── mix.exs
├── owl.sh
├── package.json
├── recipes.yml
├── test/
│ ├── api_client_test.exs
│ ├── fixtures/
│ │ └── test_recipes.yml
│ ├── helpers/
│ │ ├── api_utils_test.exs
│ │ ├── cli_utils_test.exs
│ │ └── struct_utils_test.exs
│ ├── login_flow_wrapper_test.exs
│ ├── recipes_test.exs
│ ├── response_transformer_test.exs
│ └── test_helper.exs
├── ts_src/
│ ├── login_flow.ts
│ ├── main.ts
│ ├── recipe_manager.ts
│ └── types.ts
└── tsconfig.json
================================================
FILE CONTENTS
================================================
================================================
FILE: .dockerignore
================================================
npm-debug.log
yarn.lock
_build
!_build/dev/lib/castore
deps
!deps/castore
node_modules
dist
.gitignore
================================================
FILE: .formatter.exs
================================================
# Used by "mix format"
[
inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
]
================================================
FILE: .github/workflows/tests.yml
================================================
name: Run tests
concurrency: ci_tests
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
name: Build and test
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
# cache the ASDF directory, using the values from .tool-versions
- name: ASDF cache
uses: actions/cache@v3
with:
path: ~/.asdf
key: ${{ runner.os }}-asdf-v2-${{ hashFiles('.tool-versions') }}
id: asdf-cache
# only run `asdf install` if we didn't hit the cache
- uses: asdf-vm/actions/install@v1
if: steps.asdf-cache.outputs.cache-hit != 'true'
# if we did hit the cache, set up the environment
- name: Setup ASDF environment
run: |
echo "ASDF_DIR=$HOME/.asdf" >> $GITHUB_ENV
echo "ASDF_DATA_DIR=$HOME/.asdf" >> $GITHUB_ENV
if: steps.asdf-cache.outputs.cache-hit == 'true'
- name: Reshim ASDF
run: |
echo "$ASDF_DIR/bin" >> $GITHUB_PATH
echo "$ASDF_DIR/shims" >> $GITHUB_PATH
$ASDF_DIR/bin/asdf reshim
- name: Restore dependencies cache
uses: actions/cache@v3
with:
path: deps
key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
restore-keys: ${{ runner.os }}-mix-
- name: Cache compiled build
id: cache-build
uses: actions/cache@v3
env:
cache-name: cache-compiled-build
with:
path: _build
key: ${{ runner.os }}-mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }}
restore-keys: |
${{ runner.os }}-mix-${{ env.cache-name }}-
${{ runner.os }}-mix-
- name: Install dependencies
run: |
mix local.rebar --force
mix local.hex --force
mix deps.get
- name: Compile
run: mix compile
- name: Run tests
run: mix test
================================================
FILE: .gitignore
================================================
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# Next.js build output
.next
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and *not* Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
/test-results/
/playwright-report/
/playwright/.cache/
# The directory Mix will write compiled artifacts to.
/_build/
# If you run "mix test --cover", coverage assets end up here.
/cover/
# The directory Mix downloads your dependencies sources to.
/deps/
# Where 3rd-party dependencies like ExDoc output generated docs.
/doc/
# Ignore .fetch files in case you like to edit your project deps locally.
/.fetch
# If the VM crashes, it generates a dump, let's ignore it too.
erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build").
*.ez
.DS_Store
.elixir_ls/
.owl_env.tmp
auth_cache/
results/
================================================
FILE: .tool-versions
================================================
elixir 1.14.3-otp-24
erlang 24.2.1
nodejs 19.6.0
================================================
FILE: Dockerfile
================================================
###
### First Stage - Building the Elixir app as escript
###
FROM hexpm/elixir:1.14.3-erlang-23.2.6-alpine-3.16.0 AS build
# install build dependencies
RUN apk add --no-cache build-base git
# prepare build dir
WORKDIR /app
# extend hex timeout
ENV HEX_HTTP_TIMEOUT=20
# install hex + rebar
RUN mix local.hex --force && \
mix local.rebar --force
# Copy over the mix.exs and mix.lock files to load the dependencies. If those
# files don't change, then we don't keep re-fetching and rebuilding the deps.
COPY mix.exs mix.lock ./
RUN mix deps.get && \
mix deps.compile && \
mkdir priv && \
# because of https://github.com/elixir-mint/castore/issues/35
cp _build/dev/lib/castore/priv/cacerts.pem priv/cacerts.pem
COPY lib lib
RUN mix compile && \
mix escript.build
###
### Second Stage - Setup the Runtime Environment
###
FROM node:16-bullseye-slim AS app
ENV LANG=C.UTF-8
# Install Firefox dependencies + tools
RUN sh -c 'echo "deb http://ftp.us.debian.org/debian bullseye main non-free" >> /etc/apt/sources.list.d/fonts.list' && \
DEBIAN_FRONTEND=noninteractive apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends erlang && \
# clean apt cache
rm -rf /var/lib/apt/lists/*
WORKDIR /app
RUN npm config --global set update-notifier false
ENV PLAYWRIGHT_BROWSERS_PATH=/playwright
ENV REQ_MINT_CACERTFILE=/app/bin/cacerts.pem
COPY package*.json ./
RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --include=dev --omit=optional --audit=false --progress=false --loglevel=error
RUN npm_config_ignore_scripts=1 npx playwright install-deps firefox && \
npx playwright install firefox
COPY ts_src ts_src
COPY tsconfig.json ./
RUN npm run build
COPY --from=build /app/owl ./bin/owl
COPY --from=build /app/priv/cacerts.pem ./bin/cacerts.pem
RUN mkdir -p /app/results
ENV HOME=/app
ENV MIX_ENV=dev
ENTRYPOINT ["./bin/owl"]
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2023 Omni Owl GmbH
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# OpenOwl 🦉
<a href="https://github.com/AccessOwl/open_owl/releases" target="_blank">
<img src="https://img.shields.io/github/v/release/AccessOwl/open_owl?color=white" alt="Release">
</a>
<a href="https://github.com/AccessOwl/open_owl/actions/workflows/tests.yml" target="_blank">
<img src="https://img.shields.io/github/actions/workflow/status/AccessOwl/open_owl/tests.yml?branch=main" alt="Build">
</a>
OpenOwl lets you download user lists including user permissions (and other additional data) from various SaaS applications without the need for a public API. This tool is commonly used to check for orphaned user accounts or as preparation for an access review.
This project is made with IT Ops, InfoSec and Compliance teams in mind - no developer experience needed. The [`recipes.yml`](recipes.yml) lists all supported applications. You are welcome to contribute to this project by setting up additional vendor integrations.
## How to run it
Since OpenOwl uses various technologies running it with Docker is recommended. There are additional instructions if you want to run it directly.
### Option 1: Shell Wrapper with [`Docker`](https://docs.docker.com/get-docker/) (Recommended)
The Docker image is built automatically with the first launch.
#### Show all available applications a.k.a recipes
```bash
./owl.sh recipes list
```
#### Step 1. Run `login` action
Following is an example on how to use it with [`Mezmo`](https://www.mezmo.com/)
Parameters like `OWL_USERNAME` and `OWL_PASSWORD` are passed via the environment variable.
```bash
OWL_USERNAME=someone@acme.com OWL_PASSWORD=abc123 ./owl.sh mezmo login
```
#### Step 2. Run `download_users` action
```bash
./owl.sh mezmo download_users
```
Depending on the application there might be additonal parameters required which will be listed after running the command. If that is the case, add the parameters and re-run the command.
Example for additional parameters:
In [recipes.yml](recipes.yml) you can see that there is a placeholder defined for Mezmo. Placeholders start with a `:` and look like this: `:account_id`. Depending on the recipe they are either passed as parameter or populated from other data (e.g. see Adobe). To pass the `:account_id` parameter, you prefix it with `OWL_` and upcase it. So `:account_id` becomes `OWL_ACCOUNT_ID`.
```bash
OWL_ACCOUNT_ID=YOUR_ACCOUNT_ID ./owl.sh mezmo download_users
```
### Option 2: Docker Compose
Examples:
```bash
docker compose run owl recipes list
docker compose run -e 'OWL_USERNAME=YOUR_USERNAME' -e 'OWL_PASSWORD=YOUR_PASSWORD' owl mezmo login
docker compose run -e 'OWL_ACCOUNT_ID=YOUR_ACCOUNT_ID' owl mezmo download_users
```
### Option 3: Directly
You can leverage the available [`.tool-versions`](.tool-versions) file to install requirements with [`asdf`](https://asdf-vm.com).
```bash
asdf install
```
Alternatively you can install the required Node, Elixir and Erlang version manually based on the version of the [`.tool-versions`](.tool-versions) file.
Run it with:
```bash
mix run lib/cli.exs <commands>
```
## How does it work?
OpenOwl signs in like a regular user by entering username and password (RPA via Playwright). It then uses the SaaS' internal APIs to request the list of users on your behalf.
This approach only works for SaaS applications with internal APIs. With the rise of single page apps (SPA) (think about React.js, Vue etc.), most applications can be supported. Besides SaaS applications this tool can also be used for internal apps.
## Quick Demo
<a href="http://www.youtube.com/watch?feature=player_embedded&v=0Kz2EwL7xQs" target="_blank">
<img src="http://img.youtube.com/vi/0Kz2EwL7xQs/0.jpg" alt="Watch the video" width="480" border="0" />
</a>
## Known limitations
1. User Account: The user account needs administrator rights (or a similar permissions that grants access to user lists).
2. Login: Currently only direct logins via username and password are supported. Logins via SSO (Google, Okta, Onelogin,...) will be added in the future.
3. Captchas: Login flows that include captchas are currently not supported.
## How to contribute
[Here](recipes.yml) is the list of available integrations. Open a PR to add further [recipes](recipes.yml), adjust existing ones or extend missing capabilities (like further pagination strategies) to support even more applications.
When all the required capabilities exist, a further integration can take just 30m.
### How to add a new vendor recipe
1. Check that your intended SaaS application has an internal API.
1. Open the Developer Tools/Inspector of your favourite browser.
1. Navigate to the page that shows all users.
1. In your inspector filter by XHR requests and reload the page. When you find some that belong to the same domain like your tool, the application is probably supported.
1. Use the Network tab in the inspector, find the request that includes your list of users with their permissions.
1. Copy the request as `curl`-request and paste it into a tool like [Postman](https://www.postman.com/downloads/). You can import the request by clicking *Import*, selecting *Raw text* and pasting the copied `curl`-request.
1. Execute the request in Postman and remove as many headers and parameters as possible to keep the request clean. Check that the request still works. Add the request to the [`recipes.yml`](recipes.yml).
1. When you have many users in your SaaS account, you will not see all users at once but 10, 50 or 100. Adjust the pagination parameters and try to get all data by traversing through it. Check which [pagination strategy](lib/pagination_strategies/) applies and add it to your added recipe.
1. Open the direct login page of the new vendor and find the right [selector](https://www.cuketest.com/playwright/docs/selectors) for the username and password field. Adjust the [`recipes.yml`](recipes.yml) accordingly.
1. Test that the login flow works properly and the configured action.
1. Open a PR.
## Troubleshooting
### Build Docker image
In case the Shell Wrapper cannot build the Docker image as expected, try the following command to build the image manually:
```bash
docker build -t open_owl-owl:latest .
```
## The history of OpenOwl
Knowing who has access to your SaaS tools is often guesswork at best and manually adding and deleting user accounts is tedious and error prone. The fact that SCIM/SAML are often locked up behind an enterprise-subscription is adding to the frustration. That's why Mathias and Philip decided to build [AccessOwl](https://www.accessowl.io/), a tool to automate SaaS account provisioning for employees for any tool.
OpenOwl is essentially an open-source version of the underlying technology of AccessOwl. It was created out of a discussion that having access to your own teams user data should be a basic right for anybody. No matter whether it's used for audits, to discover orphaned user accounts or run access reviews. That's why we decided to open source a core part of AccessOwl to let anybody read out crucial information out of their SaaS stack.
================================================
FILE: docker-compose.yml
================================================
services:
owl:
build: ./
volumes:
- ${PWD}/auth_cache:/app/auth_cache
- ${PWD}/results:/app/results
- ${PWD}/recipes.yml:/app/recipes.yml
================================================
FILE: lib/api_client.ex
================================================
defmodule OpenOwl.ApiClient do
import OpenOwl.Helpers.ApiUtils
@base_header [
{"accept-encoding", "gzip"},
{"accept", "*/*"}
]
@user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"
def do_request(
cookies,
session_storage,
http_method,
body,
url,
headers,
response_path,
pagination,
%{} = placeholder_params,
placeholders_from_session_storage
) do
cookie_string = filter_relevant_cookies(cookies, url) |> build_cookie_params_string()
placeholder_params =
get_params_from_session_storage(session_storage, placeholders_from_session_storage)
|> Map.merge(placeholder_params)
# required for local mix run
Application.ensure_all_started(:telemetry)
Application.ensure_all_started(:req)
connect_options =
if cacertfile = System.get_env("REQ_MINT_CACERTFILE"),
do: [transport_opts: [cacertfile: cacertfile]],
else: []
req =
Req.new(
method: http_method,
body: body,
url: url,
headers: build_headers(headers, placeholder_params, cookie_string),
path_params: placeholder_params,
connect_options: connect_options,
user_agent: @user_agent
)
case Req.request(req) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
if pagination != nil do
pagination.__struct__.handle_paginated_response(
req,
body,
pagination,
response_path,
&extract_data_from_response_body/2
)
else
{:ok, extract_data_from_response_body(body, response_path)}
end
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
end
defp extract_data_from_response_body(body, response_path) do
get_field_via_response_path(body, response_path)
end
defp build_headers(headers, placeholders, cookie_string) do
req_headers = [{"cookie", cookie_string} | @base_header]
[headers | req_headers]
|> List.flatten()
|> Enum.map(fn {key, value} -> {key, apply_placeholder_params(value, placeholders)} end)
end
end
================================================
FILE: lib/cli.ex
================================================
defmodule OpenOwl.CLI do
alias OpenOwl.Recipes
alias OpenOwl.Recipes.Recipe
alias OpenOwl.Recipes.Action
alias OpenOwl.ResponseTransformer
alias OpenOwl.LoginFlowWrapper
import OpenOwl.Helpers.ApiUtils, only: [apply_placeholder_params: 2]
import OpenOwl.Helpers.CliUtils
@owl_cli_name "./owl.sh"
@env_prefix "OWL_"
@auth_cache_folder "auth_cache"
# escript
def main(args) do
args
|> parse_args()
|> process_params()
end
# direct call via mix run
def main() do
System.argv()
|> parse_args()
|> process_params()
end
defp parse_args(args) do
{flags, commands, _} =
OptionParser.parse(args,
strict: [debug: :boolean, pwdebug: :boolean, output: :string, help: :boolean]
)
{commands, flags}
end
defp process_params({["recipes", "list"], _}) do
IO.puts("Listing recipes with actions and parameters...\n")
recipes = load_recipes()
trailing_padding = get_trailing_padding(recipes)
recipes
|> Enum.each(fn {title, %Recipe{actions: actions} = recipe} ->
title =
title
|> Atom.to_string()
|> String.capitalize()
|> then(&"#{&1}: ")
|> String.pad_trailing(trailing_padding)
login_parameters =
recipe
|> Recipes.get_required_parameters_for_recipe("login")
|> parameter_list_as_env()
|> maybe_in_brackets()
IO.puts(title <> "login #{login_parameters}")
Enum.each(actions, fn %Action{} = action ->
parameters =
recipe
|> Recipes.get_required_parameters_for_recipe(action.name)
|> parameter_list_as_env()
|> maybe_in_brackets()
IO.puts(String.pad_trailing("", trailing_padding) <> "#{action.name} #{parameters}")
end)
end)
end
defp process_params({[vendor, "login"], flags}) do
recipes = load_recipes()
# TODO: until we pull that out from a config or sth
app_env_params = get_app_env_params()
with %Recipe{} = recipe <- Recipes.get_recipe(recipes, vendor),
{:ok, %{username: username, password: password}} <-
Recipes.validate_recipe_parameters(recipe, "login", app_env_params) do
IO.puts("Starting #{vendor}: login...")
if Keyword.get(flags, :debug, false) do
IO.puts("DEBUG=true\n")
LoginFlowWrapper.get_local_cmd_command(
vendor,
apply_placeholder_params(recipe.login_url, app_env_params),
apply_placeholder_params(recipe.destination_url_pattern, app_env_params),
recipe.username_selector,
recipe.password_selector,
username,
password
)
|> IO.puts()
else
case LoginFlowWrapper.call_local_cmd(
vendor,
apply_placeholder_params(recipe.login_url, app_env_params),
apply_placeholder_params(recipe.destination_url_pattern, app_env_params),
recipe.username_selector,
recipe.password_selector,
username,
password,
Keyword.get(flags, :pwdebug, false)
) do
{:ok, _} ->
IO.puts("... authentication info saved")
IO.puts("DONE")
{:error, %{exit_code: exit_code, result: result}} ->
write_error("#{exit_code}: #{inspect(result)}")
end
end
else
{:error, missing_parameters} ->
write_error("missing parameters: #{parameter_list_as_env(missing_parameters)}")
nil ->
write_error("vendor #{vendor} not found")
end
end
defp process_params({[vendor, action_name], flags}) when vendor != "recipes" do
recipes = load_recipes()
app_env_params = get_app_env_params()
IO.puts("Starting #{vendor}: #{action_name}...")
cookies_path = Path.join([File.cwd!(), @auth_cache_folder, "#{vendor}_cookies.json"])
session_storage_path =
Path.join([File.cwd!(), @auth_cache_folder, "#{vendor}_session_storage.json"])
session_storage =
case File.read(session_storage_path) do
{:ok, data} -> Jason.decode!(data)
{:error, _} -> %{}
end
with {:ok, cookie_binary_json} <- File.read(cookies_path),
{:ok, cookies} <- Jason.decode(cookie_binary_json),
%Recipe{actions: actions} = recipe <- Recipes.get_recipe(recipes, vendor),
%Action{} = action <- Recipes.get_action_for_name(actions, action_name),
{:ok, _} <- Recipes.validate_recipe_parameters(recipe, action_name, app_env_params) do
IO.puts(
"... #{action.http_method}-request #{apply_placeholder_params(action.url, app_env_params)}..."
)
case OpenOwl.ApiClient.do_request(
cookies,
session_storage,
action.http_method,
action.body,
action.url,
action.headers,
action.response_path,
action.pagination,
app_env_params,
action.populate_placeholders_from_session_storage
) do
{:ok, response} ->
output_flag = Keyword.get(flags, :output)
filename =
if output_flag == nil, do: timebased_filename("#{vendor}.csv"), else: output_flag
path = Path.join([File.cwd!(), "results", filename])
content = ResponseTransformer.records_to_csv(response)
File.write!(path, content)
IO.puts("... wrote results to #{filename}")
IO.puts("DONE")
{:http_error, {status, body}} ->
write_error("HTTP (#{status}): #{inspect(body)}")
{:error, reason} ->
write_error("#{inspect(reason)}")
end
else
nil ->
write_error("Vendor #{vendor} or action #{action_name} not found")
{:error, missing_parameters} when is_list(missing_parameters) ->
write_error("missing parameters: #{parameter_list_as_env(missing_parameters)}")
{:error, %Jason.DecodeError{} = reason} ->
write_error("#{inspect(reason)}")
{:error, reason} ->
write_error(
"Authentication cookies file #{cookies_path} could not be loaded: #{inspect(reason)}"
)
end
end
defp process_params(_), do: print_help()
defp print_help() do
version = OpenOwl.version()
IO.puts("OpenOwl v#{version}\n")
IO.puts("Usage:")
IO.puts(
" #{@owl_cli_name} recipes list - Show list of recipes with their actions"
)
IO.puts(
"[env_vars] #{@owl_cli_name} <vendor> login - Authenticate and get required authentication"
)
IO.puts(
"[env_vars] #{@owl_cli_name} <vendor> <action> [flags] - Triggers defined action of recipes.yml\n"
)
IO.puts("Env vars:")
IO.puts(" Some commands need set environment variables. You can prepend commands with them.")
IO.puts(" For instance OWL_USERNAME=user #{@owl_cli_name} <vendor> login\n")
IO.puts("Flags:")
IO.puts(" --output - Set a custom output filename. Otherwise a timebased filename is used.")
IO.puts(" --help - Print this help message")
end
defp load_recipes() do
case Recipes.load_recipes() do
{:ok, recipes} ->
recipes
{:error, reason} ->
raise "File could not be loaded: #{inspect(reason)}"
end
end
defp get_trailing_padding(%{} = recipes) do
Enum.map(recipes, fn {title, _} -> "#{title}: " end)
|> get_trailing_padding()
end
defp get_trailing_padding(title_list) when is_list(title_list) do
title_list
|> Enum.max(fn left, right -> String.length(left) >= String.length(right) end)
|> String.length()
end
defp parameter_list_as_env(list) do
list
|> Enum.map(&to_string/1)
|> Enum.map(&"#{@env_prefix}#{String.upcase(&1)}")
|> Enum.join(", ")
end
defp maybe_in_brackets(""), do: ""
defp maybe_in_brackets(string) do
"(#{string})"
end
defp write_error(msg) do
IO.puts("ERROR: #{msg}")
System.halt(1)
end
end
OpenOwl.CLI.main()
================================================
FILE: lib/helpers/api_utils.ex
================================================
defmodule OpenOwl.Helpers.ApiUtils do
@path_separator "."
alias OpenOwl.Recipes
@doc """
Resolves a response path and get the described string field out of a map. Some APIs have a
one-element list as root. We handle it by removing this layer.
If the first argument is a list and the second is nil, the list is returned directly.
iex> get_field_via_response_path(%{}, nil)
nil
iex> get_field_via_response_path(%{}, "a.b")
nil
iex> get_field_via_response_path(%{"a" => "hit"}, nil)
nil
iex> get_field_via_response_path(%{"a" => "hit", "b" => "nob"}, "a")
"hit"
iex> get_field_via_response_path([%{"a" => "hit", "b" => "nob"}], "a")
"hit"
iex> get_field_via_response_path(%{"a" => "hit", "b" => "nob"}, "a.a1")
nil
iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => "noa1"}, "b" => "nob"}, "a.a1")
"hit"
iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => "hit2"}, "b" => "nob"}, "a.a2")
"hit2"
iex> get_field_via_response_path(%{"a" => %{"a1" => "hit", "a2" => %{"a2i" => "hit2", "a2ii" => "noa2ii"}}, "b" => "nob"}, "a.a2.a2i")
"hit2"
iex> get_field_via_response_path([%{"a" => "a1"}, %{"a" => "a2"}], nil)
[%{"a" => "a1"}, %{"a" => "a2"}]
iex> get_field_via_response_path([%{"a" => "a1"}, %{"a" => "a2"}], "a")
** (FunctionClauseError) no function clause matching in OpenOwl.Helpers.ApiUtils.get_field_via_response_path/2
"""
def get_field_via_response_path(list, nil) when is_list(list), do: list
def get_field_via_response_path([one], response_path),
do: get_field_via_response_path(one, response_path)
def get_field_via_response_path(_map, nil), do: nil
def get_field_via_response_path(%{} = map, response_path) do
path = String.split(response_path, @path_separator)
get_in(map, path)
rescue
_ -> nil
end
@doc """
Sets a specific field in a map via the passed path. The (nested) structure must exist, otherwise it
will raise. Some APIs pass a one-element list as root. Such a structure would be kept.
iex> flat_map = %{"a" => "a1", "b" => "b1"}
iex> set_field_via_path(flat_map, nil, "v")
flat_map
iex> set_field_via_path(flat_map, "", "v")
flat_map
iex> set_field_via_path(flat_map, "c", "v")
%{"a" => "a1", "b" => "b1", "c" => "v"}
iex> set_field_via_path(flat_map, "b", "v")
%{"a" => "a1", "b" => "v"}
iex> set_field_via_path([flat_map], "b", "v")
[%{"a" => "a1", "b" => "v"}]
iex> deep_map = %{"a" => "a1", "b" => "b1", "d" => %{"d1" => "d1v"}}
iex> set_field_via_path(deep_map, "d.d1", "v")
%{"a" => "a1", "b" => "b1", "d" => %{"d1" => "v"}}
iex> set_field_via_path([deep_map], "d.d1", "v")
[%{"a" => "a1", "b" => "b1", "d" => %{"d1" => "v"}}]
iex> set_field_via_path(deep_map, "d.d2", "v")
%{"a" => "a1", "b" => "b1", "d" => %{"d1" => "d1v", "d2" => "v"}}
iex> set_field_via_path(deep_map, "d.d1.deep", "v")
** (FunctionClauseError) no function clause matching in Access.get_and_update/3
"""
def set_field_via_path(%{} = map, nil, _value), do: map
def set_field_via_path(%{} = map, "", _value), do: map
def set_field_via_path([one], field_path, value),
do: set_field_via_path(one, field_path, value, true)
def set_field_via_path(%{} = map, field_path, value, list? \\ false) do
path = String.split(field_path, @path_separator)
result = put_in(map, path, value)
if list?, do: [result], else: result
end
@doc """
Filters for cookies of the passed url (domain + subdomain).
iex> cookie_domain = %{"name" => "session","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"}
iex> cookie_subdomain = %{"name" => "session","value" => "123","domain" => ".example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"}
iex> cookies = [cookie_domain, cookie_subdomain]
iex> filter_relevant_cookies(cookies, "http://example.com/a?b=2")
cookies
iex> filter_relevant_cookies(cookies, "http://s.bla.example.com/a?b=2")
cookies
"""
def filter_relevant_cookies(cookies, url) do
%URI{host: host} = URI.parse(url)
host =
host |> String.split(@path_separator) |> Enum.slice(-2..-1) |> Enum.join(@path_separator)
Enum.filter(cookies, fn cookie ->
String.contains?(cookie["domain"], host)
end)
end
@doc ~S"""
Gets the passed params (support paths separated by ".") from session storage. Searches in values
of all the session storage and merges with maps one level deeper. Also decodes JSON on first level
if necessary.
Current implementation is not flexible enough yet to support many use cases. Depending on future
requirements we will adjust it (including the interface towards recipes).
iex> get_params_from_session_storage(%{}, [])
%{}
iex> payload = %{"adobeid_ims_access_token/ONESIE1/false/AdobeID,ab.manage" => "{\"REAUTH_SCOPE\":\"reauthenticated\",\"client_id\":\"ONESIE1\",\"scope\":\"openid,AdobeID,additional_info.projectedProductContext,read_organizations,read_members,read_countries_regions,additional_info.roles,adobeio_api,read_auth_src_domains,authSources.rwd,bis.read.pi,app_policies.read,app_policies.write,client.read,publisher.read,client.scopes.read,creative_cloud,service_principals.write,aps.read.app_merchandising,aps.eval_licensesforapps,ab.manage,aps.device_activation_mgmt\",\"expire\":\"2023-01-01T13:37:00.342Z\",\"user_id\":\"thisiauser_id\",\"tokenValue\":\"longtoken\",\"sid\":\"thisissid\",\"state\":{},\"fromFragment\":false,\"impersonatorId\":\"\",\"isImpersonatedSession\":false,\"other\":\"{}\",\"pbaSatisfiedPolicies\":[\"MedSecNoEV\",\"LowSec\"]}", "adobeid_ims_profile/ONESIE1/false/AdobeID,ab.manage,additional_info" => "{\"account_type\":\"type2e\",\"utcOffset\":\"null\",\"preferred_languages\":[\"en-us\"],\"displayName\":\"Integration Account\",\"roles\":[{\"principal\":\"LONGID@AdobeOrg:123456\",\"organization\":\"LONGID@AdobeOrg\",\"named_role\":\"org_admin\",\"target\":\"LONGID@AdobeOrg\",\"target_type\":\"TRG_ORG\",\"target_data\":{}}],\"last_name\":\"Account\",\"userId\":\"someuserid.e\",\"authId\":\"someauthid@AdobeID\",\"tags\":[\"agegroup_18plus\"],\"projectedProductContext\":[],\"emailVerified\":\"true\",\"toua\":[{\"touName\":\"creative_cloud\",\"current\":true}],\"phoneNumber\":null,\"countryCode\":\"US\",\"name\":\"Integration Account\",\"mrktPerm\":\"\",\"mrktPermEmail\":null,\"first_name\":\"Integration\",\"email\":\"bruce@wayne.org\"}", "sherlockId" => "sherlockUUID"}
iex> get_params_from_session_storage(payload, [])
%{}
iex> get_params_from_session_storage(payload, ~w(client_id tokenValue notexistent))
%{client_id: "ONESIE1", tokenValue: "longtoken", notexistent: nil}
iex> get_params_from_session_storage(payload, ~w(client_id roles.organization))
%{client_id: "ONESIE1", organization: "LONGID@AdobeOrg"}
"""
def get_params_from_session_storage(session_storage, params)
when is_map(session_storage) and is_list(params) do
result =
session_storage
|> Map.values()
|> Enum.map(fn item ->
case Jason.decode(item) do
{:ok, result} -> result
{:error, _} -> item
end
end)
|> Enum.reduce(%{}, fn
item, acc when is_map(item) ->
Map.merge(acc, item)
# we ignore flat items here for the moment. We might need it later which means refactoring
# of this approach
_item, acc ->
acc
end)
|> Enum.map(fn
{key, []} ->
{key, []}
{key, value} when is_list(value) ->
{key, hd(value)}
{key, value} ->
{key, value}
end)
|> Map.new()
Enum.reduce(params, %{}, fn param, acc ->
Map.put(
acc,
get_last_response_path_part(param) |> String.to_atom(),
get_field_via_response_path(result, param)
)
end)
end
@doc """
Returns the last part of a response path that is splitted by #{@path_separator}.
iex> get_last_response_path_part("a.b.c")
"c"
iex> get_last_response_path_part("abc")
"abc"
iex> get_last_response_path_part("")
""
"""
def get_last_response_path_part(path) do
path |> String.split(@path_separator) |> Enum.reverse() |> hd()
end
@doc """
Builds a string properly formatted to send it as cookies header.
iex> cookie_a = %{"name" => "session","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"}
iex> cookie_b = %{"name" => "bla","value" => "123","domain" => "example.com","path" => "/","expires" => -1,"httpOnly" => true,"secure" => true,"sameSite" => "None"}
iex> cookies = [cookie_a, cookie_b]
iex> build_cookie_params_string([cookie_a])
"session=123"
iex> build_cookie_params_string(cookies)
"session=123; bla=123"
"""
def build_cookie_params_string(cookies) do
Enum.map_join(cookies, "; ", &(&1["name"] <> "=" <> &1["value"]))
end
@doc """
Replaces placeholder in subject using passed params.
iex> apply_placeholder_params("cool", %{org: "o1", team_id: 123})
"cool"
iex> apply_placeholder_params("cool:org", %{org: "o1", team_id: 123})
"coolo1"
iex> apply_placeholder_params("cool_:team_id_:org_", %{org: "o1", team_id: 123})
"cool_123_o1_"
iex> apply_placeholder_params("cool_:team_bla_:org_", %{_org: "o1", team_bla_id: 123})
"cool_:team_bla_:org_"
"""
def apply_placeholder_params(subject, params) do
Regex.replace(Recipes.placeholder_regex(), subject, fn whole_match, key ->
to_string(params[String.to_atom(key)] || whole_match)
end)
end
end
================================================
FILE: lib/helpers/cli_utils.ex
================================================
defmodule OpenOwl.Helpers.CliUtils do
@prefix "OWL_"
@doc """
Returns all app environment params downcased and without app prefix.
iex> get_app_env_params(%{"SHELL" => "/bin/zsh", "DOWL_BLA" => "no", "OWL_USERNAME" => "Oo", "OWL_PASSWORD" => "secret", "OWL_TEAM_ID" => "123"})
%{username: "Oo", password: "secret", team_id: "123"}
iex> get_app_env_params(%{"SHELL" => "/bin/zsh", "DOWL_BLA" => "no"})
%{}
iex> get_app_env_params(%{})
%{}
"""
def get_app_env_params(params \\ System.get_env()) do
params
|> Enum.filter(fn
{@prefix <> _, _value} -> true
_ -> false
end)
|> Enum.map(fn {key, value} -> {normalize_var(key), value} end)
|> Map.new()
end
defp normalize_var(var) do
var |> String.replace(@prefix, "") |> String.downcase() |> String.to_atom()
end
@doc """
Generates a timebased filename.
iex> datetime = ~N[2017-11-06 00:23:51.123456]
iex> timebased_filename("miro.csv", datetime)
"20171106T002351_miro.csv"
iex> timebased_filename(nil)
** (ArgumentError) filename empty
iex> timebased_filename("")
** (ArgumentError) filename empty
"""
def timebased_filename(filename, datetime \\ NaiveDateTime.utc_now())
def timebased_filename(nil, _), do: raise(ArgumentError, "filename empty")
def timebased_filename("", _), do: raise(ArgumentError, "filename empty")
def timebased_filename(filename, datetime) do
now =
datetime
|> NaiveDateTime.truncate(:second)
|> NaiveDateTime.to_iso8601(:basic)
"#{now}_#{filename}"
end
end
================================================
FILE: lib/helpers/struct_utils.ex
================================================
defmodule OpenOwl.Helpers.StructUtils do
@moduledoc """
Converts a map with strings, a map with atoms or an existing struct with all
of the necessary keys into a target struct.
"""
def to_struct(kind, attrs) do
struct = struct(kind)
map_list = Map.to_list(struct)
Enum.reduce(map_list, struct, fn
{:__struct__, _}, acc -> acc
{key, _}, acc -> fetch_value(attrs, key, acc)
end)
end
defp fetch_value(attrs, key, acc) when is_atom(key) do
case Map.fetch(attrs, key) do
{:ok, value} -> %{acc | key => value}
:error -> fetch_value(attrs, Atom.to_string(key), acc)
end
end
defp fetch_value(attrs, key, acc) do
case Map.fetch(attrs, key) do
# credo:disable-for-next-line
{:ok, value} -> %{acc | String.to_atom(key) => value}
:error -> acc
end
end
end
================================================
FILE: lib/login_flow_wrapper.ex
================================================
defmodule OpenOwl.LoginFlowWrapper do
def call_local_cmd(
slug,
url,
destination_url_pattern,
username_selector,
password_selector,
username,
password,
pw_debug?
) do
case System.cmd("npm", ["run", "start"],
env: [
{"SLUG", slug},
{"URL", url},
{"DESTINATION_URL_PATTERN", destination_url_pattern},
{"USERNAME_SELECTOR", username_selector},
{"PASSWORD_SELECTOR", password_selector},
{"USER", username},
{"PASSWORD", password},
{"PWDEBUG", if(pw_debug?, do: "1", else: "0")}
]
) do
{result, 0} ->
{:ok, result}
{result, exit_code} ->
{:error, %{exit_code: exit_code, result: result}}
end
end
@doc """
Returns the command to execute for the login flow.
## Examples
iex> get_local_cmd_command("slug", "url", "dest", "usel", "psel", "user", "password")
~s(PWDEBUG=1 SLUG=slug URL="url" DESTINATION_URL_PATTERN="dest" USERNAME_SELECTOR="usel" PASSWORD_SELECTOR="psel" USER="user" PASSWORD="password" npm run start)
"""
def get_local_cmd_command(
slug,
url,
destination_url_pattern,
username_selector,
password_selector,
username,
password
) do
env = [
"PWDEBUG=1",
"SLUG=#{slug}",
~s(URL="#{url}"),
~s(DESTINATION_URL_PATTERN="#{destination_url_pattern}"),
~s(USERNAME_SELECTOR="#{username_selector}"),
~s(PASSWORD_SELECTOR="#{password_selector}"),
~s(USER="#{username}"),
~s(PASSWORD="#{password}")
]
"#{Enum.join(env, " ")} npm run start"
end
end
================================================
FILE: lib/pagination_strategies/pagination_next_cursor_in_body_to_send_as_json.ex
================================================
defmodule OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson do
@moduledoc """
Handles APIs that return a cursor in the response body. Puts the cursur into a field of the JSON
request body using `json_body_cursor_path`.
It needs a boolean flag (`has_next_page_response_path`) to indicate whether the last page was
reached.
## Example response:
```json
{
"results": [{"id": 1}, {"id": 2}],
"pagination": {"next_cursor": "abc", "has_next_page": true}
}
```
You can use the `next_cursor_response_path` parameter to define the field that has the cursor. You
can traverse into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it
would be `pagination.next_cursor`.
"""
use OpenOwl.PaginationStrategy
alias OpenOwl.Helpers.ApiUtils
@enforce_keys [
:strategy,
:next_cursor_response_path,
:has_next_page_response_path,
:json_body_cursor_path
]
defstruct strategy: nil,
next_cursor_response_path: nil,
has_next_page_response_path: nil,
json_body_cursor_path: nil
@type t :: %__MODULE__{
strategy: :next_cursor_in_body_to_send_as_json,
next_cursor_response_path: String.t(),
has_next_page_response_path: String.t(),
json_body_cursor_path: String.t()
}
@impl true
def handle_paginated_response(
%Req.Request{} = req,
body,
%__MODULE__{} = pagination,
response_path,
data_extraction_fn,
data_acc \\ []
) do
data = data_extraction_fn.(body, response_path)
data_acc = data_acc ++ data
next_cursor = ApiUtils.get_field_via_response_path(body, pagination.next_cursor_response_path)
new_req_body =
req.body
|> Jason.decode!()
|> ApiUtils.set_field_via_path(pagination.json_body_cursor_path, next_cursor)
has_next_page? =
ApiUtils.get_field_via_response_path(body, pagination.has_next_page_response_path)
if has_next_page? do
case Req.request(req, json: new_req_body) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
handle_paginated_response(
req,
body,
pagination,
response_path,
data_extraction_fn,
data_acc
)
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
else
{:ok, data_acc}
end
end
end
================================================
FILE: lib/pagination_strategies/pagination_next_page_in_body.ex
================================================
defmodule OpenOwl.PaginationStrategy.NextPageInBody do
@moduledoc """
Handles APIs that return the next page cursor in the response body. Furthermore the cursor needs
to be passed as query parameter.
## Example response:
```json
{
"results": [{"id": 1}, {"id": 2}],
"pagination": {"next_page": 42"}
}
```
Use `page_query_param` to define the name of the query parameter.
You can use the `next_page_response_path` parameter to define the next page field. You can traverse
into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it would be
`pagination.next_page`.
"""
use OpenOwl.PaginationStrategy
alias OpenOwl.Helpers.ApiUtils
@enforce_keys [:strategy, :next_page_response_path, :page_query_param]
defstruct strategy: nil, next_page_response_path: nil, page_query_param: nil
@type t :: %__MODULE__{
strategy: :next_page_in_body,
next_page_response_path: String.t(),
page_query_param: String.t()
}
@impl true
def handle_paginated_response(
%Req.Request{} = req,
body,
%__MODULE__{} = pagination,
response_path,
data_extraction_fn,
data_acc \\ []
) do
data = data_extraction_fn.(body, response_path)
data_acc = data_acc ++ data
next_page = ApiUtils.get_field_via_response_path(body, pagination.next_page_response_path)
query_param = Keyword.new([{String.to_atom(pagination.page_query_param), next_page}])
if next_page != nil and next_page != "" do
case Req.request(req, params: query_param) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
handle_paginated_response(
req,
body,
pagination,
response_path,
data_extraction_fn,
data_acc
)
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
else
{:ok, data_acc}
end
end
end
================================================
FILE: lib/pagination_strategies/pagination_offset_params_in_url.ex
================================================
defmodule OpenOwl.PaginationStrategy.OffsetParamsInUrl do
@moduledoc """
Handles APIs that have offset params in URL query parameters. By default, it assumes `offset`
is used as parameter. The parameter must be part of the URL in the first request. Subsequent
requests will increment this parameter by the value of the `limit` query parameter. The name of
this parameter can be defined with `limit_query_param` and is `limit` by default.
The name of the passed `offset` param can be defined with `offset_query_param`.
Request loop stops when the page has no data anymore (empty result).
"""
use OpenOwl.PaginationStrategy
@enforce_keys [:strategy]
defstruct strategy: nil, offset_query_param: "offset", limit_query_param: "limit"
@type t :: %__MODULE__{
strategy: :page_params_in_url,
offset_query_param: String.t(),
limit_query_param: String.t()
}
@impl true
def handle_paginated_response(
%Req.Request{} = req,
body,
%__MODULE__{} = pagination,
response_path,
data_extraction_fn,
data_acc \\ []
) do
data = data_extraction_fn.(body, response_path)
data_acc = data_acc ++ data
%URI{query: query} = req.url
limit_query_param = pagination.limit_query_param
%{^limit_query_param => limit} = query_params = URI.decode_query(query)
query_params_string =
query_params
|> Map.update!(
pagination.offset_query_param,
&(String.to_integer(&1) + String.to_integer(limit))
)
|> URI.encode_query()
req =
update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end)
if data != [] do
case Req.request(req) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
handle_paginated_response(
req,
body,
pagination,
response_path,
data_extraction_fn,
data_acc
)
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
else
{:ok, data_acc}
end
end
end
================================================
FILE: lib/pagination_strategies/pagination_page_params_in_url.ex
================================================
defmodule OpenOwl.PaginationStrategy.PageParamsInUrl do
@moduledoc """
Handles APIs that have page params in URL query parameters. By default, it assumes `page`
is used as parameter. The parameter must be part of the URL in the first request. Subsequent
requests will increment this parameter.
The name of the passed `page` param can be defined with `page_query_param`.
Request loop stops when the page has no data anymore (empty result).
"""
use OpenOwl.PaginationStrategy
@enforce_keys [:strategy]
defstruct strategy: nil, page_query_param: "page"
@type t :: %__MODULE__{strategy: :page_params_in_url, page_query_param: String.t()}
@impl true
def handle_paginated_response(
%Req.Request{} = req,
body,
%__MODULE__{} = pagination,
response_path,
data_extraction_fn,
data_acc \\ []
) do
data = data_extraction_fn.(body, response_path)
data_acc = data_acc ++ data
%URI{query: query} = req.url
query_params_string =
query
|> URI.decode_query()
|> Map.update!(pagination.page_query_param, &(String.to_integer(&1) + 1))
|> URI.encode_query()
req =
update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end)
if data != [] do
case Req.request(req) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
handle_paginated_response(
req,
body,
pagination,
response_path,
data_extraction_fn,
data_acc
)
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
else
{:ok, data_acc}
end
end
end
================================================
FILE: lib/pagination_strategies/pagination_strategy.ex
================================================
defmodule OpenOwl.PaginationStrategy do
@type t :: module()
@type status() :: pos_integer()
@type body() :: map()
@type pagination() :: %{atom() => atom() | String.t()}
@type response_path() :: String.t()
@doc """
Casts a map to a typed struct.
"""
@callback cast(map()) :: %{
:__struct__ => atom(),
:strategy => atom(),
optional(atom()) => any()
}
@doc """
Follows the pagination strategy to accumulate all the data and returns it at once.
"""
@callback handle_paginated_response(
%Req.Request{},
body(),
pagination(),
response_path(),
fun(),
[map()]
) :: {:ok, map()} | {:http_error, {status(), body()}} | {:error, any()}
defmacro __using__(_options) do
quote do
@behaviour OpenOwl.PaginationStrategy
@impl true
def cast(attrs) do
OpenOwl.Helpers.StructUtils.to_struct(__MODULE__, attrs)
end
end
end
end
================================================
FILE: lib/pagination_strategies/pagination_url_in_body.ex
================================================
defmodule OpenOwl.PaginationStrategy.UrlInBody do
@moduledoc """
Handles APIs that return a URL in the response body where the next page can be fetched.
## Example response:
```json
{
"results": [{"id": 1}, {"id": 2}],
"nextLink": "https://api.example.com/?nextLink=456"
}
```
You can use the `next_url_response_path` parameter to define the field that has the URL. You can
traverse into nested structures with the "." syntax (e.g. with `root.deep.deeper`). Here it would
be just `nextLink`.
"""
use OpenOwl.PaginationStrategy
alias OpenOwl.Helpers.ApiUtils
@enforce_keys [:strategy, :next_url_response_path]
defstruct strategy: nil, next_url_response_path: nil
@type t :: %__MODULE__{
strategy: :url_in_body,
next_url_response_path: String.t()
}
@impl true
def handle_paginated_response(
%Req.Request{} = req,
body,
%__MODULE__{} = pagination,
response_path,
data_extraction_fn,
data_acc \\ []
) do
data = data_extraction_fn.(body, response_path)
data_acc = data_acc ++ data
next_url = ApiUtils.get_field_via_response_path(body, pagination.next_url_response_path)
if next_url != nil and next_url != "" do
case Req.request(req, url: next_url) do
{:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->
handle_paginated_response(
req,
body,
pagination,
response_path,
data_extraction_fn,
data_acc
)
{:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->
{:http_error, {status, body}}
{:error, exception} ->
{:error, exception}
end
else
{:ok, data_acc}
end
end
end
================================================
FILE: lib/recipes/action.ex
================================================
defmodule OpenOwl.Recipes.Action do
alias OpenOwl.Helpers.StructUtils
@enforce_keys [:name, :http_method, :url]
defstruct name: nil,
http_method: nil,
body: nil,
url: nil,
headers: [],
response_path: nil,
pagination: nil,
populate_placeholders_from_session_storage: []
@type t :: %__MODULE__{
name: String.t(),
http_method: :get | :post | :put | :patch | :delete,
body: String.t(),
url: String.t(),
headers: [{String.t(), String.t()}],
response_path: String.t() | nil,
pagination: OpenOwl.PaginationStrategy.t() | nil,
populate_placeholders_from_session_storage: [String.t()] | nil
}
def cast(attrs) do
http_method =
case attrs do
%{http_method: http_method} -> http_method
%{"http_method" => http_method} -> http_method
end
headers =
case attrs do
%{headers: headers} -> headers
%{"headers" => headers} -> headers
_ -> []
end
|> Enum.map(fn header -> Map.to_list(header) |> hd() end)
{pagination_strategy, pagination_attrs} =
case attrs do
%{pagination: pagination} ->
{pagination[:strategy], pagination}
%{"pagination" => pagination} ->
{pagination["strategy"], pagination}
_ ->
{nil, nil}
end
pagination_mod = modulize(pagination_strategy)
action =
StructUtils.to_struct(__MODULE__, attrs)
|> Map.put(:http_method, http_method_to_atom(http_method))
|> Map.put(:headers, headers)
if pagination_mod != nil,
do: Map.put(action, :pagination, apply(pagination_mod, :cast, [pagination_attrs])),
else: action
end
defp http_method_to_atom(http_method) do
http_method |> String.downcase() |> String.to_atom()
end
defp modulize(nil), do: nil
defp modulize(underscored_string) do
underscored_string
|> String.split("_")
|> Enum.map_join(&String.capitalize/1)
|> then(&Module.concat(OpenOwl.PaginationStrategy, &1))
end
end
================================================
FILE: lib/recipes/recipe.ex
================================================
defmodule OpenOwl.Recipes.Recipe do
alias OpenOwl.Helpers.StructUtils
alias OpenOwl.Recipes.Action
@enforce_keys [
:login_url,
:destination_url_pattern,
:username_selector,
:password_selector
]
defstruct login_url: nil,
destination_url_pattern: nil,
username_selector: nil,
password_selector: nil,
actions: []
@type t :: %__MODULE__{
login_url: String.t(),
destination_url_pattern: String.t(),
username_selector: String.t(),
password_selector: String.t(),
actions: [] | [Action.t()]
}
def cast(attrs, actions) do
recipe = StructUtils.to_struct(__MODULE__, attrs)
%{recipe | actions: Enum.map(actions, &Action.cast/1)}
end
end
================================================
FILE: lib/recipes.ex
================================================
defmodule OpenOwl.Recipes do
@recipe_file_name "recipes.yml"
@external_resource Path.join(File.cwd!(), @recipe_file_name)
@placeholder_regex ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/
alias OpenOwl.Recipes.Recipe
alias OpenOwl.Recipes.Action
def load_recipes(rel_path \\ @recipe_file_name) do
path = Path.join(File.cwd!(), rel_path)
case YamlElixir.read_from_file(path) do
{:ok, data} -> {:ok, parse_yaml_data(data)}
{:error, reason} -> {:error, reason}
end
end
def get_recipe(recipes, slug) when is_binary(slug) do
get_recipe(recipes, String.to_atom(slug))
end
def get_recipe(recipes, slug) do
Map.get(recipes, slug)
end
def get_action_for_name(actions, name) do
Enum.find(actions, &(&1.name == name))
end
@doc false
def get_required_parameters_for_recipe(
%Recipe{login_url: login_url, destination_url_pattern: destination_url_pattern},
"login"
) do
[:username, :password] ++ get_parameters([login_url, destination_url_pattern])
end
@doc false
def get_required_parameters_for_recipe(%Recipe{actions: actions}, action_name) do
with %Action{url: url, headers: headers} <- get_action_for_name(actions, action_name) do
get_parameters([url | get_header_values(headers)])
else
nil -> []
end
end
defp get_parameters(strings) do
strings
|> Enum.reduce([], fn string, acc ->
matches = Regex.scan(@placeholder_regex, string, capture: :first)
[matches | acc]
end)
|> List.flatten()
|> Enum.uniq()
|> Enum.map(&atomize_placeholder_string/1)
end
defp atomize_placeholder_string(placeholder) do
":" <> name = placeholder
String.to_atom(name)
end
defp get_header_values(headers) do
Enum.map(headers, fn {_, value} -> value end)
end
def validate_recipe_parameters(%Recipe{} = recipe, action, %{} = parameter_map) do
parameters = Map.keys(parameter_map)
required_parameters = get_required_parameters_for_recipe(recipe, action)
missing_parameters = required_parameters -- parameters
if missing_parameters == [] do
{:ok, parameter_map}
else
{:error, missing_parameters}
end
end
@doc false
def placeholder_regex() do
@placeholder_regex
end
defp parse_yaml_data(%{} = data) do
Enum.reduce(data, %{}, fn {key, value}, acc ->
Map.put(acc, String.to_atom(key), OpenOwl.Recipes.Recipe.cast(value, value["actions"]))
end)
end
end
================================================
FILE: lib/response_transformer.ex
================================================
defmodule OpenOwl.ResponseTransformer do
alias NimbleCSV.RFC4180, as: CSV
@doc """
Converts a response that consists of a list of records (maps) of the same
type to csv.
"""
def records_to_csv(records) do
records
|> records_to_list_with_header()
|> list_with_header_to_csv()
end
def list_with_header_to_csv(list_with_header) when is_list(list_with_header) do
list_with_header
|> CSV.dump_to_iodata()
|> IO.iodata_to_binary()
end
@doc false
def records_to_list_with_header([]) do
[]
end
@doc false
def records_to_list_with_header(map_list) when is_list(map_list) do
records =
map_list
|> Enum.map(fn map ->
flat_record(map)
end)
header = hd(records) |> Map.keys()
content = Enum.map(records, &Map.values/1)
[header | content]
end
@doc false
def flat_record(map) do
flat_record(map, nil)
|> List.flatten()
|> Map.new()
end
@doc false
def flat_record(map, prefix) do
map
|> Enum.map(fn
{key, value} when is_map(value) ->
flat_record(value, [key | [prefix]])
{key, value} when is_list(value) ->
if Enum.any?(value, &is_map/1) do
nil
else
{key, value}
end
{key, value} ->
field_name =
[key | [prefix]]
|> List.flatten()
|> Enum.reverse()
|> Enum.reject(&is_nil/1)
|> Enum.join(".")
{field_name, value}
end)
|> Enum.reject(&is_nil/1)
end
end
================================================
FILE: lib/saas_owl.ex
================================================
defmodule OpenOwl do
@version OpenOwl.MixProject.project() |> Keyword.fetch!(:version)
def version do
@version
end
end
================================================
FILE: mix.exs
================================================
defmodule OpenOwl.MixProject do
use Mix.Project
def project do
[
app: :open_owl,
version: "0.1.0",
elixir: "~> 1.14",
elixirc_paths: elixirc_paths(Mix.env()),
start_permanent: Mix.env() == :prod,
escript: escript(),
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications.
def application do
[
extra_applications: [:logger]
]
end
defp elixirc_paths(_), do: ["lib"]
defp escript do
[main_module: OpenOwl.CLI, name: "owl"]
end
# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:yaml_elixir, "~> 2.9"},
{:jason, "~> 1.4"},
{:nimble_csv, "~> 1.2"},
{:req, "~> 0.3.5"},
{:bypass, "~> 2.1", only: [:test]},
{:mix_test_watch, "~> 1.0", only: [:dev, :test], runtime: false}
]
end
end
================================================
FILE: owl.sh
================================================
#!/usr/bin/env sh
TMP_ENV_FILE=.owl_env.tmp
IMAGE_NAME=open_owl-owl
store_relevant_env_variables() {
printenv|grep OWL > $TMP_ENV_FILE
}
exists_docker_image() {
docker image ls -q $IMAGE_NAME:latest 2> /dev/null
}
build_docker_image() {
docker build -t $IMAGE_NAME:latest .
}
store_relevant_env_variables
if [ "$(exists_docker_image)" == "" ]; then
echo "Docker image does not exist yet, creating one..."
build_docker_image
fi
docker run --rm -it \
-v `pwd`/auth_cache:/app/auth_cache \
-v `pwd`/results:/app/results \
-v `pwd`/recipes.yml:/app/recipes.yml \
--env-file $TMP_ENV_FILE \
$IMAGE_NAME $@
result=$?
rm -f $TMP_ENV_FILE
exit $result
================================================
FILE: package.json
================================================
{
"name": "open_owl",
"version": "1.0.0",
"description": "",
"type": "module",
"scripts": {
"start": "npm run start:dev",
"start:prod": "node dist/main.js",
"start:dev": "ts-node-esm -T ts_src/main.ts",
"build": "tsc"
},
"keywords": [],
"dependencies": {
"@types/js-yaml": "^4.0.5",
"js-yaml": "^4.1.0",
"playwright": "^1.31.2"
},
"devDependencies": {
"ts-node": "^10.9.1"
}
}
================================================
FILE: recipes.yml
================================================
adobe:
login_url: https://adminconsole.adobe.com/team
destination_url_pattern: https://adminconsole.adobe.com/**/overview
username_selector: "internal:label=\"Email address\"i"
password_selector: "internal:label=\"Password\"i"
actions:
- name: download_users
http_method: GET
url: https://bps-il.adobe.io/jil-api/v2/organizations/:organization/users?filter_exclude_domain=techacct.adobe.com&page=0&page_size=20&productConfigurations=true&sort=FNAME_LNAME&sort_order=ASC
populate_placeholders_from_session_storage:
- client_id
- tokenValue
- roles.organization
headers:
- x-api-key: :client_id
- authorization: Bearer :tokenValue
pagination:
strategy: page_params_in_url
amplitude:
login_url: https://analytics.amplitude.com/login/:org_name
destination_url_pattern: https://analytics.amplitude.com/:org_name
username_selector: "input[placeholder=Email]"
password_selector: "input[placeholder=Password]"
actions:
- name: download_users
http_method: POST
body: '{"query":"query Users {\n users {\n ...userFields\n __typename\n }\n}\n\nfragment userFields on User {\n id\n alias\n avatarVersion\n blurb\n createdAt\n defaultAllProjectRole\n defaultAppId\n email\n firstName\n fullName\n hasAvatar\n hasOutstandingInvite\n isConnectedToSlack\n lastName\n loginId\n name\n orgRole\n orgTeam\n title\n pronouns\n __typename\n}\n"}'
url: https://analytics.amplitude.com/t/graphql/org/:org_id?q=Users
response_path: data.users
headers:
- content-type: application/json
- origin: https://analytics.amplitude.com
calendly:
login_url: https://calendly.com/app/login
destination_url_pattern: https://calendly.com/event_types/user/me
username_selector: "input[placeholder='email address']"
password_selector: "input[placeholder=password]"
actions:
- name: download_users
http_method: GET
url: https://calendly.com/api/organization/memberships?filter_term=&sort_field=name&sort_order=asc
response_path: results
pagination:
strategy: next_page_in_body
next_page_response_path: pagination.next_page
page_query_param: page
figma:
login_url: https://www.figma.com/login
destination_url_pattern: https://www.figma.com/files/**
username_selector: "input[placeholder=Email]"
password_selector: "input[placeholder=Password]"
actions:
- name: download_users
http_method: GET
url: https://www.figma.com/api/teams/:team_id/members
response_path: meta
loom:
login_url: https://www.loom.com/settings/workspace
destination_url_pattern: https://www.loom.com/settings/workspace
username_selector: "input[placeholder='Enter your email to continue…']"
password_selector: "#password"
actions:
- name: download_users
http_method: POST
body: '[{"operationName":"FilterWorkspaceMembers","variables":{"roles":["admin","viewer","creator","creator_lite","guest"],"status":"active","query":"","first":20,"after":""},"query":"query FilterWorkspaceMembers($query: String!, $roles: [OrganizationMemberRole!]!, $status: OrganizationMemberStatus!, $first: Int!, $after: String) {\n result: searchPaginatedWorkspaceMembers {\n ... on SearchPaginatedWorkspaceMembersResult {\n accepted(\n query: $query\n roles: $roles\n status: $status\n first: $first\n after: $after\n ) {\n edges {\n node {\n member_role\n member_status\n user {\n id\n email\n display_name\n first_name\n last_name\n createdAt\n avatars {\n thumb\n __typename\n }\n __typename\n }\n pending_downgrade {\n to_role\n status\n __typename\n }\n __typename\n }\n __typename\n }\n pageInfo {\n hasPreviousPage\n hasNextPage\n startCursor\n endCursor\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n}\n"}]'
url: https://www.loom.com/graphql
response_path: data.result.accepted.edges
headers:
- content-type: application/json
pagination:
strategy: next_cursor_in_body_to_send_as_json
next_cursor_response_path: data.result.accepted.pageInfo.endCursor
has_next_page_response_path: data.result.accepted.pageInfo.hasNextPage
json_body_cursor_path: variables.after
mezmo:
login_url: https://app.mezmo.com/account/signin
destination_url_pattern: https://app.mezmo.com/**/logs/*
username_selector: "data-testid=email"
password_selector: "data-testid=password"
actions:
- name: download_users
http_method: GET
url: https://app.mezmo.com/manage/get-team-members
response_path: users
headers:
- x-account-context: :account_id
- x-requested-with: XMLHttpRequest
miro:
login_url: https://miro.com/login
destination_url_pattern: https://miro.com/app/**
username_selector: "data-testid=mr-form-login-email-1"
password_selector: "data-testid=mr-form-login-password-1"
actions:
- name: download_users
http_method: GET
url: https://miro.com/api/v1/accounts/:team_id/users
response_path: data
pagination:
strategy: url_in_body
next_url_response_path: nextLink
================================================
FILE: test/api_client_test.exs
================================================
defmodule ApiClientTest do
use ExUnit.Case, async: true
alias OpenOwl.ApiClient
alias OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson
alias OpenOwl.PaginationStrategy.NextPageInBody
alias OpenOwl.PaginationStrategy.OffsetParamsInUrl
alias OpenOwl.PaginationStrategy.PageParamsInUrl
alias OpenOwl.PaginationStrategy.UrlInBody
# TODO: maybe add bypass to tests transparently
# https://github.com/wojtekmach/req/issues/137
setup do
bypass = Bypass.open()
[bypass: bypass, url: "http://localhost:#{bypass.port}"]
end
defp get_relevant_headers(headers) do
Enum.reject(headers, &(elem(&1, 0) in ["host", "content-length"]))
end
describe "do_request/6" do
@default_headers [
{"accept", "*/*"},
{"accept-encoding", "gzip"},
{"cookie", ""},
{"user-agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"}
]
@json_content_type {"content-type", "application/json"}
@pagination %UrlInBody{strategy: :url_in_body, next_url_response_path: "nextLink"}
for http_method <- [:get, :post, :put, :patch, :delete] do
http_method_uppercase = Atom.to_string(http_method) |> String.upcase()
test "#{http_method_uppercase} returns response body for one page when success", c do
endpoint = "/do/sth"
payload = [%{"k" => "v"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
{:ok, body, conn} = Plug.Conn.read_body(conn)
assert body == "a=b"
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload}))
end)
assert {:ok, payload} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
"a=b",
c.url <> endpoint,
[],
"data",
@pagination,
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for one page with deeper response path when success",
c do
endpoint = "/do/sth"
payload = [%{"k" => "v"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => %{"deeper" => payload}}))
end)
assert {:ok, payload} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint,
[],
"data.deeper",
@pagination,
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for one page without pagination strategy when success",
c do
endpoint = "/do/sth"
payload = [%{"k" => "v"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload}))
end)
assert {:ok, payload} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint,
[],
"data",
nil,
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with url_in_body strategy when success",
c do
endpoint1 = "/do/sth/p/1"
endpoint2 = "/do/sth/p/2"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint1, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(%{"data" => payload1, "nextLink" => %{"deeper" => c.url <> endpoint2}})
)
end)
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint2, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
# TODO: test nextLink = nil
Jason.encode!(%{"data" => payload2, "nextLink" => %{"deeper" => ""}})
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint1,
[],
"data",
%UrlInBody{strategy: :url_in_body, next_url_response_path: "nextLink.deeper"},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with next_page_in_body strategy when success",
c do
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
first_page? = not Map.has_key?(conn.query_params, "page")
if not first_page? do
assert conn.query_params == %{"page" => "2", "param" => "stays"}
end
response =
if first_page?,
do: %{"data" => payload1, "pagination" => %{"next_page" => 2}},
# TODO: test next_page = ""
else: %{"data" => payload2, "pagination" => %{"next_page" => nil}}
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint <> "?param=stays",
[],
"data",
%NextPageInBody{
strategy: :next_page_in_body,
next_page_response_path: "pagination.next_page",
page_query_param: "page"
},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with next_cursor_in_body_populate_as_placeholder strategy when success",
c do
sent_body = %{"a" => "a1", "vars" => %{"after" => "", "sth" => "keep"}}
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
req_body =
with {:ok, body, _conn} <- Plug.Conn.read_body(conn),
{:ok, body} <- Jason.decode(body) do
body
else
{:error, _} -> %{}
end
first_page? = req_body["vars"]["after"] == ""
if first_page? do
assert req_body == sent_body
assert get_relevant_headers(conn.req_headers) == @default_headers
else
assert req_body == put_in(sent_body, ["vars", "after"], "abc")
assert get_relevant_headers(conn.req_headers) ==
List.insert_at(@default_headers, 2, @json_content_type)
end
response =
if first_page?,
do: %{
"data" => payload1,
"pagination" => %{"next_cursor" => "abc", "has_next" => true}
},
else: %{
"data" => payload2,
"pagination" => %{"next_cursor" => "efg", "has_next" => false}
}
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
Jason.encode!(sent_body),
c.url <> endpoint,
[],
"data",
%NextCursorInBodyToSendAsJson{
strategy: :next_cursor_in_body_to_send_as_json,
next_cursor_response_path: "pagination.next_cursor",
has_next_page_response_path: "pagination.has_next",
json_body_cursor_path: "vars.after"
},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with page_params_in_url strategy when success",
c do
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
payload3 = []
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
page_param = conn.query_params["page"]
assert conn.query_params == %{"page" => page_param, "param" => "stays"}
response =
case page_param do
"0" -> payload1
"1" -> payload2
"2" -> payload3
end
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint <> "?page=0¶m=stays",
[],
nil,
%PageParamsInUrl{strategy: :page_params_in_url},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with page_params_in_url and custom page_query_param strategy when success",
c do
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
payload3 = []
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
page_param = conn.query_params["p"]
assert conn.query_params == %{"p" => page_param, "param" => "stays"}
response =
case page_param do
"0" -> payload1
"1" -> payload2
"2" -> payload3
end
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint <> "?p=0¶m=stays",
[],
nil,
%PageParamsInUrl{strategy: :page_params_in_url, page_query_param: "p"},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url strategy when success",
c do
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
payload3 = []
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
offset_param = conn.query_params["offset"]
limit_param = conn.query_params["limit"]
assert conn.query_params == %{
"offset" => offset_param,
"limit" => limit_param,
"param" => "stays"
}
response =
case offset_param do
"0" -> payload1
"2" -> payload2
"4" -> payload3
end
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint <> "?offset=0&limit=2¶m=stays",
[],
nil,
%OffsetParamsInUrl{strategy: :offset_params_in_url},
%{},
[]
)
end
test "#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url and custom params strategy when success",
c do
endpoint = "/do/sth"
payload1 = [%{"k" => "v1"}, %{"k" => "v2"}]
payload2 = [%{"k" => "v3"}, %{"k" => "v4"}]
payload3 = []
Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->
assert get_relevant_headers(conn.req_headers) == @default_headers
offset_param = conn.query_params["start_offset"]
limit_param = conn.query_params["size"]
assert conn.query_params == %{
"start_offset" => offset_param,
"size" => limit_param,
"param" => "stays"
}
response =
case offset_param do
"0" -> payload1
"2" -> payload2
"4" -> payload3
end
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(response)
)
end)
assert {:ok, payload1 ++ payload2} ==
ApiClient.do_request(
[],
%{},
unquote(http_method),
nil,
c.url <> endpoint <> "?start_offset=0&size=2¶m=stays",
[],
nil,
%OffsetParamsInUrl{
strategy: :offset_params_in_url,
offset_query_param: "start_offset",
limit_query_param: "size"
},
%{},
[]
)
end
end
test "replaces placeholder in passed URL and headers properly", c do
endpoint_prefix = "/do/sth:not/"
payload = [%{"k" => "v"}]
Bypass.expect(c.bypass, "GET", "#{endpoint_prefix}123", fn conn ->
assert get_relevant_headers(conn.req_headers) ==
@default_headers ++
[{"x-dynamic", "pre_123-post"}, {"x-not", ":there"}, {"x-static", "Blub 1"}]
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload}))
end)
assert {:ok, payload} ==
ApiClient.do_request(
[],
%{},
:get,
nil,
c.url <> endpoint_prefix <> ":team_id",
[
{"X-Dynamic", "pre_:team_id-post"},
{"X-Not", ":there"},
{"X-Static", "Blub 1"}
],
"data",
@pagination,
%{team_id: "123"},
[]
)
end
test "uses session_storage to populate new placeholders and replaces it in passed URL and headers properly",
c do
endpoint_prefix = "/do/sth:not/"
payload = [%{"k" => "v"}]
Bypass.expect(c.bypass, "GET", "#{endpoint_prefix}123/v2", fn conn ->
assert get_relevant_headers(conn.req_headers) ==
@default_headers ++
[
{"x-passed", "pre_123-post"},
{"x-populated", "pre_v3b"},
{"x-static", "Blub 1"}
]
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"data" => payload}))
end)
assert {:ok, payload} ==
ApiClient.do_request(
[],
%{
"whatever_1" => Jason.encode!(%{"kone" => "v1", "ktwo" => "v2"}),
"whatever_2" =>
Jason.encode!(%{"kthree" => [%{"kthree_a" => "v3a", "kthree_b" => "v3b"}]}),
"none" => "nope"
},
:get,
nil,
c.url <> endpoint_prefix <> ":team_id/:ktwo",
[
{"X-passed", "pre_:team_id-post"},
{"X-populated", "pre_:kthree_b"},
{"X-Static", "Blub 1"}
],
"data",
@pagination,
%{team_id: "123"},
["ktwo", "kthree.kthree_b"]
)
end
for status <- [400, 401, 403, 404] do
test "returns http_error when status is #{status}", c do
endpoint = "/do/sth"
payload = %{"error" => "oops"}
Bypass.expect(c.bypass, "GET", endpoint, fn conn ->
Plug.Conn.put_resp_header(conn, "content-type", "application/json")
|> Plug.Conn.send_resp(unquote(status), Jason.encode!(payload))
end)
response =
ApiClient.do_request(
[],
%{},
:get,
nil,
c.url <> endpoint,
[],
"data",
@pagination,
%{},
[]
)
assert {:http_error, {unquote(status), ^payload}} = response
end
end
end
end
================================================
FILE: test/fixtures/test_recipes.yml
================================================
adobe:
login_url: https://adminconsole.adobe.com/team
destination_url_pattern: https://adminconsole.adobe.com/**/overview
username_selector: "internal:label=\"Email address\"i"
password_selector: "internal:label=\"Password\"i"
actions:
- name: download_users
http_method: GET
url: https://adobe.io/api/sth
populate_placeholders_from_session_storage:
- client_id
- tokenValue
- roles.organization
headers:
- x-api-key: :client_id
- authorization: Bearer :tokenValue
pagination:
strategy: page_params_in_url
page_query_param: "p"
badobe:
login_url: https://adminconsole.adobe.com/team
destination_url_pattern: https://adminconsole.adobe.com/**/overview
username_selector: "internal:label=\"Email address\"i"
password_selector: "internal:label=\"Password\"i"
actions:
- name: download_users
http_method: GET
url: https://adobe.io/api/sth
pagination:
strategy: offset_params_in_url
offset_query_param: "start_offset"
limit_query_param: "size"
asana:
login_url: https://asana.com/login
destination_url_pattern: https://asana.com/app/**
username_selector: "#a"
password_selector: "#b"
actions:
- name: change_sth
http_method: POST
body: "{\"query\":\"query Users\"}"
url: https://asana.com/api/sth
headers:
- X-Dynamic: :team_id
- X-Static: Blub 1
response_path: meta.data
calendly:
login_url: https://calendly.com/app/login
destination_url_pattern: https://calendly.com/event_types/user/me
username_selector: "input[placeholder='email address']"
password_selector: "input[placeholder=password]"
actions:
- name: download_users
http_method: GET
url: https://calendly.com/api/organization/memberships
response_path: results
pagination:
strategy: next_page_in_body
next_page_response_path: pagination.next_page
page_query_param: page
loom:
login_url: https://loom.com/login
destination_url_pattern: https://loom.com/app/**
username_selector: "#a"
password_selector: "#b"
actions:
- name: download_users
http_method: GET
url: https://loom.com/api/v1
response_path: data
pagination:
strategy: next_cursor_in_body_to_send_as_json
next_cursor_response_path: pagination.next_page
has_next_page_response_path: pagination.has_next_page
json_body_cursor_path: variables.after
miro:
login_url: https://miro.com/login
destination_url_pattern: https://miro.com/app/**
username_selector: "data-testid=mr1"
password_selector: "data-testid=mr2"
actions:
- name: download_users
http_method: GET
url: https://miro.com/api/v1/accounts/:team_id/users
response_path: data
pagination:
strategy: url_in_body
next_url_response_path: nextLink
================================================
FILE: test/helpers/api_utils_test.exs
================================================
defmodule ApiUtilsTest do
use ExUnit.Case, async: true
doctest OpenOwl.Helpers.ApiUtils, import: true
end
================================================
FILE: test/helpers/cli_utils_test.exs
================================================
defmodule CliUtilsTest do
use ExUnit.Case, async: true
doctest OpenOwl.Helpers.CliUtils, import: true
end
================================================
FILE: test/helpers/struct_utils_test.exs
================================================
defmodule StructUtilsTest do
use ExUnit.Case, async: true
alias OpenOwl.Helpers.StructUtils
defmodule TestStruct do
@enforce_keys [:title]
defstruct [:id, :title]
end
defmodule AnotherTestStruct do
defstruct [:id, :title]
end
describe "to_struct/2 creates the specified struct" do
test "with passed empty Map attrs" do
assert StructUtils.to_struct(TestStruct, %{}) == %TestStruct{id: nil, title: nil}
assert StructUtils.to_struct(TestStruct, %{}) |> Map.to_list() == [
__struct__: StructUtilsTest.TestStruct,
id: nil,
title: nil
]
end
test "with passed string Map attrs" do
assert StructUtils.to_struct(TestStruct, %{"id" => "id"}) == %TestStruct{
id: "id",
title: nil
}
assert StructUtils.to_struct(TestStruct, %{"title" => "title"}) == %TestStruct{
id: nil,
title: "title"
}
end
test "with passed atom Map attrs" do
assert StructUtils.to_struct(TestStruct, %{id: "id"}) == %TestStruct{
id: "id",
title: nil
}
assert StructUtils.to_struct(TestStruct, %{title: "title"}) == %TestStruct{
id: nil,
title: "title"
}
end
test "with passed struct attrs" do
assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{}) |> Map.to_list() == [
__struct__: StructUtilsTest.TestStruct,
id: nil,
title: nil
]
assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{id: "id"}) == %TestStruct{
id: "id",
title: nil
}
assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{title: "title"}) == %TestStruct{
id: nil,
title: "title"
}
end
end
end
================================================
FILE: test/login_flow_wrapper_test.exs
================================================
defmodule LoginFlowWrapperTest do
use ExUnit.Case, async: true
doctest OpenOwl.LoginFlowWrapper, import: true
end
================================================
FILE: test/recipes_test.exs
================================================
defmodule RecipesTest do
use ExUnit.Case, async: true
alias OpenOwl.Recipes.Recipe
alias OpenOwl.Recipes.Action
alias OpenOwl.Recipes
@test_recipe1 %Recipe{
login_url: "https://example1.com",
destination_url_pattern: "dup",
username_selector: "us",
password_selector: "ps"
}
@test_recipe2 %{@test_recipe1 | login_url: "https://example2.com"}
@action1 %Action{
name: :download,
http_method: :get,
url: "https://example.com/:url_id",
headers: [{"x-1", "cool_:header_id_one"}, {"x-2", ":url_id/:header_id_two"}]
}
@action2 %Action{name: :list, http_method: :get, url: "https://example.com/:url_id"}
describe "load_recipes/1" do
test "returns parsed recipes from a yaml file" do
assert {:ok, recipes} = Recipes.load_recipes("test/fixtures/test_recipes.yml")
assert recipes == %{
adobe: %OpenOwl.Recipes.Recipe{
login_url: "https://adminconsole.adobe.com/team",
destination_url_pattern: "https://adminconsole.adobe.com/**/overview",
username_selector: "internal:label=\"Email address\"i",
password_selector: "internal:label=\"Password\"i",
actions: [
%OpenOwl.Recipes.Action{
name: "download_users",
http_method: :get,
url: "https://adobe.io/api/sth",
populate_placeholders_from_session_storage:
~w(client_id tokenValue roles.organization),
response_path: nil,
headers: [
{"x-api-key", ":client_id"},
{"authorization", "Bearer :tokenValue"}
],
pagination: %OpenOwl.PaginationStrategy.PageParamsInUrl{
strategy: "page_params_in_url",
page_query_param: "p"
}
}
]
},
badobe: %OpenOwl.Recipes.Recipe{
login_url: "https://adminconsole.adobe.com/team",
destination_url_pattern: "https://adminconsole.adobe.com/**/overview",
username_selector: "internal:label=\"Email address\"i",
password_selector: "internal:label=\"Password\"i",
actions: [
%OpenOwl.Recipes.Action{
name: "download_users",
http_method: :get,
url: "https://adobe.io/api/sth",
response_path: nil,
pagination: %OpenOwl.PaginationStrategy.OffsetParamsInUrl{
strategy: "offset_params_in_url",
offset_query_param: "start_offset",
limit_query_param: "size"
}
}
]
},
asana: %OpenOwl.Recipes.Recipe{
login_url: "https://asana.com/login",
destination_url_pattern: "https://asana.com/app/**",
username_selector: "#a",
password_selector: "#b",
actions: [
%OpenOwl.Recipes.Action{
name: "change_sth",
http_method: :post,
body: "{\"query\":\"query Users\"}",
url: "https://asana.com/api/sth",
response_path: "meta.data",
headers: [{"X-Dynamic", ":team_id"}, {"X-Static", "Blub 1"}]
}
]
},
calendly: %OpenOwl.Recipes.Recipe{
login_url: "https://calendly.com/app/login",
destination_url_pattern: "https://calendly.com/event_types/user/me",
username_selector: "input[placeholder='email address']",
password_selector: "input[placeholder=password]",
actions: [
%OpenOwl.Recipes.Action{
name: "download_users",
http_method: :get,
url: "https://calendly.com/api/organization/memberships",
response_path: "results",
pagination: %OpenOwl.PaginationStrategy.NextPageInBody{
strategy: "next_page_in_body",
next_page_response_path: "pagination.next_page",
page_query_param: "page"
},
headers: []
}
]
},
loom: %OpenOwl.Recipes.Recipe{
login_url: "https://loom.com/login",
destination_url_pattern: "https://loom.com/app/**",
username_selector: "#a",
password_selector: "#b",
actions: [
%OpenOwl.Recipes.Action{
name: "download_users",
http_method: :get,
url: "https://loom.com/api/v1",
response_path: "data",
pagination: %OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson{
strategy: "next_cursor_in_body_to_send_as_json",
next_cursor_response_path: "pagination.next_page",
has_next_page_response_path: "pagination.has_next_page",
json_body_cursor_path: "variables.after"
},
headers: []
}
]
},
miro: %OpenOwl.Recipes.Recipe{
login_url: "https://miro.com/login",
destination_url_pattern: "https://miro.com/app/**",
username_selector: "data-testid=mr1",
password_selector: "data-testid=mr2",
actions: [
%OpenOwl.Recipes.Action{
name: "download_users",
http_method: :get,
url: "https://miro.com/api/v1/accounts/:team_id/users",
response_path: "data",
pagination: %OpenOwl.PaginationStrategy.UrlInBody{
strategy: "url_in_body",
next_url_response_path: "nextLink"
},
headers: []
}
]
}
}
end
test "returns error if file was not found" do
assert {:error, %YamlElixir.FileNotFoundError{}} =
Recipes.load_recipes("test/fixtures/not_existent.yml")
end
end
test "get_recipe/2 returns recipe for a slug" do
recipes = %{test1: @test_recipe1, test2: @test_recipe2}
assert Recipes.get_recipe(%{}, :test) == nil
assert Recipes.get_recipe(recipes, :test) == nil
assert Recipes.get_recipe(recipes, :test1) == @test_recipe1
assert Recipes.get_recipe(recipes, :test2) == @test_recipe2
assert Recipes.get_recipe(recipes, "test") == nil
assert Recipes.get_recipe(recipes, "test1") == @test_recipe1
assert Recipes.get_recipe(recipes, "test2") == @test_recipe2
end
test "get_action_for_name/2 returns action for name or nil" do
actions = [@action1, @action2]
assert Recipes.get_action_for_name(actions, :not_existent) == nil
assert Recipes.get_action_for_name(actions, :download) == @action1
assert Recipes.get_action_for_name(actions, :list) == @action2
end
describe "get_required_parameters_for_recipe/2" do
test "returns required parameters for special recipe action login" do
recipe_login_param = %{@test_recipe1 | login_url: "https://example.com/:org_id/:bla"}
recipe_dest_param = %{
@test_recipe1
| destination_url_pattern: "https://example.com/*/:bla"
}
recipe_both_duplicated = %{
@test_recipe1
| login_url: "https://example.com/:org_id/:bla",
destination_url_pattern: "https://example.com/*/:bla"
}
assert Recipes.get_required_parameters_for_recipe(@test_recipe1, "login") == [
:username,
:password
]
assert Recipes.get_required_parameters_for_recipe(recipe_login_param, "login") == [
:username,
:password,
:org_id,
:bla
]
assert Recipes.get_required_parameters_for_recipe(recipe_dest_param, "login") == [
:username,
:password,
:bla
]
assert Recipes.get_required_parameters_for_recipe(recipe_both_duplicated, "login") == [
:username,
:password,
:bla,
:org_id
]
end
test "returns required parameters for dynamic recipe action" do
recipe_base = %{
@test_recipe1
| login_url: "https://example.com/:org_id",
destination_url_pattern: ":dest_id"
}
recipe_with_actions = %{recipe_base | actions: [@action1, @action2]}
# ignores login parameters
assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :not_existent) == []
assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :download) == [
:url_id,
:header_id_two,
:header_id_one
]
assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :list) == [:url_id]
end
end
describe "validate_recipe_parameters/3" do
@recipe_with_params %{
@test_recipe1
| login_url: "https://example.com/:org_id",
actions: [@action1]
}
test "returns ok with passed parameter map if all required parameters are present" do
# without params
login_parameters_pure = %{username: "u", password: "p"}
assert Recipes.validate_recipe_parameters(@test_recipe1, "login", login_parameters_pure) ==
{:ok, login_parameters_pure}
login_parameters_not_existing = Map.put(login_parameters_pure, :not_existent, 1)
assert Recipes.validate_recipe_parameters(
@test_recipe1,
"login",
login_parameters_not_existing
) ==
{:ok, login_parameters_not_existing}
login_parameters_required = Map.put(login_parameters_pure, :org_id, "1")
assert Recipes.validate_recipe_parameters(
@recipe_with_params,
"login",
login_parameters_required
) ==
{:ok, login_parameters_required}
action_parameters = %{
url_id: "1",
header_id_two: 2,
header_id_one: "a3"
}
assert Recipes.validate_recipe_parameters(
@recipe_with_params,
@action1.name,
action_parameters
) ==
{:ok, action_parameters}
end
test "returns error with list of missing parameters if not all required parameters are present" do
assert Recipes.validate_recipe_parameters(@recipe_with_params, "login", %{wrong: 1}) ==
{:error, [:username, :password, :org_id]}
assert Recipes.validate_recipe_parameters(@recipe_with_params, "login", %{
password: "p",
wrong: 1
}) ==
{:error, [:username, :org_id]}
assert Recipes.validate_recipe_parameters(@recipe_with_params, @action1.name, %{
wrong: 1,
header_id_two: "2"
}) ==
{:error, [:url_id, :header_id_one]}
end
end
test "placeholder_regex/0 returns regex for placeholders" do
assert Recipes.placeholder_regex() == ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/
end
end
================================================
FILE: test/response_transformer_test.exs
================================================
defmodule ResponseTransformerTest do
use ExUnit.Case, async: true
alias OpenOwl.ResponseTransformer
@record1 %{
"email" => "batman@wayne.org",
"emailConfirmed" => true,
"id" => 123,
"lastLoginDate" => "2022-12-13T14:07:59.371Z",
"name" => "Bruce Wayne",
"team_role" => %{
"level" => 999,
"pending" => false,
"pending_email" => nil,
"resource_type" => "team",
"role" => %{
"name" => "Heroes",
"id" => "abc"
},
"user_id" => "456"
}
}
@record2 %{@record1 | "email" => "robin@wayne.org", "name" => "Robin"}
@map_list [@record1, @record2]
describe "records_to_list_with_header/1" do
test "returns table structure with header" do
assert ResponseTransformer.records_to_list_with_header(@map_list) == [
[
"email",
"emailConfirmed",
"id",
"lastLoginDate",
"name",
"team_role.level",
"team_role.pending",
"team_role.pending_email",
"team_role.resource_type",
"team_role.role.id",
"team_role.role.name",
"team_role.user_id"
],
[
"batman@wayne.org",
true,
123,
"2022-12-13T14:07:59.371Z",
"Bruce Wayne",
999,
false,
nil,
"team",
"abc",
"Heroes",
"456"
],
[
"robin@wayne.org",
true,
123,
"2022-12-13T14:07:59.371Z",
"Robin",
999,
false,
nil,
"team",
"abc",
"Heroes",
"456"
]
]
end
test "returns empty list if input was an empty list" do
assert ResponseTransformer.records_to_list_with_header([]) == []
end
test "raises error if it is not a list" do
assert_raise FunctionClauseError, fn ->
ResponseTransformer.records_to_list_with_header(@record1)
end
end
end
describe "list_with_header_to_csv/1" do
test "returns a CSV string" do
assert ResponseTransformer.list_with_header_to_csv([]) == ""
assert ResponseTransformer.list_with_header_to_csv([~w(col_a col_b), ~w(hey you)]) ==
"col_a,col_b\r\nhey,you\r\n"
end
test "raises error if it is not a list" do
assert_raise FunctionClauseError, fn ->
ResponseTransformer.list_with_header_to_csv(%{})
end
end
end
describe "flat_record/1" do
test "returns zero-level map untouched" do
record = %{"hey" => "you", "what" => 4, "or" => true}
assert ResponseTransformer.flat_record(record) == record
end
test "ignores fields that hold lists with maps" do
record = %{
"hey" => "you",
"simple_array" => ["you", 1],
"complex_array" => [%{"sub_map" => 1}]
}
assert ResponseTransformer.flat_record(record) == %{
"hey" => "you",
"simple_array" => ["you", 1]
}
end
test "returns one-level map with concatted field names" do
record = %{"hey" => "you", "what" => 4, "deep" => %{"name" => "cracker", "good" => true}}
assert ResponseTransformer.flat_record(record) == %{
"hey" => "you",
"what" => 4,
"deep.name" => "cracker",
"deep.good" => true
}
end
test "returns multi-level map with concatted field names" do
assert ResponseTransformer.flat_record(@record1) == %{
"email" => "batman@wayne.org",
"emailConfirmed" => true,
"id" => 123,
"lastLoginDate" => "2022-12-13T14:07:59.371Z",
"name" => "Bruce Wayne",
"team_role.level" => 999,
"team_role.pending" => false,
"team_role.pending_email" => nil,
"team_role.resource_type" => "team",
"team_role.role.name" => "Heroes",
"team_role.role.id" => "abc",
"team_role.user_id" => "456"
}
end
end
end
================================================
FILE: test/test_helper.exs
================================================
ExUnit.start(exclude: :skip)
================================================
FILE: ts_src/login_flow.ts
================================================
import { firefox, Page } from 'playwright';
import path from 'path';
import * as fs from 'fs';
const authCacheFolder = 'auth_cache';
const resultFolder = 'results';
const screenshotsFolder = 'login_screenshots';
function maybeCreateScreenshotsFolder() {
fs.mkdirSync(path.join(resultFolder, screenshotsFolder), { recursive: true });
}
function clearScreenshots() {
fs.readdirSync(path.join(resultFolder, screenshotsFolder)).forEach(file => {
fs.rmSync(path.join(resultFolder, screenshotsFolder, file));
});
}
async function makeScreenshot(page: Page, slug: String) {
const screenshot = await page.screenshot();
fs.writeFileSync(path.join(resultFolder, screenshotsFolder, Date.now() + '_' + slug + '.png'), screenshot);
}
export async function loginAndSaveCookies(
slug: string,
url: string,
destination_url_pattern: string,
username_selector: string,
password_selector: string,
username: string,
password: string): Promise<void> {
maybeCreateScreenshotsFolder();
clearScreenshots();
const browser = await firefox.launch({ headless: true });
const context = await browser.newContext()
const page = await context.newPage()
await page.goto(url);
await page.fill(username_selector, username);
await makeScreenshot(page, slug);
// if password field is not visible, press actively enter so that it appears
try {
await page.waitForSelector(password_selector, { timeout: 2 });
} catch (e) {
await page.press(username_selector, 'Enter');
}
await makeScreenshot(page, slug);
await page.fill(password_selector, password);
await page.press(password_selector, 'Enter');
await makeScreenshot(page, slug);
await page.waitForURL(destination_url_pattern, { timeout: 10000 });
await makeScreenshot(page, slug);
const cookies = await context.cookies();
const cookieJson = JSON.stringify(cookies);
fs.writeFileSync(path.join(authCacheFolder, slug + '_cookies.json'), cookieJson);
const sessionStorage = await page.evaluate(() => JSON.stringify(sessionStorage));
fs.writeFileSync(path.join(authCacheFolder, slug + '_session_storage.json'), sessionStorage, 'utf-8');
clearScreenshots();
await browser.close();
}
================================================
FILE: ts_src/main.ts
================================================
import { loginAndSaveCookies } from './login_flow.js';
console.log("started...");
loginAndSaveCookies(process.env.SLUG, process.env.URL, process.env.DESTINATION_URL_PATTERN,
process.env.USERNAME_SELECTOR, process.env.PASSWORD_SELECTOR, process.env.USER,
process.env.PASSWORD);
console.log("DONE");
================================================
FILE: ts_src/recipe_manager.ts
================================================
import type { VendorTemplate, VendorAction } from "./types";
import path from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import yaml from 'js-yaml';
class RecipeManager {
recipes: { [key: string]: VendorTemplate };
constructor() {
// https://flaviocopes.com/fix-dirname-not-defined-es-module-scope/
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
let recipesPath = path.join(__dirname, '../recipes.yml');
this.recipes = yaml.load(fs.readFileSync(recipesPath).toString()) as { string: VendorTemplate }
}
getRecipes(): { [key: string]: VendorTemplate } {
return this.recipes;
}
}
export default new RecipeManager();
================================================
FILE: ts_src/types.ts
================================================
export interface VendorTemplate {
login_url: string;
destination_url_pattern: string;
username_selector: string;
password_selector: string;
actions?: VendorAction[];
}
export interface VendorAction {
name: string;
http_method: HttpMethod;
url: string;
response_path: string;
}
export enum HttpMethod {
GET = 'GET',
POST = 'POST',
PUT = 'PUT',
PATCH = 'POST',
DELETE = 'DELETE'
}
================================================
FILE: tsconfig.json
================================================
{
"compileOnSave": true,
"compilerOptions": {
"module": "ES2022",
"target": "ES2022",
"outDir": "dist",
"moduleResolution": "node",
"allowSyntheticDefaultImports": true,
"noUnusedLocals": false,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"lib": [
"ESNext"
]
},
"baseUrl": "./",
"include": [
"ts_src/**/*"
]
}
gitextract_mi7z9xmv/ ├── .dockerignore ├── .formatter.exs ├── .github/ │ └── workflows/ │ └── tests.yml ├── .gitignore ├── .tool-versions ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yml ├── lib/ │ ├── api_client.ex │ ├── cli.ex │ ├── helpers/ │ │ ├── api_utils.ex │ │ ├── cli_utils.ex │ │ └── struct_utils.ex │ ├── login_flow_wrapper.ex │ ├── pagination_strategies/ │ │ ├── pagination_next_cursor_in_body_to_send_as_json.ex │ │ ├── pagination_next_page_in_body.ex │ │ ├── pagination_offset_params_in_url.ex │ │ ├── pagination_page_params_in_url.ex │ │ ├── pagination_strategy.ex │ │ └── pagination_url_in_body.ex │ ├── recipes/ │ │ ├── action.ex │ │ └── recipe.ex │ ├── recipes.ex │ ├── response_transformer.ex │ └── saas_owl.ex ├── mix.exs ├── owl.sh ├── package.json ├── recipes.yml ├── test/ │ ├── api_client_test.exs │ ├── fixtures/ │ │ └── test_recipes.yml │ ├── helpers/ │ │ ├── api_utils_test.exs │ │ ├── cli_utils_test.exs │ │ └── struct_utils_test.exs │ ├── login_flow_wrapper_test.exs │ ├── recipes_test.exs │ ├── response_transformer_test.exs │ └── test_helper.exs ├── ts_src/ │ ├── login_flow.ts │ ├── main.ts │ ├── recipe_manager.ts │ └── types.ts └── tsconfig.json
SYMBOL INDEX (106 symbols across 28 files)
FILE: lib/api_client.ex
class OpenOwl.ApiClient (line 1) | defmodule OpenOwl.ApiClient
method do_request (line 10) | def do_request(
method extract_data_from_response_body (line 70) | defp extract_data_from_response_body(body, response_path) do
method build_headers (line 74) | defp build_headers(headers, placeholders, cookie_string) do
FILE: lib/cli.ex
class OpenOwl.CLI (line 1) | defmodule OpenOwl.CLI
method main (line 15) | def main(args) do
method main (line 22) | def main() do
method parse_args (line 28) | defp parse_args(args) do
method process_params (line 37) | defp process_params({["recipes", "list"], _}) do
method process_params (line 72) | defp process_params({[vendor, "login"], flags}) do
method process_params (line 195) | defp process_params(_), do: print_help()
method print_help (line 197) | defp print_help() do
method load_recipes (line 224) | defp load_recipes() do
method get_trailing_padding (line 234) | defp get_trailing_padding(%{} = recipes) do
method parameter_list_as_env (line 245) | defp parameter_list_as_env(list) do
method maybe_in_brackets (line 252) | defp maybe_in_brackets(""), do: ""
method maybe_in_brackets (line 254) | defp maybe_in_brackets(string) do
method write_error (line 258) | defp write_error(msg) do
FILE: lib/helpers/api_utils.ex
class OpenOwl.Helpers.ApiUtils (line 1) | defmodule OpenOwl.Helpers.ApiUtils
method get_field_via_response_path (line 46) | def get_field_via_response_path([one], response_path),
method get_field_via_response_path (line 49) | def get_field_via_response_path(_map, nil), do: nil
method get_field_via_response_path (line 51) | def get_field_via_response_path(%{} = map, response_path) do
method set_field_via_path (line 84) | def set_field_via_path(%{} = map, nil, _value), do: map
method set_field_via_path (line 85) | def set_field_via_path(%{} = map, "", _value), do: map
method set_field_via_path (line 87) | def set_field_via_path([one], field_path, value),
method set_field_via_path (line 90) | def set_field_via_path(%{} = map, field_path, value, list? \\ false) do
method filter_relevant_cookies (line 108) | def filter_relevant_cookies(cookies, url) do
method get_last_response_path_part (line 191) | def get_last_response_path_part(path) do
method build_cookie_params_string (line 206) | def build_cookie_params_string(cookies) do
method apply_placeholder_params (line 225) | def apply_placeholder_params(subject, params) do
FILE: lib/helpers/cli_utils.ex
class OpenOwl.Helpers.CliUtils (line 1) | defmodule OpenOwl.Helpers.CliUtils
method get_app_env_params (line 16) | def get_app_env_params(params \\ System.get_env()) do
method normalize_var (line 26) | defp normalize_var(var) do
method timebased_filename (line 43) | def timebased_filename(filename, datetime \\ NaiveDateTime.utc_now())
method timebased_filename (line 44) | def timebased_filename(nil, _), do: raise(ArgumentError, "filename emp...
method timebased_filename (line 45) | def timebased_filename("", _), do: raise(ArgumentError, "filename empty")
method timebased_filename (line 47) | def timebased_filename(filename, datetime) do
FILE: lib/helpers/struct_utils.ex
class OpenOwl.Helpers.StructUtils (line 1) | defmodule OpenOwl.Helpers.StructUtils
method to_struct (line 7) | def to_struct(kind, attrs) do
method fetch_value (line 25) | defp fetch_value(attrs, key, acc) do
FILE: lib/login_flow_wrapper.ex
class OpenOwl.LoginFlowWrapper (line 1) | defmodule OpenOwl.LoginFlowWrapper
method call_local_cmd (line 2) | def call_local_cmd(
method get_local_cmd_command (line 40) | def get_local_cmd_command(
FILE: lib/pagination_strategies/pagination_next_cursor_in_body_to_send_as_json.ex
class OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson (line 1) | defmodule OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson
method handle_paginated_response (line 44) | def handle_paginated_response(
FILE: lib/pagination_strategies/pagination_next_page_in_body.ex
class OpenOwl.PaginationStrategy.NextPageInBody (line 1) | defmodule OpenOwl.PaginationStrategy.NextPageInBody
method handle_paginated_response (line 34) | def handle_paginated_response(
FILE: lib/pagination_strategies/pagination_offset_params_in_url.ex
class OpenOwl.PaginationStrategy.OffsetParamsInUrl (line 1) | defmodule OpenOwl.PaginationStrategy.OffsetParamsInUrl
method handle_paginated_response (line 24) | def handle_paginated_response(
FILE: lib/pagination_strategies/pagination_page_params_in_url.ex
class OpenOwl.PaginationStrategy.PageParamsInUrl (line 1) | defmodule OpenOwl.PaginationStrategy.PageParamsInUrl
method handle_paginated_response (line 19) | def handle_paginated_response(
FILE: lib/pagination_strategies/pagination_strategy.ex
class OpenOwl.PaginationStrategy (line 1) | defmodule OpenOwl.PaginationStrategy
FILE: lib/pagination_strategies/pagination_url_in_body.ex
class OpenOwl.PaginationStrategy.UrlInBody (line 1) | defmodule OpenOwl.PaginationStrategy.UrlInBody
method handle_paginated_response (line 31) | def handle_paginated_response(
FILE: lib/recipes.ex
class OpenOwl.Recipes (line 1) | defmodule OpenOwl.Recipes
method load_recipes (line 9) | def load_recipes(rel_path \\ @recipe_file_name) do
method get_recipe (line 22) | def get_recipe(recipes, slug) do
method get_action_for_name (line 26) | def get_action_for_name(actions, name) do
method get_required_parameters_for_recipe (line 31) | def get_required_parameters_for_recipe(
method get_required_parameters_for_recipe (line 39) | def get_required_parameters_for_recipe(%Recipe{actions: actions}, acti...
method get_parameters (line 47) | defp get_parameters(strings) do
method atomize_placeholder_string (line 58) | defp atomize_placeholder_string(placeholder) do
method get_header_values (line 63) | defp get_header_values(headers) do
method validate_recipe_parameters (line 67) | def validate_recipe_parameters(%Recipe{} = recipe, action, %{} = param...
method placeholder_regex (line 80) | def placeholder_regex() do
method parse_yaml_data (line 84) | defp parse_yaml_data(%{} = data) do
FILE: lib/recipes/action.ex
class OpenOwl.Recipes.Action (line 1) | defmodule OpenOwl.Recipes.Action
method cast (line 25) | def cast(attrs) do
method http_method_to_atom (line 64) | defp http_method_to_atom(http_method) do
method modulize (line 68) | defp modulize(nil), do: nil
method modulize (line 70) | defp modulize(underscored_string) do
FILE: lib/recipes/recipe.ex
class OpenOwl.Recipes.Recipe (line 1) | defmodule OpenOwl.Recipes.Recipe
method cast (line 25) | def cast(attrs, actions) do
FILE: lib/response_transformer.ex
class OpenOwl.ResponseTransformer (line 1) | defmodule OpenOwl.ResponseTransformer
method records_to_csv (line 8) | def records_to_csv(records) do
method records_to_list_with_header (line 21) | def records_to_list_with_header([]) do
method flat_record (line 40) | def flat_record(map) do
method flat_record (line 47) | def flat_record(map, prefix) do
FILE: lib/saas_owl.ex
class OpenOwl (line 1) | defmodule OpenOwl
method version (line 4) | def version do
FILE: mix.exs
class OpenOwl.MixProject (line 1) | defmodule OpenOwl.MixProject
method project (line 4) | def project do
method application (line 17) | def application do
method elixirc_paths (line 23) | defp elixirc_paths(_), do: ["lib"]
method escript (line 25) | defp escript do
method deps (line 30) | defp deps do
FILE: test/api_client_test.exs
class ApiClientTest (line 1) | defmodule ApiClientTest
method get_relevant_headers (line 18) | defp get_relevant_headers(headers) do
FILE: test/helpers/api_utils_test.exs
class ApiUtilsTest (line 1) | defmodule ApiUtilsTest
FILE: test/helpers/cli_utils_test.exs
class CliUtilsTest (line 1) | defmodule CliUtilsTest
FILE: test/helpers/struct_utils_test.exs
class StructUtilsTest (line 1) | defmodule StructUtilsTest
class TestStruct (line 6) | defmodule TestStruct
class AnotherTestStruct (line 11) | defmodule AnotherTestStruct
FILE: test/login_flow_wrapper_test.exs
class LoginFlowWrapperTest (line 1) | defmodule LoginFlowWrapperTest
FILE: test/recipes_test.exs
class RecipesTest (line 1) | defmodule RecipesTest
FILE: test/response_transformer_test.exs
class ResponseTransformerTest (line 1) | defmodule ResponseTransformerTest
FILE: ts_src/login_flow.ts
function maybeCreateScreenshotsFolder (line 9) | function maybeCreateScreenshotsFolder() {
function clearScreenshots (line 13) | function clearScreenshots() {
function makeScreenshot (line 19) | async function makeScreenshot(page: Page, slug: String) {
function loginAndSaveCookies (line 24) | async function loginAndSaveCookies(
FILE: ts_src/recipe_manager.ts
class RecipeManager (line 7) | class RecipeManager {
method constructor (line 10) | constructor() {
method getRecipes (line 19) | getRecipes(): { [key: string]: VendorTemplate } {
FILE: ts_src/types.ts
type VendorTemplate (line 1) | interface VendorTemplate {
type VendorAction (line 9) | interface VendorAction {
type HttpMethod (line 16) | enum HttpMethod {
Condensed preview — 44 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (129K chars).
[
{
"path": ".dockerignore",
"chars": 102,
"preview": "npm-debug.log\nyarn.lock\n_build\n!_build/dev/lib/castore\ndeps\n!deps/castore\nnode_modules\ndist\n.gitignore"
},
{
"path": ".formatter.exs",
"chars": 97,
"preview": "# Used by \"mix format\"\n[\n inputs: [\"{mix,.formatter}.exs\", \"{config,lib,test}/**/*.{ex,exs}\"]\n]\n"
},
{
"path": ".github/workflows/tests.yml",
"chars": 1901,
"preview": "name: Run tests\nconcurrency: ci_tests\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ] \n\njobs:"
},
{
"path": ".gitignore",
"chars": 2229,
"preview": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\nlerna-debug.log*\n\n# Diagnostic reports (https://nodejs."
},
{
"path": ".tool-versions",
"chars": 49,
"preview": "elixir 1.14.3-otp-24\nerlang 24.2.1\nnodejs 19.6.0\n"
},
{
"path": "Dockerfile",
"chars": 1915,
"preview": "###\n### First Stage - Building the Elixir app as escript\n###\nFROM hexpm/elixir:1.14.3-erlang-23.2.6-alpine-3.16.0 AS bui"
},
{
"path": "LICENSE",
"chars": 11342,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 7078,
"preview": "# OpenOwl 🦉\n\n<a href=\"https://github.com/AccessOwl/open_owl/releases\" target=\"_blank\">\n <img src=\"https://img.shields"
},
{
"path": "docker-compose.yml",
"chars": 166,
"preview": "services:\n owl:\n build: ./\n volumes:\n - ${PWD}/auth_cache:/app/auth_cache\n - ${PWD}/results:/app/result"
},
{
"path": "lib/api_client.ex",
"chars": 2399,
"preview": "defmodule OpenOwl.ApiClient do\n import OpenOwl.Helpers.ApiUtils\n\n @base_header [\n {\"accept-encoding\", \"gzip\"},\n "
},
{
"path": "lib/cli.ex",
"chars": 8012,
"preview": "defmodule OpenOwl.CLI do\n alias OpenOwl.Recipes\n alias OpenOwl.Recipes.Recipe\n alias OpenOwl.Recipes.Action\n alias O"
},
{
"path": "lib/helpers/api_utils.ex",
"chars": 9660,
"preview": "defmodule OpenOwl.Helpers.ApiUtils do\n @path_separator \".\"\n\n alias OpenOwl.Recipes\n\n @doc \"\"\"\n Resolves a response p"
},
{
"path": "lib/helpers/cli_utils.ex",
"chars": 1557,
"preview": "defmodule OpenOwl.Helpers.CliUtils do\n @prefix \"OWL_\"\n\n @doc \"\"\"\n Returns all app environment params downcased and wi"
},
{
"path": "lib/helpers/struct_utils.ex",
"chars": 843,
"preview": "defmodule OpenOwl.Helpers.StructUtils do\n @moduledoc \"\"\"\n Converts a map with strings, a map with atoms or an existing"
},
{
"path": "lib/login_flow_wrapper.ex",
"chars": 1732,
"preview": "defmodule OpenOwl.LoginFlowWrapper do\n def call_local_cmd(\n slug,\n url,\n destination_url_pattern,\n"
},
{
"path": "lib/pagination_strategies/pagination_next_cursor_in_body_to_send_as_json.ex",
"chars": 2586,
"preview": "defmodule OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson do\n @moduledoc \"\"\"\n Handles APIs that return a curso"
},
{
"path": "lib/pagination_strategies/pagination_next_page_in_body.ex",
"chars": 2125,
"preview": "defmodule OpenOwl.PaginationStrategy.NextPageInBody do\n @moduledoc \"\"\"\n Handles APIs that return the next page cursor "
},
{
"path": "lib/pagination_strategies/pagination_offset_params_in_url.ex",
"chars": 2269,
"preview": "defmodule OpenOwl.PaginationStrategy.OffsetParamsInUrl do\n @moduledoc \"\"\"\n Handles APIs that have offset params in URL"
},
{
"path": "lib/pagination_strategies/pagination_page_params_in_url.ex",
"chars": 1857,
"preview": "defmodule OpenOwl.PaginationStrategy.PageParamsInUrl do\n @moduledoc \"\"\"\n Handles APIs that have page params in URL que"
},
{
"path": "lib/pagination_strategies/pagination_strategy.ex",
"chars": 1030,
"preview": "defmodule OpenOwl.PaginationStrategy do\n @type t :: module()\n @type status() :: pos_integer()\n @type body() :: map()\n"
},
{
"path": "lib/pagination_strategies/pagination_url_in_body.ex",
"chars": 1835,
"preview": "defmodule OpenOwl.PaginationStrategy.UrlInBody do\n @moduledoc \"\"\"\n Handles APIs that return a URL in the response body"
},
{
"path": "lib/recipes/action.ex",
"chars": 2132,
"preview": "defmodule OpenOwl.Recipes.Action do\n alias OpenOwl.Helpers.StructUtils\n\n @enforce_keys [:name, :http_method, :url]\n d"
},
{
"path": "lib/recipes/recipe.ex",
"chars": 776,
"preview": "defmodule OpenOwl.Recipes.Recipe do\n alias OpenOwl.Helpers.StructUtils\n alias OpenOwl.Recipes.Action\n\n @enforce_keys "
},
{
"path": "lib/recipes.ex",
"chars": 2465,
"preview": "defmodule OpenOwl.Recipes do\n @recipe_file_name \"recipes.yml\"\n @external_resource Path.join(File.cwd!(), @recipe_file_"
},
{
"path": "lib/response_transformer.ex",
"chars": 1523,
"preview": "defmodule OpenOwl.ResponseTransformer do\n alias NimbleCSV.RFC4180, as: CSV\n\n @doc \"\"\"\n Converts a response that consi"
},
{
"path": "lib/saas_owl.ex",
"chars": 130,
"preview": "defmodule OpenOwl do\n @version OpenOwl.MixProject.project() |> Keyword.fetch!(:version)\n\n def version do\n @version\n"
},
{
"path": "mix.exs",
"chars": 854,
"preview": "defmodule OpenOwl.MixProject do\n use Mix.Project\n\n def project do\n [\n app: :open_owl,\n version: \"0.1.0\",\n"
},
{
"path": "owl.sh",
"chars": 717,
"preview": "#!/usr/bin/env sh\n\nTMP_ENV_FILE=.owl_env.tmp\nIMAGE_NAME=open_owl-owl\n\nstore_relevant_env_variables() {\n printenv|grep O"
},
{
"path": "package.json",
"chars": 430,
"preview": "{\n \"name\": \"open_owl\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"type\": \"module\",\n \"scripts\": {\n \"start\": \"npm "
},
{
"path": "recipes.yml",
"chars": 5590,
"preview": "adobe:\n login_url: https://adminconsole.adobe.com/team\n destination_url_pattern: https://adminconsole.adobe.com/**/ove"
},
{
"path": "test/api_client_test.exs",
"chars": 19305,
"preview": "defmodule ApiClientTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.ApiClient\n alias OpenOwl.PaginationStrategy."
},
{
"path": "test/fixtures/test_recipes.yml",
"chars": 2929,
"preview": "adobe:\n login_url: https://adminconsole.adobe.com/team\n destination_url_pattern: https://adminconsole.adobe.com/**/ove"
},
{
"path": "test/helpers/api_utils_test.exs",
"chars": 110,
"preview": "defmodule ApiUtilsTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.Helpers.ApiUtils, import: true\nend\n"
},
{
"path": "test/helpers/cli_utils_test.exs",
"chars": 110,
"preview": "defmodule CliUtilsTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.Helpers.CliUtils, import: true\nend\n"
},
{
"path": "test/helpers/struct_utils_test.exs",
"chars": 1940,
"preview": "defmodule StructUtilsTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.Helpers.StructUtils\n\n defmodule TestStruct"
},
{
"path": "test/login_flow_wrapper_test.exs",
"chars": 118,
"preview": "defmodule LoginFlowWrapperTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.LoginFlowWrapper, import: true\nend\n"
},
{
"path": "test/recipes_test.exs",
"chars": 11819,
"preview": "defmodule RecipesTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.Recipes.Recipe\n alias OpenOwl.Recipes.Action\n "
},
{
"path": "test/response_transformer_test.exs",
"chars": 4429,
"preview": "defmodule ResponseTransformerTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.ResponseTransformer\n\n @record1 %{\n"
},
{
"path": "test/test_helper.exs",
"chars": 29,
"preview": "ExUnit.start(exclude: :skip)\n"
},
{
"path": "ts_src/login_flow.ts",
"chars": 2187,
"preview": "import { firefox, Page } from 'playwright';\nimport path from 'path';\nimport * as fs from 'fs';\n\nconst authCacheFolder = "
},
{
"path": "ts_src/main.ts",
"chars": 305,
"preview": "import { loginAndSaveCookies } from './login_flow.js';\n\nconsole.log(\"started...\");\n\nloginAndSaveCookies(process.env.SLUG"
},
{
"path": "ts_src/recipe_manager.ts",
"chars": 722,
"preview": "import type { VendorTemplate, VendorAction } from \"./types\";\nimport path from 'path';\nimport { fileURLToPath } from 'url"
},
{
"path": "ts_src/types.ts",
"chars": 408,
"preview": "export interface VendorTemplate {\n login_url: string;\n destination_url_pattern: string;\n username_selector: string;\n "
},
{
"path": "tsconfig.json",
"chars": 391,
"preview": "{\n \"compileOnSave\": true,\n \"compilerOptions\": {\n \"module\": \"ES2022\",\n \"target\": \"ES2022\",\n \"outDir\": \"dist\",\n"
}
]
About this extraction
This page contains the full source code of the AccessOwl/open_owl GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 44 files (117.4 KB), approximately 30.1k tokens, and a symbol index with 106 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.