Showing preview only (775K chars total). Download the full file or copy to clipboard to get everything.
Repository: spotify/zoltar
Branch: main
Commit: 384c011e9aed
Files: 180
Total size: 713.2 KB
Directory structure:
gitextract_f9bkw_4z/
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── codeql-analysis.yml
│ └── main.yml
├── .gitignore
├── LICENSE
├── NOTICE
├── README.md
├── build.sbt
├── catalog-info.yaml
├── checkstyle.xml
├── data/
│ └── iris-model/
│ ├── build.sbt
│ ├── project/
│ │ └── build.properties
│ ├── scripts/
│ │ └── publish_data.sh
│ ├── src/
│ │ ├── main/
│ │ │ └── scala/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── Iris.scala
│ │ └── test/
│ │ ├── resources/
│ │ │ └── expected_feature_spec.json
│ │ └── scala/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── IrisTest.scala
│ └── training/
│ ├── Makefile
│ ├── README.md
│ └── iris/
│ ├── iris/
│ │ ├── __init__.py
│ │ ├── tensorflow.py
│ │ └── xgboost.py
│ ├── requirements.txt
│ ├── setup.cfg
│ ├── setup.py
│ └── test-requirements.txt
├── docs/
│ └── src/
│ └── paradox/
│ ├── _template/
│ │ └── javadoc.st
│ ├── getting-started.md
│ ├── index.md
│ ├── javadoc.md
│ ├── license.md
│ ├── modules/
│ │ ├── featran.md
│ │ ├── index.md
│ │ ├── metrics.md
│ │ ├── mlengine.md
│ │ ├── tensorflow.md
│ │ └── xgboost.md
│ └── release.md
├── examples/
│ ├── apollo-service-example/
│ │ ├── SERVICE.marker
│ │ ├── findbugsexclude.xml
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── java/
│ │ │ │ └── com/
│ │ │ │ └── spotify/
│ │ │ │ └── zoltar/
│ │ │ │ └── examples/
│ │ │ │ └── apollo/
│ │ │ │ ├── App.java
│ │ │ │ ├── IrisPredictionHandler.java
│ │ │ │ ├── IrisPredictor.java
│ │ │ │ └── ModelConfig.java
│ │ │ └── resources/
│ │ │ └── zoltar-example.conf
│ │ └── test/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── apollo/
│ │ │ └── IrisPredictionHandlerTest.java
│ │ └── resources/
│ │ ├── settings.json
│ │ ├── trained_model/
│ │ │ ├── saved_model.pb
│ │ │ └── variables/
│ │ │ ├── variables.data-00000-of-00001
│ │ │ └── variables.index
│ │ └── zoltar-example.conf
│ ├── batch-predictor/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ └── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── batch/
│ │ │ ├── BatchPredictorExample.java
│ │ │ └── DummyModel.java
│ │ └── test/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── batch/
│ │ └── BatchPredictorExampleTest.java
│ ├── custom-metrics/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ └── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── metrics/
│ │ │ ├── CustomMetricsExample.java
│ │ │ └── DummyModel.java
│ │ └── test/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── metrics/
│ │ └── CustomMetricsExampleTest.java
│ └── mlengine-example/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── mlengine/
│ │ └── MlEnginePredictorExample.java
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── mlengine/
│ │ └── MlEnginePredictorExampleIT.java
│ └── resources/
│ └── settings.json
├── findbugsexclude.xml
├── pom.xml
├── project/
│ ├── build.properties
│ └── plugins.sbt
├── scripts/
│ ├── data-integration-test-a20d1bb2e128.json.enc
│ ├── deploy.sh
│ ├── libgomp/
│ │ ├── Dockerfile
│ │ └── create.sh
│ └── test.sh
├── settings.xml
├── suppressions.xml
├── zoltar-api/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ ├── Models.java
│ └── Predictors.java
├── zoltar-bom/
│ └── pom.xml
├── zoltar-core/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ ├── DefaultPredictor.java
│ ├── DefaultPredictorBuilder.java
│ ├── DefaultPredictorTimeoutScheduler.java
│ ├── FeatureExtractFns.java
│ ├── FeatureExtractor.java
│ ├── Model.java
│ ├── ModelLoader.java
│ ├── PredictFns.java
│ ├── Prediction.java
│ ├── Predictor.java
│ ├── PredictorBuilder.java
│ ├── PredictorTimeoutScheduler.java
│ ├── Vector.java
│ ├── fs/
│ │ └── FileSystemExtras.java
│ └── loaders/
│ ├── ModelMemoizer.java
│ └── Preloader.java
├── zoltar-featran/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── featran/
│ └── FeatranExtractFns.java
├── zoltar-jmh/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── jmh/
│ │ ├── BenchmarkTensorFlow.java
│ │ └── IrisHelper.java
│ ├── resources/
│ │ ├── iris.csv
│ │ ├── log4j.properties
│ │ ├── settings.json
│ │ └── trained_model/
│ │ ├── saved_model.pb
│ │ ├── trained_model.txt
│ │ └── variables/
│ │ ├── variables.data-00000-of-00001
│ │ └── variables.index
│ └── scala/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── Iris.scala
├── zoltar-metrics/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── metrics/
│ ├── FeatureExtractorMetrics.java
│ ├── Instrumentations.java
│ ├── InstrumentedFeatureExtractor.java
│ ├── InstrumentedPredictFn.java
│ ├── InstrumentedPredictorBuilder.java
│ ├── PredictFnMetrics.java
│ ├── PredictMetrics.java
│ ├── PredictorMetrics.java
│ ├── VectorMetrics.java
│ └── semantic/
│ ├── SemanticPredictMetrics.java
│ ├── SemanticPredictorMetrics.java
│ ├── SemanticVectorMetrics.java
│ └── What.java
├── zoltar-mlengine/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── mlengine/
│ ├── MlEngineLoader.java
│ ├── MlEngineModel.java
│ ├── MlEnginePredictException.java
│ └── MlEnginePredictFn.java
├── zoltar-tensorflow/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── tf/
│ ├── TensorFlowExtras.java
│ ├── TensorFlowGraphLoader.java
│ ├── TensorFlowGraphModel.java
│ ├── TensorFlowLoader.java
│ ├── TensorFlowModel.java
│ ├── TensorFlowPredictFn.java
│ └── TensorflowMetaGraphDefParsingException.java
├── zoltar-tests/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ ├── IrisHelper.java
│ │ │ └── PredictorsTest.java
│ │ └── scala/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── Iris.scala
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ ├── FeatureExtractFnsTest.java
│ │ ├── FeatureExtractorTest.java
│ │ ├── MultiPredictorTest.java
│ │ ├── PredictionTest.java
│ │ ├── PredictorBuilderTest.java
│ │ ├── PredictorTest.java
│ │ ├── VectorTest.java
│ │ ├── fs/
│ │ │ ├── FileSystemExtrasTest.java
│ │ │ ├── FileSystemExtrasTestIT.java
│ │ │ └── FileSystemExtrasTestUtils.java
│ │ ├── loaders/
│ │ │ ├── LoaderIT.java
│ │ │ ├── ModelMemoizerTest.java
│ │ │ └── PreloaderTest.java
│ │ ├── tf/
│ │ │ ├── TensorFlowExtrasTest.java
│ │ │ ├── TensorFlowGraphModelTest.java
│ │ │ └── TensorFlowModelTest.java
│ │ └── xgboost/
│ │ └── XGBoostModelTest.java
│ └── resources/
│ ├── badsubdir/
│ │ ├── 2018/
│ │ │ └── data.txt
│ │ └── z/
│ │ └── data.txt
│ ├── emptydir/
│ │ └── data.txt
│ ├── iris.csv
│ ├── iris.model
│ ├── log4j.properties
│ ├── settings.json
│ ├── settings_dummy.json
│ ├── test.txt
│ ├── testdir/
│ │ ├── 2018-01-15/
│ │ │ └── data.txt
│ │ ├── 2018-02-28/
│ │ │ └── data.txt
│ │ ├── 2018-03-01/
│ │ │ └── data.txt
│ │ └── 2018-04-01
│ ├── trained_model/
│ │ ├── saved_model.pb
│ │ ├── trained_model.txt
│ │ └── variables/
│ │ ├── variables.data-00000-of-00001
│ │ └── variables.index
│ └── trained_model.jar
└── zoltar-xgboost/
├── pom.xml
└── src/
└── main/
├── java/
│ ├── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── xgboost/
│ │ ├── XGBoostLoader.java
│ │ ├── XGBoostModel.java
│ │ └── XGBoostPredictFn.java
│ └── ml/
│ └── dmlc/
│ └── xgboost4j/
│ └── java/
│ └── GompLoader.java
└── resources/
└── lib/
└── libgomp.so.1
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: maven
directory: "/"
schedule:
interval: daily
time: "04:00"
open-pull-requests-limit: 10
================================================
FILE: .github/workflows/codeql-analysis.yml
================================================
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ main ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]
schedule:
- cron: '27 13 * * 2'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'java', 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
================================================
FILE: .github/workflows/main.yml
================================================
name: main
on: [push, pull_request]
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v2
with:
distribution: "adopt"
java-version: "11"
cache: "maven"
- run: |
mvn -Dbigquery.project.arg="-Dbigquery.project=dummy-project" \
clean \
spotless:check \
checkstyle:checkstyle \
findbugs:findbugs \
test
================================================
FILE: .gitignore
================================================
## OS
.DS_Store
## Artifacts
*.class
## Package Files
*.jar
*.war
*.ear
target
## IDE's
.idea
*.iml
.project
.settings
.classpath
.metals
.vscode
.factorypath
## Logs
*.log
## Temp
*~
.#*
.bigquery
dependency-reduced-pom.xml
created_job_ids
## Project Files
*-user.conf
## Python virtualenv
.venv
## Do not ignore jars used by unit tests
!zoltar-tests/src/test/resources/*.jar
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: NOTICE
================================================
Zoltar
Copyright 2018 Spotify AB
================================================
FILE: README.md
================================================
# Zoltar
[](https://travis-ci.org/spotify/zoltar)
[](https://codecov.io/github/spotify/zoltar?branch=master)
[](https://search.maven.org/search?q=com.spotify.zoltar)
[](./LICENSE)
Zoltar is a common library for serving TensorFlow, XGBoost and scikit-learn models in production.
See [Zoltar docs](https://spotify.github.io/zoltar) for details.
# License
Copyright 2018 Spotify AB.
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
================================================
FILE: build.sbt
================================================
lazy val noPublishSettings = Seq(
publish := {},
publishLocal := {},
publishArtifact := false
)
lazy val root = project
.in(file("."))
.settings(noPublishSettings)
.settings(name := "zoltar")
.aggregate(docs)
lazy val docs = project
.in(file("docs"))
.enablePlugins(ParadoxSitePlugin, ParadoxMaterialThemePlugin, GhpagesPlugin)
.settings(noPublishSettings)
.settings(
name := "docs",
version := "0.4.0",
paradoxProperties in Paradox ++= Map(
"javadoc.com.spotify.zoltar.base_url" -> "http://spotify.github.com/zoltar/apidocs"
),
sourceDirectory in Paradox in paradoxTheme := sourceDirectory.value / "paradox" / "_template",
ParadoxMaterialThemePlugin.paradoxMaterialThemeSettings(Paradox),
paradoxMaterialTheme in Paradox ~= {
_.withColor("white", "indigo")
.withLogo("images/logo.png")
.withCopyright("Copyright (C) 2018 Spotify AB")
.withRepository(uri("https://github.com/spotify/zoltar"))
.withSocial(uri("https://github.com/spotify"),
uri("https://twitter.com/spotifyeng"))
},
scmInfo := Some(
ScmInfo(url("https://github.com/spotify/zoltar"),
"git@github.com:spotify/zoltar.git")),
git.remoteRepo := scmInfo.value.get.connection
)
================================================
FILE: catalog-info.yaml
================================================
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: zoltar
spec:
type: resource
owner: flatmap
================================================
FILE: checkstyle.xml
================================================
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
<!--
This is a Spotified version of the google_checks.xml file, from
https://github.com/checkstyle/checkstyle/blob/master/src/main/resources/google_checks.xml,
version 3e4367941c3e9680703e8ea8400abbd5dc78e1d9 from the 15th of January 2016.
-->
<!--
Checkstyle configuration that checks the Google coding conventions from Google Java Style
that can be found at https://google.github.io/styleguide/javaguide.html.
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
To completely disable a check, just comment it out or delete it from the file.
Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
-->
<module name = "Checker">
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
<property name="fileExtensions" value="java, properties, xml"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter">
<property name="eachLine" value="true"/>
</module>
<module name="LineLength">
<property name="max" value="100"/>
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
</module>
<module name="TreeWalker">
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
</module>
<module name="AvoidEscapedUnicodeCharacters">
<property name="allowEscapesForControlCharacters" value="true"/>
<property name="allowByTailComment" value="true"/>
<property name="allowNonPrintableEscapes" value="true"/>
</module>
<module name="AvoidStarImport"/>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<module name="OneTopLevelClass"/>
<module name="NoLineWrap"/>
<module name="EmptyBlock">
<property name="option" value="TEXT"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
</module>
<module name="NeedBraces"/>
<module name="LeftCurly"/>
<module name="RightCurly">
<property name="id" value="RightCurlySame"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/>
</module>
<module name="RightCurly">
<property name="id" value="RightCurlyAlone"/>
<property name="option" value="alone"/>
<property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
</module>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyTypes" value="true"/>
<property name="allowEmptyLoops" value="true"/>
<message key="ws.notFollowed"
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
<message key="ws.notPreceded"
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
</module>
<module name="OneStatementPerLine"/>
<module name="MultipleVariableDeclarations"/>
<module name="ArrayTypeStyle"/>
<module name="MissingSwitchDefault"/>
<module name="FallThrough"/>
<module name="UpperEll"/>
<module name="ModifierOrder"/>
<module name="EmptyLineSeparator">
<property name="allowNoEmptyLineBetweenFields" value="true"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapDot"/>
<property name="tokens" value="DOT"/>
<property name="option" value="nl"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapComma"/>
<property name="tokens" value="COMMA"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
<property name="id" value="SeparatorWrapEllipsis"/>
<property name="tokens" value="ELLIPSIS"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
<property name="id" value="SeparatorWrapArrayDeclarator"/>
<property name="tokens" value="ARRAY_DECLARATOR"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapMethodRef"/>
<property name="tokens" value="METHOD_REF"/>
<property name="option" value="nl"/>
</module>
<module name="PackageName">
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
<message key="name.invalidPattern"
value="Package name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="TypeName">
<message key="name.invalidPattern"
value="Type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MemberName">
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
<message key="name.invalidPattern"
value="Member name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern"
value="Parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="CatchParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern"
value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="LocalVariableName">
<property name="tokens" value="VARIABLE_DEF"/>
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<property name="allowOneCharVarInForLoop" value="true"/>
<message key="name.invalidPattern"
value="Local variable name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ClassTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Class type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Method type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="InterfaceTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Interface type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="NoFinalizer"/>
<module name="GenericWhitespace">
<message key="ws.followed"
value="GenericWhitespace ''{0}'' is followed by whitespace."/>
<message key="ws.preceded"
value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
<message key="ws.illegalFollow"
value="GenericWhitespace ''{0}'' should followed by whitespace."/>
<message key="ws.notPreceded"
value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
</module>
<module name="Indentation">
<property name="basicOffset" value="2"/>
<property name="braceAdjustment" value="0"/>
<property name="caseIndent" value="2"/>
<property name="throwsIndent" value="4"/>
<property name="lineWrappingIndentation" value="4"/>
<property name="arrayInitIndent" value="2"/>
</module>
<module name="AbbreviationAsWordInName">
<property name="ignoreFinal" value="false"/>
<property name="allowedAbbreviationLength" value="1"/>
<property name="allowedAbbreviations" value="IT"/>
</module>
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="true"/>
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
</module>
<module name="MethodParamPad"/>
<module name="NoWhitespaceBefore">
<property name="tokens"
value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
<property name="allowLineBreaks" value="true"/>
</module>
<module name="ParenPad"/>
<module name="OperatorWrap">
<property name="option" value="NL"/>
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationMostCases"/>
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationVariables"/>
<property name="tokens" value="VARIABLE_DEF"/>
<property name="allowSamelineMultipleAnnotations" value="true"/>
</module>
<module name="NonEmptyAtclauseDescription"/>
<module name="JavadocTagContinuationIndentation"/>
<module name="SummaryJavadoc">
<property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
</module>
<module name="JavadocParagraph"/>
<module name="AtclauseOrder">
<property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
<property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
</module>
<module name="JavadocMethod">
<property name="scope" value="public"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="allowedAnnotations" value="Override, Test"/>
</module>
<module name="MissingJavadocMethod">
<property name="scope" value="public"/>
<property name="minLineCount" value="2"/>
<property name="allowedAnnotations" value="Override, Test"/>
</module>
<module name="MethodName">
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
<message key="name.invalidPattern"
value="Method name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="SingleLineJavadoc">
<property name="ignoreInlineTags" value="false"/>
</module>
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
<!-- Required by SuppressWarningsFilter below -->
<module name="SuppressWarningsHolder"/>
<!-- Allow turning checks off in the code -->
<!-- See http://checkstyle.sourceforge.net/config.html#SuppressionCommentFilter -->
<module name="SuppressionCommentFilter"/>
</module>
<!-- Allow turning checks off in the code using @SuppressWarnings annotation -->
<!-- See http://checkstyle.sourceforge.net/config_filters.html#SuppressWarningsFilter -->
<module name="SuppressWarningsFilter"/>
<!-- Location is overridable in a project. For mechanism, see -->
<!-- http://checkstyle.sourceforge.net/config_filters.html#SuppressionFilter -->
<module name="SuppressionFilter">
<property name="file" value="suppressions.xml"/>
<property name="optional" value="true"/>
</module>
</module>
================================================
FILE: data/iris-model/build.sbt
================================================
inThisBuild(scalaVersion := "2.11.12")
lazy val root = project
.in(file("."))
.settings(
libraryDependencies := Seq(
"com.spotify" %% "scio-core" % "0.5.7",
"com.spotify" %% "scio-bigquery" % "0.5.7",
"com.spotify" %% "scio-tensorflow" % "0.5.7",
"com.spotify" %% "scio-test" % "0.5.7" % Test,
"org.scalatest" %% "scalatest" % "3.0.5" % Test
),
addCompilerPlugin(
"org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full
)
)
================================================
FILE: data/iris-model/project/build.properties
================================================
sbt.version=1.3.6
================================================
FILE: data/iris-model/scripts/publish_data.sh
================================================
#!/bin/bash
set -o nounset
set -o errexit
set -o pipefail
DEST_TABLE=${1:-"data-integration-test:zoltar.iris"}
TEMP_DIR=$(mktemp -d)
wget --directory-prefix=$TEMP_DIR https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data
sed '$d' "$TEMP_DIR/iris.data" > "$TEMP_DIR/iris_clean.data"
bq load $DEST_TABLE "$TEMP_DIR/iris_clean.data" sepal_length:float,sepal_width:float,petal_length:float,petal_width:float,class_name:string
================================================
FILE: data/iris-model/src/main/scala/com/spotify/zoltar/Iris.scala
================================================
/*
* Copyright 2018 Spotify AB.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.spotify.zoltar
import com.spotify.featran._
import com.spotify.featran.scio._
import com.spotify.featran.tensorflow._
import com.spotify.featran.transformers._
import com.spotify.scio._
import com.spotify.scio.bigquery._
import com.spotify.scio.tensorflow._
import org.tensorflow.example.{Example => TFExample}
object IrisFeaturesSpec {
@BigQueryType.fromTable("data-integration-test:zoltar.iris")
class Record
case class Iris(sepalLength: Option[Double],
sepalWidth: Option[Double],
petalLength: Option[Double],
petalWidth: Option[Double],
className: Option[String])
object Iris {
def apply(record: Record): Iris =
Iris(record.petal_length,
record.petal_width,
record.sepal_length,
record.sepal_width,
record.class_name)
}
val irisFeaturesSpec: FeatureSpec[Iris] = FeatureSpec
.of[Iris]
.optional(_.petalLength)(StandardScaler("petal_length", withMean = true))
.optional(_.petalWidth)(StandardScaler("petal_width", withMean = true))
.optional(_.sepalLength)(StandardScaler("sepal_length", withMean = true))
.optional(_.sepalWidth)(StandardScaler("sepal_width", withMean = true))
val irisLabelSpec: FeatureSpec[Iris] = FeatureSpec
.of[Iris]
.optional(_.className)(OneHotEncoder("class_name"))
val irisSpec = MultiFeatureSpec(irisFeaturesSpec, irisLabelSpec)
}
object IrisFeaturesJob {
def main(cmdLineArgs: Array[String]): Unit = {
import IrisFeaturesSpec._
val (sc, args) = ContextAndArgs(cmdLineArgs)
val data = sc.typedBigQuery[Record]().map(Iris(_))
val extractedFeatures = irisSpec.extract(data)
val (train, test) = extractedFeatures
.featureValues[TFExample]
.randomSplit(.9)
extractedFeatures.featureSettings.saveAsTextFile(
args("output") + "/settings")
train.saveAsTfExampleFile(args("output") + "/train", extractedFeatures)
test.saveAsTfExampleFile(args("output") + "/eval", extractedFeatures)
sc.close()
}
}
================================================
FILE: data/iris-model/src/test/resources/expected_feature_spec.json
================================================
{"version":1,"features":[{"name":"petal_length","kind":"FloatList","tags":{"multispec-id":"0"}},{"name":"petal_width","kind":"FloatList","tags":{"multispec-id":"0"}},{"name":"sepal_length","kind":"FloatList","tags":{"multispec-id":"0"}},{"name":"sepal_width","kind":"FloatList","tags":{"multispec-id":"0"}},{"name":"class_name_iris_setosa","kind":"FloatList","tags":{"multispec-id":"1"}},{"name":"class_name_iris_versicolor","kind":"FloatList","tags":{"multispec-id":"1"}},{"name":"class_name_iris_virginica","kind":"FloatList","tags":{"multispec-id":"1"}}],"compression":"UNCOMPRESSED"}
================================================
FILE: data/iris-model/src/test/scala/com/spotify/zoltar/IrisTest.scala
================================================
/*
* Copyright 2018 Spotify AB.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.spotify.zoltar
import java.nio.charset.Charset
import java.util.concurrent.ThreadLocalRandom
import com.spotify.scio.tensorflow.TFExampleIO
import com.spotify.scio.testing._
import com.spotify.zoltar.IrisFeaturesSpec.Record
import org.apache.commons.io.IOUtils
import org.tensorflow.example.Example
class IrisTest extends PipelineSpec {
private val classes = Seq("iris-setosa", "iris-virginica", "iris-versicolor")
private def rndD = ThreadLocalRandom.current().nextDouble(42.0D)
private val input = (1 to 1000).par
.map(
_ =>
Record(
Some(rndD),
Some(rndD),
Some(rndD),
Some(rndD),
Some(classes(ThreadLocalRandom.current().nextInt(3)))
)
)
.seq
// scalastyle:off line.size.limit
private val expectedFeatureSpec = IOUtils.toString(
this.getClass.getResourceAsStream("/expected_feature_spec.json"),
Charset.defaultCharset()
)
"IrisJob" should "work" in {
JobTest[IrisFeaturesJob.type]
.args("--output=out")
.input(BigQueryIO[Record](Record.table), input)
.output(TextIO("out/train/_tf_record_spec.json"))(
_ should containSingleValue(expectedFeatureSpec)
)
.output(TextIO("out/eval/_tf_record_spec.json"))(
_ should containSingleValue(expectedFeatureSpec)
)
.output(TFExampleIO("out/train"))(
_ should satisfy[Example](_.size === 900 +- 50)
)
.output(TFExampleIO("out/eval"))(
_ should satisfy[Example](_.size === 100 +- 50)
)
.output(TextIO("out/settings"))(_ should haveSize(1))
.run()
// scalastyle:on line.size.limit
}
}
================================================
FILE: data/iris-model/training/Makefile
================================================
SHELL := /bin/bash
ID := $(shell date +%Y-%m-%d--%H-%M-%S)
USERNAME ?= $(shell whoami)
JOB_DIR ?= gs://data-integration-test-us/zoltar/iris/trained/$(USERNAME)/$(ID)
TRAINING_SET ?= gs://data-integration-test-us/zoltar/iris
EXTRA_FLAGS ?=
IRIS_PYTHON_PKG := iris/zoltar_model_train.egg-info
.PHONY: help clean iris-tensorflow iris-xgboost
.DEFAULT_GOAL := help
help:
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[32m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
$(IRIS_PYTHON_PKG):
cd iris; pip install -r requirements.txt
clean:
@rm -r $(IRIS_PYTHON_PKG)
iris-tensorflow: $(IRIS_PYTHON_PKG) ## Train iris tensorflow model
iris-tensorflow \
--job_dir=$(JOB_DIR) \
--training_set=$(TRAINING_SET) \
--batch_size=12 \
$(EXTRA_FLAGS)
iris-xgboost: $(IRIS_PYTHON_PKG) ## Train iris xgboost model
iris-xgboost \
--job_dir=$(JOB_DIR) \
--training_set=$(TRAINING_SET) \
--batch_size=12 \
$(EXTRA_FLAGS)
================================================
FILE: data/iris-model/training/README.md
================================================
# Training
## Iris
```bash
make
iris-tensorflow Train iris tensorflow model
iris-xgboost Train iris xgboost model
```
================================================
FILE: data/iris-model/training/iris/iris/__init__.py
================================================
================================================
FILE: data/iris-model/training/iris/iris/tensorflow.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2018 Spotify AB.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
from __future__ import absolute_import
import tensorflow as tf
from os.path import join as pjoin
from spotify_tensorflow import Datasets, Trainer
FLAGS = tf.flags.FLAGS
tf.logging.set_verbosity(tf.logging.INFO)
def train(_):
config = Trainer.get_default_run_config()
training_data_dir = pjoin(FLAGS.training_set, FLAGS.train_subdir)
feature_context = Datasets.get_context(training_data_dir)
(feature_names, label_names) = feature_context.multispec_feature_groups
features = [tf.feature_column.numeric_column(x) for x in feature_names]
def split_features_label_fn(spec):
# Canned TF's LinearClassifier requires label to be a single integer, Featran gives us
# one hot encoding for class, thus we need to convert one hot encoding to single integer
labels = tf.concat([[spec.pop(l)] for l in label_names], axis=0)
label = tf.argmax(labels, axis=0)
# Get the rest of the features out of the spec
split_features = {k: spec[k] for k in spec.viewkeys() - set(label_names)}
return split_features, label
classifier = tf.estimator.LinearClassifier(feature_columns=features,
n_classes=3,
config=config)
Trainer.run(estimator=classifier,
split_features_label_fn=split_features_label_fn,
run_config=config)
input_recv = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_context.features)
classifier.export_savedmodel(FLAGS.job_dir + "/export", input_recv)
def main():
tf.app.run(main=train)
if __name__ == "__main__":
main()
================================================
FILE: data/iris-model/training/iris/iris/xgboost.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2018 Spotify AB.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
from __future__ import absolute_import
import logging
import numpy as np
import tensorflow as tf
import xgboost as xgb
from collections import defaultdict
from os.path import join as pjoin
from spotify_tensorflow.dataset import Datasets
from tensorflow.python.lib.io import file_io
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_integer('rounds', 50, 'Number of Rounds')
flags.DEFINE_string('local_dir', "/tmp/features", 'GCS Train Path')
def transform_dataset(ctx, dataset):
(feature_names, label_names) = ctx.multispec_feature_groups
data = defaultdict(list)
for key, values in dataset.iteritems():
if key in feature_names:
data["features"].append(values)
else:
data["labels"].append(values)
return zip(*data["features"]), np.argmax(zip(*data["labels"]), axis=1)
def train(_):
training_dir = pjoin(FLAGS.training_set, FLAGS.train_subdir)
feature_context = Datasets.get_context(training_dir)
(feature_names, label_names) = feature_context.multispec_feature_groups
training_dataset = Datasets.dict.read_dataset(training_dir)
(feature_train_data, labels_train_data) = transform_dataset(feature_context,
training_dataset)
params = {
'objective': 'multi:softprob',
'verbose': False,
'num_class': len(label_names),
'max_depth': 6,
'nthread': 4,
'silent': 1
}
xg_train = xgb.DMatrix(feature_train_data,
label=labels_train_data)
xg_model = xgb.train(params, xg_train, FLAGS.rounds)
model_path = pjoin(FLAGS.local_dir, "iterator.model")
xg_model.save_model(model_path)
output_path = pjoin(FLAGS.training_set, "xgboost/iterator.model")
file_io.copy(model_path, output_path, overwrite=True)
def main():
tf.app.run(main=train)
if __name__ == "__main__":
main()
================================================
FILE: data/iris-model/training/iris/requirements.txt
================================================
# dependencies are read from setup.py.
-e .
================================================
FILE: data/iris-model/training/iris/setup.cfg
================================================
[metadata]
name = zoltar_model_train
long_description = file: README.md
url = https://github.com/spotify/zoltar
license = Apache-2
requires-python = >=2.7
classifier =
Development Status :: 3 - Alpha
Intended Audience :: Developers
Intended Audience :: Information Technology
License :: OSI Approved :: Apache Software License
Operating System :: OS Independent
Programming Language :: Python
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.2
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
keywords =
tensorflow
xgboost
[options]
packages = find:
[global]
setup-hooks =
pbr.hooks.setup_hook
[flake8]
max-line-length = 100
format = pylint
statistics = 1
show-source = 1
exclude = build,.git,.circleci,.coverage,.eggs,.gitignore,.idea,coverage,dist,*.egg*,venv
inline-quotes = "
multiline-quotes = "
accept-encodings = utf-8
import-order-style = pep8
[nosetests]
with-coverage = 1
cover-package = find:
cover-inclusive = 1
cover-xml = 1
cover-xml-file = coverage/coverage.xml
================================================
FILE: data/iris-model/training/iris/setup.py
================================================
from setuptools import setup
setup(
install_requires=[
"numpy==1.14.0",
"tensorflow==1.15.2",
"xgboost==0.6a2",
"spotify-tensorflow==0.2.11"
],
dependency_links=[
"https://pypi.spotify.net/spotify/production"
],
entry_points={
'console_scripts': [
'iris-tensorflow = iris.tensorflow:main',
'iris-xgboost = iris.xgboost:main'
]
}
)
================================================
FILE: data/iris-model/training/iris/test-requirements.txt
================================================
coverage
flake8>=3.4.1
flake8-quotes>=0.12.0
flake8-coding>=1.3.0
flake8-import-order>=0.15
nose>=1.3.7
================================================
FILE: docs/src/paradox/_template/javadoc.st
================================================
<!DOCTYPE html>
<html>
<head>
<!-- HTML meta refresh URL redirection -->
<meta http-equiv="refresh" content="0; url=apidocs/index.html">
</head>
<body>
<p>The page has moved to: <a href="apidocs/index.html">this page</a></p>
</body>
</html>
================================================
FILE: docs/src/paradox/getting-started.md
================================================
# Getting Started
To start using `Zoltar` you need to add this dependency to your project.
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-api"
version="$project.version$"
}
@@@ note {.help title="Using Maven?" }
You might want to include as it might help you resolve some dependency conflicts.
Maven
: @@snip [pom.xml](../../../examples/apollo-service-example/pom.xml) { #bom_example }
@@@
See @ref:[Modules](modules/index.md) for extra integrations.
================================================
FILE: docs/src/paradox/index.md
================================================
# Zoltar
[](https://travis-ci.org/spotify/zoltar)
[](https://codecov.io/github/spotify/zoltar?branch=master)
[](https://search.maven.org/#search%7Cga%7C1%7Ccom.spotify.zoltar)
[](./LICENSE)
Common library for serving TensorFlow and XGBoost models in production.
Zoltar is a library that helps load predictive machine learning models in a JVM. It provides several key abstractions which you can implement to help your service load a serialized model, featurize input data, submit feature vectors to the model, and serve the model's predictions. Below is a quick overview of these abstractions which can be found in @github[zoltar-core](../../../zoltar-core/src/main/java/com/spotify/zoltar).
## Features
* TensorFlow model loading and prediction.
* XGBoost model loading and prediction.
* Filesystems:
* Local
* Google Cloud Storage
* [Featran](https://github.com/spotify/featran) featurization library integration.
* Easy integration with [Apollo](https://github.com/spotify/apollo) services.
* Easy integration with [Semantic Metrics](https://github.com/spotify/semantic-metrics).
## Abstractions
@javadoc[Predictor](com.spotify.zoltar.Predictor):
The core functionality of Zoltar. This object loads a model and calls functions to featurize input vectors and submit them for prediction. To these ends, a predictor is composed of a ModelLoader, FeatureExtractor, and PredictFn.
@javadoc[PredictFn](com.spotify.zoltar.PredictFns):
A function that submits a feature vector to a model for prediction.
@javadoc[Prediction](com.spotify.zoltar.Prediction):
A wrapper around a single feature vector and its predicted output.
@javadoc[ModelLoader](com.spotify.zoltar.ModelLoader):
An object that loads an XGBoost or TensorFlow model from a supported filesystem.
@javadoc[Model](com.spotify.zoltar.Model):
The Java object that houses the predictive model itself.
@javadoc[FeatureExtractor](com.spotify.zoltar.FeatureExtractor):
Takes an input vector and applies a FeatureExtractFn to it.
@javadoc[FeatureExtractFn](com.spotify.zoltar.FeatureExtractFns):
A function that takes a raw input vector and extracts a feature vector from it.
For more details, take a look at the source code and follow the documentation in the comments. If you're using TensorFlow or XGBoost, you can find model specific implementations of these abstractions in @javadoc[zoltar-tensorflow](com.spotify.zoltar.tf.package-summary) and @javadoc[zoltar-xgboost](com.spotify.zoltar.xgboost.package-summary) respectively.
We've also provided an @github[example](../../../examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo) service built on the [Apollo](https://github.com/spotify/apollo) framework. It demonstrates how Zoltar components are implemented and organized. We highly recommend you walk through this example before you try integrating Zoltar into your applications.
@@@index
* [Getting Started](getting-started.md)
* [Modules](modules/index.md)
* [Release](release.md)
* [License](license.md)
* [Javadoc](javadoc.md)
@@@
================================================
FILE: docs/src/paradox/javadoc.md
================================================
---
layout: javadoc
---
# Api Documentation
================================================
FILE: docs/src/paradox/license.md
================================================
# License
Copyright 2018 Spotify AB.
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
================================================
FILE: docs/src/paradox/modules/featran.md
================================================
# Featran
`Featran` dependency is not included by default. To use it you need to include an extra dependency.
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-featran"
version="$project.version$"
}
================================================
FILE: docs/src/paradox/modules/index.md
================================================
# Modules
@@@ index
* [TensorFlow](tensorflow.md)
* [XGBoost](xgboost.md)
* [Featran](featran.md)
* [Metrics](metrics.md)
* [Google Cloud ML Engine](mlengine.md)
@@@
@@ toc { depth=2 }
================================================
FILE: docs/src/paradox/modules/metrics.md
================================================
# Metrics
`Zoltar` provides a way to attach and extend metrics around @javadoc[Predictor](com.spotify.zoltar.Predictor).
`zoltar-metrics` takes advantage of the [semantic-metrics](https://github.com/spotify/semantic-metrics)
to be able to capture meaningful metrics.
## Getting Started
Add `zoltar-metrics` dependency to your project:
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-metrics"
version="$project.version$"
}
Create a semantic registry:
@@snip [SemanticMetricRegistry](../../../../examples/custom-metrics/src/test/java/com/spotify/zoltar/examples/metrics/CustomMetricsExampleTest.java) { #SemanticMetricRegistry }
Attach it to the @javadoc[PredictorBuilder](com.spotify.zoltar.PredictorBuilder):
@@snip [PredictorMetrics](../../../../examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/CustomMetricsExample.java) { #PredictorMetrics }
@@snip [PredictorBuilderWithMetrics](../../../../examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/CustomMetricsExample.java) { #PredictorBuilderWithMetrics }
## Example
Follow this @github[example](../../../../examples/custom-metrics) to see how you can create
new set of metrics and attach them.
================================================
FILE: docs/src/paradox/modules/mlengine.md
================================================
# Google Cloud ML Engine
## Getting started
To use `Google Cloud ML Engine`, you need to deploy your model first!
Deployment might depend on the type of model you are using:
* [TensorFlow](https://cloud.google.com/ml-engine/docs/tensorflow/deploying-models).
* [scikit-learn](https://cloud.google.com/ml-engine/docs/scikit/quickstart#deploy_models_and_versions) or [XGBoost](https://cloud.google.com/ml-engine/docs/scikit/quickstart#deploy_models_and_versions).
## Usage
Include the following dependency:
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-mlengine"
version="$project.version$"
}
Replace your existing @javadoc[ModelLoader](com.spotify.zoltar.ModelLoader) with
@javadoc[MlEngineLoader](com.spotify.zoltar.mlengine.MlEngineLoader)
@@snip [MlEnginePredictorExample.java](../../../../examples/mlengine-example/src/main/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExample.java) { #MlEngineLoader }
## Example
Follow this @github[example](../../../../examples/mlengine-example)
to create a @javadoc[Predictor](com.spotify.zoltar.Predictor) that uses `Google Cloud ML Engine` deployed models.
================================================
FILE: docs/src/paradox/modules/tensorflow.md
================================================
# TensorFlow
`TensorFlow` dependency is not included by default. To use it you need to include an extra dependency.
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-tensorflow"
version="$project.version$"
}
================================================
FILE: docs/src/paradox/modules/xgboost.md
================================================
# XGBoost
`XGBoost` dependency is not included by default. To use it you need to include an extra dependency.
@@dependency[Maven,Gradle,sbt] {
group="com.spotify"
artifact="zoltar-xgboost"
version="$project.version$"
}
================================================
FILE: docs/src/paradox/release.md
================================================
# Release Instructions
These instructions are based on the [instructions](http://central.sonatype.org/pages/ossrh-guide.html)
for deploying to the Central Repository using [Maven](http://central.sonatype.org/pages/apache-maven.html).
Note: this section only applies for internal Spotify developers.
You will need the following:
- Sign up for a Sonatype account [here](https://issues.sonatype.org/secure/Signup!default.jspa)
- Ask for permissions to push to com.spotify domain like in this [ticket](https://issues.sonatype.org/browse/OSSRH-20689)
- [GPG set up on the machine you're deploying from](http://central.sonatype.org/pages/working-with-pgp-signatures.html)
Once you've got that in place, you should be able to do deployment using the following commands:
```
# setup credentials
export SONATYPE_USERNAME=<your Sonatype username>
export SONATYPE_PASSWORD=<your Sonatype password>
# deploy snapshot version (to test signing)
mvn clean deploy -Prelease --settings settings.xml
# make and deploy a release
mvn release:clean release:prepare release:perform -Prelease --settings settings.xml
```
Then update https://github.com/spotify/zoltar/releases with release notes!
================================================
FILE: examples/apollo-service-example/SERVICE.marker
================================================
(This marker file causes Maven to build this artifact as a service)
================================================
FILE: examples/apollo-service-example/findbugsexclude.xml
================================================
<?xml version="1.0"?>
<FindBugsFilter>
<Match>
<!--
<Or> This element combines Match clauses as disjuncts. I.e., you can put two Method elements in an Or clause in order to match either method.
You can specify a package name as a string.
<Package name="com.spotify.ml-serving-example.path.to.eg.data.objects"/>
Or multiple packages as a regex. The below filter will match any package containing the substring "proto".
<Package name="~.*\.proto"/>
See more findbugs filter methods: http://findbugs.sourceforge.net/manual/filter.html
</Or>
-->
</Match>
</FindBugsFilter>
================================================
FILE: examples/apollo-service-example/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8" ?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.5.7-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>apollo-service-example</artifactId>
<properties>
<service.mainClass>com.spotify.zoltar.examples.apollo.App</service.mainClass>
<findbugs.excludeFilterFile>findbugsexclude.xml</findbugs.excludeFilterFile>
<maven.install.skip>true</maven.install.skip>
<checkstyle.violationSeverity>warning</checkstyle.violationSeverity>
<jacoco.skip>true</jacoco.skip>
</properties>
<dependencyManagement>
<dependencies>
<!-- #bom_example -->
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.5.7-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- #bom_example -->
<dependency>
<groupId>com.spotify</groupId>
<artifactId>apollo-bom</artifactId>
<version>1.15.7</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.10.5</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>apollo-http-service</artifactId>
<exclusions>
<exclusion>
<artifactId>auto-value-annotations</artifactId>
<groupId>com.google.auto.value</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-api</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-core</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-featran</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-metrics</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tests</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tensorflow</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>com.spotify</groupId>
<artifactId>apollo-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>auto-value-annotations</artifactId>
<groupId>com.google.auto.value</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.4</version>
<executions>
<execution>
<id>create-revision</id>
<phase>validate</phase>
<goals>
<goal>create</goal>
</goals>
<configuration>
<getRevisionOnlyOnce>true</getRevisionOnlyOnce>
<doCheck>false</doCheck>
<doUpdate>false</doUpdate>
<shortRevisionLength>7</shortRevisionLength>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>service</id>
<activation>
<file>
<exists>SERVICE.marker</exists>
</file>
</activation>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.codehaus.gmaven</groupId>
<artifactId>groovy-maven-plugin</artifactId>
<version>2.0</version>
<executions>
<execution>
<id>root-pom-service-build</id>
<phase>prepare-package</phase>
<goals>
<goal>execute</goal>
</goals>
<configuration>
<source>
log.info(' _____________________________________________ ')
log.info('/\\ \\ ')
log.info('\\_| == SERVICE marker file detected ======= | ')
log.info(' | | ')
log.info(' | -> Assuming service module | ')
log.info(' | -> Dependencies in target/lib | ')
log.info(' | -> Runnable jar with classpath manifest | ')
log.info(' | -> Docker image being built and pushed | ')
log.info(' | _________________________________________|_')
log.info(' \\_/__________________________________________/')
</source>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>default</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
<configuration>
<useBaseVersion>false</useBaseVersion>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>true</overWriteSnapshots>
<includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<executions>
<execution>
<id>create-timestamp</id>
<phase>validate</phase>
<goals>
<goal>create-timestamp</goal>
</goals>
<configuration>
<timezone>UTC</timezone>
<timestampFormat>yyyyMMdd'T'HHmmss</timestampFormat>
<timestampPropertyName>buildTimestamp</timestampPropertyName>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>default-jar</id>
<phase>prepare-package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<archive>
<addMavenDescriptor>true</addMavenDescriptor>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>${service.mainClass}</mainClass>
</manifest>
<manifestEntries>
<Implementation-Version>${project.version}-${buildTimestamp}-${buildNumber}</Implementation-Version>
</manifestEntries>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
================================================
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/App.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.apollo;
import java.io.IOException;
import java.util.stream.Stream;
import okio.ByteString;
import com.typesafe.config.Config;
import com.spotify.apollo.Environment;
import com.spotify.apollo.Response;
import com.spotify.apollo.core.Service;
import com.spotify.apollo.httpservice.HttpService;
import com.spotify.apollo.httpservice.LoadingException;
import com.spotify.apollo.route.AsyncHandler;
import com.spotify.apollo.route.Route;
import com.spotify.metrics.core.MetricId;
import com.spotify.metrics.core.SemanticMetricRegistry;
import com.spotify.metrics.ffwd.FastForwardReporter;
import com.spotify.zoltar.IrisFeaturesSpec.Iris;
import com.spotify.zoltar.Predictor;
import com.spotify.zoltar.metrics.PredictorMetrics;
import com.spotify.zoltar.metrics.semantic.SemanticPredictorMetrics;
/** Application entry point. */
public class App {
private static final String SERVICE_NAME = "zoltar-example";
private App() {}
static void configure(final Environment environment) {
final Config config = environment.config();
final SemanticMetricRegistry metricRegistry = environment.resolve(SemanticMetricRegistry.class);
final MetricId serviceMetricId = MetricId.build().tagged("service", SERVICE_NAME);
final PredictorMetrics metrics =
SemanticPredictorMetrics.create(metricRegistry, serviceMetricId);
try {
// Optional: check out https://github.com/spotify/semantic-metrics#provided-plugins
final FastForwardReporter reporter = FastForwardReporter.forRegistry(metricRegistry).build();
reporter.start();
} catch (final IOException e) {
throw new RuntimeException(e.getMessage());
}
final Predictor<Iris, Long> predictor;
try {
final ModelConfig irisModelConfig = ModelConfig.from(config.getConfig("iris"));
predictor = IrisPredictor.create(irisModelConfig, metrics);
} catch (final Exception e) {
throw new RuntimeException("Could not load model with config");
}
final IrisPredictionHandler irisPredictionHandler = IrisPredictionHandler.create(predictor);
final Stream<Route<AsyncHandler<Response<ByteString>>>> routes =
irisPredictionHandler.routes().map(r -> r.withPrefix("/v1"));
environment.routingEngine().registerRoutes(routes);
}
/**
* Runs the app locally.
*
* <p>$ curl http://localhost:8080/v1/predict/5.8-2.7-5.1-1.9 Lengths seperated by "-"
*/
public static void main(final String... args) throws LoadingException {
final Service service = HttpService.usingAppInit(App::configure, SERVICE_NAME).build();
HttpService.boot(service, args);
}
}
================================================
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandler.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.apollo;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Stream;
import okio.ByteString;
import scala.Option;
import com.google.common.collect.ImmutableMap;
import com.spotify.apollo.RequestContext;
import com.spotify.apollo.Response;
import com.spotify.apollo.Status;
import com.spotify.apollo.route.AsyncHandler;
import com.spotify.apollo.route.Middlewares;
import com.spotify.apollo.route.Route;
import com.spotify.zoltar.IrisFeaturesSpec.Iris;
import com.spotify.zoltar.Prediction;
import com.spotify.zoltar.Predictor;
/** Route endpoints. */
final class IrisPredictionHandler {
private static final Map<Long, String> idToClass =
ImmutableMap.of(
0L, "Iris-setosa",
1L, "Iris-versicolor",
2L, "Iris-virginica");
private IrisPredictionHandler(final Predictor<Iris, Long> predictor) {
this.predictor = predictor;
}
private final Predictor<Iris, Long> predictor;
static IrisPredictionHandler create(final Predictor<Iris, Long> predictor) {
return new IrisPredictionHandler(predictor);
}
Stream<Route<AsyncHandler<Response<ByteString>>>> routes() {
final Stream<Route<AsyncHandler<Response<ByteString>>>> routes =
Stream.of(
Route.async("GET", "/predict/<features>", this::predict)
.withDocString("Prediction handler", "Predicts the type of the iris"));
return routes.map(r -> r.withMiddleware(Middlewares.apolloDefaults()));
}
/**
* Prediction endpoint. Takes a request in a create of a String containing iris features `-`
* separated, and returns a response in a form of a predicted iris class.
*/
CompletionStage<Response<ByteString>> predict(final RequestContext context) {
return Optional.ofNullable(context.pathArgs().get("features"))
.map(f -> f.split("-"))
.filter(features -> features.length == 4)
.map(this::predict)
.map(p -> p.thenApply(ByteString::encodeUtf8))
.map(p -> p.thenApply(Response::forPayload))
.orElse(CompletableFuture.completedFuture(Response.forStatus(Status.BAD_REQUEST)));
}
/**
* Prediction endpoint. Takes a request in a create of a String containing iris features `-`
* separated, and returns a response in a form of a predicted iris class.
*/
CompletionStage<String> predict(final String[] features) {
final Iris featureData =
new Iris(
Option.apply(Double.parseDouble(features[0])),
Option.apply(Double.parseDouble(features[1])),
Option.apply(Double.parseDouble(features[2])),
Option.apply(Double.parseDouble(features[3])),
Option.empty());
return predictor
.predict(featureData)
.thenApply(
ps -> {
return ps.stream()
.findFirst()
.map(Prediction::value)
.map(idToClass::get)
.orElseThrow(() -> new RuntimeException("we expect a prediction"));
});
}
}
================================================
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictor.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.apollo;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.stream.Collectors;
import org.tensorflow.example.Example;
import com.spotify.featran.FeatureSpec;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.IrisFeaturesSpec;
import com.spotify.zoltar.IrisFeaturesSpec.Iris;
import com.spotify.zoltar.Predictor;
import com.spotify.zoltar.Predictors;
import com.spotify.zoltar.featran.FeatranExtractFns;
import com.spotify.zoltar.metrics.PredictorMetrics;
/** Iris prediction meat and potatoes. */
public final class IrisPredictor {
/** Configure Iris prediction, should be called at the service startup/configuration stage. */
public static Predictor<Iris, Long> create(
final ModelConfig modelConfig, final PredictorMetrics metrics) throws IOException {
final FeatureSpec<Iris> irisFeatureSpec = IrisFeaturesSpec.irisFeaturesSpec();
final String settings = new String(Files.readAllBytes(Paths.get(modelConfig.settingsUri())));
final ExtractFn<Iris, Example> extractFn = FeatranExtractFns.example(irisFeatureSpec, settings);
final String[] ops = new String[] {"linear/head/predictions/class_ids"};
return Predictors.tensorFlow(
modelConfig.modelUri().toString(),
extractFn,
tensors ->
Arrays.stream(tensors.get(ops[0]).longValue()).boxed().collect(Collectors.toList()),
ops,
metrics);
}
}
================================================
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/ModelConfig.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.apollo;
import java.net.URI;
import java.net.URISyntaxException;
import com.google.auto.value.AutoValue;
import com.typesafe.config.Config;
/** Immutable object that contains model / feature extraction configuration properties. */
@AutoValue
public abstract class ModelConfig {
/** model URI path. */
public abstract URI modelUri();
/** settings URI path. */
public abstract URI settingsUri();
/** Creates a {@link ModelConfig} create a {@link Config}. */
public static ModelConfig from(final Config config) throws URISyntaxException {
URI modelUri = URI.create(config.getString("model"));
if (modelUri.getScheme() == null) {
modelUri = ModelConfig.class.getResource(config.getString("model")).toURI();
}
URI settingsUri = URI.create(config.getString("settings"));
if (settingsUri.getScheme() == null) {
settingsUri = ModelConfig.class.getResource(config.getString("settings")).toURI();
}
return create(modelUri, settingsUri);
}
public static ModelConfig create(final URI modelUri, final URI settingsUri) {
return new AutoValue_ModelConfig(modelUri, settingsUri);
}
}
================================================
FILE: examples/apollo-service-example/src/main/resources/zoltar-example.conf
================================================
# Configuration for ml-serving-example in production mode
# Over-all Apollo settings.
# https://github.com/spotify/apollo/blob/master/apollo-environment/README.md
apollo {
logIncomingRequests: true
logOutgoingRequests: true
}
# HTTP server settings.
http.server {
port: 8080
allowAllHeaders: true
}
# Meta API settings.
_meta {
expose-config: true
}
#Model Configuration
iris {
settings: "gs://data-integration-test-us/zoltar/iris/settings/part-00000-of-00001.txt"
model: "gs://data-integration-test-us/zoltar/iris/trained/regadas/2018-04-16--14-47-55/export/1523904529/"
}
================================================
FILE: examples/apollo-service-example/src/test/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandlerTest.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.apollo;
import static org.hamcrest.core.Is.is;
import okio.ByteString;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import com.spotify.apollo.Response;
import com.spotify.apollo.test.ServiceHelper;
public class IrisPredictionHandlerTest {
@Rule
public final ServiceHelper serviceHelper = ServiceHelper.create(App::configure, "zoltar-example");
@Test
public void shouldPredict() throws Exception {
final Response<ByteString> replyFuture =
serviceHelper.request("GET", "/v1/predict/5.3-2.7-2.0-1.9").toCompletableFuture().get();
final String reply = replyFuture.payload().get().utf8();
Assert.assertThat(reply, is("Iris-versicolor"));
}
}
================================================
FILE: examples/apollo-service-example/src/test/resources/settings.json
================================================
[
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "petal_length",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"petal_length"
],
"aggregators": "3.7586666666666675,1.7585291834055217"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "petal_width",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"petal_width"
],
"aggregators": "1.1986666666666665,0.7606126185881715"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "sepal_length",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"sepal_length"
],
"aggregators": "5.843333333333332,0.8253012917851413"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "sepal_width",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"sepal_width"
],
"aggregators": "3.0540000000000007,0.43214658007054363"
},
{
"cls": "com.spotify.featran.transformers.OneHotEncoder",
"name": "class_name",
"params": {},
"featureNames": [
"class_name_Iris-setosa",
"class_name_Iris-versicolor",
"class_name_Iris-virginica"
],
"aggregators": "label:Iris-setosa,label:Iris-versicolor,label:Iris-virginica"
}
]
================================================
FILE: examples/apollo-service-example/src/test/resources/zoltar-example.conf
================================================
# Configuration for ml-serving-example in production mode
# Over-all Apollo settings.
# https://github.com/spotify/apollo/blob/master/apollo-environment/README.md
apollo {
logIncomingRequests: true
logOutgoingRequests: true
}
# HTTP server settings.
http.server {
port: 8080
allowAllHeaders: true
}
# Meta API settings.
_meta {
expose-config: true
}
#Model Configuration
iris {
settings: "/settings.json"
model: "/trained_model"
}
================================================
FILE: examples/batch-predictor/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.6.1-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>batch-predictor</artifactId>
<properties>
<findbugs.excludeFilterFile>findbugsexclude.xml</findbugs.excludeFilterFile>
<maven.install.skip>true</maven.install.skip>
<checkstyle.violationSeverity>warning</checkstyle.violationSeverity>
<jacoco.skip>true</jacoco.skip>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.6.1-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-api</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tests</artifactId>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/BatchPredictorExample.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.batch;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.ModelLoader;
import com.spotify.zoltar.PredictFns.PredictFn;
import com.spotify.zoltar.Prediction;
import com.spotify.zoltar.Predictor;
import com.spotify.zoltar.PredictorBuilder;
import com.spotify.zoltar.Predictors;
/** Example showing a batch predictor. */
class BatchPredictorExample implements Predictor<Integer, Float> {
private final PredictorBuilder<DummyModel, Integer, Float, Float> predictorBuilder;
BatchPredictorExample() throws InterruptedException, ExecutionException, TimeoutException {
final ModelLoader<DummyModel> modelLoader =
ModelLoader.preload(ModelLoader.loaded(new DummyModel()), Duration.ofMinutes(1));
final ExtractFn<Integer, Float> extractFn = ExtractFn.lift(input -> (float) input / 10);
final PredictFn<DummyModel, Integer, Float, Float> predictFn =
(model, vectors) -> {
return vectors
.stream()
.map(vector -> Prediction.create(vector.input(), vector.value() * 2))
.collect(Collectors.toList());
};
// We build the PredictorBuilder as usual
predictorBuilder = Predictors.newBuilder(modelLoader, extractFn, predictFn);
}
@Override
public CompletionStage<List<Prediction<Integer, Float>>> predict(
final ScheduledExecutorService scheduler, final Duration timeout, final Integer... input) {
return predictorBuilder.predictor().predict(scheduler, timeout, input);
}
}
================================================
FILE: examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/DummyModel.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.batch;
import com.spotify.zoltar.Model;
class DummyModel implements Model<Object> {
@Override
public Id id() {
return Id.create("dummy");
}
@Override
public Object instance() {
return new Object();
}
@Override
public void close() {}
}
================================================
FILE: examples/batch-predictor/src/test/java/com/spotify/zoltar/examples/batch/BatchPredictorExampleTest.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.batch;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.junit.Test;
import com.spotify.zoltar.Prediction;
public class BatchPredictorExampleTest {
@SuppressWarnings("unchecked")
@Test
public void testCustomMetrics()
throws InterruptedException, ExecutionException, TimeoutException {
final BatchPredictorExample example = new BatchPredictorExample();
final Integer[] batch = new Integer[] {3, 1, -4, -42, 42, -10};
final List<Float> predictions =
example
.predict(batch)
.toCompletableFuture()
.join()
.stream()
.map(Prediction::value)
.collect(Collectors.toList());
final List<Float> expected =
Arrays.stream(batch).map(v -> (float) v / 10 * 2).collect(Collectors.toList());
assertThat(predictions, is(expected));
}
}
================================================
FILE: examples/custom-metrics/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.6.1-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>custom-metrics</artifactId>
<properties>
<findbugs.excludeFilterFile>findbugsexclude.xml</findbugs.excludeFilterFile>
<maven.install.skip>true</maven.install.skip>
<checkstyle.violationSeverity>warning</checkstyle.violationSeverity>
<jacoco.skip>true</jacoco.skip>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.6.1-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-api</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-metrics</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tests</artifactId>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/CustomMetricsExample.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.metrics;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.google.auto.value.AutoValue;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.spotify.metrics.core.MetricId;
import com.spotify.metrics.core.SemanticMetricRegistry;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.FeatureExtractor;
import com.spotify.zoltar.Model;
import com.spotify.zoltar.ModelLoader;
import com.spotify.zoltar.PredictFns.PredictFn;
import com.spotify.zoltar.Prediction;
import com.spotify.zoltar.Predictor;
import com.spotify.zoltar.PredictorBuilder;
import com.spotify.zoltar.Predictors;
import com.spotify.zoltar.Vector;
import com.spotify.zoltar.metrics.FeatureExtractorMetrics;
import com.spotify.zoltar.metrics.Instrumentations;
import com.spotify.zoltar.metrics.PredictFnMetrics;
import com.spotify.zoltar.metrics.PredictMetrics;
import com.spotify.zoltar.metrics.PredictorMetrics;
import com.spotify.zoltar.metrics.VectorMetrics;
import com.spotify.zoltar.metrics.semantic.SemanticPredictorMetrics;
/** Example showing how to add custom metrics to a Predictor. */
class CustomMetricsExample implements Predictor<Integer, Float> {
private final PredictorBuilder<DummyModel, Integer, Float, Float> predictorBuilder;
/** Define a class containing all the additional metrics we want to register. */
@AutoValue
abstract static class CustomMetrics {
abstract Counter negativePredictCount();
abstract Counter negativeExtractCount();
static CustomMetrics create(final SemanticMetricRegistry registry, final MetricId metricId) {
final MetricId predictCountId = metricId.tagged("what", "negativePredictCount");
final MetricId extractCountId = metricId.tagged("what", "negativeExtractCount");
final Counter negativePredictCount = registry.counter(predictCountId);
final Counter negativeExtractCount = registry.counter(extractCountId);
return new AutoValue_CustomMetricsExample_CustomMetrics(
negativePredictCount, negativeExtractCount);
}
}
/**
* Define an implementation of PredictorMetrics, which will hold metrics for feature extraction
* and prediction. In this example we don't make use of the model ID.
*/
@AutoValue
abstract static class CustomPredictorMetrics implements PredictorMetrics<Integer, Float, Float> {
abstract LoadingCache<Model.Id, CustomMetrics> metricsCache();
static CustomPredictorMetrics create(
final SemanticMetricRegistry registry, final MetricId metricId) {
final LoadingCache<Model.Id, CustomMetrics> metersCache =
CacheBuilder.newBuilder()
.build(
new CacheLoader<Model.Id, CustomMetrics>() {
@Override
public CustomMetrics load(final Model.Id id) {
return CustomMetrics.create(registry, metricId.tagged("model", id.value()));
}
});
return new AutoValue_CustomMetricsExample_CustomPredictorMetrics(metersCache);
}
@Override
public PredictFnMetrics<Integer, Float> predictFnMetrics() {
return id -> {
final CustomMetrics metrics = metricsCache().getUnchecked(id);
final Counter negativePredictCounter = metrics.negativePredictCount();
return NegativePredictMetrics.create(negativePredictCounter);
};
}
@Override
public FeatureExtractorMetrics<Integer, Float> featureExtractorMetrics() {
return id -> {
final CustomMetrics metrics = metricsCache().getUnchecked(id);
final Counter negativeExtractCounter = metrics.negativeExtractCount();
return NegativeExtractMetrics.create(negativeExtractCounter);
};
}
}
/** To define a metric you want to measure for prediction, implement PredictMetrics. */
@AutoValue
abstract static class NegativePredictMetrics implements PredictMetrics<Integer, Float> {
abstract Counter negativePredictCount();
static NegativePredictMetrics create(final Counter negativeCount) {
return new AutoValue_CustomMetricsExample_NegativePredictMetrics(negativeCount);
}
/** Here, given a list of predictions, we want to measure the number of negatives. */
@Override
public void prediction(final List<Prediction<Integer, Float>> predictions) {
negativePredictCount().inc(predictions.stream().filter(x -> x.value() < 0).count());
}
}
/** To define a metric you want to measure for feature extraction, implement VectorMetrics. */
@AutoValue
abstract static class NegativeExtractMetrics implements VectorMetrics<Integer, Float> {
abstract Counter negativeExtractCount();
static NegativeExtractMetrics create(final Counter negativeCount) {
return new AutoValue_CustomMetricsExample_NegativeExtractMetrics(negativeCount);
}
/**
* Here, given a list of feature extraction results, we want to count the number of negatives.
*/
@Override
public void extraction(final List<Vector<Integer, Float>> vectors) {
negativeExtractCount().inc(vectors.stream().filter(v -> v.value() < 0).count());
}
}
CustomMetricsExample(final SemanticMetricRegistry metricRegistry, final MetricId metricId)
throws InterruptedException, ExecutionException, TimeoutException {
final ModelLoader<DummyModel> modelLoader =
ModelLoader.preload(ModelLoader.loaded(new DummyModel()), Duration.ofMinutes(1));
final ExtractFn<Integer, Float> extractFn = ExtractFn.lift(input -> (float) input / 10);
final PredictFn<DummyModel, Integer, Float, Float> predictFn =
(model, vectors) -> {
return vectors
.stream()
.map(vector -> Prediction.create(vector.input(), vector.value() * 2))
.collect(Collectors.toList());
};
final FeatureExtractor<DummyModel, Integer, Float> featureExtractor =
FeatureExtractor.create(extractFn);
// We build the PredictorBuilder as usual, compose with the built-in metrics, and then compose
// with our custom metrics.
// #PredictorMetrics
final PredictorMetrics<Integer, Float, Float> predictorMetrics =
SemanticPredictorMetrics.create(metricRegistry, metricId);
// #PredictorMetrics
final PredictorMetrics<Integer, Float, Float> customMetrics =
CustomPredictorMetrics.create(metricRegistry, metricId);
predictorBuilder =
// #PredictorBuilderWithMetrics
Predictors.newBuilder(modelLoader, featureExtractor, predictFn, predictorMetrics)
// #PredictorBuilderWithMetrics
.with(Instrumentations.predictor(customMetrics));
}
@Override
public CompletionStage<List<Prediction<Integer, Float>>> predict(
final ScheduledExecutorService scheduler, final Duration timeout, final Integer... input) {
return predictorBuilder.predictor().predict(scheduler, timeout, input);
}
}
================================================
FILE: examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/DummyModel.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.metrics;
import com.spotify.zoltar.Model;
class DummyModel implements Model<Object> {
@Override
public Id id() {
return Id.create("dummy");
}
@Override
public Object instance() {
return new Object();
}
@Override
public void close() {}
}
================================================
FILE: examples/custom-metrics/src/test/java/com/spotify/zoltar/examples/metrics/CustomMetricsExampleTest.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.metrics;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.junit.Assert;
import org.junit.Test;
import com.spotify.metrics.core.MetricId;
import com.spotify.metrics.core.SemanticMetricRegistry;
public class CustomMetricsExampleTest {
@Test
public void testCustomMetrics()
throws InterruptedException, ExecutionException, TimeoutException {
// #SemanticMetricRegistry
final SemanticMetricRegistry registry = new SemanticMetricRegistry();
final MetricId metricId = MetricId.build().tagged("service", "my-application");
// #SemanticMetricRegistry
final CustomMetricsExample example = new CustomMetricsExample(registry, metricId);
example.predict(3, 1, -4, -42, 42, -10).toCompletableFuture().join();
registry.getCounters().values().forEach(counter -> Assert.assertEquals(3, counter.getCount()));
}
}
================================================
FILE: examples/mlengine-example/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.6.1-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>mlengine-example</artifactId>
<properties>
<maven.install.skip>true</maven.install.skip>
<checkstyle.violationSeverity>warning</checkstyle.violationSeverity>
<jacoco.skip>true</jacoco.skip>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.6.1-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-api</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-core</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-featran</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tests</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-mlengine</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>completable-futures</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: examples/mlengine-example/src/main/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExample.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.mlengine;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.tensorflow.proto.example.Example;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
import com.spotify.featran.FeatureSpec;
import com.spotify.futures.CompletableFutures;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.IrisFeaturesSpec;
import com.spotify.zoltar.IrisFeaturesSpec.Iris;
import com.spotify.zoltar.Models;
import com.spotify.zoltar.Prediction;
import com.spotify.zoltar.Predictor;
import com.spotify.zoltar.Predictors;
import com.spotify.zoltar.featran.FeatranExtractFns;
import com.spotify.zoltar.mlengine.MlEngineLoader;
import com.spotify.zoltar.mlengine.MlEnginePredictException;
import com.spotify.zoltar.mlengine.MlEnginePredictFn;
/** Cloud Machine Learning Engine online prediction example. */
public final class MlEnginePredictorExample implements Predictor<Iris, Integer> {
private final Predictor<Iris, Integer> predictor;
private MlEnginePredictorExample(final Predictor<Iris, Integer> predictor) {
this.predictor = predictor;
}
/** Runs a simple Iris prediction against the running Iris ml-engine instance. */
public static MlEnginePredictorExample create(
final String projectId, final String modelId, final String versionId) throws Exception {
// #MlEngineLoader
final MlEngineLoader mlEngineLoader = Models.mlEngine(projectId, modelId, versionId);
// #MlEngineLoader
final FeatureSpec<Iris> irisFeatureSpec = IrisFeaturesSpec.irisFeaturesSpec();
final URI settingsUri = MlEnginePredictorExample.class.getResource("/settings.json").toURI();
final String settings = new String(Files.readAllBytes(Paths.get(settingsUri)));
final ExtractFn<Iris, Example> extractFn = FeatranExtractFns.example(irisFeatureSpec, settings);
final MlEnginePredictFn<Iris, Example, Integer> predictFn =
(model, vectors) -> {
final List<CompletableFuture<Prediction<Iris, Integer>>> predictions =
vectors
.stream()
.map(
vector ->
CompletableFuture.supplyAsync(
() -> {
try {
final List<Example> input =
Collections.singletonList(vector.value());
final MlEnginePrediction result =
model
.predictExamples(input)
.values(MlEnginePrediction.class)
.get(0);
final int max =
IntStream.range(0, result.scores().size())
.reduce(
(i, j) -> {
final BigDecimal ci = result.scores().get(i);
final BigDecimal cj = result.scores().get(j);
return ci.compareTo(cj) > 0 ? i : j;
})
.getAsInt();
return result.classes().get(max);
} catch (IOException
| MlEnginePredictException
| ExecutionException e) {
throw new RuntimeException(e);
}
})
.thenApply(v -> Prediction.create(vector.input(), v)))
.collect(Collectors.toList());
return CompletableFutures.allAsList(predictions);
};
final Predictor<Iris, Integer> predictor =
Predictors.newBuilder(mlEngineLoader, extractFn, predictFn).predictor();
return new MlEnginePredictorExample(predictor);
}
@Override
public CompletionStage<List<Prediction<Iris, Integer>>> predict(
final ScheduledExecutorService scheduler, final Duration timeout, final Iris... input) {
return predictor.predict(scheduler, timeout, input);
}
@AutoValue
abstract static class MlEnginePrediction {
public abstract List<Integer> classes();
public abstract List<BigDecimal> scores();
@JsonCreator
static MlEnginePrediction create(
@JsonProperty("classes") final List<Integer> classes,
@JsonProperty("scores") final List<BigDecimal> scores) {
return new AutoValue_MlEnginePredictorExample_MlEnginePrediction(classes, scores);
}
}
}
================================================
FILE: examples/mlengine-example/src/test/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExampleIT.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar.examples.mlengine;
import static org.junit.Assert.assertFalse;
import java.util.List;
import org.junit.Test;
import com.spotify.zoltar.IrisFeaturesSpec.Iris;
import com.spotify.zoltar.Prediction;
import scala.Option;
public class MlEnginePredictorExampleIT {
@Test
public void testPrediction() throws Exception {
final String projectId = "data-integration-test";
final String modelId = "iristf";
final String versionId = "v1";
final MlEnginePredictorExample predictor =
MlEnginePredictorExample.create(projectId, modelId, versionId);
final Iris input =
new Iris(
Option.apply(5.1),
Option.apply(3.5),
Option.apply(1.4),
Option.apply(0.2),
Option.apply("Iris-setosa"));
final List<Prediction<Iris, Integer>> predictions =
predictor.predict(input).toCompletableFuture().get();
assertFalse(predictions.isEmpty());
}
}
================================================
FILE: examples/mlengine-example/src/test/resources/settings.json
================================================
[
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "petal_length",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"petal_length"
],
"aggregators": "3.7586666666666675,1.7585291834055217"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "petal_width",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"petal_width"
],
"aggregators": "1.1986666666666665,0.7606126185881715"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "sepal_length",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"sepal_length"
],
"aggregators": "5.843333333333332,0.8253012917851413"
},
{
"cls": "com.spotify.featran.transformers.StandardScaler",
"name": "sepal_width",
"params": {
"withStd": "true",
"withMean": "true"
},
"featureNames": [
"sepal_width"
],
"aggregators": "3.0540000000000007,0.43214658007054363"
},
{
"cls": "com.spotify.featran.transformers.OneHotEncoder",
"name": "class_name",
"params": {},
"featureNames": [
"class_name_Iris-setosa",
"class_name_Iris-versicolor",
"class_name_Iris-virginica"
],
"aggregators": "label:Iris-setosa,label:Iris-versicolor,label:Iris-virginica"
}
]
================================================
FILE: findbugsexclude.xml
================================================
<?xml version="1.0"?>
<FindBugsFilter>
<Match>
<Package name="~.*\.proto"/>
</Match>
<Match>
<Class name="~.*\.*Builder.*"/>
</Match>
<Match>
<Class name="~.*\.*Test.*"/>
</Match>
<Match>
<Class name="~.*\.*IT.*"/>
</Match>
</FindBugsFilter>
================================================
FILE: pom.xml
================================================
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>zoltar-parent</name>
<artifactId>zoltar-parent</artifactId>
<version>0.6.1-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>com.spotify</groupId>
<artifactId>foss-root</artifactId>
<version>15</version>
</parent>
<modules>
<module>zoltar-bom</module>
<module>zoltar-core</module>
<module>zoltar-tensorflow</module>
<module>zoltar-xgboost</module>
<module>zoltar-featran</module>
<module>zoltar-tests</module>
<module>zoltar-api</module>
<module>zoltar-metrics</module>
<module>zoltar-mlengine</module>
<module>zoltar-jmh</module>
<!-- <module>examples/apollo-service-example</module> -->
<module>examples/custom-metrics</module>
<module>examples/mlengine-example</module>
<module>examples/batch-predictor</module>
</modules>
<properties>
<findbugs.excludeFilterFile>findbugsexclude.xml</findbugs.excludeFilterFile>
<checkstyle.violationSeverity>error</checkstyle.violationSeverity>
</properties>
<scm>
<url>https://github.com/spotify/zoltar</url>
<connection>scm:git:git@github.com:spotify/zoltar.git</connection>
<developerConnection>scm:git:git@github.com:spotify/zoltar.git</developerConnection>
<tag>HEAD</tag>
</scm>
<developers>
<developer>
<name>Filipe Regadas</name>
<email>filiperegadas@gmail.com</email>
<url>https://twitter.com/regadas</url>
</developer>
</developers>
<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
<repository>
<id>ossrh</id>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.6</version>
<configuration>
<excludes>
<exclude>**/*Builder.*</exclude>
<exclude>**/*Value.*</exclude>
<exclude>**/*Factory.*</exclude>
<exclude>**/di/*</exclude>
<exclude>**/proto/*</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<configLocation>checkstyle.xml</configLocation>
<failsOnError>true</failsOnError>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<version>3.0.5</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.3.2</version>
<configuration>
<failOnError>false</failOnError>
<links>
<link>https://www.tensorflow.org/api_docs/java/reference</link>
<link>http://dmlc.ml/docs/javadocs</link>
</links>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<executions>
<execution>
<id>check-file-header</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>2.22.1</version>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>apply</goal>
</goals>
</execution>
</executions>
<configuration>
<licenseHeader>
<content>
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
</content>
</licenseHeader>
<java>
<googleJavaFormat>
<version>1.6</version>
<style>GOOGLE</style>
</googleJavaFormat>
<removeUnusedImports />
<importOrder>
<order>java,javax,org,com,com.spotify,</order>
</importOrder>
</java>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>quick</id>
<activation>
<property>
<name>quick</name>
</property>
</activation>
<properties>
<checkstyle.skip>true</checkstyle.skip>
<findbugs.skip>true</findbugs.skip>
<jacoco.skip>true</jacoco.skip>
</properties>
</profile>
</profiles>
</project>
================================================
FILE: project/build.properties
================================================
sbt.version=1.1.5
================================================
FILE: project/plugins.sbt
================================================
addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.2")
addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.3.3")
addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.4.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.2")
================================================
FILE: scripts/deploy.sh
================================================
#!/bin/bash
echo 'VERSION=${project.version}' | mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate | grep '^VERSION' | grep -q '\-SNAPSHOT'
if [ $? -eq 0 ] && [ "$TRAVIS_BRANCH" == "master" ]; then
mvn deploy --settings settings.xml -DskipTests=true
fi
================================================
FILE: scripts/libgomp/Dockerfile
================================================
FROM debian:sid-slim
RUN apt-get update && apt-get install -y libgomp1
================================================
FILE: scripts/libgomp/create.sh
================================================
#!/usr/bin/env bash
# Builds a debian docker image with libgomp1 installed and copies the binary to the output path.
set -o errexit
set -o nounset
set -o pipefail
DIR=$(dirname "$0")
OUTPUT_PATH=$DIR/../../zoltar-xgboost/src/main/resources/lib/libgomp.so.1
IMAGE_NAME=debian-libgomp1
docker build -t $IMAGE_NAME $DIR
docker cp -L $(docker create $IMAGE_NAME):/lib/x86_64-linux-gnu/libgomp.so.1 $OUTPUT_PATH
================================================
FILE: scripts/test.sh
================================================
#!/usr/bin/env bash
GOAL=test
if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ]; then
GOAL=verify
openssl aes-256-cbc \
-K $encrypted_cfd4364d84ec_key \
-iv $encrypted_cfd4364d84ec_iv \
-in scripts/data-integration-test-a20d1bb2e128.json.enc \
-out scripts/data-integration-test-a20d1bb2e128.json \
-d
fi
mvn -Dbigquery.project.arg="-Dbigquery.project=dummy-project" \
clean \
spotless:check \
checkstyle:checkstyle \
findbugs:findbugs \
$GOAL
================================================
FILE: settings.xml
================================================
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository/>
<interactiveMode/>
<usePluginRegistry/>
<offline/>
<pluginGroups/>
<proxies/>
<servers>
<server>
<id>ossrh</id>
<username>${env.SONATYPE_USERNAME}</username>
<password>${env.SONATYPE_PASSWORD}</password>
</server>
</servers>
</settings>
================================================
FILE: suppressions.xml
================================================
<?xml version="1.0"?>
<!DOCTYPE suppressions PUBLIC
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
<suppressions>
<suppress files="AutoValue.*\.java" checks="[a-zA-Z0-9]*"/>
</suppressions>
================================================
FILE: zoltar-api/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.6.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>zoltar-api</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.6.1-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-core</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-metrics</artifactId>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tensorflow</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-xgboost</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-featran</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>featran-java_2.12</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-mlengine</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
================================================
FILE: zoltar-api/src/main/java/com/spotify/zoltar/Models.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import javax.annotation.Nullable;
import org.tensorflow.Graph;
import org.tensorflow.proto.framework.ConfigProto;
import com.spotify.zoltar.mlengine.MlEngineLoader;
import com.spotify.zoltar.tf.TensorFlowGraphLoader;
import com.spotify.zoltar.tf.TensorFlowLoader;
import com.spotify.zoltar.tf.TensorFlowModel;
import com.spotify.zoltar.xgboost.XGBoostLoader;
/**
* This class consists exclusively of static methods that return Model Loaders.
*
* <p>This is the public entry point for get a Model.
*/
public final class Models {
private Models() {}
/**
* Returns a XGBoost model loader given the serialized model stored in the model URI.
*
* @param modelUri should point to serialized XGBoost model file, can be a URI to a local
* filesystem, resource, GCS etc.
*/
public static XGBoostLoader xgboost(final String modelUri) {
return XGBoostLoader.create(modelUri);
}
/**
* Returns a XGBoost model loader given the serialized model stored in the model URI.
*
* @param id model id @{link Model.Id}.
* @param modelUri should point to serialized XGBoost model file, can be a URI to a local
* filesystem, resource, GCS etc.
*/
public static XGBoostLoader xgboost(final Model.Id id, final String modelUri) {
return XGBoostLoader.create(id, modelUri);
}
/**
* Returns a TensorFlow model loader based on a saved model.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
*/
public static TensorFlowLoader tensorFlow(final String modelUri) {
return TensorFlowLoader.create(modelUri);
}
/**
* Returns a TensorFlow model loader based on a saved model.
*
* @param id model id @{link Model.Id}.
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
*/
public static TensorFlowLoader tensorFlow(final Model.Id id, final String modelUri) {
return TensorFlowLoader.create(id, modelUri);
}
/**
* Returns a TensorFlow model loader based on a saved model.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param options TensorFlow options, see {@link TensorFlowModel.Options}.
*/
public static TensorFlowLoader tensorFlow(
final String modelUri, final TensorFlowModel.Options options) {
return TensorFlowLoader.create(modelUri, options);
}
/**
* Returns a TensorFlow model loader based on a saved model.
*
* @param id model id @{link Model.Id}.
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param options TensorFlow options, see {@link TensorFlowModel.Options}.
*/
public static TensorFlowLoader tensorFlow(
final Model.Id id, final String modelUri, final TensorFlowModel.Options options) {
return TensorFlowLoader.create(id, modelUri, options);
}
/**
* Returns a TensorFlow model loader based on a serialized TensorFlow {@link Graph}.
*
* @param modelUri should point to a serialized TensorFlow {@link org.tensorflow.Graph} file on
* local filesystem, resource, GCS etc.
* @param config optional TensorFlow {@link ConfigProto} config.
* @param prefix optional prefix that will be prepended to names in the graph.
*/
public static TensorFlowGraphLoader tensorFlowGraph(
final String modelUri, @Nullable final ConfigProto config, @Nullable final String prefix) {
return TensorFlowGraphLoader.create(modelUri, config, prefix);
}
/**
* Returns a TensorFlow model loader based on a serialized TensorFlow {@link Graph}.
*
* @param id model id @{link Model.Id}.
* @param modelUri should point to a serialized TensorFlow {@link org.tensorflow.Graph} file on
* local filesystem, resource, GCS etc.
* @param config optional TensorFlow {@link ConfigProto} config.
* @param prefix optional prefix that will be prepended to names in the graph.
*/
public static TensorFlowGraphLoader tensorFlowGraph(
final Model.Id id,
final String modelUri,
@Nullable final ConfigProto config,
@Nullable final String prefix) {
return TensorFlowGraphLoader.create(id, modelUri, config, prefix);
}
/**
* Returns a TensorFlow model loader based on a serialized TensorFlow {@link Graph}.
*
* @param graphDef byte array representing the TensorFlow {@link Graph} definition.
* @param config optional TensorFlow {@link ConfigProto} config.
* @param prefix optional prefix that will be prepended to names in the graph.
*/
public static TensorFlowGraphLoader tensorFlowGraph(
final byte[] graphDef, @Nullable final ConfigProto config, @Nullable final String prefix) {
return TensorFlowGraphLoader.create(graphDef, config, prefix);
}
/**
* Returns a TensorFlow model loader based on a serialized TensorFlow {@link Graph}.
*
* @param id model id @{link Model.Id}.
* @param graphDef byte array representing the TensorFlow {@link Graph} definition.
* @param config optional TensorFlow {@link ConfigProto} config.
* @param prefix optional prefix that will be prepended to names in the graph.
*/
public static TensorFlowGraphLoader tensorFlowGraph(
final Model.Id id,
final byte[] graphDef,
@Nullable final ConfigProto config,
@Nullable final String prefix) {
return TensorFlowGraphLoader.create(id, graphDef, config, prefix);
}
/**
* Returns a Cloud ML Engine backed model.
*
* @param id model id. Id needs to be in the following format: <code>
* projects/$projectId/models/$modelId/version/$versionId</code>
*/
public static MlEngineLoader mlEngine(final Model.Id id) {
return MlEngineLoader.create(id);
}
/**
* Returns a Cloud Ml Engine backed model.
*
* @param projectId Google project id.
* @param modelId Model id.
* @param versionId Model version id.
*/
public static MlEngineLoader mlEngine(
final String projectId, final String modelId, final String versionId) {
return MlEngineLoader.create(projectId, modelId, versionId);
}
}
================================================
FILE: zoltar-api/src/main/java/com/spotify/zoltar/Predictors.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.tensorflow.Tensor;
import org.tensorflow.proto.example.Example;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.PredictFns.AsyncPredictFn;
import com.spotify.zoltar.PredictFns.PredictFn;
import com.spotify.zoltar.metrics.Instrumentations;
import com.spotify.zoltar.metrics.PredictorMetrics;
import com.spotify.zoltar.tf.TensorFlowLoader;
import com.spotify.zoltar.tf.TensorFlowModel;
import com.spotify.zoltar.tf.TensorFlowPredictFn;
/**
* This class consists exclusively of static methods that return {@link PredictorBuilder} or {@link
* Predictor}.
*
* <p>This is the public entry point for Predictors.
*/
public final class Predictors {
private Predictors() {}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor}and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return DefaultPredictorBuilder.create(modelLoader, extractFn, predictFn);
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* AsyncPredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return DefaultPredictorBuilder.create(modelLoader, extractFn, predictFn);
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return DefaultPredictorBuilder.create(modelLoader, featureExtractor, predictFn);
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return DefaultPredictorBuilder.create(modelLoader, featureExtractor, predictFn);
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor}and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn,
final PredictorMetrics<InputT, VectorT, ValueT> metrics) {
return DefaultPredictorBuilder.create(modelLoader, extractFn, predictFn)
.with(Instrumentations.predictor(metrics));
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* AsyncPredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn,
final PredictorMetrics<InputT, VectorT, ValueT> metrics) {
return DefaultPredictorBuilder.create(modelLoader, extractFn, predictFn)
.with(Instrumentations.predictor(metrics));
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn,
final PredictorMetrics<InputT, VectorT, ValueT> metrics) {
return DefaultPredictorBuilder.create(modelLoader, featureExtractor, predictFn)
.with(Instrumentations.predictor(metrics));
}
/**
* Returns a PredictorBuilder given a {@link Model}, {@link FeatureExtractor} and a {@link
* PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
PredictorBuilder<ModelT, InputT, VectorT, ValueT> newBuilder(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn,
final PredictorMetrics<InputT, VectorT, ValueT> metrics) {
return DefaultPredictorBuilder.create(modelLoader, featureExtractor, predictFn)
.with(Instrumentations.predictor(metrics));
}
/**
* Returns a TensorFlow Predictor.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param outTensorExtractor function to extract the output value from a {@link Tensor}.
* @param fetchOps operations to fetch.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
final String modelUri,
final ExtractFn<InputT, Example> extractFn,
final Function<Map<String, Tensor>, List<ValueT>> outTensorExtractor,
final String... fetchOps) {
return tensorFlow(modelUri, FeatureExtractor.create(extractFn), outTensorExtractor, fetchOps);
}
/**
* Returns a TensorFlow Predictor.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param outTensorExtractor function to extract the output value from a {@link Tensor}.
* @param fetchOps operations to fetch.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
final String modelUri,
final ExtractFn<InputT, Example> extractFn,
final Function<Map<String, Tensor>, List<ValueT>> outTensorExtractor,
final String[] fetchOps,
final PredictorMetrics<InputT, Example, ValueT> metrics) {
return tensorFlow(
modelUri, FeatureExtractor.create(extractFn), outTensorExtractor, fetchOps, metrics);
}
/**
* Returns a TensorFlow Predictor.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param outTensorExtractor function to extract the output value from a {@link Tensor}.
* @param fetchOps operations to fetch.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
final String modelUri,
final FeatureExtractor<TensorFlowModel, InputT, Example> featureExtractor,
final Function<Map<String, Tensor>, List<ValueT>> outTensorExtractor,
final String... fetchOps) {
final ModelLoader<TensorFlowModel> modelLoader = TensorFlowLoader.create(modelUri);
final TensorFlowPredictFn<InputT, Example, ValueT> predictFn =
TensorFlowPredictFn.example(outTensorExtractor, fetchOps);
return newBuilder(modelLoader, featureExtractor, predictFn).predictor();
}
/**
* Returns a TensorFlow Predictor.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param outTensorExtractor function to extract the output value from a {@link Tensor}.
* @param fetchOps operations to fetch.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
final String modelUri,
final FeatureExtractor<TensorFlowModel, InputT, Example> featureExtractor,
final Function<Map<String, Tensor>, List<ValueT>> outTensorExtractor,
final String[] fetchOps,
final PredictorMetrics<InputT, Example, ValueT> metrics) {
final ModelLoader<TensorFlowModel> modelLoader = TensorFlowLoader.create(modelUri);
final TensorFlowPredictFn<InputT, Example, ValueT> predictFn =
TensorFlowPredictFn.example(outTensorExtractor, fetchOps);
return newBuilder(modelLoader, featureExtractor, predictFn, metrics).predictor();
}
/**
* Returns a TensorFlow Predictor. Assumes feature extraction is embedded in the model via
* Tensorflow Transform, so no extractFn is needed and the input type must be Example.
*
* @param modelUri should point to a directory of the saved TensorFlow {@link
* org.tensorflow.SavedModelBundle}, can be a URI to a local filesystem, resource, GCS etc.
* @param outTensorExtractor function to extract the output value from a {@link Tensor}.
* @param fetchOps operations to fetch.
* @param metrics a predictor metrics implementation {@link
* com.spotify.zoltar.metrics.semantic.SemanticPredictMetrics}.
* @param <ValueT> type of the prediction result.
*/
public static <ValueT> Predictor<Example, ValueT> tensorFlow(
final String modelUri,
final Function<Map<String, Tensor>, List<ValueT>> outTensorExtractor,
final String[] fetchOps,
final PredictorMetrics<Example, Example, ValueT> metrics) {
return tensorFlow(modelUri, ExtractFn.identity(), outTensorExtractor, fetchOps, metrics);
}
}
================================================
FILE: zoltar-bom/pom.xml
================================================
<?xml version="1.0" encoding="utf-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.spotify</groupId>
<artifactId>zoltar-parent</artifactId>
<version>0.6.1-SNAPSHOT</version>
</parent>
<name>Zoltar Bill of Materials</name>
<artifactId>zoltar-bom</artifactId>
<packaging>pom</packaging>
<properties>
<apiclient.version>1.32.1</apiclient.version>
<boringssl.version>2.0.38.Final</boringssl.version>
<errorprone.version>2.11.0</errorprone.version>
<featran.version>0.8.0-RC2</featran.version>
<google.auth.version>0.26.0</google.auth.version>
<grpc.version>1.39.0</grpc.version>
<gson.version>2.9.0</gson.version>
<guava.version>30.1-jre</guava.version>
<httpclient.version>1.39.2</httpclient.version>
<jackson.version>2.13.2</jackson.version>
<joda.version>2.10.10</joda.version>
<netty.version>4.1.52.Final</netty.version>
<opencensus.version>0.28.3</opencensus.version>
<proto.version>3.19.4</proto.version>
<scala.version>2.12.13</scala.version>
<scio.version>0.5.7</scio.version>
<tensorFlow.version>0.4.1</tensorFlow.version>
<xgboost.version>1.3.1</xgboost.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bom</artifactId>
<version>0.158.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-api</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-core</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-featran</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-metrics</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tensorflow</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-tests</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-xgboost</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-mlengine</artifactId>
<version>0.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.spotify.metrics</groupId>
<artifactId>semantic-metrics-core</artifactId>
<version>1.1.11</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
<version>1.7.5</version>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>3.0.2</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>featran-core_2.12</artifactId>
<version>${featran.version}</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>featran-java_2.12</artifactId>
<version>${featran.version}</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>featran-tensorflow_2.12</artifactId>
<version>${featran.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${proto.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>${proto.version}</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>futures-extra</artifactId>
<version>4.3.1</version>
</dependency>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>completable-futures</artifactId>
<version>0.3.4</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>${joda.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.tensorflow</groupId>
<artifactId>tensorflow-core-platform</artifactId>
<version>${tensorFlow.version}</version>
</dependency>
<dependency>
<groupId>ml.dmlc</groupId>
<artifactId>xgboost4j_2.12</artifactId>
<version>${xgboost.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<!-- initial reason: breeze 1.0-RC2 from featran 0.1.19 -->
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.typelevel</groupId>
<artifactId>cats-kernel_2.12</artifactId>
<version>2.4.2</version>
</dependency>
<dependency>
<groupId>org.typelevel</groupId>
<artifactId>machinist_2.12</artifactId>
<version>0.6.8</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-all</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-auth</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>${boringssl.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-credentials</artifactId>
<version>${google.auth.version}</version>
</dependency>
<dependency>
<groupId>com.google.auth</groupId>
<artifactId>google-auth-library-oauth2-http</artifactId>
<version>${google.auth.version}</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
<version>${opencensus.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_2.12</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
<version>2.8</version>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<version>${errorprone.version}</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-jackson2</artifactId>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>${apiclient.version}</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-storage</artifactId>
<version>v1-rev20210127-1.32.1</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-ml</artifactId>
<version>v1-rev20210212-1.31.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
</dependency>
</dependencies>
</dependencyManagement>
<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>
https://oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
</distributionManagement>
</project>
================================================
FILE: zoltar-core/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>zoltar-parent</artifactId>
<groupId>com.spotify</groupId>
<version>0.6.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>zoltar-core</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.spotify</groupId>
<artifactId>zoltar-bom</artifactId>
<version>0.6.1-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-nio</artifactId>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
<exclusion>
<artifactId>proto-google-common-protos</artifactId>
<groupId>com.google.api.grpc</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<artifactId>proto-google-common-protos</artifactId>
<groupId>com.google.api.grpc</groupId>
<version>2.8.3</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictor.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.spotify.zoltar.PredictFns.AsyncPredictFn;
import com.spotify.zoltar.PredictFns.PredictFn;
/**
* Allows E2E prediction given a recipe made of a {@link Model}, {@link FeatureExtractor} and a
* {@link PredictFn}.
*
* @param <InputT> type of the feature extraction input.
* @param <ValueT> type of the prediction output.
*/
@FunctionalInterface
interface DefaultPredictor<InputT, ValueT> extends Predictor<InputT, ValueT> {
/**
* Returns a predictor given a {@link Model}, {@link FeatureExtractor} and a {@link
* AsyncPredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
static <ModelT extends Model<?>, InputT, VectorT, ValueT> DefaultPredictor<InputT, ValueT> create(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return (scheduler, timeout, inputs) -> {
final CompletableFuture<List<Prediction<InputT, ValueT>>> future =
modelLoader
.get()
.thenCompose(
model -> {
try {
return predictFn.apply(model, featureExtractor.extract(model, inputs));
} catch (final Exception e) {
throw new CompletionException(e);
}
})
.toCompletableFuture();
final ScheduledFuture<?> schedule =
scheduler.schedule(
() -> {
future.completeExceptionally(new TimeoutException());
},
timeout.toMillis(),
TimeUnit.MILLISECONDS);
future.whenComplete((r, t) -> schedule.cancel(true));
return future;
};
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorBuilder.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import com.google.auto.value.AutoValue;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
import com.spotify.zoltar.PredictFns.AsyncPredictFn;
import com.spotify.zoltar.PredictFns.PredictFn;
/**
* Entry point for prediction. Default implementation of a PredictorBuilder that holds the necessary
* info to build a {@link Predictor}.
*
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@AutoValue
abstract class DefaultPredictorBuilder<ModelT extends Model<?>, InputT, VectorT, ValueT>
implements PredictorBuilder<ModelT, InputT, VectorT, ValueT> {
/**
* Returns a context given a {@link Model}, {@link FeatureExtractor} and a {@link PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
DefaultPredictorBuilder<ModelT, InputT, VectorT, ValueT> create(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return create(modelLoader, FeatureExtractor.create(extractFn), AsyncPredictFn.lift(predictFn));
}
/**
* Returns a context given a {@link Model}, {@link FeatureExtractor} and a {@link AsyncPredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param extractFn a feature extract function to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
DefaultPredictorBuilder<ModelT, InputT, VectorT, ValueT> create(
final ModelLoader<ModelT> modelLoader,
final ExtractFn<InputT, VectorT> extractFn,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return create(modelLoader, FeatureExtractor.create(extractFn), predictFn);
}
/**
* Returns a context given a {@link Model}, {@link FeatureExtractor} and a {@link PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link PredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
DefaultPredictorBuilder<ModelT, InputT, VectorT, ValueT> create(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final PredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return create(modelLoader, featureExtractor, AsyncPredictFn.lift(predictFn));
}
/**
* Returns a context given a {@link Model}, {@link FeatureExtractor} and a {@link PredictFn}.
*
* @param modelLoader model loader that loads the model to perform prediction on.
* @param featureExtractor a feature extractor to use to transform input into extracted features.
* @param predictFn a prediction function to perform prediction with {@link AsyncPredictFn}.
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to the {@link FeatureExtractor}.
* @param <VectorT> type of the output from {@link FeatureExtractor}.
* @param <ValueT> type of the prediction result.
*/
@SuppressWarnings("checkstyle:LineLength")
public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
DefaultPredictorBuilder<ModelT, InputT, VectorT, ValueT> create(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return new AutoValue_DefaultPredictorBuilder<>(
modelLoader,
featureExtractor,
predictFn,
DefaultPredictor.create(modelLoader, featureExtractor, predictFn));
}
public abstract ModelLoader<ModelT> modelLoader();
public abstract FeatureExtractor<ModelT, InputT, VectorT> featureExtractor();
public abstract AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn();
public abstract Predictor<InputT, ValueT> predictor();
@Override
public DefaultPredictorBuilder<ModelT, InputT, VectorT, ValueT> with(
final ModelLoader<ModelT> modelLoader,
final FeatureExtractor<ModelT, InputT, VectorT> featureExtractor,
final AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn) {
return create(modelLoader, featureExtractor, predictFn);
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorTimeoutScheduler.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
/**
* Default {@link Predictor} {@link ScheduledExecutorService} supplier implementation for timeout
* scheduling.
*/
final class DefaultPredictorTimeoutScheduler implements PredictorTimeoutScheduler {
private static final ScheduledExecutorService SCHEDULER =
Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors());
private DefaultPredictorTimeoutScheduler() {}
public static DefaultPredictorTimeoutScheduler create() {
return new DefaultPredictorTimeoutScheduler();
}
@Override
public ScheduledExecutorService scheduler() {
return SCHEDULER;
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractFns.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import com.google.common.collect.ImmutableList;
/**
* Feature extraction functions. Functions used to transform raw input into extracted features,
* should be used together with {@link FeatureExtractor}.
*
* @see FeatureExtractor
*/
public interface FeatureExtractFns {
/**
* Generic feature extraction function, takes multiple raw input and should return extracted
* features of user defined type.
*
* @param <InputT> type of the input to feature extraction.
* @param <VectorT> type of feature extraction result.
*/
@FunctionalInterface
interface ExtractFn<InputT, VectorT> {
static <InputT, VectorT> ExtractFn<InputT, VectorT> lift(final Function<InputT, VectorT> fn) {
return inputs -> {
final List<VectorT> result = new ArrayList<>();
for (InputT inputT : inputs) {
result.add(fn.apply(inputT));
}
return result;
};
}
static <InputT> ExtractFn<InputT, InputT> identity() {
return Arrays::asList;
}
/** Functional interface. Perform feature extraction. */
List<VectorT> apply(InputT... inputs) throws Exception;
default <C extends ExtractFn<InputT, VectorT>> C with(
final Function<ExtractFn<InputT, VectorT>, C> fn) {
return fn.apply(this);
}
}
/**
* Generic feature extraction function, takes a batch of raw inputs and should return extracted
* features of user defined type for each input.
*
* @deprecated Use {@link ExtractFn} instead
* @param <InputT> type of the input to feature extraction.
* @param <VectorT> type of feature extraction result.
*/
@FunctionalInterface
@Deprecated
interface BatchExtractFn<InputT, VectorT> extends ExtractFn<List<InputT>, List<VectorT>> {
@SuppressWarnings("unchecked")
static <InputT, VectorT> BatchExtractFn<InputT, VectorT> lift(
final ExtractFn<InputT, VectorT> fn) {
return inputs -> {
final ImmutableList.Builder<List<VectorT>> output = ImmutableList.builder();
for (final List<InputT> batch : inputs) {
final InputT[] objects = batch.toArray((InputT[]) new Object[0]);
output.add(fn.apply(objects));
}
return output.build();
};
}
static <InputT, VectorT> BatchExtractFn<InputT, VectorT> lift(
final Function<InputT, VectorT> fn) {
return lift(ExtractFn.lift(fn));
}
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractor.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.function.Function;
import com.spotify.zoltar.FeatureExtractFns.ExtractFn;
/**
* Functional interface for feature extraction. Should be used together with {@link Predictor}. In
* most cases you should use the static factory methods.
*
* @param <ModelT> underlying type of the {@link Model}.
* @param <InputT> type of the input to feature extraction.
* @param <VectorT> type of feature extraction result.
*/
@FunctionalInterface
public interface FeatureExtractor<ModelT extends Model<?>, InputT, VectorT> {
/**
* Creates an extractor given a generic {@link ExtractFn}, consider using <a
* href="https://github.com/spotify/featran">Featran</a> and FeatranExtractFns whenever possible.
*
* @param fn {@link ExtractFn} extraction function
* @param <InputT> type of the input to feature extraction.
* @param <VectorT> type of feature extraction result.
*/
@SuppressWarnings("checkstyle:LineLength")
static <ModelT extends Model<?>, InputT, VectorT>
FeatureExtractor<ModelT, InputT, VectorT> create(final ExtractFn<InputT, VectorT> fn) {
return (model, inputs) -> {
final List<Vector<InputT, VectorT>> result = new ArrayList<>();
final Iterator<InputT> i1 = Arrays.asList(inputs).iterator();
final Iterator<VectorT> i2 = fn.apply(inputs).iterator();
while (i1.hasNext() && i2.hasNext()) {
result.add(Vector.create(i1.next(), i2.next()));
}
return result;
};
}
/** Functional interface. Perform the feature extraction given the input. */
List<Vector<InputT, VectorT>> extract(ModelT model, InputT... input) throws Exception;
default <C extends FeatureExtractor<ModelT, InputT, VectorT>> C with(
final Function<FeatureExtractor<ModelT, InputT, VectorT>, C> fn) {
return fn.apply(this);
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Model.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import com.google.auto.value.AutoValue;
/**
* Model interface. In most cases you can just use the prebaked implementations.
*
* @param <UnderlyingT> the underlying type of the model.
*/
public interface Model<UnderlyingT> extends AutoCloseable {
/** value class to define model id. */
@AutoValue
abstract class Id {
public abstract String value();
public static Id create(final String value) {
return new AutoValue_Model_Id(value);
}
}
Id id();
/**
* Returns an instance of the underlying model. This could be for example TensorFlow's graph,
* session or XGBoost's booster.
*/
UnderlyingT instance();
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/ModelLoader.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.io.Closeable;
import java.io.IOException;
import java.time.Duration;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Function;
/**
* Handles the model loading logic.
*
* @param <M> Underlying model instance.
*/
@FunctionalInterface
public interface ModelLoader<M extends Model<?>> extends Closeable {
/** default executor. */
ExecutorService DEFAULT_EXECUTOR_SERVICE = Executors.newWorkStealingPool();
/** Supplier of {@link Model}. Can throw {@link Exception} when the supplier is invoked. */
@FunctionalInterface
interface ThrowableSupplier<M extends Model<?>> {
M get() throws Exception;
}
/**
* PreLoader is model loader that calls {@link ModelLoader#get()} allowing model preloading.
*
* @param <M> Model instance type.
*/
@FunctionalInterface
interface PreLoader<M extends Model<?>> extends ModelLoader<M> {
/**
* Returns a blocking {@link ModelLoader}. Blocks till the model is loaded or a {@link Duration}
* is met.
*
* @param supplier model supplier.
* @param duration Amount of time that it should wait, if necessary, for model to be loaded.
* @param executor the executor to use for asynchronous execution.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> PreLoader<M> preload(
final ThrowableSupplier<M> supplier, final Duration duration, final Executor executor)
throws InterruptedException, ExecutionException, TimeoutException {
return preload(ModelLoader.load(supplier, executor), duration);
}
/**
* Returns a blocking {@link PreLoader}. Blocks till the model is loaded or a {@link Duration}
* is met.
*
* @param loader model loader.
* @param duration Amount of time that it should wait, if necessary, for model to be loaded.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> PreLoader<M> preload(
final ModelLoader<M> loader, final Duration duration)
throws InterruptedException, ExecutionException, TimeoutException {
return ModelLoader.loaded(loader.get(duration))::get;
}
/**
* Returns a blocking {@link PreLoader}. Blocks till the model is loaded or a {@link Duration}
* is met.
*
* @param duration Amount of time that it should wait, if necessary, for model to be loaded.
*/
static <M extends Model<?>> Function<ModelLoader<M>, PreLoader<M>> preload(
final Duration duration) {
return loader -> {
try {
return preload(loader, duration);
} catch (final Exception e) {
final CompletableFuture<M> failed = new CompletableFuture<>();
failed.completeExceptionally(e);
return () -> failed;
}
};
}
}
/**
* ConsLoader is a constant {@link ModelLoader}.
*
* @param <M> Model instance type.
*/
@FunctionalInterface
interface ConsLoader<M extends Model<?>> extends ModelLoader<M> {
/**
* Creates a {@link ModelLoader} with an already loaded model.
*
* @param model Underlying model instance.
*/
static <M extends Model<?>> ConsLoader<M> cons(final M model) {
final CompletableFuture<M> m = CompletableFuture.completedFuture(model);
return () -> m;
}
}
/**
* Creates a {@link ModelLoader} with an already loaded model.
*
* @param model Underlying model instance.
*/
static <M extends Model<?>> ModelLoader<M> loaded(final M model) {
return ConsLoader.cons(model);
}
/**
* Returns a blocking {@link ModelLoader}. Blocks till the model is loaded or a {@link Duration}
* is met.
*
* @param supplier model supplier.
* @param duration Amount of time that it should wait, if necessary, for model to be loaded.
* @param executor the executor to use for asynchronous execution.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> PreLoader<M> preload(
final ThrowableSupplier<M> supplier, final Duration duration, final Executor executor)
throws InterruptedException, ExecutionException, TimeoutException {
return PreLoader.preload(supplier, duration, executor);
}
/**
* Returns a blocking {@link PreLoader}. Blocks till the model is loaded or a {@link Duration} is
* met.
*
* @param loader model loader.
* @param duration Amount of time that it should wait, if necessary, for model to be loaded.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> ModelLoader<M> preload(
final ModelLoader<M> loader, final Duration duration)
throws InterruptedException, ExecutionException, TimeoutException {
return PreLoader.preload(loader, duration);
}
/**
* Create a {@link ModelLoader} that loads the supplied model asynchronously.
*
* @param supplier model supplier.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> ModelLoader<M> load(
final ThrowableSupplier<M> supplier, final Executor executor) {
final CompletableFuture<M> future =
CompletableFuture.supplyAsync(
() -> {
try {
return supplier.get();
} catch (final Exception e) {
throw new CompletionException(e);
}
},
executor);
return () -> future;
}
/**
* Create a {@link ModelLoader} that loads the supplied model asynchronously.
*
* @param supplier model supplier.
* @param <M> Underlying model instance.
*/
static <M extends Model<?>> ModelLoader<M> load(final ThrowableSupplier<M> supplier) {
return load(supplier, DEFAULT_EXECUTOR_SERVICE);
}
/**
* Lifts a supplier into a {@link ModelLoader}.
*
* @param supplier model supplier.
* @param <M> Underlying model instance.
*/
@Deprecated
static <M extends Model<?>> ModelLoader<M> lift(final ThrowableSupplier<M> supplier) {
return load(supplier);
}
/**
* Get's the underlying model instance.
*
* @return Model instance
*/
CompletionStage<M> get();
/**
* Get's the underlying model instance.
*
* @return Model instance
*/
default M get(final Duration duration)
throws InterruptedException, ExecutionException, TimeoutException {
return get().toCompletableFuture().get(duration.toMillis(), TimeUnit.MILLISECONDS);
}
@Deprecated
default <L extends ModelLoader<M>> L with(final Function<ModelLoader<M>, L> fn) {
return compose(fn);
}
default <L extends ModelLoader<M>> L compose(final Function<ModelLoader<M>, L> fn) {
return fn.apply(this);
}
@Override
default void close() throws IOException {
DEFAULT_EXECUTOR_SERVICE.shutdown();
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/PredictFns.java
================================================
/*
* Copyright (C) 2019 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.spotify.zoltar;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.function.Function;
/**
* Prediction function interfaces. Describe how to perform predictions on a model given input
* vectors. See prebaked implementations for prebaked models.
*
* @see Predictor
*/
public interface PredictFns {
/**
* Asynchronous prediction functional interface. Allows to define prediction function via lambda,
* prediction is happens asynchronously.
*
* @param <InputT> type of the feature extraction input.
* @param <VectorT> type of the feature extraction output.
* @param <ValueT> type of the prediction output.
*/
@FunctionalInterface
interface AsyncPredictFn<ModelT extends Model<?>, InputT, VectorT, ValueT> {
/**
* Lifts prediction function to asynchronous prediction function.
*
* @param <ModelT> type of the {@link Model}
* @param <InputT> type of the feature extraction input.
* @param <VectorT> type of the feature extraction output.
* @param <ValueT> type of the prediction output.
*/
@SuppressWarnings("checkstyle:LineLength")
static <ModelT extends Model<?>, InputT, VectorT, ValueT>
AsyncPredictFn<ModelT, InputT, VectorT, ValueT> lift(
final PredictFn<ModelT, InputT, VectorT, ValueT> fn) {
return (model, vectors) ->
CompletableFuture.supplyAsync(
() -> {
try {
return fn.apply(model, vectors);
} catch (final Exception e) {
throw new RuntimeException(e.getCause());
}
});
}
/**
* The functional interface. Your function/lambda takes model and features after extractions as
* input, should perform a asynchronous prediction and return the "future" of predictions.
*
* <p>Note: if you have a synchronous implementation of prediction function you can use {@link
* AsyncPredictFn#lift} to make it asynchronous.
*
* @param model model to perform prediction on.
* @param vectors extracted features.
* @return {@link CompletionStage} of predictions ({@link Prediction}).
*/
CompletionStage<List<Prediction<InputT, ValueT>>> apply(
ModelT model, List<Vector<InputT, VectorT>> vectors);
default <C extends AsyncPredictFn<ModelT, InputT, VectorT, ValueT>> C with(
final Function<AsyncPredictFn<ModelT, InputT, VectorT, ValueT>, C> fn) {
return fn.apply(this);
}
}
/**
* Synchronous prediction functional interface. Allows to define prediction function via lambda.
*
* @param <InputT> type of the feature extraction input.
* @param <VectorT> type of the feature extraction output.
* @param <ValueT> type of the prediction output.
*/
@FunctionalInterface
interface PredictFn<ModelT extends Model<?>, InputT, VectorT, ValueT> {
/**
* The functional interface. Your function/lambda takes model and features after extractions as
* input, should perform a prediction and return the predictions.
*
* @param model model to perform prediction on.
* @param vectors extracted features.
* @return predictions ({@link Prediction}).
*/
List<Prediction<InputT, ValueT>> apply(ModelT model, List<Vector<InputT, VectorT>> vectors)
throws Exception;
default <C extends PredictFn<ModelT, InputT, VectorT, ValueT>> C with(
final Function<PredictFn<ModelT, InputT, VectorT, ValueT>, C> fn) {
return fn.apply(this);
}
}
}
================================================
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Prediction.java
================================================
/*
gitextract_f9bkw_4z/
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── codeql-analysis.yml
│ └── main.yml
├── .gitignore
├── LICENSE
├── NOTICE
├── README.md
├── build.sbt
├── catalog-info.yaml
├── checkstyle.xml
├── data/
│ └── iris-model/
│ ├── build.sbt
│ ├── project/
│ │ └── build.properties
│ ├── scripts/
│ │ └── publish_data.sh
│ ├── src/
│ │ ├── main/
│ │ │ └── scala/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── Iris.scala
│ │ └── test/
│ │ ├── resources/
│ │ │ └── expected_feature_spec.json
│ │ └── scala/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── IrisTest.scala
│ └── training/
│ ├── Makefile
│ ├── README.md
│ └── iris/
│ ├── iris/
│ │ ├── __init__.py
│ │ ├── tensorflow.py
│ │ └── xgboost.py
│ ├── requirements.txt
│ ├── setup.cfg
│ ├── setup.py
│ └── test-requirements.txt
├── docs/
│ └── src/
│ └── paradox/
│ ├── _template/
│ │ └── javadoc.st
│ ├── getting-started.md
│ ├── index.md
│ ├── javadoc.md
│ ├── license.md
│ ├── modules/
│ │ ├── featran.md
│ │ ├── index.md
│ │ ├── metrics.md
│ │ ├── mlengine.md
│ │ ├── tensorflow.md
│ │ └── xgboost.md
│ └── release.md
├── examples/
│ ├── apollo-service-example/
│ │ ├── SERVICE.marker
│ │ ├── findbugsexclude.xml
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── java/
│ │ │ │ └── com/
│ │ │ │ └── spotify/
│ │ │ │ └── zoltar/
│ │ │ │ └── examples/
│ │ │ │ └── apollo/
│ │ │ │ ├── App.java
│ │ │ │ ├── IrisPredictionHandler.java
│ │ │ │ ├── IrisPredictor.java
│ │ │ │ └── ModelConfig.java
│ │ │ └── resources/
│ │ │ └── zoltar-example.conf
│ │ └── test/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── apollo/
│ │ │ └── IrisPredictionHandlerTest.java
│ │ └── resources/
│ │ ├── settings.json
│ │ ├── trained_model/
│ │ │ ├── saved_model.pb
│ │ │ └── variables/
│ │ │ ├── variables.data-00000-of-00001
│ │ │ └── variables.index
│ │ └── zoltar-example.conf
│ ├── batch-predictor/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ └── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── batch/
│ │ │ ├── BatchPredictorExample.java
│ │ │ └── DummyModel.java
│ │ └── test/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── batch/
│ │ └── BatchPredictorExampleTest.java
│ ├── custom-metrics/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ └── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ └── examples/
│ │ │ └── metrics/
│ │ │ ├── CustomMetricsExample.java
│ │ │ └── DummyModel.java
│ │ └── test/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── metrics/
│ │ └── CustomMetricsExampleTest.java
│ └── mlengine-example/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ └── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── mlengine/
│ │ └── MlEnginePredictorExample.java
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── examples/
│ │ └── mlengine/
│ │ └── MlEnginePredictorExampleIT.java
│ └── resources/
│ └── settings.json
├── findbugsexclude.xml
├── pom.xml
├── project/
│ ├── build.properties
│ └── plugins.sbt
├── scripts/
│ ├── data-integration-test-a20d1bb2e128.json.enc
│ ├── deploy.sh
│ ├── libgomp/
│ │ ├── Dockerfile
│ │ └── create.sh
│ └── test.sh
├── settings.xml
├── suppressions.xml
├── zoltar-api/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ ├── Models.java
│ └── Predictors.java
├── zoltar-bom/
│ └── pom.xml
├── zoltar-core/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ ├── DefaultPredictor.java
│ ├── DefaultPredictorBuilder.java
│ ├── DefaultPredictorTimeoutScheduler.java
│ ├── FeatureExtractFns.java
│ ├── FeatureExtractor.java
│ ├── Model.java
│ ├── ModelLoader.java
│ ├── PredictFns.java
│ ├── Prediction.java
│ ├── Predictor.java
│ ├── PredictorBuilder.java
│ ├── PredictorTimeoutScheduler.java
│ ├── Vector.java
│ ├── fs/
│ │ └── FileSystemExtras.java
│ └── loaders/
│ ├── ModelMemoizer.java
│ └── Preloader.java
├── zoltar-featran/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── featran/
│ └── FeatranExtractFns.java
├── zoltar-jmh/
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── jmh/
│ │ ├── BenchmarkTensorFlow.java
│ │ └── IrisHelper.java
│ ├── resources/
│ │ ├── iris.csv
│ │ ├── log4j.properties
│ │ ├── settings.json
│ │ └── trained_model/
│ │ ├── saved_model.pb
│ │ ├── trained_model.txt
│ │ └── variables/
│ │ ├── variables.data-00000-of-00001
│ │ └── variables.index
│ └── scala/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── Iris.scala
├── zoltar-metrics/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── metrics/
│ ├── FeatureExtractorMetrics.java
│ ├── Instrumentations.java
│ ├── InstrumentedFeatureExtractor.java
│ ├── InstrumentedPredictFn.java
│ ├── InstrumentedPredictorBuilder.java
│ ├── PredictFnMetrics.java
│ ├── PredictMetrics.java
│ ├── PredictorMetrics.java
│ ├── VectorMetrics.java
│ └── semantic/
│ ├── SemanticPredictMetrics.java
│ ├── SemanticPredictorMetrics.java
│ ├── SemanticVectorMetrics.java
│ └── What.java
├── zoltar-mlengine/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── mlengine/
│ ├── MlEngineLoader.java
│ ├── MlEngineModel.java
│ ├── MlEnginePredictException.java
│ └── MlEnginePredictFn.java
├── zoltar-tensorflow/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── com/
│ └── spotify/
│ └── zoltar/
│ └── tf/
│ ├── TensorFlowExtras.java
│ ├── TensorFlowGraphLoader.java
│ ├── TensorFlowGraphModel.java
│ ├── TensorFlowLoader.java
│ ├── TensorFlowModel.java
│ ├── TensorFlowPredictFn.java
│ └── TensorflowMetaGraphDefParsingException.java
├── zoltar-tests/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── spotify/
│ │ │ └── zoltar/
│ │ │ ├── IrisHelper.java
│ │ │ └── PredictorsTest.java
│ │ └── scala/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── Iris.scala
│ └── test/
│ ├── java/
│ │ └── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ ├── FeatureExtractFnsTest.java
│ │ ├── FeatureExtractorTest.java
│ │ ├── MultiPredictorTest.java
│ │ ├── PredictionTest.java
│ │ ├── PredictorBuilderTest.java
│ │ ├── PredictorTest.java
│ │ ├── VectorTest.java
│ │ ├── fs/
│ │ │ ├── FileSystemExtrasTest.java
│ │ │ ├── FileSystemExtrasTestIT.java
│ │ │ └── FileSystemExtrasTestUtils.java
│ │ ├── loaders/
│ │ │ ├── LoaderIT.java
│ │ │ ├── ModelMemoizerTest.java
│ │ │ └── PreloaderTest.java
│ │ ├── tf/
│ │ │ ├── TensorFlowExtrasTest.java
│ │ │ ├── TensorFlowGraphModelTest.java
│ │ │ └── TensorFlowModelTest.java
│ │ └── xgboost/
│ │ └── XGBoostModelTest.java
│ └── resources/
│ ├── badsubdir/
│ │ ├── 2018/
│ │ │ └── data.txt
│ │ └── z/
│ │ └── data.txt
│ ├── emptydir/
│ │ └── data.txt
│ ├── iris.csv
│ ├── iris.model
│ ├── log4j.properties
│ ├── settings.json
│ ├── settings_dummy.json
│ ├── test.txt
│ ├── testdir/
│ │ ├── 2018-01-15/
│ │ │ └── data.txt
│ │ ├── 2018-02-28/
│ │ │ └── data.txt
│ │ ├── 2018-03-01/
│ │ │ └── data.txt
│ │ └── 2018-04-01
│ ├── trained_model/
│ │ ├── saved_model.pb
│ │ ├── trained_model.txt
│ │ └── variables/
│ │ ├── variables.data-00000-of-00001
│ │ └── variables.index
│ └── trained_model.jar
└── zoltar-xgboost/
├── pom.xml
└── src/
└── main/
├── java/
│ ├── com/
│ │ └── spotify/
│ │ └── zoltar/
│ │ └── xgboost/
│ │ ├── XGBoostLoader.java
│ │ ├── XGBoostModel.java
│ │ └── XGBoostPredictFn.java
│ └── ml/
│ └── dmlc/
│ └── xgboost4j/
│ └── java/
│ └── GompLoader.java
└── resources/
└── lib/
└── libgomp.so.1
SYMBOL INDEX (484 symbols across 83 files)
FILE: data/iris-model/training/iris/iris/tensorflow.py
function train (line 30) | def train(_):
function main (line 59) | def main():
FILE: data/iris-model/training/iris/iris/xgboost.py
function transform_dataset (line 40) | def transform_dataset(ctx, dataset):
function train (line 53) | def train(_):
function main (line 83) | def main():
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/App.java
class App (line 41) | public class App {
method App (line 45) | private App() {}
method configure (line 47) | static void configure(final Environment environment) {
method main (line 84) | public static void main(final String... args) throws LoadingException {
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandler.java
class IrisPredictionHandler (line 40) | final class IrisPredictionHandler {
method IrisPredictionHandler (line 48) | private IrisPredictionHandler(final Predictor<Iris, Long> predictor) {
method create (line 54) | static IrisPredictionHandler create(final Predictor<Iris, Long> predic...
method routes (line 58) | Stream<Route<AsyncHandler<Response<ByteString>>>> routes() {
method predict (line 70) | CompletionStage<Response<ByteString>> predict(final RequestContext con...
method predict (line 84) | CompletionStage<String> predict(final String[] features) {
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictor.java
class IrisPredictor (line 36) | public final class IrisPredictor {
method create (line 39) | public static Predictor<Iris, Long> create(
FILE: examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/ModelConfig.java
class ModelConfig (line 25) | @AutoValue
method modelUri (line 29) | public abstract URI modelUri();
method settingsUri (line 32) | public abstract URI settingsUri();
method from (line 35) | public static ModelConfig from(final Config config) throws URISyntaxEx...
method create (line 50) | public static ModelConfig create(final URI modelUri, final URI setting...
FILE: examples/apollo-service-example/src/test/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandlerTest.java
class IrisPredictionHandlerTest (line 29) | public class IrisPredictionHandlerTest {
method shouldPredict (line 34) | @Test
FILE: examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/BatchPredictorExample.java
class BatchPredictorExample (line 35) | class BatchPredictorExample implements Predictor<Integer, Float> {
method BatchPredictorExample (line 39) | BatchPredictorExample() throws InterruptedException, ExecutionExceptio...
method predict (line 57) | @Override
FILE: examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/DummyModel.java
class DummyModel (line 20) | class DummyModel implements Model<Object> {
method id (line 21) | @Override
method instance (line 26) | @Override
method close (line 31) | @Override
FILE: examples/batch-predictor/src/test/java/com/spotify/zoltar/examples/batch/BatchPredictorExampleTest.java
class BatchPredictorExampleTest (line 31) | public class BatchPredictorExampleTest {
method testCustomMetrics (line 33) | @SuppressWarnings("unchecked")
FILE: examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/CustomMetricsExample.java
class CustomMetricsExample (line 53) | class CustomMetricsExample implements Predictor<Integer, Float> {
class CustomMetrics (line 58) | @AutoValue
method negativePredictCount (line 61) | abstract Counter negativePredictCount();
method negativeExtractCount (line 63) | abstract Counter negativeExtractCount();
method create (line 65) | static CustomMetrics create(final SemanticMetricRegistry registry, f...
class CustomPredictorMetrics (line 80) | @AutoValue
method metricsCache (line 83) | abstract LoadingCache<Model.Id, CustomMetrics> metricsCache();
method create (line 85) | static CustomPredictorMetrics create(
method predictFnMetrics (line 100) | @Override
method featureExtractorMetrics (line 109) | @Override
class NegativePredictMetrics (line 120) | @AutoValue
method negativePredictCount (line 123) | abstract Counter negativePredictCount();
method create (line 125) | static NegativePredictMetrics create(final Counter negativeCount) {
method prediction (line 130) | @Override
class NegativeExtractMetrics (line 137) | @AutoValue
method negativeExtractCount (line 140) | abstract Counter negativeExtractCount();
method create (line 142) | static NegativeExtractMetrics create(final Counter negativeCount) {
method extraction (line 149) | @Override
method CustomMetricsExample (line 155) | CustomMetricsExample(final SemanticMetricRegistry metricRegistry, fina...
method predict (line 187) | @Override
FILE: examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/DummyModel.java
class DummyModel (line 20) | class DummyModel implements Model<Object> {
method id (line 21) | @Override
method instance (line 26) | @Override
method close (line 31) | @Override
FILE: examples/custom-metrics/src/test/java/com/spotify/zoltar/examples/metrics/CustomMetricsExampleTest.java
class CustomMetricsExampleTest (line 27) | public class CustomMetricsExampleTest {
method testCustomMetrics (line 29) | @Test
FILE: examples/mlengine-example/src/main/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExample.java
class MlEnginePredictorExample (line 54) | public final class MlEnginePredictorExample implements Predictor<Iris, I...
method MlEnginePredictorExample (line 58) | private MlEnginePredictorExample(final Predictor<Iris, Integer> predic...
method create (line 63) | public static MlEnginePredictorExample create(
method predict (line 120) | @Override
class MlEnginePrediction (line 126) | @AutoValue
method classes (line 129) | public abstract List<Integer> classes();
method scores (line 131) | public abstract List<BigDecimal> scores();
method create (line 133) | @JsonCreator
FILE: examples/mlengine-example/src/test/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExampleIT.java
class MlEnginePredictorExampleIT (line 29) | public class MlEnginePredictorExampleIT {
method testPrediction (line 31) | @Test
FILE: zoltar-api/src/main/java/com/spotify/zoltar/Models.java
class Models (line 34) | public final class Models {
method Models (line 36) | private Models() {}
method xgboost (line 44) | public static XGBoostLoader xgboost(final String modelUri) {
method xgboost (line 55) | public static XGBoostLoader xgboost(final Model.Id id, final String mo...
method tensorFlow (line 65) | public static TensorFlowLoader tensorFlow(final String modelUri) {
method tensorFlow (line 76) | public static TensorFlowLoader tensorFlow(final Model.Id id, final Str...
method tensorFlow (line 87) | public static TensorFlowLoader tensorFlow(
method tensorFlow (line 100) | public static TensorFlowLoader tensorFlow(
method tensorFlowGraph (line 113) | public static TensorFlowGraphLoader tensorFlowGraph(
method tensorFlowGraph (line 127) | public static TensorFlowGraphLoader tensorFlowGraph(
method tensorFlowGraph (line 142) | public static TensorFlowGraphLoader tensorFlowGraph(
method tensorFlowGraph (line 155) | public static TensorFlowGraphLoader tensorFlowGraph(
method mlEngine (line 169) | public static MlEngineLoader mlEngine(final Model.Id id) {
method mlEngine (line 180) | public static MlEngineLoader mlEngine(
FILE: zoltar-api/src/main/java/com/spotify/zoltar/Predictors.java
class Predictors (line 40) | public final class Predictors {
method Predictors (line 42) | private Predictors() {}
method newBuilder (line 56) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 77) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 98) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 119) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 142) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 167) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 192) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 217) | @SuppressWarnings("checkstyle:LineLength")
method tensorFlow (line 239) | public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
method tensorFlow (line 260) | public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
method tensorFlow (line 281) | public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
method tensorFlow (line 306) | public static <InputT, ValueT> Predictor<InputT, ValueT> tensorFlow(
method tensorFlow (line 331) | public static <ValueT> Predictor<Example, ValueT> tensorFlow(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictor.java
type DefaultPredictor (line 35) | @FunctionalInterface
method create (line 50) | static <ModelT extends Model<?>, InputT, VectorT, ValueT> DefaultPredi...
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorBuilder.java
class DefaultPredictorBuilder (line 33) | @AutoValue
method create (line 48) | @SuppressWarnings("checkstyle:LineLength")
method create (line 68) | @SuppressWarnings("checkstyle:LineLength")
method create (line 88) | @SuppressWarnings("checkstyle:LineLength")
method create (line 108) | @SuppressWarnings("checkstyle:LineLength")
method modelLoader (line 121) | public abstract ModelLoader<ModelT> modelLoader();
method featureExtractor (line 123) | public abstract FeatureExtractor<ModelT, InputT, VectorT> featureExtra...
method predictFn (line 125) | public abstract AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predic...
method predictor (line 127) | public abstract Predictor<InputT, ValueT> predictor();
method with (line 129) | @Override
FILE: zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorTimeoutScheduler.java
class DefaultPredictorTimeoutScheduler (line 25) | final class DefaultPredictorTimeoutScheduler implements PredictorTimeout...
method DefaultPredictorTimeoutScheduler (line 30) | private DefaultPredictorTimeoutScheduler() {}
method create (line 32) | public static DefaultPredictorTimeoutScheduler create() {
method scheduler (line 36) | @Override
FILE: zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractFns.java
type FeatureExtractFns (line 31) | public interface FeatureExtractFns {
type ExtractFn (line 40) | @FunctionalInterface
method lift (line 43) | static <InputT, VectorT> ExtractFn<InputT, VectorT> lift(final Funct...
method identity (line 53) | static <InputT> ExtractFn<InputT, InputT> identity() {
method apply (line 58) | List<VectorT> apply(InputT... inputs) throws Exception;
method with (line 60) | default <C extends ExtractFn<InputT, VectorT>> C with(
type BatchExtractFn (line 74) | @FunctionalInterface
method lift (line 78) | @SuppressWarnings("unchecked")
method lift (line 92) | static <InputT, VectorT> BatchExtractFn<InputT, VectorT> lift(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractor.java
type FeatureExtractor (line 34) | @FunctionalInterface
method create (line 45) | @SuppressWarnings("checkstyle:LineLength")
method extract (line 60) | List<Vector<InputT, VectorT>> extract(ModelT model, InputT... input) t...
method with (line 62) | default <C extends FeatureExtractor<ModelT, InputT, VectorT>> C with(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Model.java
type Model (line 25) | public interface Model<UnderlyingT> extends AutoCloseable {
class Id (line 28) | @AutoValue
method value (line 30) | public abstract String value();
method create (line 32) | public static Id create(final String value) {
method id (line 37) | Id id();
method instance (line 43) | UnderlyingT instance();
FILE: zoltar-core/src/main/java/com/spotify/zoltar/ModelLoader.java
type ModelLoader (line 37) | @FunctionalInterface
type ThrowableSupplier (line 43) | @FunctionalInterface
method get (line 46) | M get() throws Exception;
type PreLoader (line 54) | @FunctionalInterface
method preload (line 66) | static <M extends Model<?>> PreLoader<M> preload(
method preload (line 80) | static <M extends Model<?>> PreLoader<M> preload(
method preload (line 92) | static <M extends Model<?>> Function<ModelLoader<M>, PreLoader<M>> p...
type ConsLoader (line 112) | @FunctionalInterface
method cons (line 120) | static <M extends Model<?>> ConsLoader<M> cons(final M model) {
method loaded (line 131) | static <M extends Model<?>> ModelLoader<M> loaded(final M model) {
method preload (line 144) | static <M extends Model<?>> PreLoader<M> preload(
method preload (line 158) | static <M extends Model<?>> ModelLoader<M> preload(
method load (line 170) | static <M extends Model<?>> ModelLoader<M> load(
method load (line 192) | static <M extends Model<?>> ModelLoader<M> load(final ThrowableSupplie...
method lift (line 202) | @Deprecated
method get (line 212) | CompletionStage<M> get();
method get (line 219) | default M get(final Duration duration)
method with (line 224) | @Deprecated
method compose (line 229) | default <L extends ModelLoader<M>> L compose(final Function<ModelLoade...
method close (line 233) | @Override
FILE: zoltar-core/src/main/java/com/spotify/zoltar/PredictFns.java
type PredictFns (line 29) | public interface PredictFns {
type AsyncPredictFn (line 39) | @FunctionalInterface
method lift (line 50) | @SuppressWarnings("checkstyle:LineLength")
method apply (line 76) | CompletionStage<List<Prediction<InputT, ValueT>>> apply(
method with (line 79) | default <C extends AsyncPredictFn<ModelT, InputT, VectorT, ValueT>> ...
type PredictFn (line 92) | @FunctionalInterface
method apply (line 103) | List<Prediction<InputT, ValueT>> apply(ModelT model, List<Vector<Inp...
method with (line 106) | default <C extends PredictFn<ModelT, InputT, VectorT, ValueT>> C with(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Prediction.java
class Prediction (line 26) | @AutoValue
method input (line 30) | public abstract InputT input();
method value (line 33) | public abstract ValueT value();
method create (line 36) | public static <InputT, ValueT> Prediction<InputT, ValueT> create(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Predictor.java
type Predictor (line 34) | @FunctionalInterface
method timeoutScheduler (line 38) | default PredictorTimeoutScheduler timeoutScheduler() {
method predict (line 51) | @SuppressWarnings("checkstyle:LineLength")
method predict (line 56) | default CompletionStage<List<Prediction<InputT, ValueT>>> predict(
method predict (line 62) | default CompletionStage<List<Prediction<InputT, ValueT>>> predict(fina...
FILE: zoltar-core/src/main/java/com/spotify/zoltar/PredictorBuilder.java
type PredictorBuilder (line 30) | public interface PredictorBuilder<ModelT extends Model<?>, InputT, Vecto...
method modelLoader (line 32) | ModelLoader<ModelT> modelLoader();
method featureExtractor (line 34) | FeatureExtractor<ModelT, InputT, VectorT> featureExtractor();
method predictFn (line 36) | AsyncPredictFn<ModelT, InputT, VectorT, ValueT> predictFn();
method predictor (line 38) | Predictor<InputT, ValueT> predictor();
method with (line 40) | PredictorBuilder<ModelT, InputT, VectorT, ValueT> with(
method with (line 45) | @SuppressWarnings("unchecked")
method with (line 51) | @SuppressWarnings("unchecked")
method with (line 57) | @SuppressWarnings("unchecked")
method with (line 63) | default <C extends PredictorBuilder<ModelT, InputT, VectorT, ValueT>> ...
FILE: zoltar-core/src/main/java/com/spotify/zoltar/PredictorTimeoutScheduler.java
type PredictorTimeoutScheduler (line 21) | @FunctionalInterface
method scheduler (line 24) | ScheduledExecutorService scheduler();
FILE: zoltar-core/src/main/java/com/spotify/zoltar/Vector.java
class Vector (line 26) | @AutoValue
method input (line 30) | public abstract InputT input();
method value (line 33) | public abstract ValueT value();
method create (line 36) | public static <InputT, ValueT> Vector<InputT, ValueT> create(
FILE: zoltar-core/src/main/java/com/spotify/zoltar/fs/FileSystemExtras.java
class FileSystemExtras (line 36) | public final class FileSystemExtras {
method FileSystemExtras (line 39) | private FileSystemExtras() {}
method getLatestDate (line 45) | public static Optional<String> getLatestDate(final String src) throws ...
method path (line 54) | public static Path path(final URI uri) throws IOException {
method downloadIfNonLocal (line 85) | public static URI downloadIfNonLocal(final URI path) throws IOException {
method copyDir (line 95) | static Path copyDir(final Path src, final Path dest, final boolean ove...
FILE: zoltar-core/src/main/java/com/spotify/zoltar/loaders/ModelMemoizer.java
type ModelMemoizer (line 30) | @Deprecated
method memoize (line 41) | static <M extends Model<?>> ModelMemoizer<M> memoize(final ModelLoader...
FILE: zoltar-core/src/main/java/com/spotify/zoltar/loaders/Preloader.java
type Preloader (line 31) | @Deprecated
method preload (line 36) | static <M extends Model<?>> Function<ModelLoader<M>, Preloader<M>> pre...
method preload (line 46) | static <M extends Model<?>> Function<ModelLoader<M>, Preloader<M>> pre...
method preloadAsync (line 65) | static <M extends Model<?>> Function<ModelLoader<M>, Preloader<M>> pre...
FILE: zoltar-featran/src/main/java/com/spotify/zoltar/featran/FeatranExtractFns.java
class FeatranExtractFns (line 36) | public final class FeatranExtractFns {
method FeatranExtractFns (line 38) | private FeatranExtractFns() {}
method doubles (line 48) | public static <InputT> ExtractFn<InputT, double[]> doubles(
method doubles (line 61) | public static <InputT> ExtractFn<InputT, double[]> doubles(
method example (line 74) | public static <InputT> ExtractFn<InputT, Example> example(
method example (line 87) | public static <InputT> ExtractFn<InputT, Example> example(
method floats (line 100) | public static <InputT> ExtractFn<InputT, float[]> floats(
method floats (line 113) | public static <InputT> ExtractFn<InputT, float[]> floats(
method sparseFloats (line 126) | public static <InputT> ExtractFn<InputT, FloatSparseArray> sparseFloats(
method sparseFloats (line 139) | public static <InputT> ExtractFn<InputT, FloatSparseArray> sparseFloats(
method sparseDoubles (line 152) | public static <InputT> ExtractFn<InputT, DoubleSparseArray> sparseDoub...
method sparseDoubles (line 165) | public static <InputT> ExtractFn<InputT, DoubleSparseArray> sparseDoub...
method labeledPoints (line 178) | public static <InputT> ExtractFn<InputT, LabeledPoint> labeledPoints(
method labeledPoints (line 191) | public static <InputT> ExtractFn<InputT, LabeledPoint> labeledPoints(
method sparseLabeledPoints (line 204) | public static <InputT> ExtractFn<InputT, SparseLabeledPoint> sparseLab...
method sparseLabeledPoints (line 217) | public static <InputT> ExtractFn<InputT, SparseLabeledPoint> sparseLab...
FILE: zoltar-jmh/src/main/java/com/spotify/zoltar/jmh/BenchmarkTensorFlow.java
class BenchmarkTensorFlow (line 57) | @OutputTimeUnit(TimeUnit.MILLISECONDS)
method main (line 71) | public static void main(final String[] args) throws RunnerException {
method setup (line 79) | @Setup
method predict (line 86) | @Benchmark
method shutdown (line 92) | @TearDown
method extractFn (line 97) | private static ExtractFn<Iris, Example> extractFn() throws IOException...
method predictor (line 105) | private static Predictor<Iris, Long> predictor() throws Exception {
FILE: zoltar-jmh/src/main/java/com/spotify/zoltar/jmh/IrisHelper.java
class IrisHelper (line 28) | public class IrisHelper {
method IrisHelper (line 30) | private IrisHelper() {}
method fromCsvString (line 32) | private static IrisFeaturesSpec.Iris fromCsvString(final String featur...
method getIrisData (line 43) | public static Iris[] getIrisData() {
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/FeatureExtractorMetrics.java
type FeatureExtractorMetrics (line 23) | @FunctionalInterface
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/Instrumentations.java
class Instrumentations (line 29) | public final class Instrumentations {
method Instrumentations (line 31) | private Instrumentations() {}
method predictor (line 37) | @SuppressWarnings("checkstyle:LineLength")
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedFeatureExtractor.java
type InstrumentedFeatureExtractor (line 32) | @FunctionalInterface
method create (line 37) | @SuppressWarnings("checkstyle:LineLength")
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedPredictFn.java
type InstrumentedPredictFn (line 33) | @FunctionalInterface
method create (line 38) | @SuppressWarnings("checkstyle:LineLength")
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedPredictorBuilder.java
class InstrumentedPredictorBuilder (line 37) | @AutoValue
method predictorBuilder (line 41) | public abstract PredictorBuilder<ModelT, InputT, VectorT, ValueT> pred...
method metrics (line 43) | public abstract PredictorMetrics<InputT, VectorT, ValueT> metrics();
method create (line 46) | @SuppressWarnings("checkstyle:LineLength")
method modelLoader (line 72) | @Override
method featureExtractor (line 77) | @Override
method predictFn (line 82) | @Override
method predictor (line 87) | @Override
method with (line 92) | @Override
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictFnMetrics.java
type PredictFnMetrics (line 23) | @FunctionalInterface
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictMetrics.java
type PredictMetrics (line 23) | public interface PredictMetrics<InputT, ValueT> {
method prediction (line 25) | void prediction(List<Prediction<InputT, ValueT>> predictions);
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictorMetrics.java
type PredictorMetrics (line 19) | public interface PredictorMetrics<InputT, VectorT, PredictionT> {
method predictFnMetrics (line 22) | PredictFnMetrics<InputT, PredictionT> predictFnMetrics();
method featureExtractorMetrics (line 28) | FeatureExtractorMetrics<InputT, VectorT> featureExtractorMetrics();
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/VectorMetrics.java
type VectorMetrics (line 23) | public interface VectorMetrics<InputT, ValueT> {
method extraction (line 25) | void extraction(List<Vector<InputT, ValueT>> vectors);
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticPredictMetrics.java
class SemanticPredictMetrics (line 28) | @AutoValue
method predictDurationTimer (line 32) | abstract Timer.Context predictDurationTimer();
method predictRateCounter (line 34) | abstract Meter predictRateCounter();
method create (line 36) | static <InputT, ValueT> SemanticPredictMetrics<InputT, ValueT> create(
method prediction (line 41) | @Override
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticPredictorMetrics.java
class SemanticPredictorMetrics (line 39) | @AutoValue
method metricsCache (line 43) | abstract LoadingCache<Id, Metrics> metricsCache();
method create (line 46) | public static <InputT, VectorT, ValueT> SemanticPredictorMetrics<Input...
method predictFnMetrics (line 61) | @Override
method featureExtractorMetrics (line 72) | @Override
class Metrics (line 83) | @AutoValue
method predictDurationTimer (line 86) | abstract Timer predictDurationTimer();
method predictRateCounter (line 88) | abstract Meter predictRateCounter();
method extractDurationTimer (line 90) | abstract Timer extractDurationTimer();
method extractRateCounter (line 92) | abstract Meter extractRateCounter();
method create (line 94) | static Metrics create(final SemanticMetricRegistry registry, final M...
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticVectorMetrics.java
class SemanticVectorMetrics (line 28) | @AutoValue
method extractDurationTimer (line 32) | abstract Timer.Context extractDurationTimer();
method extractRateCounter (line 34) | abstract Meter extractRateCounter();
method create (line 36) | static <InputT, ValueT> SemanticVectorMetrics<InputT, ValueT> create(
method extraction (line 41) | @Override
FILE: zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/What.java
type What (line 19) | public enum What {
method What (line 31) | What(final String tag) {
method tag (line 35) | public String tag() {
FILE: zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEngineLoader.java
type MlEngineLoader (line 22) | @FunctionalInterface
method create (line 31) | static MlEngineLoader create(final String projectId, final String mode...
method create (line 42) | static MlEngineLoader create(
method create (line 55) | static MlEngineLoader create(final Model.Id id) {
method create (line 64) | static MlEngineLoader create(final ThrowableSupplier<MlEngineModel> su...
FILE: zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEngineModel.java
class MlEngineModel (line 52) | @AutoValue
method create (line 63) | static MlEngineModel create(final String projectId, final String modelId)
method create (line 76) | static MlEngineModel create(final String projectId, final String model...
method create (line 91) | public static MlEngineModel create(final Model.Id id)
method httpTransport (line 107) | abstract HttpTransport httpTransport();
method predict (line 114) | public Predictions predict(final List<?> data) throws IOException, MlE...
method predictExamples (line 134) | public Predictions predictExamples(final List<Example> examples)
method close (line 152) | @Override
class Response (line 158) | @AutoValue
method content (line 161) | abstract GoogleApiHttpBody content();
method from (line 163) | static Response from(final GoogleApiHttpBody content) {
method predictions (line 173) | @SuppressWarnings("unchecked")
method error (line 186) | public Optional<String> error() {
class Predictions (line 191) | @AutoValue
method values (line 203) | public abstract List<Object> values();
method values (line 210) | public <T> List<T> values(final Class<T> klass) throws ExecutionEx...
method create (line 220) | static Predictions create(final List<Object> values) {
FILE: zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEnginePredictException.java
class MlEnginePredictException (line 19) | public final class MlEnginePredictException extends Exception {
method MlEnginePredictException (line 27) | public MlEnginePredictException(final String message) {
FILE: zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEnginePredictFn.java
type MlEnginePredictFn (line 27) | @FunctionalInterface
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowExtras.java
class TensorFlowExtras (line 28) | public class TensorFlowExtras {
method TensorFlowExtras (line 30) | private TensorFlowExtras() {}
method runAndExtract (line 41) | public static <A> A runAndExtract(
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowGraphLoader.java
type TensorFlowGraphLoader (line 30) | @FunctionalInterface
method create (line 41) | static TensorFlowGraphLoader create(
method create (line 55) | static TensorFlowGraphLoader create(
method create (line 70) | static TensorFlowGraphLoader create(
method create (line 82) | static TensorFlowGraphLoader create(
method create (line 95) | static TensorFlowGraphLoader create(
method create (line 111) | static TensorFlowGraphLoader create(
method create (line 124) | static TensorFlowGraphLoader create(final ThrowableSupplier<TensorFlow...
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowGraphModel.java
class TensorFlowGraphModel (line 44) | @AutoValue
method create (line 59) | public static TensorFlowGraphModel create(
method create (line 75) | public static TensorFlowGraphModel create(
method create (line 95) | public static TensorFlowGraphModel create(
method create (line 111) | public static TensorFlowGraphModel create(
method create (line 127) | public static TensorFlowGraphModel create(
method create (line 146) | public static TensorFlowGraphModel create(
method close (line 168) | @Override
method graph (line 181) | public abstract Graph graph();
method instance (line 184) | @Override
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowLoader.java
type TensorFlowLoader (line 24) | @FunctionalInterface
method create (line 33) | static TensorFlowLoader create(final String modelUri) {
method create (line 44) | static TensorFlowLoader create(final Model.Id id, final String modelUr...
method create (line 55) | static TensorFlowLoader create(final String modelUri, final TensorFlow...
method create (line 67) | static TensorFlowLoader create(
method create (line 81) | static TensorFlowLoader create(
method create (line 94) | static TensorFlowLoader create(final ThrowableSupplier<TensorFlowModel...
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowModel.java
class TensorFlowModel (line 43) | @AutoValue
method create (line 56) | public static TensorFlowModel create(final URI modelResource) throws I...
method create (line 65) | public static TensorFlowModel create(final Model.Id id, final URI mode...
method create (line 76) | public static TensorFlowModel create(final URI modelResource, final Op...
method create (line 87) | public static TensorFlowModel create(
method create (line 98) | public static TensorFlowModel create(
method close (line 129) | @Override
method instance (line 137) | @Override
method options (line 141) | public abstract Options options();
method metaGraphDefinition (line 143) | public abstract MetaGraphDef metaGraphDefinition();
method signatureDefinition (line 145) | public abstract SignatureDef signatureDefinition();
method inputsNameMap (line 147) | public abstract Map<String, String> inputsNameMap();
method outputsNameMap (line 149) | public abstract Map<String, String> outputsNameMap();
class Options (line 152) | @AutoValue
method tags (line 159) | public abstract List<String> tags();
method builder (line 161) | public static Builder builder() {
class Builder (line 166) | @AutoValue.Builder
method tags (line 169) | public abstract Builder tags(List<String> tags);
method build (line 171) | public abstract Options build();
method toNameMap (line 175) | private static Map<String, String> toNameMap(final Map<String, TensorI...
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowPredictFn.java
type TensorFlowPredictFn (line 47) | @FunctionalInterface
method example (line 57) | static <InputT, ValueT> TensorFlowPredictFn<InputT, Example, ValueT> e...
method exampleBatch (line 102) | @Deprecated
FILE: zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorflowMetaGraphDefParsingException.java
class TensorflowMetaGraphDefParsingException (line 19) | public final class TensorflowMetaGraphDefParsingException extends Except...
method TensorflowMetaGraphDefParsingException (line 20) | public TensorflowMetaGraphDefParsingException(final String message, fi...
FILE: zoltar-tests/src/main/java/com/spotify/zoltar/IrisHelper.java
class IrisHelper (line 25) | public class IrisHelper {
method IrisHelper (line 27) | private IrisHelper() {}
method fromCsvString (line 29) | private static IrisFeaturesSpec.Iris fromCsvString(final String featur...
method getIrisTestData (line 40) | public static IrisFeaturesSpec.Iris[] getIrisTestData() throws Excepti...
FILE: zoltar-tests/src/main/java/com/spotify/zoltar/PredictorsTest.java
class PredictorsTest (line 23) | public final class PredictorsTest {
method PredictorsTest (line 25) | private PredictorsTest() {}
method newBuilder (line 39) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 60) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 81) | @SuppressWarnings("checkstyle:LineLength")
method newBuilder (line 102) | @SuppressWarnings("checkstyle:LineLength")
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/FeatureExtractFnsTest.java
class FeatureExtractFnsTest (line 30) | public class FeatureExtractFnsTest {
method batchExtractFn (line 32) | @SuppressWarnings("unchecked")
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/FeatureExtractorTest.java
class FeatureExtractorTest (line 29) | public class FeatureExtractorTest {
method emptyExtract (line 31) | @Test
method nonEmptyExtract (line 39) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/MultiPredictorTest.java
class MultiPredictorTest (line 31) | public class MultiPredictorTest {
method testMultiplePredictors (line 33) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/PredictionTest.java
class PredictionTest (line 24) | public class PredictionTest {
method predictionSerializable (line 26) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/PredictorBuilderTest.java
class PredictorBuilderTest (line 32) | public class PredictorBuilderTest {
class DummyModel (line 34) | static class DummyModel implements Model<Object> {
method id (line 36) | @Override
method instance (line 41) | @Override
method close (line 46) | @Override
class IdentityPredictorBuilder (line 50) | static final class IdentityPredictorBuilder<ModelT extends Model<?>, I...
method IdentityPredictorBuilder (line 55) | IdentityPredictorBuilder(
method decorate (line 60) | public static <ModelT extends Model<?>, InputT, VectorT, ValueT>
method modelLoader (line 68) | @Override
method featureExtractor (line 73) | @Override
method predictFn (line 78) | @Override
method predictor (line 83) | @Override
method with (line 88) | @Override
method identityDecoration (line 98) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/PredictorTest.java
class PredictorTest (line 38) | public class PredictorTest {
class DummyModel (line 40) | static class DummyModel implements Model<Object> {
method id (line 42) | @Override
method instance (line 47) | @Override
method close (line 52) | @Override
method timeout (line 56) | @Test
method empty (line 82) | @Test
method nonEmpty (line 97) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/VectorTest.java
class VectorTest (line 24) | public class VectorTest {
method vectorSerializable (line 26) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTest.java
class FileSystemExtrasTest (line 37) | public class FileSystemExtrasTest {
method localPath (line 39) | @Test
method jarPath (line 48) | @Test
method gcsPath (line 54) | @Test
method invalidGcsBucketUri (line 60) | @Test(expected = IllegalArgumentException.class)
method getLatestDateTest (line 66) | @Test
method getLatestDateTestInputDoesNotExist (line 72) | @Test(expected = IOException.class)
method getLatestDateTestInputNotADirectory (line 77) | @Test(expected = IOException.class)
method getLatestDateTestInputEmptyDir (line 82) | @Test(expected = IOException.class)
method getLatestDateTestInputBadSubdirs (line 87) | @Test(expected = IOException.class)
method noCopyIfDefaultFileSystem (line 92) | @Test
method downloadIfNonLocalWithJar (line 98) | @Test
method copyDirectoryFromJar (line 105) | @Test
method copyDirectory (line 114) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTestIT.java
class FileSystemExtrasTestIT (line 26) | public class FileSystemExtrasTestIT {
method downloadGcsBucket (line 28) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTestUtils.java
class FileSystemExtrasTestUtils (line 28) | public final class FileSystemExtrasTestUtils {
method checkCopiedDirectory (line 30) | public static void checkCopiedDirectory(final File file, String... ite...
method jarUri (line 40) | public static URI jarUri() {
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/loaders/LoaderIT.java
class LoaderIT (line 37) | public class LoaderIT {
method tfLoaderGcs (line 41) | @Test
method tfLoaderGcsTrailingSlash (line 50) | @Test
method tfLoaderJar (line 58) | @Test
method tfLoaderJarTrailingSlash (line 66) | @Test(expected = ExecutionException.class)
method tfGraphLoader (line 77) | @Test
method tfGraphLoaderTrailingSlash (line 85) | @Test(expected = ExecutionException.class)
method xgBoostLoader (line 94) | @Test
method xgBoostLoaderTrailingSlash (line 102) | @Test(expected = ExecutionException.class)
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/loaders/ModelMemoizerTest.java
class ModelMemoizerTest (line 31) | public class ModelMemoizerTest {
class DummyModel (line 33) | static class DummyModel implements Model<Object> {
method DummyModel (line 36) | public DummyModel() {
method id (line 40) | @Override
method instance (line 45) | @Override
method close (line 51) | @Override
method getIncrementValue (line 54) | public int getIncrementValue() {
method memoize (line 59) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/loaders/PreloaderTest.java
class PreloaderTest (line 30) | public class PreloaderTest {
class DummyModel (line 32) | static class DummyModel implements Model<Object> {
method DummyModel (line 34) | public DummyModel() {}
method id (line 36) | @Override
method instance (line 41) | @Override
method close (line 46) | @Override
method preload (line 50) | @Test
method preloadTimeout (line 58) | @Test
method preloadAsync (line 73) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowExtrasTest.java
class TensorFlowExtrasTest (line 35) | public class TensorFlowExtrasTest {
method createDummyGraph (line 40) | private static Graph createDummyGraph() {
method testExtract1 (line 78) | @Test
method testExtract2a (line 99) | @Test
method testExtract2b (line 121) | @Test
method assertScalar (line 143) | private void assertScalar(final Tensor jt, final Double value) {
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowGraphModelTest.java
class TensorFlowGraphModelTest (line 52) | public class TensorFlowGraphModelTest {
method createADummyTFGraph (line 61) | private Path createADummyTFGraph() throws IOException {
method testDefaultId (line 90) | @Test
method testCustomId (line 101) | @Test
method testDummyLoadOfTensorFlowGraph (line 112) | @Test
method testDummyLoadOfTensorFlowGraphWithPrefix (line 131) | @Test
method testModelInference (line 156) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowModelTest.java
class TensorFlowModelTest (line 60) | public class TensorFlowModelTest {
method getTFIrisPredictor (line 62) | public static Predictor<Iris, Long> getTFIrisPredictor() throws Except...
method testDefaultId (line 82) | @Test
method testMetaGraphDefinition (line 92) | @Test
method testDefaultSignatureInputAndOutputNameMapping (line 107) | @Test
method testProvidedSignatureDefInputAndOutputNameMapping (line 135) | @Test
method testCustomId (line 168) | @Test
method testModelInference (line 179) | @Test
method optionsSerializable (line 207) | @Test
FILE: zoltar-tests/src/test/java/com/spotify/zoltar/xgboost/XGBoostModelTest.java
class XGBoostModelTest (line 55) | public class XGBoostModelTest {
method getXGBoostIrisPredictor (line 57) | public static Predictor<Iris, Long> getXGBoostIrisPredictor() throws E...
method testDefaultId (line 100) | @Test
method testCustomId (line 110) | @Test
method testLoadingModel (line 121) | @Test
method testModelPrediction (line 127) | @Test
FILE: zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostLoader.java
type XGBoostLoader (line 24) | @FunctionalInterface
method create (line 34) | static XGBoostLoader create(final String modelUri) {
method create (line 45) | static XGBoostLoader create(final Model.Id id, final String modelUri) {
method create (line 54) | static XGBoostLoader create(final ThrowableSupplier<XGBoostModel> supp...
FILE: zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostModel.java
class XGBoostModel (line 38) | @SuppressWarnings("checkstyle:AbbreviationAsWordInName")
method create (line 48) | public static XGBoostModel create(final URI modelUri) throws IOExcepti...
method create (line 57) | public static XGBoostModel create(final Model.Id id, final URI modelUr...
method instance (line 68) | public abstract Booster instance();
method close (line 71) | @Override
FILE: zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostPredictFn.java
type XGBoostPredictFn (line 27) | @SuppressWarnings("checkstyle:AbbreviationAsWordInName")
FILE: zoltar-xgboost/src/main/java/ml/dmlc/xgboost4j/java/GompLoader.java
class GompLoader (line 27) | public class GompLoader {
method isLinux (line 29) | private static boolean isLinux() {
method loadGomp (line 33) | private static void loadGomp() throws IOException {
method load (line 39) | private static void load(final String name) throws IOException {
method getLoadedLibraries (line 49) | @SuppressWarnings("unchecked")
method start (line 62) | public static void start() throws IOException {
Condensed preview — 180 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (769K chars).
[
{
"path": ".github/dependabot.yml",
"chars": 157,
"preview": "version: 2\nupdates:\n - package-ecosystem: maven\n directory: \"/\"\n schedule:\n interval: daily\n time: \"04:"
},
{
"path": ".github/workflows/codeql-analysis.yml",
"chars": 2331,
"preview": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# Y"
},
{
"path": ".github/workflows/main.yml",
"chars": 499,
"preview": "name: main\non: [push, pull_request]\n\njobs:\n build:\n name: Build\n runs-on: ubuntu-latest\n steps:\n - uses: "
},
{
"path": ".gitignore",
"chars": 387,
"preview": "## OS\n.DS_Store\n\n## Artifacts\n*.class\n\n## Package Files\n*.jar\n*.war\n*.ear\ntarget\n\n## IDE's \n.idea\n*.iml\n.project\n.settin"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "NOTICE",
"chars": 33,
"preview": "Zoltar\nCopyright 2018 Spotify AB\n"
},
{
"path": "README.md",
"chars": 781,
"preview": "# Zoltar\n\n[](https://travis-ci.org/spotify/zoltar"
},
{
"path": "build.sbt",
"chars": 1287,
"preview": "lazy val noPublishSettings = Seq(\n publish := {},\n publishLocal := {},\n publishArtifact := false\n)\n\nlazy val root = p"
},
{
"path": "catalog-info.yaml",
"chars": 114,
"preview": "apiVersion: backstage.io/v1alpha1\nkind: Resource\nmetadata:\n name: zoltar\nspec:\n type: resource\n owner: flatmap\n"
},
{
"path": "checkstyle.xml",
"chars": 13524,
"preview": "<?xml version=\"1.0\"?>\n<!DOCTYPE module PUBLIC\n \"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN\"\n \"htt"
},
{
"path": "data/iris-model/build.sbt",
"chars": 492,
"preview": "inThisBuild(scalaVersion := \"2.11.12\")\n\nlazy val root = project\n .in(file(\".\"))\n .settings(\n libraryDependencies :="
},
{
"path": "data/iris-model/project/build.properties",
"chars": 18,
"preview": "sbt.version=1.3.6\n"
},
{
"path": "data/iris-model/scripts/publish_data.sh",
"chars": 446,
"preview": "#!/bin/bash\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\nDEST_TABLE=${1:-\"data-integration-test:zoltar.iris\"}\nTEMP_DI"
},
{
"path": "data/iris-model/src/main/scala/com/spotify/zoltar/Iris.scala",
"chars": 2673,
"preview": "/*\n * Copyright 2018 Spotify AB.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not us"
},
{
"path": "data/iris-model/src/test/resources/expected_feature_spec.json",
"chars": 587,
"preview": "{\"version\":1,\"features\":[{\"name\":\"petal_length\",\"kind\":\"FloatList\",\"tags\":{\"multispec-id\":\"0\"}},{\"name\":\"petal_width\",\"k"
},
{
"path": "data/iris-model/src/test/scala/com/spotify/zoltar/IrisTest.scala",
"chars": 2263,
"preview": "/*\n * Copyright 2018 Spotify AB.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not us"
},
{
"path": "data/iris-model/training/Makefile",
"chars": 944,
"preview": "SHELL := /bin/bash\nID := $(shell date +%Y-%m-%d--%H-%M-%S)\nUSERNAME ?= $(shell whoami)\nJOB_DIR ?= gs://data-integration-"
},
{
"path": "data/iris-model/training/README.md",
"chars": 154,
"preview": "# Training\n\n## Iris\n\n```bash\nmake\n\niris-tensorflow Train iris tensorflow model\niris-xgboost "
},
{
"path": "data/iris-model/training/iris/iris/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "data/iris-model/training/iris/iris/tensorflow.py",
"chars": 2302,
"preview": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright 2018 Spotify AB.\n#\n# Licensed under the Apache License, Vers"
},
{
"path": "data/iris-model/training/iris/iris/xgboost.py",
"chars": 2544,
"preview": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright 2018 Spotify AB.\n#\n# Licensed under the Apache License, Vers"
},
{
"path": "data/iris-model/training/iris/requirements.txt",
"chars": 44,
"preview": "# dependencies are read from setup.py.\n-e .\n"
},
{
"path": "data/iris-model/training/iris/setup.cfg",
"chars": 1125,
"preview": "[metadata]\nname = zoltar_model_train\nlong_description = file: README.md\nurl = https://github.com/spotify/zoltar\nlicense "
},
{
"path": "data/iris-model/training/iris/setup.py",
"chars": 436,
"preview": "from setuptools import setup\n\nsetup(\n install_requires=[\n \"numpy==1.14.0\",\n \"tensorflow==1.15.2\",\n "
},
{
"path": "data/iris-model/training/iris/test-requirements.txt",
"chars": 104,
"preview": "coverage\nflake8>=3.4.1\nflake8-quotes>=0.12.0\nflake8-coding>=1.3.0\nflake8-import-order>=0.15\nnose>=1.3.7\n"
},
{
"path": "docs/src/paradox/_template/javadoc.st",
"chars": 245,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <!-- HTML meta refresh URL redirection -->\n <meta http-equiv=\"refresh\" content=\"0; url="
},
{
"path": "docs/src/paradox/getting-started.md",
"chars": 489,
"preview": "# Getting Started\n\nTo start using `Zoltar` you need to add this dependency to your project.\n\n@@dependency[Maven,Gradle,s"
},
{
"path": "docs/src/paradox/index.md",
"chars": 3350,
"preview": "# Zoltar\n\n[](https://travis-ci.org/spotify/zoltar"
},
{
"path": "docs/src/paradox/javadoc.md",
"chars": 44,
"preview": "---\nlayout: javadoc\n---\n\n# Api Documentation"
},
{
"path": "docs/src/paradox/license.md",
"chars": 129,
"preview": "# License\n\nCopyright 2018 Spotify AB.\n\nLicensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LI"
},
{
"path": "docs/src/paradox/modules/featran.md",
"chars": 227,
"preview": "# Featran\n\n`Featran` dependency is not included by default. To use it you need to include an extra dependency.\n\n@@depend"
},
{
"path": "docs/src/paradox/modules/index.md",
"chars": 188,
"preview": "# Modules\n\n@@@ index\n\n* [TensorFlow](tensorflow.md)\n* [XGBoost](xgboost.md)\n* [Featran](featran.md)\n* [Metrics](metrics."
},
{
"path": "docs/src/paradox/modules/metrics.md",
"chars": 1235,
"preview": "# Metrics\n\n`Zoltar` provides a way to attach and extend metrics around @javadoc[Predictor](com.spotify.zoltar.Predictor)"
},
{
"path": "docs/src/paradox/modules/mlengine.md",
"chars": 1155,
"preview": "# Google Cloud ML Engine\n\n## Getting started\n\nTo use `Google Cloud ML Engine`, you need to deploy your model first! \nDep"
},
{
"path": "docs/src/paradox/modules/tensorflow.md",
"chars": 235,
"preview": "# TensorFlow\n\n`TensorFlow` dependency is not included by default. To use it you need to include an extra dependency.\n\n@@"
},
{
"path": "docs/src/paradox/modules/xgboost.md",
"chars": 228,
"preview": "# XGBoost\n\n`XGBoost` dependency is not included by default. To use it you need to include an extra dependency.\n \n@@depen"
},
{
"path": "docs/src/paradox/release.md",
"chars": 1181,
"preview": "# Release Instructions\n\nThese instructions are based on the [instructions](http://central.sonatype.org/pages/ossrh-guide"
},
{
"path": "examples/apollo-service-example/SERVICE.marker",
"chars": 68,
"preview": "(This marker file causes Maven to build this artifact as a service)\n"
},
{
"path": "examples/apollo-service-example/findbugsexclude.xml",
"chars": 619,
"preview": "<?xml version=\"1.0\"?>\n<FindBugsFilter>\n <Match>\n <!--\n <Or> This element combines Match clauses as disjuncts. I.e"
},
{
"path": "examples/apollo-service-example/pom.xml",
"chars": 8996,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/"
},
{
"path": "examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/App.java",
"chars": 3250,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandler.java",
"chars": 3740,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/IrisPredictor.java",
"chars": 2119,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/apollo-service-example/src/main/java/com/spotify/zoltar/examples/apollo/ModelConfig.java",
"chars": 1775,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/apollo-service-example/src/main/resources/zoltar-example.conf",
"chars": 593,
"preview": "# Configuration for ml-serving-example in production mode\n\n# Over-all Apollo settings.\n# https://github.com/spotify/apol"
},
{
"path": "examples/apollo-service-example/src/test/java/com/spotify/zoltar/examples/apollo/IrisPredictionHandlerTest.java",
"chars": 1337,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/apollo-service-example/src/test/resources/settings.json",
"chars": 1450,
"preview": "[\n {\n \"cls\": \"com.spotify.featran.transformers.StandardScaler\",\n \"name\": \"petal_length\",\n \"params\": {\n \"w"
},
{
"path": "examples/apollo-service-example/src/test/resources/zoltar-example.conf",
"chars": 450,
"preview": "# Configuration for ml-serving-example in production mode\n\n# Over-all Apollo settings.\n# https://github.com/spotify/apol"
},
{
"path": "examples/batch-predictor/pom.xml",
"chars": 1944,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/BatchPredictorExample.java",
"chars": 2426,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/batch-predictor/src/main/java/com/spotify/zoltar/examples/batch/DummyModel.java",
"chars": 898,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/batch-predictor/src/test/java/com/spotify/zoltar/examples/batch/BatchPredictorExampleTest.java",
"chars": 1707,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/custom-metrics/pom.xml",
"chars": 2061,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/CustomMetricsExample.java",
"chars": 7905,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/custom-metrics/src/main/java/com/spotify/zoltar/examples/metrics/DummyModel.java",
"chars": 900,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/custom-metrics/src/test/java/com/spotify/zoltar/examples/metrics/CustomMetricsExampleTest.java",
"chars": 1535,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/mlengine-example/pom.xml",
"chars": 2354,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "examples/mlengine-example/src/main/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExample.java",
"chars": 5993,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/mlengine-example/src/test/java/com/spotify/zoltar/examples/mlengine/MlEnginePredictorExampleIT.java",
"chars": 1570,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "examples/mlengine-example/src/test/resources/settings.json",
"chars": 1450,
"preview": "[\n {\n \"cls\": \"com.spotify.featran.transformers.StandardScaler\",\n \"name\": \"petal_length\",\n \"params\": {\n \"w"
},
{
"path": "findbugsexclude.xml",
"chars": 274,
"preview": "<?xml version=\"1.0\"?>\n<FindBugsFilter>\n <Match>\n <Package name=\"~.*\\.proto\"/>\n </Match>\n <Match>\n <Class name=\""
},
{
"path": "pom.xml",
"chars": 6484,
"preview": "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocat"
},
{
"path": "project/build.properties",
"chars": 18,
"preview": "sbt.version=1.1.5\n"
},
{
"path": "project/plugins.sbt",
"chars": 251,
"preview": "addSbtPlugin(\"com.typesafe.sbt\" % \"sbt-site\" % \"1.3.2\")\naddSbtPlugin(\"com.lightbend.paradox\" % \"sbt-paradox\" % \"0.3.3\")\n"
},
{
"path": "scripts/deploy.sh",
"chars": 270,
"preview": "#!/bin/bash\n\necho 'VERSION=${project.version}' | mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate | grep '^"
},
{
"path": "scripts/libgomp/Dockerfile",
"chars": 72,
"preview": "FROM debian:sid-slim\n\nRUN apt-get update && apt-get install -y libgomp1\n"
},
{
"path": "scripts/libgomp/create.sh",
"chars": 412,
"preview": "#!/usr/bin/env bash\n\n# Builds a debian docker image with libgomp1 installed and copies the binary to the output path.\n\ns"
},
{
"path": "scripts/test.sh",
"chars": 490,
"preview": "#!/usr/bin/env bash\n\nGOAL=test\n\nif [ \"${TRAVIS_SECURE_ENV_VARS}\" = \"true\" ]; then\n GOAL=verify\n\n openssl aes-256-c"
},
{
"path": "settings.xml",
"chars": 546,
"preview": "<settings xmlns=\"http://maven.apache.org/SETTINGS/1.0.0\"\n xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xsi:s"
},
{
"path": "suppressions.xml",
"chars": 255,
"preview": "<?xml version=\"1.0\"?>\n\n<!DOCTYPE suppressions PUBLIC\n \"-//Puppy Crawl//DTD Suppressions 1.1//EN\"\n \"http://www.pupp"
},
{
"path": "zoltar-api/pom.xml",
"chars": 1950,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-api/src/main/java/com/spotify/zoltar/Models.java",
"chars": 7060,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-api/src/main/java/com/spotify/zoltar/Predictors.java",
"chars": 17096,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-bom/pom.xml",
"chars": 11538,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-core/pom.xml",
"chars": 1964,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictor.java",
"chars": 3141,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorBuilder.java",
"chars": 6549,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/DefaultPredictorTimeoutScheduler.java",
"chars": 1326,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractFns.java",
"chars": 3145,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/FeatureExtractor.java",
"chars": 2546,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/Model.java",
"chars": 1281,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/ModelLoader.java",
"chars": 7731,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/PredictFns.java",
"chars": 4216,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/Prediction.java",
"chars": 1268,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/Predictor.java",
"chars": 2590,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/PredictorBuilder.java",
"chars": 2521,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/PredictorTimeoutScheduler.java",
"chars": 883,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/Vector.java",
"chars": 1296,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/fs/FileSystemExtras.java",
"chars": 4589,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/loaders/ModelMemoizer.java",
"chars": 2021,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-core/src/main/java/com/spotify/zoltar/loaders/Preloader.java",
"chars": 2348,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-featran/pom.xml",
"chars": 1201,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-featran/src/main/java/com/spotify/zoltar/featran/FeatranExtractFns.java",
"chars": 8451,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-jmh/pom.xml",
"chars": 3787,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-jmh/src/main/java/com/spotify/zoltar/jmh/BenchmarkTensorFlow.java",
"chars": 4212,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-jmh/src/main/java/com/spotify/zoltar/jmh/IrisHelper.java",
"chars": 1705,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-jmh/src/main/resources/iris.csv",
"chars": 345800,
"preview": "5.1,3.5,1.4,0.2,Iris-setosa\n4.9,3.0,1.4,0.2,Iris-setosa\n4.7,3.2,1.3,0.2,Iris-setosa\n4.6,3.1,1.5,0.2,Iris-setosa\n5.0,3.6,"
},
{
"path": "zoltar-jmh/src/main/resources/log4j.properties",
"chars": 227,
"preview": "log4j.rootLogger=INFO,stdout\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.layout=org.ap"
},
{
"path": "zoltar-jmh/src/main/resources/settings.json",
"chars": 1450,
"preview": "[\n {\n \"cls\": \"com.spotify.featran.transformers.StandardScaler\",\n \"name\": \"petal_length\",\n \"params\": {\n \"w"
},
{
"path": "zoltar-jmh/src/main/resources/trained_model/trained_model.txt",
"chars": 34,
"preview": "# Some dummy info about the model?"
},
{
"path": "zoltar-jmh/src/main/scala/com/spotify/zoltar/Iris.scala",
"chars": 1531,
"preview": "/*\n * Copyright 2018 Spotify AB.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not us"
},
{
"path": "zoltar-metrics/pom.xml",
"chars": 1193,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/FeatureExtractorMetrics.java",
"chars": 892,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/Instrumentations.java",
"chars": 1596,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedFeatureExtractor.java",
"chars": 1923,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedPredictFn.java",
"chars": 2038,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/InstrumentedPredictorBuilder.java",
"chars": 4008,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictFnMetrics.java",
"chars": 884,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictMetrics.java",
"chars": 887,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/PredictorMetrics.java",
"chars": 1129,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/VectorMetrics.java",
"chars": 870,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticPredictMetrics.java",
"chars": 1586,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticPredictorMetrics.java",
"chars": 4249,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/SemanticVectorMetrics.java",
"chars": 1562,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-metrics/src/main/java/com/spotify/zoltar/metrics/semantic/What.java",
"chars": 1160,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-mlengine/pom.xml",
"chars": 1917,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEngineLoader.java",
"chars": 2166,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEngineModel.java",
"chars": 7742,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEnginePredictException.java",
"chars": 1088,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-mlengine/src/main/java/com/spotify/zoltar/mlengine/MlEnginePredictFn.java",
"chars": 1118,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/pom.xml",
"chars": 2454,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowExtras.java",
"chars": 1893,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowGraphLoader.java",
"chars": 5102,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowGraphModel.java",
"chars": 6719,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowLoader.java",
"chars": 3824,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowModel.java",
"chars": 6007,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorFlowPredictFn.java",
"chars": 5294,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tensorflow/src/main/java/com/spotify/zoltar/tf/TensorflowMetaGraphDefParsingException.java",
"chars": 926,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/pom.xml",
"chars": 4113,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-tests/src/main/java/com/spotify/zoltar/IrisHelper.java",
"chars": 1571,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/main/java/com/spotify/zoltar/PredictorsTest.java",
"chars": 5351,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/main/scala/com/spotify/zoltar/Iris.scala",
"chars": 1531,
"preview": "/*\n * Copyright 2018 Spotify AB.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not us"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/FeatureExtractFnsTest.java",
"chars": 1467,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/FeatureExtractorTest.java",
"chars": 1529,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/MultiPredictorTest.java",
"chars": 3048,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/PredictionTest.java",
"chars": 1024,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/PredictorBuilderTest.java",
"chars": 3979,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/PredictorTest.java",
"chars": 4097,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/VectorTest.java",
"chars": 1000,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTest.java",
"chars": 4400,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTestIT.java",
"chars": 1245,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/fs/FileSystemExtrasTestUtils.java",
"chars": 1495,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/loaders/LoaderIT.java",
"chars": 4643,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/loaders/ModelMemoizerTest.java",
"chars": 1885,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/loaders/PreloaderTest.java",
"chars": 2459,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowExtrasTest.java",
"chars": 4285,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowGraphModelTest.java",
"chars": 7821,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/tf/TensorFlowModelTest.java",
"chars": 8259,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/java/com/spotify/zoltar/xgboost/XGBoostModelTest.java",
"chars": 6073,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-tests/src/test/resources/badsubdir/2018/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/badsubdir/z/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/emptydir/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/iris.csv",
"chars": 4549,
"preview": "5.1,3.5,1.4,0.2,Iris-setosa\n4.9,3.0,1.4,0.2,Iris-setosa\n4.7,3.2,1.3,0.2,Iris-setosa\n4.6,3.1,1.5,0.2,Iris-setosa\n5.0,3.6,"
},
{
"path": "zoltar-tests/src/test/resources/log4j.properties",
"chars": 227,
"preview": "log4j.rootLogger=INFO,stdout\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.layout=org.ap"
},
{
"path": "zoltar-tests/src/test/resources/settings.json",
"chars": 1450,
"preview": "[\n {\n \"cls\": \"com.spotify.featran.transformers.StandardScaler\",\n \"name\": \"petal_length\",\n \"params\": {\n \"w"
},
{
"path": "zoltar-tests/src/test/resources/settings_dummy.json",
"chars": 126,
"preview": "[{\"cls\":\"com.spotify.featran.transformers.Identity\",\"name\":\"feature\",\"params\":{},\"featureNames\":[\"feature\"],\"aggregators"
},
{
"path": "zoltar-tests/src/test/resources/test.txt",
"chars": 4,
"preview": "test"
},
{
"path": "zoltar-tests/src/test/resources/testdir/2018-01-15/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/testdir/2018-02-28/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/testdir/2018-03-01/data.txt",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/testdir/2018-04-01",
"chars": 0,
"preview": ""
},
{
"path": "zoltar-tests/src/test/resources/trained_model/trained_model.txt",
"chars": 34,
"preview": "# Some dummy info about the model?"
},
{
"path": "zoltar-xgboost/pom.xml",
"chars": 1685,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostLoader.java",
"chars": 1995,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostModel.java",
"chars": 2244,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-xgboost/src/main/java/com/spotify/zoltar/xgboost/XGBoostPredictFn.java",
"chars": 1156,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
},
{
"path": "zoltar-xgboost/src/main/java/ml/dmlc/xgboost4j/java/GompLoader.java",
"chars": 2368,
"preview": "/*\n * Copyright (C) 2019 Spotify AB\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not"
}
]
// ... and 13 more files (download for full content)
About this extraction
This page contains the full source code of the spotify/zoltar GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 180 files (713.2 KB), approximately 334.5k tokens, and a symbol index with 484 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.