Showing preview only (6,692K chars total). Download the full file or copy to clipboard to get everything.
Repository: googledatalab/pydatalab
Branch: master
Commit: 8bf007da3e43
Files: 438
Total size: 6.3 MB
Directory structure:
gitextract_nogfp4_v/
├── .build-bot.json
├── .coveragerc
├── .gitignore
├── .travis.yml
├── CONTRIBUTING.md
├── LICENSE.txt
├── README.md
├── datalab/
│ ├── README
│ ├── __init__.py
│ ├── bigquery/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ ├── _csv_options.py
│ │ ├── _dataset.py
│ │ ├── _dialect.py
│ │ ├── _federated_table.py
│ │ ├── _job.py
│ │ ├── _parser.py
│ │ ├── _query.py
│ │ ├── _query_job.py
│ │ ├── _query_results_table.py
│ │ ├── _query_stats.py
│ │ ├── _sampling.py
│ │ ├── _schema.py
│ │ ├── _table.py
│ │ ├── _udf.py
│ │ ├── _utils.py
│ │ ├── _view.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _bigquery.py
│ ├── context/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ ├── _context.py
│ │ ├── _project.py
│ │ ├── _utils.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _projects.py
│ ├── data/
│ │ ├── __init__.py
│ │ ├── _csv.py
│ │ ├── _sql_module.py
│ │ ├── _sql_statement.py
│ │ ├── _utils.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _sql.py
│ ├── kernel/
│ │ └── __init__.py
│ ├── notebook/
│ │ ├── __init__.py
│ │ └── static/
│ │ ├── bigquery.css
│ │ ├── bigquery.ts
│ │ ├── charting.css
│ │ ├── charting.ts
│ │ ├── element.ts
│ │ ├── extern/
│ │ │ ├── d3.parcoords.css
│ │ │ ├── d3.parcoords.js
│ │ │ ├── lantern-browser.html
│ │ │ ├── parcoords-LICENSE.txt
│ │ │ ├── sylvester-LICENSE.txt
│ │ │ └── sylvester.js
│ │ ├── job.css
│ │ ├── job.ts
│ │ ├── parcoords.ts
│ │ ├── style.ts
│ │ └── visualization.ts
│ ├── stackdriver/
│ │ ├── __init__.py
│ │ ├── commands/
│ │ │ ├── __init__.py
│ │ │ └── _monitoring.py
│ │ └── monitoring/
│ │ ├── __init__.py
│ │ ├── _group.py
│ │ ├── _metric.py
│ │ ├── _query.py
│ │ ├── _query_metadata.py
│ │ ├── _resource.py
│ │ └── _utils.py
│ ├── storage/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ ├── _bucket.py
│ │ ├── _item.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _storage.py
│ └── utils/
│ ├── __init__.py
│ ├── _async.py
│ ├── _dataflow_job.py
│ ├── _gcp_job.py
│ ├── _http.py
│ ├── _iterator.py
│ ├── _job.py
│ ├── _json_encoder.py
│ ├── _lambda_job.py
│ ├── _lru_cache.py
│ ├── _utils.py
│ └── commands/
│ ├── __init__.py
│ ├── _chart.py
│ ├── _chart_data.py
│ ├── _commands.py
│ ├── _csv.py
│ ├── _extension.py
│ ├── _html.py
│ ├── _job.py
│ ├── _modules.py
│ └── _utils.py
├── docs/
│ ├── .nojekyll
│ ├── Makefile
│ ├── README
│ ├── conf.py
│ ├── datalab Commands.rst
│ ├── datalab.bigquery.rst
│ ├── datalab.context.rst
│ ├── datalab.data.rst
│ ├── datalab.stackdriver.monitoring.rst
│ ├── datalab.storage.rst
│ ├── gen-magic-rst.ipy
│ ├── google.datalab Commands.rst
│ ├── google.datalab.bigquery.rst
│ ├── google.datalab.data.rst
│ ├── google.datalab.ml.rst
│ ├── google.datalab.rst
│ ├── google.datalab.stackdriver.monitoring.rst
│ ├── google.datalab.storage.rst
│ ├── index.rst
│ ├── make.bat
│ ├── mltoolbox.classification.dnn.rst
│ ├── mltoolbox.classification.linear.rst
│ ├── mltoolbox.image.classification.rst
│ ├── mltoolbox.regression.dnn.rst
│ └── mltoolbox.regression.linear.rst
├── externs/
│ └── ts/
│ └── require/
│ └── require.d.ts
├── google/
│ ├── __init__.py
│ └── datalab/
│ ├── __init__.py
│ ├── _context.py
│ ├── _job.py
│ ├── bigquery/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ ├── _csv_options.py
│ │ ├── _dataset.py
│ │ ├── _external_data_source.py
│ │ ├── _job.py
│ │ ├── _parser.py
│ │ ├── _query.py
│ │ ├── _query_job.py
│ │ ├── _query_output.py
│ │ ├── _query_results_table.py
│ │ ├── _query_stats.py
│ │ ├── _sampling.py
│ │ ├── _schema.py
│ │ ├── _table.py
│ │ ├── _udf.py
│ │ ├── _utils.py
│ │ ├── _view.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _bigquery.py
│ ├── commands/
│ │ ├── __init__.py
│ │ └── _datalab.py
│ ├── contrib/
│ │ ├── __init__.py
│ │ ├── bigquery/
│ │ │ ├── __init__.py
│ │ │ ├── commands/
│ │ │ │ ├── __init__.py
│ │ │ │ └── _bigquery.py
│ │ │ └── operators/
│ │ │ ├── __init__.py
│ │ │ ├── _bq_execute_operator.py
│ │ │ ├── _bq_extract_operator.py
│ │ │ └── _bq_load_operator.py
│ │ ├── mlworkbench/
│ │ │ ├── __init__.py
│ │ │ ├── _archive.py
│ │ │ ├── _local_predict.py
│ │ │ ├── _prediction_explainer.py
│ │ │ ├── _shell_process.py
│ │ │ └── commands/
│ │ │ ├── __init__.py
│ │ │ └── _ml.py
│ │ └── pipeline/
│ │ ├── __init__.py
│ │ ├── _pipeline.py
│ │ ├── airflow/
│ │ │ ├── __init__.py
│ │ │ └── _airflow.py
│ │ ├── commands/
│ │ │ ├── __init__.py
│ │ │ └── _pipeline.py
│ │ └── composer/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ └── _composer.py
│ ├── data/
│ │ ├── __init__.py
│ │ └── _csv_file.py
│ ├── kernel/
│ │ └── __init__.py
│ ├── ml/
│ │ ├── __init__.py
│ │ ├── _cloud_models.py
│ │ ├── _cloud_training_config.py
│ │ ├── _confusion_matrix.py
│ │ ├── _dataset.py
│ │ ├── _fasets.py
│ │ ├── _feature_slice_view.py
│ │ ├── _job.py
│ │ ├── _metrics.py
│ │ ├── _summary.py
│ │ ├── _tensorboard.py
│ │ └── _util.py
│ ├── notebook/
│ │ ├── __init__.py
│ │ └── static/
│ │ ├── bigquery.css
│ │ ├── bigquery.ts
│ │ ├── charting.css
│ │ ├── charting.ts
│ │ ├── element.ts
│ │ ├── extern/
│ │ │ ├── d3.parcoords.css
│ │ │ ├── d3.parcoords.js
│ │ │ ├── facets-jupyter.html
│ │ │ ├── lantern-browser.html
│ │ │ ├── parcoords-LICENSE.txt
│ │ │ ├── sylvester-LICENSE.txt
│ │ │ └── sylvester.js
│ │ ├── job.css
│ │ ├── job.ts
│ │ ├── parcoords.ts
│ │ ├── style.ts
│ │ └── visualization.ts
│ ├── stackdriver/
│ │ ├── __init__.py
│ │ ├── commands/
│ │ │ ├── __init__.py
│ │ │ └── _monitoring.py
│ │ └── monitoring/
│ │ ├── __init__.py
│ │ ├── _group.py
│ │ ├── _metric.py
│ │ ├── _query.py
│ │ ├── _query_metadata.py
│ │ ├── _resource.py
│ │ └── _utils.py
│ ├── storage/
│ │ ├── __init__.py
│ │ ├── _api.py
│ │ ├── _bucket.py
│ │ ├── _object.py
│ │ └── commands/
│ │ ├── __init__.py
│ │ └── _storage.py
│ └── utils/
│ ├── __init__.py
│ ├── _async.py
│ ├── _dataflow_job.py
│ ├── _gcp_job.py
│ ├── _http.py
│ ├── _iterator.py
│ ├── _json_encoder.py
│ ├── _lambda_job.py
│ ├── _lru_cache.py
│ ├── _utils.py
│ ├── commands/
│ │ ├── __init__.py
│ │ ├── _chart.py
│ │ ├── _chart_data.py
│ │ ├── _commands.py
│ │ ├── _csv.py
│ │ ├── _html.py
│ │ ├── _job.py
│ │ └── _utils.py
│ └── facets/
│ ├── __init__.py
│ ├── base_feature_statistics_generator.py
│ ├── base_generic_feature_statistics_generator.py
│ ├── feature_statistics_generator.py
│ ├── feature_statistics_pb2.py
│ └── generic_feature_statistics_generator.py
├── install-no-virtualenv.sh
├── install-virtualenv.sh
├── legacy_tests/
│ ├── _util/
│ │ ├── __init__.py
│ │ ├── http_tests.py
│ │ ├── lru_cache_tests.py
│ │ └── util_tests.py
│ ├── bigquery/
│ │ ├── __init__.py
│ │ ├── api_tests.py
│ │ ├── dataset_tests.py
│ │ ├── federated_table_tests.py
│ │ ├── jobs_tests.py
│ │ ├── parser_tests.py
│ │ ├── query_tests.py
│ │ ├── sampling_tests.py
│ │ ├── schema_tests.py
│ │ ├── table_tests.py
│ │ ├── udf_tests.py
│ │ └── view_tests.py
│ ├── data/
│ │ ├── __init__.py
│ │ └── sql_tests.py
│ ├── kernel/
│ │ ├── __init__.py
│ │ ├── bigquery_tests.py
│ │ ├── chart_data_tests.py
│ │ ├── chart_tests.py
│ │ ├── commands_tests.py
│ │ ├── html_tests.py
│ │ ├── module_tests.py
│ │ ├── sql_tests.py
│ │ ├── storage_tests.py
│ │ └── utils_tests.py
│ ├── main.py
│ ├── stackdriver/
│ │ ├── __init__.py
│ │ ├── commands/
│ │ │ ├── __init__.py
│ │ │ └── monitoring_tests.py
│ │ └── monitoring/
│ │ ├── __init__.py
│ │ ├── group_tests.py
│ │ ├── metric_tests.py
│ │ ├── query_metadata_tests.py
│ │ ├── query_tests.py
│ │ ├── resource_tests.py
│ │ └── utils_tests.py
│ └── storage/
│ ├── __init__.py
│ ├── api_tests.py
│ ├── bucket_tests.py
│ └── item_tests.py
├── release.sh
├── setup.cfg
├── setup.py
├── solutionbox/
│ ├── image_classification/
│ │ ├── mltoolbox/
│ │ │ ├── __init__.py
│ │ │ └── image/
│ │ │ ├── __init__.py
│ │ │ └── classification/
│ │ │ ├── __init__.py
│ │ │ ├── _api.py
│ │ │ ├── _cloud.py
│ │ │ ├── _inceptionlib.py
│ │ │ ├── _local.py
│ │ │ ├── _model.py
│ │ │ ├── _predictor.py
│ │ │ ├── _preprocess.py
│ │ │ ├── _trainer.py
│ │ │ ├── _util.py
│ │ │ ├── setup.py
│ │ │ └── task.py
│ │ └── setup.py
│ ├── ml_workbench/
│ │ ├── setup.py
│ │ ├── tensorflow/
│ │ │ ├── __init__.py
│ │ │ ├── analyze.py
│ │ │ ├── setup.py
│ │ │ ├── trainer/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── feature_analysis.py
│ │ │ │ ├── feature_transforms.py
│ │ │ │ └── task.py
│ │ │ └── transform.py
│ │ ├── test_tensorflow/
│ │ │ ├── run_all.sh
│ │ │ ├── test_analyze.py
│ │ │ ├── test_cloud_workflow.py
│ │ │ ├── test_feature_transforms.py
│ │ │ ├── test_training.py
│ │ │ └── test_transform.py
│ │ ├── test_xgboost/
│ │ │ ├── run_all.sh
│ │ │ ├── test_analyze.py
│ │ │ └── test_transform.py
│ │ └── xgboost/
│ │ ├── __init__.py
│ │ ├── analyze.py
│ │ ├── setup.py
│ │ ├── trainer/
│ │ │ ├── __init__.py
│ │ │ ├── feature_analysis.py
│ │ │ ├── feature_transforms.py
│ │ │ └── task.py
│ │ └── transform.py
│ └── structured_data/
│ ├── build.sh
│ ├── mltoolbox/
│ │ ├── __init__.py
│ │ ├── _structured_data/
│ │ │ ├── __init__.py
│ │ │ ├── __version__.py
│ │ │ ├── _package.py
│ │ │ ├── master_setup.py
│ │ │ ├── prediction/
│ │ │ │ ├── __init__.py
│ │ │ │ └── predict.py
│ │ │ ├── preprocess/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── cloud_preprocess.py
│ │ │ │ └── local_preprocess.py
│ │ │ └── trainer/
│ │ │ ├── __init__.py
│ │ │ ├── task.py
│ │ │ └── util.py
│ │ ├── classification/
│ │ │ ├── __init__.py
│ │ │ ├── dnn/
│ │ │ │ ├── __init__.py
│ │ │ │ └── _classification_dnn.py
│ │ │ └── linear/
│ │ │ ├── __init__.py
│ │ │ └── _classification_linear.py
│ │ └── regression/
│ │ ├── __init__.py
│ │ ├── dnn/
│ │ │ ├── __init__.py
│ │ │ └── _regression_dnn.py
│ │ └── linear/
│ │ ├── __init__.py
│ │ └── _regression_linear.py
│ ├── setup.py
│ └── test_mltoolbox/
│ ├── __init__.py
│ ├── e2e_functions.py
│ ├── test_datalab_e2e.py
│ ├── test_package_functions.py
│ ├── test_sd_preprocess.py
│ └── test_sd_trainer.py
├── tests/
│ ├── _util/
│ │ ├── __init__.py
│ │ ├── commands_tests.py
│ │ ├── feature_statistics_generator_test.py
│ │ ├── generic_feature_statistics_generator_test.py
│ │ ├── http_tests.py
│ │ ├── lru_cache_tests.py
│ │ └── util_tests.py
│ ├── bigquery/
│ │ ├── __init__.py
│ │ ├── api_tests.py
│ │ ├── dataset_tests.py
│ │ ├── external_data_source_tests.py
│ │ ├── jobs_tests.py
│ │ ├── operator_tests.py
│ │ ├── parser_tests.py
│ │ ├── pipeline_tests.py
│ │ ├── query_tests.py
│ │ ├── sampling_tests.py
│ │ ├── schema_tests.py
│ │ ├── table_tests.py
│ │ ├── udf_tests.py
│ │ └── view_tests.py
│ ├── context_tests.py
│ ├── integration/
│ │ └── storage_test.py
│ ├── kernel/
│ │ ├── __init__.py
│ │ ├── bigquery_tests.py
│ │ ├── chart_data_tests.py
│ │ ├── chart_tests.py
│ │ ├── html_tests.py
│ │ ├── pipeline_tests.py
│ │ ├── storage_tests.py
│ │ └── utils_tests.py
│ ├── main.py
│ ├── ml/
│ │ ├── __init__.py
│ │ ├── confusion_matrix_tests.py
│ │ ├── dataset_tests.py
│ │ ├── facets_tests.py
│ │ ├── metrics_tests.py
│ │ ├── summary_tests.py
│ │ └── tensorboard_tests.py
│ ├── ml_workbench/
│ │ ├── __init__.py
│ │ └── all_tests.py
│ ├── mltoolbox_structured_data/
│ │ ├── __init__.py
│ │ ├── dl_interface_tests.py
│ │ ├── sd_e2e_tests.py
│ │ └── traininglib_tests.py
│ ├── mlworkbench_magic/
│ │ ├── __init__.py
│ │ ├── archive_tests.py
│ │ ├── explainer_tests.py
│ │ ├── local_predict_tests.py
│ │ ├── ml_tests.py
│ │ └── shell_process_tests.py
│ ├── pipeline/
│ │ ├── __init__.py
│ │ ├── airflow_tests.py
│ │ ├── composer_api_tests.py
│ │ ├── composer_tests.py
│ │ └── pipeline_tests.py
│ ├── stackdriver/
│ │ ├── __init__.py
│ │ ├── commands/
│ │ │ ├── __init__.py
│ │ │ └── monitoring_tests.py
│ │ └── monitoring/
│ │ ├── __init__.py
│ │ ├── group_tests.py
│ │ ├── metric_tests.py
│ │ ├── query_metadata_tests.py
│ │ ├── query_tests.py
│ │ ├── resource_tests.py
│ │ └── utils_tests.py
│ └── storage/
│ ├── __init__.py
│ ├── api_tests.py
│ ├── bucket_tests.py
│ └── object_tests.py
└── tox.ini
================================================
FILE CONTENTS
================================================
================================================
FILE: .coveragerc
================================================
# .coveragerc to control coverage.py
[run]
[report]
include = */site-packages/google/datalab/*
================================================
FILE: .gitignore
================================================
*.pyc
*.pyi
*.map
*.egg-info
*.iml
.idea
.DS_Store
MANIFEST
build
.coverage
dist
datalab.magics.rst
datalab/notebook/static/*.js
google/datalab/notebook/static/*.js
# Test files
.tox/
.cache/
================================================
FILE: .travis.yml
================================================
language: python
dist: trusty
sudo: false
matrix:
include:
- python: 2.7
env: TOX_ENV=py27
- python: 3.5
env: TOX_ENV=py35
- python: 2.7
env: TOX_ENV=flake8
- python: 2.7
env: TOX_ENV=coveralls
before_install:
- npm install -g typescript@3.0.3
- tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts
# We use tox for actually running tests.
- pip install --upgrade pip tox
script:
# tox reads its configuration from tox.ini.
- tox -e $TOX_ENV
================================================
FILE: CONTRIBUTING.md
================================================
Want to contribute? Great! First, read this page (including the small print at the end).
### Before you contribute
Before we can use your code, you must sign the
[Google Individual Contributor License Agreement]
(https://cla.developers.google.com/about/google-individual)
(CLA), which you can do online. The CLA is necessary mainly because you own the
copyright to your changes, even after your contribution becomes part of our
codebase, so we need your permission to use and distribute your code. We also
need to be sure of various other things—for instance that you'll tell us if you
know that your code infringes on other people's patents. You don't have to sign
the CLA until after you've submitted your code for review and a member has
approved it, but you must do it before we can put your code into our codebase.
Before you start working on a larger contribution, you should get in touch with
us first through the issue tracker with your idea so that we can help out and
possibly guide you. Coordinating up front makes it much easier to avoid
frustration later on.
### Code reviews
All submissions, including submissions by project members, require review. We
use Github pull requests for this purpose.
### Running tests
We use [`tox`](https://tox.readthedocs.io/) for running our tests. To run tests
before sending out a pull request, just
[install tox](https://tox.readthedocs.io/en/latest/install.html) and run
```shell
$ tox
```
to run tests under all supported environments. (This will skip any environments
for which no interpreter is available.) `tox -l` will provide a list of all
supported environments.
`tox` will run all tests referenced by `tests/main.py` and
`legacy_tests/main.py`.
### The small print
Contributions made by corporations are covered by a different agreement than
the one above, the
[Software Grant and Corporate Contributor License Agreement]
(https://cla.developers.google.com/about/google-corporate).
================================================
FILE: LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# Google Cloud DataLab
Datalab is deprecated. [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench) provides a notebook-based environment that offers capabilities beyond Datalab. We recommend that you use Vertex AI Workbench for new projects and [migrate your Datalab notebooks to Vertex AI Workbench](https://cloud.google.com/datalab/docs/resources/troubleshooting#migrate). For more information, see [Deprecation information](https://cloud.google.com/datalab/docs/resources/deprecation). To get help migrating Datalab projects to Vertex AI Workbench see [Get help](https://cloud.google.com/datalab/docs/resources/support#get-help).
================================================
FILE: datalab/README
================================================
Everything under datalab namespace is actively maintained but no new features are being added. Please use corresponding libraries under google.datalab namespace (source code under google/datalab directory).
To migrate existing code that relies on datalab namespace, since most API interfaces are the same between google.datalab and datalab, usually you just need to change the import namespace. The magic interface is different for bigquery though (%%sql --> %%bq).
For more details please see https://github.com/googledatalab/pydatalab/wiki/%60datalab%60-to-%60google.datalab%60-Migration-Guide.
================================================
FILE: datalab/__init__.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
================================================
FILE: datalab/bigquery/__init__.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Google Cloud Platform library - BigQuery Functionality."""
from __future__ import absolute_import
from ._csv_options import CSVOptions
from ._dataset import Dataset, Datasets
from ._dialect import Dialect
from ._federated_table import FederatedTable
from ._job import Job
from ._query import Query
from ._query_job import QueryJob
from ._query_results_table import QueryResultsTable
from ._query_stats import QueryStats
from ._sampling import Sampling
from ._schema import Schema
from ._table import Table, TableMetadata
from ._udf import UDF
from ._utils import TableName, DatasetName
from ._view import View
__all__ = ['CSVOptions', 'Dataset', 'Datasets', 'Dialect', 'FederatedTable', 'Query', 'QueryJob',
'QueryResultsTable', 'QueryStats', 'Sampling', 'Schema', 'Table', 'TableMetadata',
'UDF', 'TableName', 'DatasetName', 'View']
def wait_any(jobs, timeout=None):
""" Return when at least one of the specified jobs has completed or timeout expires.
Args:
jobs: a list of Jobs to wait on.
timeout: a timeout in seconds to wait for. None (the default) means no timeout.
Returns:
Once at least one job completes, a list of all completed jobs.
If the call times out then an empty list will be returned.
"""
return Job.wait_any(jobs, timeout)
def wait_all(jobs, timeout=None):
""" Return when all of the specified jobs have completed or timeout expires.
Args:
jobs: a single Job or list of Jobs to wait on.
timeout: a timeout in seconds to wait for. None (the default) means no timeout.
Returns:
A list of completed Jobs. If the call timed out this will be shorter than the
list of jobs supplied as a parameter.
"""
return Job.wait_all(jobs, timeout)
================================================
FILE: datalab/bigquery/_api.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery HTTP API wrapper."""
from __future__ import absolute_import
from __future__ import unicode_literals
from past.builtins import basestring
from builtins import object
import datalab.utils
import datalab.bigquery
class Api(object):
"""A helper class to issue BigQuery HTTP requests."""
# TODO(nikhilko): Use named placeholders in these string templates.
_ENDPOINT = 'https://www.googleapis.com/bigquery/v2'
_JOBS_PATH = '/projects/%s/jobs/%s'
_QUERIES_PATH = '/projects/%s/queries/%s'
_DATASETS_PATH = '/projects/%s/datasets/%s'
_TABLES_PATH = '/projects/%s/datasets/%s/tables/%s%s'
_TABLEDATA_PATH = '/projects/%s/datasets/%s/tables/%s%s/data'
_DEFAULT_TIMEOUT = 60000
def __init__(self, context):
"""Initializes the BigQuery helper with context information.
Args:
context: a Context object providing project_id and credentials.
"""
self._credentials = context.credentials
self._project_id = context.project_id
@property
def project_id(self):
"""The project_id associated with this API client."""
return self._project_id
def jobs_insert_load(self, source, table_name, append=False, overwrite=False, create=False,
source_format='CSV', field_delimiter=',', allow_jagged_rows=False,
allow_quoted_newlines=False, encoding='UTF-8', ignore_unknown_values=False,
max_bad_records=0, quote='"', skip_leading_rows=0):
""" Issues a request to load data from GCS to a BQ table
Args:
source: the URL of the source bucket(s). Can include wildcards, and can be a single
string argument or a list.
table_name: a tuple representing the full name of the destination table.
append: if True append onto existing table contents.
overwrite: if True overwrite existing table contents.
create: if True, create the table if it doesn't exist
source_format: the format of the data; default 'CSV'. Other options are DATASTORE_BACKUP
or NEWLINE_DELIMITED_JSON.
field_delimiter: The separator for fields in a CSV file. BigQuery converts the string to
ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data
as raw binary (default ',').
allow_jagged_rows: If True, accept rows in CSV files that are missing trailing optional
columns; the missing values are treated as nulls (default False).
allow_quoted_newlines: If True, allow quoted data sections in CSV files that contain newline
characters (default False).
encoding: The character encoding of the data, either 'UTF-8' (the default) or 'ISO-8859-1'.
ignore_unknown_values: If True, accept rows that contain values that do not match the schema;
the unknown values are ignored (default False).
max_bad_records: The maximum number of bad records that are allowed (and ignored) before
returning an 'invalid' error in the Job result (default 0).
quote: The value used to quote data sections in a CSV file; default '"'. If your data does
not contain quoted sections, set the property value to an empty string. If your data
contains quoted newline characters, you must also enable allow_quoted_newlines.
skip_leading_rows: A number of rows at the top of a CSV file to skip (default 0).
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._JOBS_PATH % (table_name.project_id, ''))
if isinstance(source, basestring):
source = [source]
write_disposition = 'WRITE_EMPTY'
if overwrite:
write_disposition = 'WRITE_TRUNCATE'
if append:
write_disposition = 'WRITE_APPEND'
data = {
'kind': 'bigquery#job',
'configuration': {
'load': {
'sourceUris': source,
'destinationTable': {
'projectId': table_name.project_id,
'datasetId': table_name.dataset_id,
'tableId': table_name.table_id
},
'createDisposition': 'CREATE_IF_NEEDED' if create else 'CREATE_NEVER',
'writeDisposition': write_disposition,
'sourceFormat': source_format,
'ignoreUnknownValues': ignore_unknown_values,
'maxBadRecords': max_bad_records,
}
}
}
if source_format == 'CSV':
load_config = data['configuration']['load']
load_config.update({
'fieldDelimiter': field_delimiter,
'allowJaggedRows': allow_jagged_rows,
'allowQuotedNewlines': allow_quoted_newlines,
'quote': quote,
'encoding': encoding,
'skipLeadingRows': skip_leading_rows
})
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def jobs_insert_query(self, sql, code=None, imports=None, table_name=None, append=False,
overwrite=False, dry_run=False, use_cache=True, batch=True,
allow_large_results=False, table_definitions=None, dialect=None,
billing_tier=None):
"""Issues a request to insert a query job.
Args:
sql: the SQL string representing the query to execute.
code: code for Javascript UDFs, if any.
imports: a list of GCS URLs containing additional Javascript UDF support code, if any.
table_name: None for an anonymous table, or a name parts tuple for a long-lived table.
append: if True, append to the table if it is non-empty; else the request will fail if table
is non-empty unless overwrite is True.
overwrite: if the table already exists, truncate it instead of appending or raising an
Exception.
dry_run: whether to actually execute the query or just dry run it.
use_cache: whether to use past query results or ignore cache. Has no effect if destination is
specified.
batch: whether to run this as a batch job (lower priority) or as an interactive job (high
priority, more expensive).
allow_large_results: whether to allow large results (slower with some restrictions but
can handle big jobs).
table_definitions: a list of JSON external table definitions for any external tables
referenced in the query.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._JOBS_PATH % (self._project_id, ''))
if dialect is None:
dialect = datalab.bigquery.Dialect.default().bq_dialect
data = {
'kind': 'bigquery#job',
'configuration': {
'query': {
'query': sql,
'useQueryCache': use_cache,
'allowLargeResults': allow_large_results,
'useLegacySql': dialect == 'legacy'
},
'dryRun': dry_run,
'priority': 'BATCH' if batch else 'INTERACTIVE',
},
}
query_config = data['configuration']['query']
resources = []
if code:
resources.extend([{'inlineCode': fragment} for fragment in code])
if imports:
resources.extend([{'resourceUri': uri} for uri in imports])
query_config['userDefinedFunctionResources'] = resources
if table_definitions:
query_config['tableDefinitions'] = table_definitions
if table_name:
query_config['destinationTable'] = {
'projectId': table_name.project_id,
'datasetId': table_name.dataset_id,
'tableId': table_name.table_id
}
if append:
query_config['writeDisposition'] = "WRITE_APPEND"
elif overwrite:
query_config['writeDisposition'] = "WRITE_TRUNCATE"
if billing_tier:
query_config['maximumBillingTier'] = billing_tier
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def jobs_query_results(self, job_id, project_id, page_size, timeout, start_index=0):
"""Issues a request to the jobs/getQueryResults method.
Args:
job_id: the id of job from a previously executed query.
project_id: the project id to use to fetch the results; use None for the default project.
page_size: limit to the number of rows to fetch.
timeout: duration (in milliseconds) to wait for the query to complete.
start_index: the index of the row (0-based) at which to start retrieving the page of result
rows.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
if timeout is None:
timeout = Api._DEFAULT_TIMEOUT
if project_id is None:
project_id = self._project_id
args = {
'maxResults': page_size,
'timeoutMs': timeout,
'startIndex': start_index
}
url = Api._ENDPOINT + (Api._QUERIES_PATH % (project_id, job_id))
return datalab.utils.Http.request(url, args=args, credentials=self._credentials)
def jobs_get(self, job_id, project_id=None):
"""Issues a request to retrieve information about a job.
Args:
job_id: the id of the job
project_id: the project id to use to fetch the results; use None for the default project.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
if project_id is None:
project_id = self._project_id
url = Api._ENDPOINT + (Api._JOBS_PATH % (project_id, job_id))
return datalab.utils.Http.request(url, credentials=self._credentials)
def datasets_insert(self, dataset_name, friendly_name=None, description=None):
"""Issues a request to create a dataset.
Args:
dataset_name: the name of the dataset to create.
friendly_name: (optional) the friendly name for the dataset
description: (optional) a description for the dataset
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._DATASETS_PATH % (dataset_name.project_id, ''))
data = {
'kind': 'bigquery#dataset',
'datasetReference': {
'projectId': dataset_name.project_id,
'datasetId': dataset_name.dataset_id
},
}
if friendly_name:
data['friendlyName'] = friendly_name
if description:
data['description'] = description
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def datasets_delete(self, dataset_name, delete_contents):
"""Issues a request to delete a dataset.
Args:
dataset_name: the name of the dataset to delete.
delete_contents: if True, any tables in the dataset will be deleted. If False and the
dataset is non-empty an exception will be raised.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._DATASETS_PATH % dataset_name)
args = {}
if delete_contents:
args['deleteContents'] = True
return datalab.utils.Http.request(url, method='DELETE', args=args,
credentials=self._credentials, raw_response=True)
def datasets_update(self, dataset_name, dataset_info):
"""Updates the Dataset info.
Args:
dataset_name: the name of the dataset to update as a tuple of components.
dataset_info: the Dataset resource with updated fields.
"""
url = Api._ENDPOINT + (Api._DATASETS_PATH % dataset_name)
return datalab.utils.Http.request(url, method='PUT', data=dataset_info,
credentials=self._credentials)
def datasets_get(self, dataset_name):
"""Issues a request to retrieve information about a dataset.
Args:
dataset_name: the name of the dataset
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._DATASETS_PATH % dataset_name)
return datalab.utils.Http.request(url, credentials=self._credentials)
def datasets_list(self, project_id=None, max_results=0, page_token=None):
"""Issues a request to list the datasets in the project.
Args:
project_id: the project id to use to fetch the results; use None for the default project.
max_results: an optional maximum number of tables to retrieve.
page_token: an optional token to continue the retrieval.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
if project_id is None:
project_id = self._project_id
url = Api._ENDPOINT + (Api._DATASETS_PATH % (project_id, ''))
args = {}
if max_results != 0:
args['maxResults'] = max_results
if page_token is not None:
args['pageToken'] = page_token
return datalab.utils.Http.request(url, args=args, credentials=self._credentials)
def tables_get(self, table_name):
"""Issues a request to retrieve information about a table.
Args:
table_name: a tuple representing the full name of the table.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._TABLES_PATH % table_name)
return datalab.utils.Http.request(url, credentials=self._credentials)
def tables_list(self, dataset_name, max_results=0, page_token=None):
"""Issues a request to retrieve a list of tables.
Args:
dataset_name: the name of the dataset to enumerate.
max_results: an optional maximum number of tables to retrieve.
page_token: an optional token to continue the retrieval.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT +\
(Api._TABLES_PATH % (dataset_name.project_id, dataset_name.dataset_id, '', ''))
args = {}
if max_results != 0:
args['maxResults'] = max_results
if page_token is not None:
args['pageToken'] = page_token
return datalab.utils.Http.request(url, args=args, credentials=self._credentials)
def tables_insert(self, table_name, schema=None, query=None, friendly_name=None,
description=None):
"""Issues a request to create a table or view in the specified dataset with the specified id.
A schema must be provided to create a Table, or a query must be provided to create a View.
Args:
table_name: the name of the table as a tuple of components.
schema: the schema, if this is a Table creation.
query: the query, if this is a View creation.
friendly_name: an optional friendly name.
description: an optional description.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + \
(Api._TABLES_PATH % (table_name.project_id, table_name.dataset_id, '', ''))
data = {
'kind': 'bigquery#table',
'tableReference': {
'projectId': table_name.project_id,
'datasetId': table_name.dataset_id,
'tableId': table_name.table_id
}
}
if schema:
data['schema'] = {'fields': schema}
if query:
data['view'] = {'query': query}
if friendly_name:
data['friendlyName'] = friendly_name
if description:
data['description'] = description
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def tabledata_insert_all(self, table_name, rows):
"""Issues a request to insert data into a table.
Args:
table_name: the name of the table as a tuple of components.
rows: the data to populate the table, as a list of dictionaries.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._TABLES_PATH % table_name) + "/insertAll"
data = {
'kind': 'bigquery#tableDataInsertAllRequest',
'rows': rows
}
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def tabledata_list(self, table_name, start_index=None, max_results=None, page_token=None):
""" Retrieves the contents of a table.
Args:
table_name: the name of the table as a tuple of components.
start_index: the index of the row at which to start retrieval.
max_results: an optional maximum number of rows to retrieve.
page_token: an optional token to continue the retrieval.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._TABLEDATA_PATH % table_name)
args = {}
if start_index:
args['startIndex'] = start_index
if max_results:
args['maxResults'] = max_results
if page_token is not None:
args['pageToken'] = page_token
return datalab.utils.Http.request(url, args=args, credentials=self._credentials)
def table_delete(self, table_name):
"""Issues a request to delete a table.
Args:
table_name: the name of the table as a tuple of components.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._TABLES_PATH % table_name)
return datalab.utils.Http.request(url, method='DELETE', credentials=self._credentials,
raw_response=True)
def table_extract(self, table_name, destination, format='CSV', compress=True,
field_delimiter=',', print_header=True):
"""Exports the table to GCS.
Args:
table_name: the name of the table as a tuple of components.
destination: the destination URI(s). Can be a single URI or a list.
format: the format to use for the exported data; one of CSV, NEWLINE_DELIMITED_JSON or AVRO.
Defaults to CSV.
compress: whether to compress the data on export. Compression is not supported for
AVRO format. Defaults to False.
field_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
print_header: for CSV exports, whether to include an initial header line. Default true.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.
"""
url = Api._ENDPOINT + (Api._JOBS_PATH % (table_name.project_id, ''))
if isinstance(destination, basestring):
destination = [destination]
data = {
# 'projectId': table_name.project_id, # Code sample shows this but it is not in job
# reference spec. Filed as b/19235843
'kind': 'bigquery#job',
'configuration': {
'extract': {
'sourceTable': {
'projectId': table_name.project_id,
'datasetId': table_name.dataset_id,
'tableId': table_name.table_id,
},
'compression': 'GZIP' if compress else 'NONE',
'fieldDelimiter': field_delimiter,
'printHeader': print_header,
'destinationUris': destination,
'destinationFormat': format,
}
}
}
return datalab.utils.Http.request(url, data=data, credentials=self._credentials)
def table_update(self, table_name, table_info):
"""Updates the Table info.
Args:
table_name: the name of the table to update as a tuple of components.
table_info: the Table resource with updated fields.
"""
url = Api._ENDPOINT + (Api._TABLES_PATH % table_name)
return datalab.utils.Http.request(url, method='PUT', data=table_info,
credentials=self._credentials)
================================================
FILE: datalab/bigquery/_csv_options.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements CSV options for External Tables and Table loads from GCS."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import object
class CSVOptions(object):
def __init__(self, delimiter=',', skip_leading_rows=0, encoding='utf-8', quote='"',
allow_quoted_newlines=False, allow_jagged_rows=False):
""" Initialize an instance of CSV options.
Args:
delimiter: The separator for fields in a CSV file. BigQuery converts the string to
ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data
as raw binary (default ',').
skip_leading_rows: A number of rows at the top of a CSV file to skip (default 0).
encoding: The character encoding of the data, either 'utf-8' (the default) or 'iso-8859-1'.
quote: The value used to quote data sections in a CSV file; default '"'. If your data does
not contain quoted sections, set the property value to an empty string. If your data
contains quoted newline characters, you must also enable allow_quoted_newlines.
allow_quoted_newlines: If True, allow quoted data sections in CSV files that contain newline
characters (default False).
allow_jagged_rows: If True, accept rows in CSV files that are missing trailing optional
columns; the missing values are treated as nulls (default False).
"""
encoding_upper = encoding.upper()
if encoding_upper != 'UTF-8' and encoding_upper != 'ISO-8859-1':
raise Exception("Invalid source encoding %s" % encoding)
self._delimiter = delimiter
self._skip_leading_rows = skip_leading_rows
self._encoding = encoding
self._quote = quote
self._allow_quoted_newlines = allow_quoted_newlines
self._allow_jagged_rows = allow_jagged_rows
@property
def delimiter(self):
return self._delimiter
@property
def skip_leading_rows(self):
return self._skip_leading_rows
@property
def encoding(self):
return self._encoding
@property
def quote(self):
return self._quote
@property
def allow_quoted_newlines(self):
return self._allow_quoted_newlines
@property
def allow_jagged_rows(self):
return self._allow_jagged_rows
def _to_query_json(self):
""" Return the options as a dictionary to be used as JSON in a query job. """
return {
'quote': self._quote,
'fieldDelimiter': self._delimiter,
'encoding': self._encoding.upper(),
'skipLeadingRows': self._skip_leading_rows,
'allowQuotedNewlines': self._allow_quoted_newlines,
'allowJaggedRows': self._allow_jagged_rows
}
================================================
FILE: datalab/bigquery/_dataset.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements Dataset, and related Dataset BigQuery APIs."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import object
import datalab.context
import datalab.utils
from . import _api
from . import _table
from . import _utils
from . import _view
class Dataset(object):
"""Represents a list of BigQuery tables in a dataset."""
def __init__(self, name, context=None):
"""Initializes an instance of a Dataset.
Args:
name: the name of the dataset, as a string or (project_id, dataset_id) tuple.
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
Raises:
Exception if the name is invalid.
"""
if context is None:
context = datalab.context.Context.default()
self._context = context
self._api = _api.Api(context)
self._name_parts = _utils.parse_dataset_name(name, self._api.project_id)
self._full_name = '%s:%s' % self._name_parts
self._info = None
try:
self._info = self._get_info()
except datalab.utils.RequestException:
pass
@property
def name(self):
"""The DatasetName named tuple (project_id, dataset_id) for the dataset."""
return self._name_parts
@property
def description(self):
"""The description of the dataset, if any.
Raises:
Exception if the dataset exists but the metadata for the dataset could not be retrieved.
"""
self._get_info()
return self._info['description'] if self._info else None
@property
def friendly_name(self):
"""The friendly name of the dataset, if any.
Raises:
Exception if the dataset exists but the metadata for the dataset could not be retrieved.
"""
self._get_info()
return self._info['friendlyName'] if self._info else None
def _get_info(self):
try:
if self._info is None:
self._info = self._api.datasets_get(self._name_parts)
return self._info
except datalab.utils.RequestException as e:
if e.status == 404:
return None
raise e
except Exception as e:
raise e
def exists(self):
""" Checks if the dataset exists.
Returns:
True if the dataset exists; False otherwise.
Raises:
Exception if the dataset exists but the metadata for the dataset could not be retrieved.
"""
self._get_info()
return self._info is not None
def delete(self, delete_contents=False):
"""Issues a request to delete the dataset.
Args:
delete_contents: if True, any tables and views in the dataset will be deleted. If False
and the dataset is non-empty an exception will be raised.
Returns:
None on success.
Raises:
Exception if the delete fails (including if table was nonexistent).
"""
if not self.exists():
raise Exception('Cannot delete non-existent dataset %s' % self._full_name)
try:
self._api.datasets_delete(self._name_parts, delete_contents=delete_contents)
except Exception as e:
raise e
self._info = None
return None
def create(self, friendly_name=None, description=None):
"""Creates the Dataset with the specified friendly name and description.
Args:
friendly_name: (optional) the friendly name for the dataset if it is being created.
description: (optional) a description for the dataset if it is being created.
Returns:
The Dataset.
Raises:
Exception if the Dataset could not be created.
"""
if not self.exists():
try:
response = self._api.datasets_insert(self._name_parts,
friendly_name=friendly_name,
description=description)
except Exception as e:
raise e
if 'selfLink' not in response:
raise Exception("Could not create dataset %s" % self._full_name)
return self
def update(self, friendly_name=None, description=None):
""" Selectively updates Dataset information.
Args:
friendly_name: if not None, the new friendly name.
description: if not None, the new description.
Returns:
"""
self._get_info()
if self._info:
if friendly_name:
self._info['friendlyName'] = friendly_name
if description:
self._info['description'] = description
try:
self._api.datasets_update(self._name_parts, self._info)
except Exception as e:
raise e
finally:
self._info = None # need a refresh
def _retrieve_items(self, page_token, item_type):
try:
list_info = self._api.tables_list(self._name_parts, page_token=page_token)
except Exception as e:
raise e
tables = list_info.get('tables', [])
contents = []
if len(tables):
try:
for info in tables:
if info['type'] != item_type:
continue
if info['type'] == 'TABLE':
item = _table.Table((info['tableReference']['projectId'],
info['tableReference']['datasetId'],
info['tableReference']['tableId']), self._context)
else:
item = _view.View((info['tableReference']['projectId'],
info['tableReference']['datasetId'],
info['tableReference']['tableId']), self._context)
contents.append(item)
except KeyError:
raise Exception('Unexpected item list response')
page_token = list_info.get('nextPageToken', None)
return contents, page_token
def _retrieve_tables(self, page_token, _):
return self._retrieve_items(page_token=page_token, item_type='TABLE')
def _retrieve_views(self, page_token, _):
return self._retrieve_items(page_token=page_token, item_type='VIEW')
def tables(self):
""" Returns an iterator for iterating through the Tables in the dataset. """
return iter(datalab.utils.Iterator(self._retrieve_tables))
def views(self):
""" Returns an iterator for iterating through the Views in the dataset. """
return iter(datalab.utils.Iterator(self._retrieve_views))
def __iter__(self):
""" Returns an iterator for iterating through the Tables in the dataset. """
return self.tables()
def __str__(self):
"""Returns a string representation of the dataset using its specified name.
Returns:
The string representation of this object.
"""
return self._full_name
def __repr__(self):
"""Returns a representation for the dataset for showing in the notebook.
"""
return 'Dataset %s' % self._full_name
class Datasets(object):
""" Iterator class for enumerating the datasets in a project. """
def __init__(self, project_id=None, context=None):
""" Initialize the Datasets object.
Args:
project_id: the ID of the project whose datasets you want to list. If None defaults
to the project in the context.
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
"""
if context is None:
context = datalab.context.Context.default()
self._context = context
self._api = _api.Api(context)
self._project_id = project_id if project_id else self._api.project_id
def _retrieve_datasets(self, page_token, count):
try:
list_info = self._api.datasets_list(self._project_id, max_results=count,
page_token=page_token)
except Exception as e:
raise e
datasets = list_info.get('datasets', [])
if len(datasets):
try:
datasets = [Dataset((info['datasetReference']['projectId'],
info['datasetReference']['datasetId']), self._context)
for info in datasets]
except KeyError:
raise Exception('Unexpected response from server.')
page_token = list_info.get('nextPageToken', None)
return datasets, page_token
def __iter__(self):
""" Returns an iterator for iterating through the Datasets in the project.
"""
return iter(datalab.utils.Iterator(self._retrieve_datasets))
================================================
FILE: datalab/bigquery/_dialect.py
================================================
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Google Cloud Platform library - BigQuery SQL Dialect"""
from __future__ import absolute_import
class Dialect(object):
"""
Represents the default BigQuery SQL dialect
"""
_global_dialect = None
def __init__(self, bq_dialect):
self._global_dialect = bq_dialect
@property
def bq_dialect(self):
"""Retrieves the value of the bq_dialect property.
Returns:
The default BigQuery SQL dialect
"""
return self._global_dialect
def set_bq_dialect(self, bq_dialect):
""" Set the default BigQuery SQL dialect"""
if bq_dialect in ['legacy', 'standard']:
self._global_dialect = bq_dialect
@staticmethod
def default():
"""Retrieves the default BigQuery SQL dialect, creating it if necessary.
Returns:
An initialized and shared instance of a Dialect object.
"""
if Dialect._global_dialect is None:
Dialect._global_dialect = Dialect('legacy')
return Dialect._global_dialect
================================================
FILE: datalab/bigquery/_federated_table.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements External Table functionality."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import object
from . import _csv_options
class FederatedTable(object):
@staticmethod
def from_storage(source, source_format='csv', csv_options=None, ignore_unknown_values=False,
max_bad_records=0, compressed=False, schema=None):
""" Create an external table for a GCS object.
Args:
source: the URL of the source objects(s). Can include a wildcard '*' at the end of the item
name. Can be a single source or a list.
source_format: the format of the data, 'csv' or 'json'; default 'csv'.
csv_options: For CSV files, the options such as quote character and delimiter.
ignore_unknown_values: If True, accept rows that contain values that do not match the schema;
the unknown values are ignored (default False).
max_bad_records: The maximum number of bad records that are allowed (and ignored) before
returning an 'invalid' error in the Job result (default 0).
compressed: whether the data is GZ compressed or not (default False). Note that compressed
data can be used as a federated table but cannot be loaded into a BQ Table.
schema: the schema of the data. This is required for this table to be used as a federated
table or to be loaded using a Table object that itself has no schema (default None).
"""
result = FederatedTable()
# Do some sanity checking and concert some params from friendly form to form used by BQ.
if source_format == 'csv':
result._bq_source_format = 'CSV'
if csv_options is None:
csv_options = _csv_options.CSVOptions() # use defaults
elif source_format == 'json':
if csv_options:
raise Exception('CSV options are not support for JSON tables')
result._bq_source_format = 'NEWLINE_DELIMITED_JSON'
else:
raise Exception("Invalid source format %s" % source_format)
result._source = source if isinstance(source, list) else [source]
result._source_format = source_format
result._csv_options = csv_options
result._ignore_unknown_values = ignore_unknown_values
result._max_bad_records = max_bad_records
result._compressed = compressed
result._schema = schema
return result
def __init__(self):
""" Create an external table reference. Do not call this directly; use factory method(s). """
# Do some sanity checking and concert some params from friendly form to form used by BQ.
self._bq_source_format = None
self._source = None
self._source_format = None
self._csv_options = None
self._ignore_unknown_values = None
self._max_bad_records = None
self._compressed = None
self._schema = None
@property
def schema(self):
return self._schema
def _to_query_json(self):
""" Return the table as a dictionary to be used as JSON in a query job. """
json = {
'compression': 'GZIP' if self._compressed else 'NONE',
'ignoreUnknownValues': self._ignore_unknown_values,
'maxBadRecords': self._max_bad_records,
'sourceFormat': self._bq_source_format,
'sourceUris': self._source,
}
if self._source_format == 'csv' and self._csv_options:
json['csvOptions'] = {}
json['csvOptions'].update(self._csv_options._to_query_json())
if self._schema:
json['schema'] = {'fields': self._schema._bq_schema}
return json
================================================
FILE: datalab/bigquery/_job.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery Job functionality."""
from __future__ import absolute_import
from __future__ import unicode_literals
from __future__ import division
import datetime
import datalab.utils
from . import _api
class Job(datalab.utils.GCPJob):
"""Represents a BigQuery Job.
"""
def __init__(self, job_id, context):
"""Initializes an instance of a Job.
Args:
job_id: the BigQuery job ID corresponding to this job.
context: a Context object providing project_id and credentials.
"""
super(Job, self).__init__(job_id, context)
def _create_api(self, context):
return _api.Api(context)
def _refresh_state(self):
""" Get the state of a job. If the job is complete this does nothing;
otherwise it gets a refreshed copy of the job resource.
"""
# TODO(gram): should we put a choke on refreshes? E.g. if the last call was less than
# a second ago should we return the cached value?
if self._is_complete:
return
try:
response = self._api.jobs_get(self._job_id)
except Exception as e:
raise e
if 'status' in response:
status = response['status']
if 'state' in status and status['state'] == 'DONE':
self._end_time = datetime.datetime.utcnow()
self._is_complete = True
self._process_job_status(status)
if 'statistics' in response:
statistics = response['statistics']
start_time = statistics.get('creationTime', None)
end_time = statistics.get('endTime', None)
if start_time and end_time and end_time >= start_time:
self._start_time = datetime.datetime.fromtimestamp(float(start_time) / 1000.0)
self._end_time = datetime.datetime.fromtimestamp(float(end_time) / 1000.0)
def _process_job_status(self, status):
if 'errorResult' in status:
error_result = status['errorResult']
location = error_result.get('location', None)
message = error_result.get('message', None)
reason = error_result.get('reason', None)
self._fatal_error = datalab.utils.JobError(location, message, reason)
if 'errors' in status:
self._errors = []
for error in status['errors']:
location = error.get('location', None)
message = error.get('message', None)
reason = error.get('reason', None)
self._errors.append(datalab.utils.JobError(location, message, reason))
================================================
FILE: datalab/bigquery/_parser.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery related data parsing helpers."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from builtins import zip
from builtins import str
from builtins import object
import datetime
class Parser(object):
"""A set of helper functions to parse data in BigQuery responses."""
def __init__(self):
pass
@staticmethod
def parse_row(schema, data):
"""Parses a row from query results into an equivalent object.
Args:
schema: the array of fields defining the schema of the data.
data: the JSON row from a query result.
Returns:
The parsed row object.
"""
def parse_value(data_type, value):
"""Parses a value returned from a BigQuery response.
Args:
data_type: the type of the value as specified by the schema.
value: the raw value to return (before casting to data_type).
Returns:
The value cast to the data_type.
"""
if value is not None:
if value == 'null':
value = None
elif data_type == 'INTEGER':
value = int(value)
elif data_type == 'FLOAT':
value = float(value)
elif data_type == 'TIMESTAMP':
value = datetime.datetime.utcfromtimestamp(float(value))
elif data_type == 'BOOLEAN':
value = value == 'true'
elif (type(value) != str):
# TODO(gram): Handle nested JSON records
value = str(value)
return value
row = {}
if data is None:
return row
for i, (field, schema_field) in enumerate(zip(data['f'], schema)):
val = field['v']
name = schema_field['name']
data_type = schema_field['type']
repeated = True if 'mode' in schema_field and schema_field['mode'] == 'REPEATED' else False
if repeated and val is None:
row[name] = []
elif data_type == 'RECORD':
sub_schema = schema_field['fields']
if repeated:
row[name] = [Parser.parse_row(sub_schema, v['v']) for v in val]
else:
row[name] = Parser.parse_row(sub_schema, val)
elif repeated:
row[name] = [parse_value(data_type, v['v']) for v in val]
else:
row[name] = parse_value(data_type, val)
return row
@staticmethod
def parse_timestamp(value):
"""Parses a timestamp.
Args:
value: the number of milliseconds since epoch.
"""
return datetime.datetime.utcfromtimestamp(float(value) / 1000.0)
================================================
FILE: datalab/bigquery/_query.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements Query BigQuery API."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import object
import datalab.context
import datalab.data
import datalab.utils
from . import _api
from . import _federated_table
from . import _query_job
from . import _sampling
from . import _udf
from . import _utils
class Query(object):
"""Represents a Query object that encapsulates a BigQuery SQL query.
This object can be used to execute SQL queries and retrieve results.
"""
@staticmethod
def sampling_query(sql, context, fields=None, count=5, sampling=None, udfs=None,
data_sources=None):
"""Returns a sampling Query for the SQL object.
Args:
sql: the SQL statement (string) or Query object to sample.
context: a Context object providing project_id and credentials.
fields: an optional list of field names to retrieve.
count: an optional count of rows to retrieve which is used if a specific
sampling is not specified.
sampling: an optional sampling strategy to apply to the table.
udfs: array of UDFs referenced in the SQL.
data_sources: dictionary of federated (external) tables referenced in the SQL.
Returns:
A Query object for sampling the table.
"""
return Query(_sampling.Sampling.sampling_query(sql, fields, count, sampling), context=context,
udfs=udfs, data_sources=data_sources)
def __init__(self, sql, context=None, values=None, udfs=None, data_sources=None, **kwargs):
"""Initializes an instance of a Query object.
Note that either values or kwargs may be used, but not both.
Args:
sql: the BigQuery SQL query string to execute, or a SqlStatement object. The latter will
have any variable references replaced before being associated with the Query (i.e.
once constructed the SQL associated with a Query is static).
It is possible to have variable references in a query string too provided the variables
are passed as keyword arguments to this constructor.
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
values: a dictionary used to expand variables if passed a SqlStatement or a string with
variable references.
udfs: array of UDFs referenced in the SQL.
data_sources: dictionary of federated (external) tables referenced in the SQL.
kwargs: arguments to use when expanding the variables if passed a SqlStatement
or a string with variable references.
Raises:
Exception if expansion of any variables failed.
"""
if context is None:
context = datalab.context.Context.default()
self._context = context
self._api = _api.Api(context)
self._data_sources = data_sources
self._udfs = udfs
if data_sources is None:
data_sources = {}
self._results = None
self._code = None
self._imports = []
if values is None:
values = kwargs
self._sql = datalab.data.SqlModule.expand(sql, values)
# We need to take care not to include the same UDF code twice so we use sets.
udfs = set(udfs if udfs else [])
for value in list(values.values()):
if isinstance(value, _udf.UDF):
udfs.add(value)
included_udfs = set([])
tokens = datalab.data.tokenize(self._sql)
udf_dict = {udf.name: udf for udf in udfs}
for i, token in enumerate(tokens):
# Find the preceding and following non-whitespace tokens
prior = i - 1
while prior >= 0 and tokens[prior].isspace():
prior -= 1
if prior < 0:
continue
next = i + 1
while next < len(tokens) and tokens[next].isspace():
next += 1
if next >= len(tokens):
continue
uprior = tokens[prior].upper()
if uprior != 'FROM' and uprior != 'JOIN':
continue
# Check for external tables.
if tokens[next] not in "[('\"":
if token not in data_sources:
if values and token in values:
if isinstance(values[token], _federated_table.FederatedTable):
data_sources[token] = values[token]
# Now check for UDF calls.
if uprior != 'FROM' or tokens[next] != '(':
continue
# We have a 'FROM token (' sequence.
if token in udf_dict:
udf = udf_dict[token]
if token not in included_udfs:
included_udfs.add(token)
if self._code is None:
self._code = []
self._code.append(udf.code)
if udf.imports:
self._imports.extend(udf.imports)
fields = ', '.join([f[0] for f in udf._outputs])
tokens[i] = '(SELECT %s FROM %s' % (fields, token)
# Find the closing parenthesis and add the additional one now needed.
num_paren = 0
j = i + 1
while j < len(tokens):
if tokens[j] == '(':
num_paren += 1
elif tokens[j] == ')':
num_paren -= 1
if num_paren == 0:
tokens[j] = '))'
break
j += 1
self._external_tables = None
if len(data_sources):
self._external_tables = {}
for name, table in list(data_sources.items()):
if table.schema is None:
raise Exception('Referenced external table %s has no known schema' % name)
self._external_tables[name] = table._to_query_json()
self._sql = ''.join(tokens)
def _repr_sql_(self):
"""Creates a SQL representation of this object.
Returns:
The SQL representation to use when embedding this object into other SQL.
"""
return '(%s)' % self._sql
def __str__(self):
"""Creates a string representation of this object.
Returns:
The string representation of this object (the unmodified SQL).
"""
return self._sql
def __repr__(self):
"""Creates a friendly representation of this object.
Returns:
The friendly representation of this object (the unmodified SQL).
"""
return self._sql
@property
def sql(self):
""" Get the SQL for the query. """
return self._sql
@property
def scripts(self):
""" Get the code for any Javascript UDFs used in the query. """
return self._code
def results(self, use_cache=True, dialect=None, billing_tier=None):
"""Retrieves table of results for the query. May block if the query must be executed first.
Args:
use_cache: whether to use cached results or not. Ignored if append is specified.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryResultsTable containing the result set.
Raises:
Exception if the query could not be executed or query response was malformed.
"""
if not use_cache or (self._results is None):
self.execute(use_cache=use_cache, dialect=dialect, billing_tier=billing_tier)
return self._results.results
def extract(self, storage_uris, format='csv', csv_delimiter=',', csv_header=True,
compress=False, use_cache=True, dialect=None, billing_tier=None):
"""Exports the query results to GCS.
Args:
storage_uris: the destination URI(s). Can be a single URI or a list.
format: the format to use for the exported data; one of 'csv', 'json', or 'avro'
(default 'csv').
csv_delimiter: for csv exports, the field delimiter to use (default ',').
csv_header: for csv exports, whether to include an initial header line (default True).
compress: whether to compress the data on export. Compression is not supported for
AVRO format (default False).
use_cache: whether to use cached results or not (default True).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A Job object for the export Job if it was completed successfully; else None.
Raises:
An Exception if the query or extract failed.
"""
return self.results(use_cache=use_cache, dialect=dialect,
billing_tier=billing_tier).extract(storage_uris, format=format,
csv_delimiter=csv_delimiter,
csv_header=csv_header,
compress=compress)
@datalab.utils.async_method
def extract_async(self, storage_uris, format='csv', csv_delimiter=',', csv_header=True,
compress=False, use_cache=True, dialect=None, billing_tier=None):
"""Exports the query results to GCS. Returns a Job immediately.
Note that there are two jobs that may need to be run sequentially, one to run the query,
and the second to extract the resulting table. These are wrapped by a single outer Job.
If the query has already been executed and you would prefer to get a Job just for the
extract, you can can call extract_async on the QueryResultsTable instead; i.e.:
query.results().extract_async(...)
Args:
storage_uris: the destination URI(s). Can be a single URI or a list.
format: the format to use for the exported data; one of 'csv', 'json', or 'avro'
(default 'csv').
csv_delimiter: for CSV exports, the field delimiter to use (default ',').
csv_header: for CSV exports, whether to include an initial header line (default True).
compress: whether to compress the data on export. Compression is not supported for
AVRO format (default False).
use_cache: whether to use cached results or not (default True).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A Job for the combined (execute, extract) task that will in turn return the Job object for
the completed extract task when done; else None.
Raises:
An Exception if the query failed.
"""
return self.extract(storage_uris, format=format, csv_delimiter=csv_delimiter,
csv_header=csv_header, use_cache=use_cache, compress=compress,
dialect=dialect, billing_tier=billing_tier)
def to_dataframe(self, start_row=0, max_rows=None, use_cache=True, dialect=None,
billing_tier=None):
""" Exports the query results to a Pandas dataframe.
Args:
start_row: the row of the table at which to start the export (default 0).
max_rows: an upper limit on the number of rows to export (default None).
use_cache: whether to use cached results or not (default True).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A Pandas dataframe containing the table data.
"""
return self.results(use_cache=use_cache, dialect=dialect, billing_tier=billing_tier) \
.to_dataframe(start_row=start_row, max_rows=max_rows)
def to_file(self, path, format='csv', csv_delimiter=',', csv_header=True, use_cache=True,
dialect=None, billing_tier=None):
"""Save the results to a local file in CSV format.
Args:
path: path on the local filesystem for the saved results.
format: the format to use for the exported data; currently only 'csv' is supported.
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
use_cache: whether to use cached results or not.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
The path to the local file.
Raises:
An Exception if the operation failed.
"""
self.results(use_cache=use_cache, dialect=dialect, billing_tier=billing_tier) \
.to_file(path, format=format, csv_delimiter=csv_delimiter, csv_header=csv_header)
return path
@datalab.utils.async_method
def to_file_async(self, path, format='csv', csv_delimiter=',', csv_header=True, use_cache=True,
dialect=None, billing_tier=None):
"""Save the results to a local file in CSV format. Returns a Job immediately.
Args:
path: path on the local filesystem for the saved results.
format: the format to use for the exported data; currently only 'csv' is supported.
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
use_cache: whether to use cached results or not.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A Job for the save that returns the path to the local file on completion.
Raises:
An Exception if the operation failed.
"""
return self.to_file(path, format=format, csv_delimiter=csv_delimiter, csv_header=csv_header,
use_cache=use_cache, dialect=dialect, billing_tier=billing_tier)
def sample(self, count=5, fields=None, sampling=None, use_cache=True, dialect=None,
billing_tier=None):
"""Retrieves a sampling of rows for the query.
Args:
count: an optional count of rows to retrieve which is used if a specific
sampling is not specified (default 5).
fields: the list of fields to sample (default None implies all).
sampling: an optional sampling strategy to apply to the table.
use_cache: whether to use cached results or not (default True).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryResultsTable containing a sampling of the result set.
Raises:
Exception if the query could not be executed or query response was malformed.
"""
return Query.sampling_query(self._sql, self._context, count=count, fields=fields,
sampling=sampling, udfs=self._udfs,
data_sources=self._data_sources).results(use_cache=use_cache,
dialect=dialect,
billing_tier=billing_tier)
def execute_dry_run(self, dialect=None, billing_tier=None):
"""Dry run a query, to check the validity of the query and return some useful statistics.
Args:
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A dict with 'cacheHit' and 'totalBytesProcessed' fields.
Raises:
An exception if the query was malformed.
"""
try:
query_result = self._api.jobs_insert_query(self._sql, self._code, self._imports,
dry_run=True,
table_definitions=self._external_tables,
dialect=dialect, billing_tier=billing_tier)
except Exception as e:
raise e
return query_result['statistics']['query']
def execute_async(self, table_name=None, table_mode='create', use_cache=True,
priority='interactive', allow_large_results=False, dialect=None,
billing_tier=None):
""" Initiate the query and return a QueryJob.
Args:
table_name: the result table name as a string or TableName; if None (the default), then a
temporary table will be used.
table_mode: one of 'create', 'overwrite' or 'append'. If 'create' (the default), the request
will fail if the table exists.
use_cache: whether to use past query results or ignore cache. Has no effect if destination is
specified (default True).
priority:one of 'batch' or 'interactive' (default). 'interactive' jobs should be scheduled
to run quickly but are subject to rate limits; 'batch' jobs could be delayed by as much
as three hours but are not rate-limited.
allow_large_results: whether to allow large results; i.e. compressed data over 100MB. This is
slower and requires a table_name to be specified) (default False).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryJob.
Raises:
Exception if query could not be executed.
"""
batch = priority == 'low'
append = table_mode == 'append'
overwrite = table_mode == 'overwrite'
if table_name is not None:
table_name = _utils.parse_table_name(table_name, self._api.project_id)
try:
query_result = self._api.jobs_insert_query(self._sql, self._code, self._imports,
table_name=table_name,
append=append,
overwrite=overwrite,
use_cache=use_cache,
batch=batch,
allow_large_results=allow_large_results,
table_definitions=self._external_tables,
dialect=dialect,
billing_tier=billing_tier)
except Exception as e:
raise e
if 'jobReference' not in query_result:
raise Exception('Unexpected response from server')
job_id = query_result['jobReference']['jobId']
if not table_name:
try:
destination = query_result['configuration']['query']['destinationTable']
table_name = (destination['projectId'], destination['datasetId'], destination['tableId'])
except KeyError:
# The query was in error
raise Exception(_utils.format_query_errors(query_result['status']['errors']))
return _query_job.QueryJob(job_id, table_name, self._sql, context=self._context)
def execute(self, table_name=None, table_mode='create', use_cache=True, priority='interactive',
allow_large_results=False, dialect=None, billing_tier=None):
""" Initiate the query, blocking until complete and then return the results.
Args:
table_name: the result table name as a string or TableName; if None (the default), then a
temporary table will be used.
table_mode: one of 'create', 'overwrite' or 'append'. If 'create' (the default), the request
will fail if the table exists.
use_cache: whether to use past query results or ignore cache. Has no effect if destination is
specified (default True).
priority:one of 'batch' or 'interactive' (default). 'interactive' jobs should be scheduled
to run quickly but are subject to rate limits; 'batch' jobs could be delayed by as much
as three hours but are not rate-limited.
allow_large_results: whether to allow large results; i.e. compressed data over 100MB. This is
slower and requires a table_name to be specified) (default False).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
The QueryResultsTable for the query.
Raises:
Exception if query could not be executed.
"""
job = self.execute_async(table_name=table_name, table_mode=table_mode, use_cache=use_cache,
priority=priority, allow_large_results=allow_large_results,
dialect=dialect, billing_tier=billing_tier)
self._results = job.wait()
return self._results
def to_view(self, view_name):
""" Create a View from this Query.
Args:
view_name: the name of the View either as a string or a 3-part tuple
(projectid, datasetid, name).
Returns:
A View for the Query.
"""
# Do the import here to avoid circular dependencies at top-level.
from . import _view
return _view.View(view_name, self._context).create(self._sql)
================================================
FILE: datalab/bigquery/_query_job.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery query job functionality."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from . import _job
from . import _query_results_table
class QueryJob(_job.Job):
""" Represents a BigQuery Query Job. """
def __init__(self, job_id, table_name, sql, context):
""" Initializes a QueryJob object.
Args:
job_id: the ID of the query job.
table_name: the name of the table where the query results will be stored.
sql: the SQL statement that was executed for the query.
context: the Context object providing project_id and credentials that was used
when executing the query.
"""
super(QueryJob, self).__init__(job_id, context)
self._sql = sql
self._table = _query_results_table.QueryResultsTable(table_name, context, self,
is_temporary=True)
self._bytes_processed = None
self._cache_hit = None
self._total_rows = None
@property
def bytes_processed(self):
""" The number of bytes processed, or None if the job is not complete. """
return self._bytes_processed
@property
def total_rows(self):
""" The total number of rows in the result, or None if not complete. """
return self._total_rows
@property
def cache_hit(self):
""" Whether the query results were obtained from the cache or not, or None if not complete. """
return self._cache_hit
@property
def sql(self):
""" The SQL statement that was executed for the query. """
return self._sql
def wait(self, timeout=None):
""" Wait for the job to complete, or a timeout to happen.
This is more efficient than the version in the base Job class, in that we can
use a call that blocks for the poll duration rather than a sleep. That means we
shouldn't block unnecessarily long and can also poll less.
Args:
timeout: how long to wait (in seconds) before giving up; default None which means no timeout.
Returns:
The QueryJob
"""
poll = 30
while not self._is_complete:
try:
query_result = self._api.jobs_query_results(self._job_id,
project_id=self._context.project_id,
page_size=0,
timeout=poll * 1000)
except Exception as e:
raise e
if query_result['jobComplete']:
if 'totalBytesProcessed' in query_result:
self._bytes_processed = int(query_result['totalBytesProcessed'])
self._cache_hit = query_result.get('cacheHit', None)
if 'totalRows' in query_result:
self._total_rows = int(query_result['totalRows'])
break
if timeout is not None:
timeout -= poll
if timeout <= 0:
break
self._refresh_state()
return self
@property
def results(self):
""" Get the table used for the results of the query. If the query is incomplete, this blocks.
Raises:
Exception if we timed out waiting for results or the query failed.
"""
self.wait()
if self.failed:
raise Exception('Query failed: %s' % str(self.errors))
return self._table
================================================
FILE: datalab/bigquery/_query_results_table.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery query job results table functionality."""
from __future__ import absolute_import
from __future__ import unicode_literals
from . import _table
class QueryResultsTable(_table.Table):
""" A subclass of Table specifically for Query results.
The primary differences are the additional properties job_id and sql.
"""
def __init__(self, name, context, job, is_temporary=False):
"""Initializes an instance of a Table object.
Args:
name: the name of the table either as a string or a 3-part tuple (projectid, datasetid, name).
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
job: the QueryJob associated with these results.
is_temporary: if True, this is a short-lived table for intermediate results (default False).
"""
super(QueryResultsTable, self).__init__(name, context)
self._job = job
self._is_temporary = is_temporary
def __repr__(self):
"""Returns a representation for the dataset for showing in the notebook.
"""
if self._is_temporary:
return 'QueryResultsTable %s' % self.job_id
else:
return super(QueryResultsTable, self).__repr__()
@property
def job(self):
""" The QueryJob object that caused the table to be populated. """
return self._job
@property
def job_id(self):
""" The ID of the query job that caused the table to be populated. """
return self._job.id
@property
def sql(self):
""" The SQL statement for the query that populated the table. """
return self._job.sql
@property
def is_temporary(self):
""" Whether this is a short-lived table or not. """
return self._is_temporary
================================================
FILE: datalab/bigquery/_query_stats.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements representation of BigQuery query job dry run results."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import object
class QueryStats(object):
"""A wrapper for statistics returned by a dry run query. Useful so we can get an HTML
representation in a notebook.
"""
def __init__(self, total_bytes, is_cached):
self.total_bytes = float(total_bytes)
self.is_cached = is_cached
def _repr_html_(self):
self.total_bytes = QueryStats._size_formatter(self.total_bytes)
return """
<p>Dry run information: %s to process, results %s</p>
""" % (self.total_bytes, "cached" if self.is_cached else "not cached")
@staticmethod
def _size_formatter(byte_num, suf='B'):
for mag in ['', 'K', 'M', 'G', 'T']:
if byte_num < 1000.0:
if suf == 'B': # Don't do fractional bytes
return "%5d%s%s" % (int(byte_num), mag, suf)
return "%3.1f%s%s" % (byte_num, mag, suf)
byte_num /= 1000.0
return "%.1f%s%s".format(byte_num, 'P', suf)
================================================
FILE: datalab/bigquery/_sampling.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Sampling for BigQuery."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from builtins import object
class Sampling(object):
"""Provides common sampling strategies.
Sampling strategies can be used for sampling tables or queries.
They are implemented as functions that take in a SQL statement representing the table or query
that should be sampled, and return a new SQL statement that limits the result set in some manner.
"""
def __init__(self):
pass
@staticmethod
def _create_projection(fields):
"""Creates a projection for use in a SELECT statement.
Args:
fields: the list of fields to be specified.
"""
if (fields is None) or (len(fields) == 0):
return '*'
return ','.join(fields)
@staticmethod
def default(fields=None, count=5):
"""Provides a simple default sampling strategy which limits the result set by a count.
Args:
fields: an optional list of field names to retrieve.
count: optional number of rows to limit the sampled results to.
Returns:
A sampling function that can be applied to get a random sampling.
"""
projection = Sampling._create_projection(fields)
return lambda sql: 'SELECT %s FROM (%s) LIMIT %d' % (projection, sql, count)
@staticmethod
def sorted(field_name, ascending=True, fields=None, count=5):
"""Provides a sampling strategy that picks from an ordered set of rows.
Args:
field_name: the name of the field to sort the rows by.
ascending: whether to sort in ascending direction or not.
fields: an optional list of field names to retrieve.
count: optional number of rows to limit the sampled results to.
Returns:
A sampling function that can be applied to get the initial few rows.
"""
direction = '' if ascending else ' DESC'
projection = Sampling._create_projection(fields)
return lambda sql: 'SELECT %s FROM (%s) ORDER BY %s%s LIMIT %d' % (projection, sql, field_name,
direction, count)
@staticmethod
def sampling_query(sql, fields=None, count=5, sampling=None):
"""Returns a sampling query for the SQL object.
Args:
sql: the SQL object to sample
fields: an optional list of field names to retrieve.
count: an optional count of rows to retrieve which is used if a specific
sampling is not specified.
sampling: an optional sampling strategy to apply to the table.
Returns:
A SQL query string for sampling the input sql.
"""
if sampling is None:
sampling = Sampling.default(count=count, fields=fields)
return sampling(sql)
@staticmethod
def hashed(field_name, percent, fields=None, count=0):
"""Provides a sampling strategy based on hashing and selecting a percentage of data.
Args:
field_name: the name of the field to hash.
percent: the percentage of the resulting hashes to select.
fields: an optional list of field names to retrieve.
count: optional maximum count of rows to pick.
Returns:
A sampling function that can be applied to get a hash-based sampling.
"""
def _hashed_sampling(sql):
projection = Sampling._create_projection(fields)
sql = 'SELECT %s FROM (%s) WHERE ABS(HASH(%s)) %% 100 < %d' % \
(projection, sql, field_name, percent)
if count != 0:
sql = '%s LIMIT %d' % (sql, count)
return sql
return _hashed_sampling
@staticmethod
def random(percent, fields=None, count=0):
"""Provides a sampling strategy that picks a semi-random set of rows.
Args:
percent: the percentage of the resulting hashes to select.
fields: an optional list of field names to retrieve.
count: maximum number of rows to limit the sampled results to (default 5).
Returns:
A sampling function that can be applied to get some random rows. In order for this to
provide a good random sample percent should be chosen to be ~count/#rows where #rows
is the number of rows in the object (query, view or table) being sampled.
The rows will be returned in order; i.e. the order itself is not randomized.
"""
def _random_sampling(sql):
projection = Sampling._create_projection(fields)
sql = 'SELECT %s FROM (%s) WHERE rand() < %f' % (projection, sql, (float(percent) / 100.0))
if count != 0:
sql = '%s LIMIT %d' % (sql, count)
return sql
return _random_sampling
================================================
FILE: datalab/bigquery/_schema.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements Table and View Schema APIs."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from builtins import range
from past.builtins import basestring
from builtins import object
import datetime
import pandas
class Schema(list):
"""Represents the schema of a BigQuery table as a flattened list of objects representing fields.
Each field object has name, data_type, mode and description properties. Nested fields
get flattened with their full-qualified names. So a Schema that has an object A with nested
field B will be represented as [(name: 'A', ...), (name: 'A.b', ...)].
"""
class Field(object):
""" Represents a single field in a Table schema.
This has the properties:
- name: the flattened, full-qualified name of the field.
- data_type: the type of the field as a string ('INTEGER', 'BOOLEAN', 'FLOAT', 'STRING'
or 'TIMESTAMP').
- mode: the mode of the field; 'NULLABLE' by default.
- description: a description of the field, if known; empty string by default.
"""
# TODO(gram): consider renaming data_type member to type. Yes, it shadows top-level
# name but that is what we are using in __str__ and __getitem__ and is what is used in BQ.
# The shadowing is unlikely to cause problems.
def __init__(self, name, data_type, mode='NULLABLE', description=''):
self.name = name
self.data_type = data_type
self.mode = mode
self.description = description
def _repr_sql_(self):
"""Returns a representation of the field for embedding into a SQL statement.
Returns:
A formatted field name for use within SQL statements.
"""
return self.name
def __eq__(self, other):
""" Compare two schema field objects for equality (ignoring description). """
return self.name == other.name and self.data_type == other.data_type\
and self.mode == other.mode
def __str__(self):
""" Returns the schema field as a string form of a dictionary. """
return "{ 'name': '%s', 'type': '%s', 'mode':'%s', 'description': '%s' }" % \
(self.name, self.data_type, self.mode, self.description)
def __repr__(self):
""" Returns the schema field as a string form of a dictionary. """
return str(self)
def __getitem__(self, item):
# TODO(gram): Currently we need this for a Schema object to work with the Parser object.
# Eventually if we change Parser to only work with Schema (and not also with the
# schema dictionaries in query results) we can remove this.
if item == 'name':
return self.name
if item == 'type':
return self.data_type
if item == 'mode':
return self.mode
if item == 'description':
return self.description
@staticmethod
def _from_dataframe(dataframe, default_type='STRING'):
"""
Infer a BigQuery table schema from a Pandas dataframe. Note that if you don't explicitly set
the types of the columns in the dataframe, they may be of a type that forces coercion to
STRING, so even though the fields in the dataframe themselves may be numeric, the type in the
derived schema may not be. Hence it is prudent to make sure the Pandas dataframe is typed
correctly.
Args:
dataframe: The DataFrame.
default_type : The default big query type in case the type of the column does not exist in
the schema. Defaults to 'STRING'.
Returns:
A list of dictionaries containing field 'name' and 'type' entries, suitable for use in a
BigQuery Tables resource schema.
"""
type_mapping = {
'i': 'INTEGER',
'b': 'BOOLEAN',
'f': 'FLOAT',
'O': 'STRING',
'S': 'STRING',
'U': 'STRING',
'M': 'TIMESTAMP'
}
fields = []
for column_name, dtype in dataframe.dtypes.iteritems():
fields.append({'name': column_name,
'type': type_mapping.get(dtype.kind, default_type)})
return fields
@staticmethod
def from_dataframe(dataframe, default_type='STRING'):
"""
Infer a BigQuery table schema from a Pandas dataframe. Note that if you don't explicitly set
the types of the columns in the dataframe, they may be of a type that forces coercion to
STRING, so even though the fields in the dataframe themselves may be numeric, the type in the
derived schema may not be. Hence it is prudent to make sure the Pandas dataframe is typed
correctly.
Args:
dataframe: The DataFrame.
default_type : The default big query type in case the type of the column does not exist in
the schema. Defaults to 'STRING'.
Returns:
A Schema.
"""
return Schema(Schema._from_dataframe(dataframe, default_type=default_type))
@staticmethod
def _get_field_entry(name, value):
entry = {'name': name}
if isinstance(value, datetime.datetime):
_type = 'TIMESTAMP'
elif isinstance(value, bool):
_type = 'BOOLEAN'
elif isinstance(value, float):
_type = 'FLOAT'
elif isinstance(value, int):
_type = 'INTEGER'
elif isinstance(value, dict) or isinstance(value, list):
_type = 'RECORD'
entry['fields'] = Schema._from_record(value)
else:
_type = 'STRING'
entry['type'] = _type
return entry
@staticmethod
def _from_dict_record(data):
"""
Infer a BigQuery table schema from a dictionary. If the dictionary has entries that
are in turn OrderedDicts these will be turned into RECORD types. Ideally this will
be an OrderedDict but it is not required.
Args:
data: The dict to infer a schema from.
Returns:
A list of dictionaries containing field 'name' and 'type' entries, suitable for use in a
BigQuery Tables resource schema.
"""
return [Schema._get_field_entry(name, value) for name, value in list(data.items())]
@staticmethod
def _from_list_record(data):
"""
Infer a BigQuery table schema from a list of values.
Args:
data: The list of values.
Returns:
A list of dictionaries containing field 'name' and 'type' entries, suitable for use in a
BigQuery Tables resource schema.
"""
return [Schema._get_field_entry('Column%d' % (i + 1), value) for i, value in enumerate(data)]
@staticmethod
def _from_record(data):
"""
Infer a BigQuery table schema from a list of fields or a dictionary. The typeof the elements
is used. For a list, the field names are simply 'Column1', 'Column2', etc.
Args:
data: The list of fields or dictionary.
Returns:
A list of dictionaries containing field 'name' and 'type' entries, suitable for use in a
BigQuery Tables resource schema.
"""
if isinstance(data, dict):
return Schema._from_dict_record(data)
elif isinstance(data, list):
return Schema._from_list_record(data)
else:
raise Exception('Cannot create a schema from record %s' % str(data))
@staticmethod
def from_record(source):
"""
Infers a table/view schema from a single record that can contain a list of fields or a
dictionary of fields. The type of the elements is used for the types in the schema. For a
dict, key names are used for column names while for a list, the field names are simply named
'Column1', 'Column2', etc. Note that if using a dict you may want to use an OrderedDict
to ensure column ordering is deterministic.
Args:
source: The list of field values or dictionary of key/values.
Returns:
A Schema for the data.
"""
# TODO(gram): may want to allow an optional second argument which is a list of field
# names; could be useful for the record-containing-list case.
return Schema(Schema._from_record(source))
@staticmethod
def from_data(source):
"""Infers a table/view schema from its JSON representation, a list of records, or a Pandas
dataframe.
Args:
source: the Pandas Dataframe, a dictionary representing a record, a list of heterogeneous
data (record) or homogeneous data (list of records) from which to infer the schema, or
a definition of the schema as a list of dictionaries with 'name' and 'type' entries
and possibly 'mode' and 'description' entries. Only used if no data argument was provided.
'mode' can be 'NULLABLE', 'REQUIRED' or 'REPEATED'. For the allowed types, see:
https://cloud.google.com/bigquery/preparing-data-for-bigquery#datatypes
Note that there is potential ambiguity when passing a list of lists or a list of
dicts between whether that should be treated as a list of records or a single record
that is a list. The heuristic used is to check the length of the entries in the
list; if they are equal then a list of records is assumed. To avoid this ambiguity
you can instead use the Schema.from_record method which assumes a single record,
in either list of values or dictionary of key-values form.
Returns:
A Schema for the data.
"""
if isinstance(source, pandas.DataFrame):
bq_schema = Schema._from_dataframe(source)
elif isinstance(source, list):
if len(source) == 0:
bq_schema = source
elif all(isinstance(d, dict) for d in source):
if all('name' in d and 'type' in d for d in source):
# It looks like a bq_schema; use it as-is.
bq_schema = source
elif all(len(d) == len(source[0]) for d in source):
bq_schema = Schema._from_dict_record(source[0])
else:
raise Exception(('Cannot create a schema from heterogeneous list %s; perhaps you meant ' +
'to use Schema.from_record?') % str(source))
elif isinstance(source[0], list) and \
all([isinstance(l, list) and len(l) == len(source[0]) for l in source]):
# A list of lists all of the same length; treat first entry as a list record.
bq_schema = Schema._from_record(source[0])
else:
# A heterogeneous list; treat as a record.
raise Exception(('Cannot create a schema from heterogeneous list %s; perhaps you meant ' +
'to use Schema.from_record?') % str(source))
elif isinstance(source, dict):
raise Exception(('Cannot create a schema from dict %s; perhaps you meant to use ' +
'Schema.from_record?') % str(source))
else:
raise Exception('Cannot create a schema from %s' % str(source))
return Schema(bq_schema)
def __init__(self, definition=None):
"""Initializes a Schema from its raw JSON representation, a Pandas Dataframe, or a list.
Args:
definition: a definition of the schema as a list of dictionaries with 'name' and 'type'
entries and possibly 'mode' and 'description' entries. Only used if no data argument was
provided. 'mode' can be 'NULLABLE', 'REQUIRED' or 'REPEATED'. For the allowed types, see:
https://cloud.google.com/bigquery/preparing-data-for-bigquery#datatypes
"""
super(Schema, self).__init__()
self._map = {}
self._bq_schema = definition
self._populate_fields(definition)
def __getitem__(self, key):
"""Provides ability to lookup a schema field by position or by name.
"""
if isinstance(key, basestring):
return self._map.get(key, None)
# noinspection PyCallByClass
return list.__getitem__(self, key)
def _add_field(self, name, data_type, mode='NULLABLE', description=''):
field = Schema.Field(name, data_type, mode, description)
self.append(field)
self._map[name] = field
def find(self, name):
""" Get the index of a field in the flattened list given its (fully-qualified) name.
Args:
name: the fully-qualified name of the field.
Returns:
The index of the field, if found; else -1.
"""
for i in range(0, len(self)):
if self[i].name == name:
return i
return -1
def _populate_fields(self, data, prefix=''):
for field_data in data:
name = prefix + field_data['name']
data_type = field_data['type']
self._add_field(name, data_type, field_data.get('mode', None),
field_data.get('description', None))
if data_type == 'RECORD':
# Recurse into the nested fields, using this field's name as a prefix.
self._populate_fields(field_data.get('fields'), name + '.')
def __str__(self):
""" Returns a string representation of the non-flattened form of the schema. """
# TODO(gram): We should probably return the flattened form. There was a reason why this is
# not but I don't remember what it was. Figure that out and fix this.
return str(self._bq_schema)
def __eq__(self, other):
""" Compares two schema for equality. """
other_map = other._map
if len(self._map) != len(other_map):
return False
for name in self._map.keys():
if name not in other_map:
return False
if not self._map[name] == other_map[name]:
return False
return True
def __ne__(self, other):
""" Compares two schema for inequality. """
return not(self.__eq__(other))
================================================
FILE: datalab/bigquery/_table.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements Table, and related Table BigQuery APIs."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from builtins import str
from past.utils import old_div
from builtins import object
import calendar
import codecs
import csv
import datetime
import pandas
import time
import traceback
import uuid
import sys
import datalab.context
import datalab.utils
from . import _api
from . import _csv_options
from . import _job
from . import _parser
from . import _schema
from . import _utils
# import of Query is at end of module as we have a circular dependency of
# Query.execute().results -> Table and Table.sample() -> Query
class TableMetadata(object):
"""Represents metadata about a BigQuery table."""
def __init__(self, table, info):
"""Initializes a TableMetadata instance.
Args:
table: the Table object this belongs to.
info: The BigQuery information about this table as a Python dictionary.
"""
self._table = table
self._info = info
@property
def created_on(self):
"""The creation timestamp."""
timestamp = self._info.get('creationTime')
return _parser.Parser.parse_timestamp(timestamp)
@property
def description(self):
"""The description of the table if it exists."""
return self._info.get('description', '')
@property
def expires_on(self):
"""The timestamp for when the table will expire, or None if unknown."""
timestamp = self._info.get('expirationTime', None)
if timestamp is None:
return None
return _parser.Parser.parse_timestamp(timestamp)
@property
def friendly_name(self):
"""The friendly name of the table if it exists."""
return self._info.get('friendlyName', '')
@property
def modified_on(self):
"""The timestamp for when the table was last modified."""
timestamp = self._info.get('lastModifiedTime')
return _parser.Parser.parse_timestamp(timestamp)
@property
def rows(self):
"""The number of rows within the table, or -1 if unknown. """
return int(self._info['numRows']) if 'numRows' in self._info else -1
@property
def size(self):
"""The size of the table in bytes, or -1 if unknown. """
return int(self._info['numBytes']) if 'numBytes' in self._info else -1
def refresh(self):
""" Refresh the metadata. """
self._info = self._table._load_info()
class Table(object):
"""Represents a Table object referencing a BigQuery table. """
# Allowed characters in a BigQuery table column name
_VALID_COLUMN_NAME_CHARACTERS = '_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
# When fetching table contents, the max number of rows to fetch per HTTP request
_DEFAULT_PAGE_SIZE = 1024
# Milliseconds per week
_MSEC_PER_WEEK = 7 * 24 * 3600 * 1000
def __init__(self, name, context=None):
"""Initializes an instance of a Table object. The Table need not exist yet.
Args:
name: the name of the table either as a string or a 3-part tuple (projectid, datasetid, name).
If a string, it must have the form '<project>:<dataset>.<table>' or '<dataset>.<table>'.
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
Raises:
Exception if the name is invalid.
"""
if context is None:
context = datalab.context.Context.default()
self._context = context
self._api = _api.Api(context)
self._name_parts = _utils.parse_table_name(name, self._api.project_id)
self._full_name = '%s:%s.%s%s' % self._name_parts
self._info = None
self._cached_page = None
self._cached_page_index = 0
self._schema = None
@property
def name(self):
"""The TableName named tuple (project_id, dataset_id, table_id, decorator) for the table."""
return self._name_parts
@property
def job(self):
""" For tables resulting from executing queries, the job that created the table.
Default is None for a Table object; this is overridden by QueryResultsTable.
"""
return None
@property
def is_temporary(self):
""" Whether this is a short-lived table or not. Always False for non-QueryResultsTables. """
return False
def _load_info(self):
"""Loads metadata about this table."""
if self._info is None:
try:
self._info = self._api.tables_get(self._name_parts)
except Exception as e:
raise e
@property
def metadata(self):
"""Retrieves metadata about the table.
Returns:
A TableMetadata object.
Raises
Exception if the request could not be executed or the response was malformed.
"""
self._load_info()
return TableMetadata(self, self._info)
def exists(self):
"""Checks if the table exists.
Returns:
True if the table exists; False otherwise.
Raises:
Exception if there was an error requesting information about the table.
"""
try:
info = self._api.tables_get(self._name_parts)
except datalab.utils.RequestException as e:
if e.status == 404:
return False
raise e
except Exception as e:
raise e
self._info = info
return True
def is_listable(self):
""" Determine if the table can be listed.
Returns:
True is the Table can be listed; False otherwise.
"""
self._load_info()
return 'type' not in self._info or 'MODEL' != self._info['type']
def delete(self):
""" Delete the table.
Returns:
True if the Table no longer exists; False otherwise.
"""
try:
self._api.table_delete(self._name_parts)
except datalab.utils.RequestException:
# TODO(gram): May want to check the error reasons here and if it is not
# because the file didn't exist, return an error.
pass
except Exception as e:
raise e
return not self.exists()
def create(self, schema, overwrite=False):
""" Create the table with the specified schema.
Args:
schema: the schema to use to create the table. Should be a list of dictionaries, each
containing at least a pair of entries, 'name' and 'type'.
See https://cloud.google.com/bigquery/docs/reference/v2/tables#resource
overwrite: if True, delete the table first if it exists. If False and the table exists,
creation will fail and raise an Exception.
Returns:
The Table instance.
Raises:
Exception if the table couldn't be created or already exists and truncate was False.
"""
if overwrite and self.exists():
self.delete()
if not isinstance(schema, _schema.Schema):
# Convert to a Schema object
schema = _schema.Schema(schema)
try:
response = self._api.tables_insert(self._name_parts, schema=schema._bq_schema)
except Exception as e:
raise e
if 'selfLink' in response:
self._schema = schema
return self
raise Exception("Table %s could not be created as it already exists" % self._full_name)
def sample(self, fields=None, count=5, sampling=None, use_cache=True, dialect=None,
billing_tier=None):
"""Retrieves a sampling of data from the table.
Args:
fields: an optional list of field names to retrieve.
count: an optional count of rows to retrieve which is used if a specific
sampling is not specified.
sampling: an optional sampling strategy to apply to the table.
use_cache: whether to use cached results or not.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryResultsTable object containing the resulting data.
Raises:
Exception if the sample query could not be executed or query response was malformed.
"""
# Do import here to avoid top-level circular dependencies.
from . import _query
sql = self._repr_sql_()
return _query.Query.sampling_query(sql, context=self._context, count=count, fields=fields,
sampling=sampling).results(use_cache=use_cache,
dialect=dialect,
billing_tier=billing_tier)
@staticmethod
def _encode_dict_as_row(record, column_name_map):
""" Encode a dictionary representing a table row in a form suitable for streaming to BQ.
This includes encoding timestamps as ISO-compatible strings and removing invalid
characters from column names.
Args:
record: a Python dictionary representing the table row.
column_name_map: a dictionary mapping dictionary keys to column names. This is initially
empty and built up by this method when it first encounters each column, then used as a
cache subsequently.
Returns:
The sanitized dictionary.
"""
for k in list(record.keys()):
v = record[k]
# If the column is a date, convert to ISO string.
if isinstance(v, pandas.Timestamp) or isinstance(v, datetime.datetime):
v = record[k] = record[k].isoformat()
# If k has invalid characters clean it up
if k not in column_name_map:
column_name_map[k] = ''.join(c for c in k if c in Table._VALID_COLUMN_NAME_CHARACTERS)
new_k = column_name_map[k]
if k != new_k:
record[new_k] = v
del record[k]
return record
def insert_data(self, data, include_index=False, index_name=None):
""" Insert the contents of a Pandas DataFrame or a list of dictionaries into the table.
The insertion will be performed using at most 500 rows per POST, and at most 10 POSTs per
second, as BigQuery has some limits on streaming rates.
Args:
data: the DataFrame or list to insert.
include_index: whether to include the DataFrame or list index as a column in the BQ table.
index_name: for a list, if include_index is True, this should be the name for the index.
If not specified, 'Index' will be used.
Returns:
The table.
Raises:
Exception if the table doesn't exist, the table's schema differs from the data's schema,
or the insert failed.
"""
# TODO(gram): we could create the Table here is it doesn't exist using a schema derived
# from the data. IIRC we decided not to but doing so seems less unwieldy that having to
# create it first and then validate the schema against it itself.
# There are BigQuery limits on the streaming API:
#
# max_rows_per_post = 500
# max_bytes_per_row = 20000
# max_rows_per_second = 10000
# max_bytes_per_post = 1000000
# max_bytes_per_second = 10000000
#
# It is non-trivial to enforce these here, and the max bytes per row is not something we
# can really control. As an approximation we enforce the 500 row limit
# with a 0.05 sec POST interval (to enforce the 10,000 rows per sec limit).
max_rows_per_post = 500
post_interval = 0.05
# TODO(gram): add different exception types for each failure case.
if not self.exists():
raise Exception('Table %s does not exist.' % self._full_name)
data_schema = _schema.Schema.from_data(data)
if isinstance(data, list):
if include_index:
if not index_name:
index_name = 'Index'
data_schema._add_field(index_name, 'INTEGER')
table_schema = self.schema
# Do some validation of the two schema to make sure they are compatible.
for data_field in data_schema:
name = data_field.name
table_field = table_schema[name]
if table_field is None:
raise Exception('Table does not contain field %s' % name)
data_type = data_field.data_type
table_type = table_field.data_type
if table_type != data_type:
raise Exception('Field %s in data has type %s but in table has type %s' %
(name, data_type, table_type))
total_rows = len(data)
total_pushed = 0
job_id = uuid.uuid4().hex
rows = []
column_name_map = {}
is_dataframe = isinstance(data, pandas.DataFrame)
if is_dataframe:
# reset_index creates a new dataframe so we don't affect the original. reset_index(drop=True)
# drops the original index and uses an integer range.
gen = data.reset_index(drop=not include_index).iterrows()
else:
gen = enumerate(data)
for index, row in gen:
if is_dataframe:
row = row.to_dict()
elif include_index:
row[index_name] = index
rows.append({
'json': self._encode_dict_as_row(row, column_name_map),
'insertId': job_id + str(index)
})
total_pushed += 1
if (total_pushed == total_rows) or (len(rows) == max_rows_per_post):
try:
response = self._api.tabledata_insert_all(self._name_parts, rows)
except Exception as e:
raise e
if 'insertErrors' in response:
raise Exception('insertAll failed: %s' % response['insertErrors'])
time.sleep(post_interval) # Streaming API is rate-limited
rows = []
# Block until data is ready
while True:
self._info = self._api.tables_get(self._name_parts)
if 'streamingBuffer' not in self._info or \
'estimatedRows' not in self._info['streamingBuffer'] or \
int(self._info['streamingBuffer']['estimatedRows']) > 0:
break
time.sleep(2)
return self
def _init_job_from_response(self, response):
""" Helper function to create a Job instance from a response. """
job = None
if response and 'jobReference' in response:
job = _job.Job(job_id=response['jobReference']['jobId'], context=self._context)
return job
def extract_async(self, destination, format='csv', csv_delimiter=',', csv_header=True,
compress=False):
"""Starts a job to export the table to GCS.
Args:
destination: the destination URI(s). Can be a single URI or a list.
format: the format to use for the exported data; one of 'csv', 'json', or 'avro'
(default 'csv').
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
compress: whether to compress the data on export. Compression is not supported for
AVRO format. Defaults to False.
Returns:
A Job object for the export Job if it was started successfully; else None.
"""
format = format.upper()
if format == 'JSON':
format = 'NEWLINE_DELIMITED_JSON'
try:
response = self._api.table_extract(self._name_parts, destination, format, compress,
csv_delimiter, csv_header)
return self._init_job_from_response(response)
except Exception as e:
raise datalab.utils.JobError(location=traceback.format_exc(), message=str(e),
reason=str(type(e)))
def extract(self, destination, format='csv', csv_delimiter=',', csv_header=True, compress=False):
"""Exports the table to GCS; blocks until complete.
Args:
destination: the destination URI(s). Can be a single URI or a list.
format: the format to use for the exported data; one of 'csv', 'json', or 'avro'
(default 'csv').
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
compress: whether to compress the data on export. Compression is not supported for
AVRO format. Defaults to False.
Returns:
A Job object for the completed export Job if it was started successfully; else None.
"""
job = self.extract_async(destination, format=format, csv_delimiter=csv_delimiter,
csv_header=csv_header, compress=compress)
if job is not None:
job.wait()
return job
def load_async(self, source, mode='create', source_format='csv', csv_options=None,
ignore_unknown_values=False, max_bad_records=0):
""" Starts importing a table from GCS and return a Future.
Args:
source: the URL of the source objects(s). Can include a wildcard '*' at the end of the item
name. Can be a single source or a list.
mode: one of 'create', 'append', or 'overwrite'. 'append' or 'overwrite' will fail if the
table does not already exist, while 'create' will fail if it does. The default is
'create'. If 'create' the schema will be inferred if necessary.
source_format: the format of the data, 'csv' or 'json'; default 'csv'.
csv_options: if source format is 'csv', additional options as a CSVOptions object.
ignore_unknown_values: If True, accept rows that contain values that do not match the schema;
the unknown values are ignored (default False).
max_bad_records: the maximum number of bad records that are allowed (and ignored) before
returning an 'invalid' error in the Job result (default 0).
Returns:
A Job object for the import if it was started successfully or None if not.
Raises:
Exception if the load job failed to be started or invalid arguments were supplied.
"""
if source_format == 'csv':
source_format = 'CSV'
elif source_format == 'json':
source_format = 'NEWLINE_DELIMITED_JSON'
else:
raise Exception("Invalid source format %s" % source_format)
if not(mode == 'create' or mode == 'append' or mode == 'overwrite'):
raise Exception("Invalid mode %s" % mode)
if csv_options is None:
csv_options = _csv_options.CSVOptions()
try:
response = self._api.jobs_insert_load(source, self._name_parts,
append=(mode == 'append'),
overwrite=(mode == 'overwrite'),
create=(mode == 'create'),
source_format=source_format,
field_delimiter=csv_options.delimiter,
allow_jagged_rows=csv_options.allow_jagged_rows,
allow_quoted_newlines=csv_options.allow_quoted_newlines,
encoding=csv_options.encoding.upper(),
ignore_unknown_values=ignore_unknown_values,
max_bad_records=max_bad_records,
quote=csv_options.quote,
skip_leading_rows=csv_options.skip_leading_rows)
except Exception as e:
raise e
return self._init_job_from_response(response)
def load(self, source, mode='create', source_format='csv', csv_options=None,
ignore_unknown_values=False, max_bad_records=0):
""" Load the table from GCS.
Args:
source: the URL of the source objects(s). Can include a wildcard '*' at the end of the item
name. Can be a single source or a list.
mode: one of 'create', 'append', or 'overwrite'. 'append' or 'overwrite' will fail if the
table does not already exist, while 'create' will fail if it does. The default is
'create'. If 'create' the schema will be inferred if necessary.
source_format: the format of the data, 'csv' or 'json'; default 'csv'.
csv_options: if source format is 'csv', additional options as a CSVOptions object.
ignore_unknown_values: if True, accept rows that contain values that do not match the schema;
the unknown values are ignored (default False).
max_bad_records: the maximum number of bad records that are allowed (and ignored) before
returning an 'invalid' error in the Job result (default 0).
Returns:
A Job object for the completed load Job if it was started successfully; else None.
"""
job = self.load_async(source,
mode=mode,
source_format=source_format,
csv_options=csv_options,
ignore_unknown_values=ignore_unknown_values,
max_bad_records=max_bad_records)
if job is not None:
job.wait()
return job
def _get_row_fetcher(self, start_row=0, max_rows=None, page_size=_DEFAULT_PAGE_SIZE):
""" Get a function that can retrieve a page of rows.
The function returned is a closure so that it can have a signature suitable for use
by Iterator.
Args:
start_row: the row to start fetching from; default 0.
max_rows: the maximum number of rows to fetch (across all calls, not per-call). Default
is None which means no limit.
page_size: the maximum number of results to fetch per page; default 1024.
Returns:
A function that can be called repeatedly with a page token and running count, and that
will return an array of rows and a next page token; when the returned page token is None
the fetch is complete.
"""
if not start_row:
start_row = 0
elif start_row < 0: # We are measuring from the table end
if self.length >= 0:
start_row += self.length
else:
raise Exception('Cannot use negative indices for table of unknown length')
schema = self.schema._bq_schema
name_parts = self._name_parts
def _retrieve_rows(page_token, count):
page_rows = []
if max_rows and count >= max_rows:
page_token = None
else:
if max_rows and page_size > (max_rows - count):
max_results = max_rows - count
else:
max_results = page_size
try:
if page_token:
response = self._api.tabledata_list(name_parts, page_token=page_token,
max_results=max_results)
else:
response = self._api.tabledata_list(name_parts, start_index=start_row,
max_results=max_results)
except Exception as e:
raise e
page_token = response['pageToken'] if 'pageToken' in response else None
if 'rows' in response:
page_rows = response['rows']
rows = []
for row_dict in page_rows:
rows.append(_parser.Parser.parse_row(schema, row_dict))
return rows, page_token
return _retrieve_rows
def range(self, start_row=0, max_rows=None):
""" Get an iterator to iterate through a set of table rows.
Args:
start_row: the row of the table at which to start the iteration (default 0)
max_rows: an upper limit on the number of rows to iterate through (default None)
Returns:
A row iterator.
"""
fetcher = self._get_row_fetcher(start_row=start_row, max_rows=max_rows)
return iter(datalab.utils.Iterator(fetcher))
def to_dataframe(self, start_row=0, max_rows=None):
""" Exports the table to a Pandas dataframe.
Args:
start_row: the row of the table at which to start the export (default 0)
max_rows: an upper limit on the number of rows to export (default None)
Returns:
A Pandas dataframe containing the table data.
"""
fetcher = self._get_row_fetcher(start_row=start_row, max_rows=max_rows)
count = 0
page_token = None
df = None
while True:
page_rows, page_token = fetcher(page_token, count)
if len(page_rows):
count += len(page_rows)
if df is None:
df = pandas.DataFrame.from_records(page_rows)
else:
df = df.append(page_rows, ignore_index=True)
if not page_token:
break
# Need to reorder the dataframe to preserve column ordering
ordered_fields = [field.name for field in self.schema]
return df[ordered_fields] if df is not None else pandas.DataFrame()
def to_file(self, destination, format='csv', csv_delimiter=',', csv_header=True):
"""Save the results to a local file in CSV format.
Args:
destination: path on the local filesystem for the saved results.
format: the format to use for the exported data; currently only 'csv' is supported.
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
Raises:
An Exception if the operation failed.
"""
f = codecs.open(destination, 'w', 'utf-8')
fieldnames = []
for column in self.schema:
fieldnames.append(column.name)
if sys.version_info[0] == 2:
csv_delimiter = csv_delimiter.encode('unicode_escape')
writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter=csv_delimiter)
if csv_header:
writer.writeheader()
for row in self:
writer.writerow(row)
f.close()
@datalab.utils.async_method
def to_file_async(self, destination, format='csv', csv_delimiter=',', csv_header=True):
"""Start saving the results to a local file in CSV format and return a Job for completion.
Args:
destination: path on the local filesystem for the saved results.
format: the format to use for the exported data; currently only 'csv' is supported.
csv_delimiter: for CSV exports, the field delimiter to use. Defaults to ','
csv_header: for CSV exports, whether to include an initial header line. Default true.
Returns:
A Job for the async save operation.
Raises:
An Exception if the operation failed.
"""
self.to_file(destination, format=format, csv_delimiter=csv_delimiter, csv_header=csv_header)
@property
def schema(self):
"""Retrieves the schema of the table.
Returns:
A Schema object containing a list of schema fields and associated metadata.
Raises
Exception if the request could not be executed or the response was malformed.
"""
if not self._schema:
try:
self._load_info()
self._schema = _schema.Schema(self._info['schema']['fields'])
except KeyError:
raise Exception('Unexpected table response: missing schema')
return self._schema
def update(self, friendly_name=None, description=None, expiry=None, schema=None):
""" Selectively updates Table information.
Any parameters that are omitted or None are not updated.
Args:
friendly_name: if not None, the new friendly name.
description: if not None, the new description.
expiry: if not None, the new expiry time, either as a DateTime or milliseconds since epoch.
schema: if not None, the new schema: either a list of dictionaries or a Schema.
"""
self._load_info()
if friendly_name is not None:
self._info['friendlyName'] = friendly_name
if description is not None:
self._info['description'] = description
if expiry is not None:
if isinstance(expiry, datetime.datetime):
expiry = calendar.timegm(expiry.utctimetuple()) * 1000
self._info['expirationTime'] = expiry
if schema is not None:
if isinstance(schema, _schema.Schema):
schema = schema._bq_schema
self._info['schema'] = {'fields': schema}
try:
self._api.table_update(self._name_parts, self._info)
except datalab.utils.RequestException:
# The cached metadata is out of sync now; abandon it.
self._info = None
except Exception as e:
raise e
def _repr_sql_(self):
"""Returns a representation of the table for embedding into a SQL statement.
Returns:
A formatted table name for use within SQL statements.
"""
return '[' + self._full_name + ']'
def __repr__(self):
"""Returns a representation for the table for showing in the notebook.
"""
return 'Table %s' % self._full_name
def __str__(self):
"""Returns a string representation of the table using its specified name.
Returns:
The string representation of this object.
"""
return self._full_name
@property
def length(self):
""" Get the length of the table (number of rows). We don't use __len__ as this may
return -1 for 'unknown'.
"""
return self.metadata.rows
def __iter__(self):
""" Get an iterator for the table.
"""
return self.range(start_row=0)
def __getitem__(self, item):
""" Get an item or a slice of items from the table. This uses a small cache
to reduce the number of calls to tabledata.list.
Note: this is a useful function to have, and supports some current usage like
query.results()[0], but should be used with care.
"""
if isinstance(item, slice):
# Just treat this as a set of calls to __getitem__(int)
result = []
i = item.start
step = item.step if item.step else 1
while i < item.stop:
result.append(self[i])
i += step
return result
# Handle the integer index case.
if item < 0:
if self.length >= 0:
item += self.length
else:
raise Exception('Cannot use negative indices for table of unknown length')
if not self._cached_page \
or self._cached_page_index > item \
or self._cached_page_index + len(self._cached_page) <= item:
# cache a new page. To get the start row we round to the nearest multiple of the page
# size.
first = old_div(item, self._DEFAULT_PAGE_SIZE) * self._DEFAULT_PAGE_SIZE
count = self._DEFAULT_PAGE_SIZE
if self.length >= 0:
remaining = self.length - first
if count > remaining:
count = remaining
fetcher = self._get_row_fetcher(start_row=first, max_rows=count, page_size=count)
self._cached_page_index = first
self._cached_page, _ = fetcher(None, 0)
return self._cached_page[item - self._cached_page_index]
@staticmethod
def _convert_decorator_time(when):
if isinstance(when, datetime.datetime):
value = 1000 * (when - datetime.datetime.utcfromtimestamp(0)).total_seconds()
elif isinstance(when, datetime.timedelta):
value = when.total_seconds() * 1000
if value > 0:
raise Exception("Invalid snapshot relative when argument: %s" % str(when))
else:
raise Exception("Invalid snapshot when argument type: %s" % str(when))
if value < -Table._MSEC_PER_WEEK:
raise Exception("Invalid snapshot relative when argument: must be within 7 days: %s"
% str(when))
if value > 0:
now = 1000 * (datetime.datetime.utcnow() -
datetime.datetime.utcfromtimestamp(0)).total_seconds()
# Check that an abs value is not more than 7 days in the past and is
# not in the future
if not ((now - Table._MSEC_PER_WEEK) < value < now):
raise Exception("Invalid snapshot absolute when argument: %s" % str(when))
return int(value)
def snapshot(self, at):
""" Return a new Table which is a snapshot of this table at the specified time.
Args:
at: the time of the snapshot. This can be a Python datetime (absolute) or timedelta
(relative to current time). The result must be after the table was created and no more
than seven days in the past. Passing None will get a reference the oldest snapshot.
Note that using a datetime will get a snapshot at an absolute point in time, while
a timedelta will provide a varying snapshot; any queries issued against such a Table
will be done against a snapshot that has an age relative to the execution time of the
query.
Returns:
A new Table object referencing the snapshot.
Raises:
An exception if this Table is already decorated, or if the time specified is invalid.
"""
if self._name_parts.decorator != '':
raise Exception("Cannot use snapshot() on an already decorated table")
value = Table._convert_decorator_time(at)
return Table("%s@%s" % (self._full_name, str(value)), context=self._context)
def window(self, begin, end=None):
""" Return a new Table limited to the rows added to this Table during the specified time range.
Args:
begin: the start time of the window. This can be a Python datetime (absolute) or timedelta
(relative to current time). The result must be after the table was created and no more
than seven days in the past.
Note that using a relative value will provide a varying snapshot, not a fixed
snapshot; any queries issued against such a Table will be done against a snapshot
that has an age relative to the execution time of the query.
end: the end time of the snapshot; if None, then the current time is used. The types and
interpretation of values is as for start.
Returns:
A new Table object referencing the window.
Raises:
An exception if this Table is already decorated, or if the time specified is invalid.
"""
if self._name_parts.decorator != '':
raise Exception("Cannot use window() on an already decorated table")
start = Table._convert_decorator_time(begin)
if end is None:
if isinstance(begin, datetime.timedelta):
end = datetime.timedelta(0)
else:
end = datetime.datetime.utcnow()
stop = Table._convert_decorator_time(end)
# Both values must have the same sign
if (start > 0 >= stop) or (stop > 0 >= start):
raise Exception("window: Between arguments must both be absolute or relative: %s, %s" %
(str(begin), str(end)))
# start must be less than stop
if start > stop:
raise Exception("window: Between arguments: begin must be before end: %s, %s" %
(str(begin), str(end)))
return Table("%s@%s-%s" % (self._full_name, str(start), str(stop)), context=self._context)
def to_query(self, fields=None):
""" Return a Query for this Table.
Args:
fields: the fields to return. If None, all fields will be returned. This can be a string
which will be injected into the Query after SELECT, or a list of field names.
Returns:
A Query object that will return the specified fields from the records in the Table.
"""
# Do import here to avoid top-level circular dependencies.
from . import _query
if fields is None:
fields = '*'
elif isinstance(fields, list):
fields = ','.join(fields)
return _query.Query('SELECT %s FROM %s' % (fields, self._repr_sql_()), context=self._context)
================================================
FILE: datalab/bigquery/_udf.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Google Cloud Platform library - BigQuery UDF Functionality."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from builtins import object
import json
class UDF(object):
"""Represents a BigQuery UDF declaration.
"""
@property
def name(self):
return self._name
@property
def imports(self):
return self._imports
@property
def code(self):
return self._code
def __init__(self, inputs, outputs, name, implementation, support_code=None, imports=None):
"""Initializes a Function object from its pieces.
Args:
inputs: a list of string field names representing the schema of input.
outputs: a list of name/type tuples representing the schema of the output.
name: the name of the javascript function
implementation: a javascript function implementing the logic.
support_code: additional javascript code that the function can use.
imports: a list of GCS URLs or files containing further support code.
Raises:
Exception if the name is invalid.
"""
self._outputs = outputs
self._name = name
self._implementation = implementation
self._support_code = support_code
self._imports = imports
self._code = UDF._build_js(inputs, outputs, name, implementation, support_code)
@staticmethod
def _build_js(inputs, outputs, name, implementation, support_code):
"""Creates a BigQuery SQL UDF javascript object.
Args:
inputs: a list of (name, type) tuples representing the schema of input.
outputs: a list of (name, type) tuples representing the schema of the output.
name: the name of the function
implementation: a javascript function defining the UDF logic.
support_code: additional javascript code that the function can use.
"""
# Construct a comma-separated list of input field names
# For example, field1,field2,...
input_fields = json.dumps([f[0] for f in inputs])
# Construct a json representation of the output schema
# For example, [{'name':'field1','type':'string'},...]
output_fields = [{'name': f[0], 'type': f[1]} for f in outputs]
output_fields = json.dumps(output_fields, sort_keys=True)
# Build the JS from the individual bits with proper escaping of the implementation
if support_code is None:
support_code = ''
return ('{code}\n{name}={implementation};\nbigquery.defineFunction(\'{name}\', {inputs}, '
'{outputs}, {name});').format(code=support_code, name=name,
implementation=implementation, inputs=str(input_fields),
outputs=str(output_fields))
================================================
FILE: datalab/bigquery/_utils.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Useful common utility functions."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from past.builtins import basestring
import collections
import re
DatasetName = collections.namedtuple('DatasetName', ['project_id', 'dataset_id'])
""" A namedtuple for Dataset names.
Args:
project_id: the project id for the dataset.
dataset_id: the dataset id for the dataset.
"""
TableName = collections.namedtuple('TableName',
['project_id', 'dataset_id', 'table_id', 'decorator'])
""" A namedtuple for Table names.
Args:
project_id: the project id for the table.
dataset_id: the dataset id for the table.
table_id: the table id for the table.
decorator: the optional decorator for the table (for windowing/snapshot-ing).
"""
# Absolute project-qualified name pattern: <project>:<dataset>
_ABS_DATASET_NAME_PATTERN = r'^([a-z\d\-_\.:]+)\:(\w+)$'
# Relative name pattern: <dataset>
_REL_DATASET_NAME_PATTERN = r'^(\w+)$'
# Absolute project-qualified name pattern: <project>:<dataset>.<table>
_ABS_TABLE_NAME_PATTERN = r'^([a-z\d\-_\.:]+)\:(\w+)\.(\w+)(@[\d\-]+)?$'
# Relative name pattern: <dataset>.<table>
_REL_TABLE_NAME_PATTERN = r'^(\w+)\.(\w+)(@[\d\-]+)?$'
# Table-only name pattern: <table>. Includes an optional decorator.
_TABLE_NAME_PATTERN = r'^(\w+)(@[\d\-]+)$'
def parse_dataset_name(name, project_id=None):
"""Parses a dataset name into its individual parts.
Args:
name: the name to parse, or a tuple, dictionary or array containing the parts.
project_id: the expected project ID. If the name does not contain a project ID,
this will be used; if the name does contain a project ID and it does not match
this, an exception will be thrown.
Returns:
A DatasetName named tuple for the dataset.
Raises:
Exception: raised if the name doesn't match the expected formats or a project_id was
specified that does not match that in the name.
"""
_project_id = _dataset_id = None
if isinstance(name, basestring):
# Try to parse as absolute name first.
m = re.match(_ABS_DATASET_NAME_PATTERN, name, re.IGNORECASE)
if m is not None:
_project_id, _dataset_id = m.groups()
else:
# Next try to match as a relative name implicitly scoped within current project.
m = re.match(_REL_DATASET_NAME_PATTERN, name)
if m is not None:
groups = m.groups()
_dataset_id = groups[0]
elif isinstance(name, dict):
try:
_dataset_id = name['dataset_id']
_project_id = name['project_id']
except KeyError:
pass
else:
# Try treat as an array or tuple
if len(name) == 2:
# Treat as a tuple or array.
_project_id, _dataset_id = name
elif len(name) == 1:
_dataset_id = name[0]
if not _dataset_id:
raise Exception('Invalid dataset name: ' + str(name))
if not _project_id:
_project_id = project_id
return DatasetName(_project_id, _dataset_id)
def parse_table_name(name, project_id=None, dataset_id=None):
"""Parses a table name into its individual parts.
Args:
name: the name to parse, or a tuple, dictionary or array containing the parts.
project_id: the expected project ID. If the name does not contain a project ID,
this will be used; if the name does contain a project ID and it does not match
this, an exception will be thrown.
dataset_id: the expected dataset ID. If the name does not contain a dataset ID,
this will be used; if the name does contain a dataset ID and it does not match
this, an exception will be thrown.
Returns:
A TableName named tuple consisting of the full name and individual name parts.
Raises:
Exception: raised if the name doesn't match the expected formats, or a project_id and/or
dataset_id was provided that does not match that in the name.
"""
_project_id = _dataset_id = _table_id = _decorator = None
if isinstance(name, basestring):
# Try to parse as absolute name first.
m = re.match(_ABS_TABLE_NAME_PATTERN, name, re.IGNORECASE)
if m is not None:
_project_id, _dataset_id, _table_id, _decorator = m.groups()
else:
# Next try to match as a relative name implicitly scoped within current project.
m = re.match(_REL_TABLE_NAME_PATTERN, name)
if m is not None:
groups = m.groups()
_project_id, _dataset_id, _table_id, _decorator =\
project_id, groups[0], groups[1], groups[2]
else:
# Finally try to match as a table name only.
m = re.match(_TABLE_NAME_PATTERN, name)
if m is not None:
groups = m.groups()
_project_id, _dataset_id, _table_id, _decorator =\
project_id, dataset_id, groups[0], groups[1]
elif isinstance(name, dict):
try:
_table_id = name['table_id']
_dataset_id = name['dataset_id']
_project_id = name['project_id']
except KeyError:
pass
else:
# Try treat as an array or tuple
if len(name) == 4:
_project_id, _dataset_id, _table_id, _decorator = name
elif len(name) == 3:
_project_id, _dataset_id, _table_id = name
elif len(name) == 2:
_dataset_id, _table_id = name
if not _table_id:
raise Exception('Invalid table name: ' + str(name))
if not _project_id:
_project_id = project_id
if not _dataset_id:
_dataset_id = dataset_id
if not _decorator:
_decorator = ''
return TableName(_project_id, _dataset_id, _table_id, _decorator)
def format_query_errors(errors):
return '\n'.join(['%s: %s' % (error['reason'], error['message']) for error in errors])
================================================
FILE: datalab/bigquery/_view.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Implements BigQuery Views."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from builtins import object
import datalab.context
from . import _query
from . import _table
# Query import is at end to avoid issues with circular dependencies.
class View(object):
""" An implementation of a BigQuery View. """
# Views in BigQuery are virtual tables, but it is useful to have a mixture of both Table and
# Query semantics; our version thus internally has a BaseTable and a Query (for materialization;
# not the same as the view query), and exposes a number of the same APIs as Table and Query
# through wrapper functions around these.
def __init__(self, name, context=None):
"""Initializes an instance of a View object.
Args:
name: the name of the view either as a string or a 3-part tuple
(projectid, datasetid, name). If a string, it must have the form
'<project>:<dataset>.<view>' or '<dataset>.<view>'.
context: an optional Context object providing project_id and credentials. If a specific
project id or credentials are unspecified, the default ones configured at the global
level are used.
Raises:
Exception if the name is invalid.
"""
if context is None:
context = datalab.context.Context.default()
self._context = context
self._table = _table.Table(name, context=context)
self._materialization = _query.Query('SELECT * FROM %s' % self._repr_sql_(), context=context)
def __str__(self):
"""The full name for the view as a string."""
return str(self._table)
@property
def name(self):
"""The name for the view as a named tuple."""
return self._table.name
@property
def description(self):
"""The description of the view if it exists."""
return self._table.metadata.description
@property
def friendly_name(self):
"""The friendly name of the view if it exists."""
return self._table.metadata.friendly_name
@property
def query(self):
"""The Query that defines the view."""
if not self.exists():
return None
self._table._load_info()
if 'view' in self._table._info and 'query' in self._table._info['view']:
return _query.Query(self._table._info['view']['query'], context=self._context)
return None
def exists(self):
"""Whether the view's Query has been executed and the view is available or not."""
return self._table.exists()
def delete(self):
"""Removes the view if it exists."""
self._table.delete()
def create(self, query):
""" Creates the view with the specified query.
Args:
query: the query to use to for the View; either a string containing a SQL query or
a Query object.
Returns:
The View instance.
Raises:
Exception if the view couldn't be created or already exists and overwrite was False.
"""
if isinstance(query, _query.Query):
query = query.sql
try:
response = self._table._api.tables_insert(self._table.name, query=query)
except Exception as e:
raise e
if 'selfLink' in response:
return self
raise Exception("View %s could not be created as it already exists" % str(self))
def sample(self, fields=None, count=5, sampling=None, use_cache=True, dialect=None,
billing_tier=None):
"""Retrieves a sampling of data from the view.
Args:
fields: an optional list of field names to retrieve.
count: an optional count of rows to retrieve which is used if a specific
sampling is not specified.
sampling: an optional sampling strategy to apply to the view.
use_cache: whether to use cached results or not.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryResultsTable object containing the resulting data.
Raises:
Exception if the sample query could not be executed or the query response was malformed.
"""
return self._table.sample(fields=fields, count=count, sampling=sampling, use_cache=use_cache,
dialect=dialect, billing_tier=billing_tier)
@property
def schema(self):
"""Retrieves the schema of the table.
Returns:
A Schema object containing a list of schema fields and associated metadata.
Raises
Exception if the request could not be executed or the response was malformed.
"""
return self._table.schema
def update(self, friendly_name=None, description=None, query=None):
""" Selectively updates View information.
Any parameters that are None (the default) are not applied in the update.
Args:
friendly_name: if not None, the new friendly name.
description: if not None, the new description.
query: if not None, a new query string for the View.
"""
self._table._load_info()
if query is not None:
if isinstance(query, _query.Query):
query = query.sql
self._table._info['view'] = {'query': query}
self._table.update(friendly_name=friendly_name, description=description)
def results(self, use_cache=True, dialect=None, billing_tier=None):
"""Materialize the view synchronously.
If you require more control over the execution, use execute() or execute_async().
Args:
use_cache: whether to use cached results or not.
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryResultsTable containing the result set.
Raises:
Exception if the query could not be executed or query response was malformed.
"""
return self._materialization.results(use_cache=use_cache, dialect=dialect,
billing_tier=billing_tier)
def execute_async(self, table_name=None, table_mode='create', use_cache=True, priority='high',
allow_large_results=False, dialect=None, billing_tier=None):
"""Materialize the View asynchronously.
Args:
table_name: the result table name; if None, then a temporary table will be used.
table_mode: one of 'create', 'overwrite' or 'append'. If 'create' (the default), the request
will fail if the table exists.
use_cache: whether to use past query results or ignore cache. Has no effect if destination is
specified (default True).
priority:one of 'low' or 'high' (default). Note that 'high' is more expensive, but is
better suited to exploratory analysis.
allow_large_results: whether to allow large results; i.e. compressed data over 100MB. This is
slower and requires a table_name to be specified) (default False).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryJob for the materialization
Raises:
Exception (KeyError) if View could not be materialized.
"""
return self._materialization.execute_async(table_name=table_name, table_mode=table_mode,
use_cache=use_cache, priority=priority,
allow_large_results=allow_large_results,
dialect=dialect, billing_tier=billing_tier)
def execute(self, table_name=None, table_mode='create', use_cache=True, priority='high',
allow_large_results=False, dialect=None, billing_tier=None):
"""Materialize the View synchronously.
Args:
table_name: the result table name; if None, then a temporary table will be used.
table_mode: one of 'create', 'overwrite' or 'append'. If 'create' (the default), the request
will fail if the table exists.
use_cache: whether to use past query results or ignore cache. Has no effect if destination is
specified (default True).
priority:one of 'low' or 'high' (default). Note that 'high' is more expensive, but is
better suited to exploratory analysis.
allow_large_results: whether to allow large results; i.e. compressed data over 100MB. This is
slower and requires a table_name to be specified) (default False).
dialect : {'legacy', 'standard'}, default 'legacy'
'legacy' : Use BigQuery's legacy SQL dialect.
'standard' : Use BigQuery's standard SQL (beta), which is
compliant with the SQL 2011 standard.
billing_tier: Limits the billing tier for this job. Queries that have resource
usage beyond this tier will fail (without incurring a charge). If unspecified, this
will be set to your project default. This can also be used to override your
project-wide default billing tier on a per-query basis.
Returns:
A QueryJob for the materialization
Raises:
Exception (KeyError) if View could not be materialized.
"""
return self._materialization.execute(table_name=table_name, table_mode=table_mode,
use_cache=use_cache, priority=priority,
allow_large_results=allow_large_results,
dialect=dialect, billing_tier=billing_tier)
def _repr_sql_(self):
"""Returns a representation of the view for embedding into a SQL statement.
Returns:
A formatted table name for use within SQL statements.
"""
return '[' + str(self) + ']'
def __repr__(self):
"""Returns a representation for the view for showing in the notebook.
"""
return 'View %s\n%s' % (self._table, self.query)
================================================
FILE: datalab/bigquery/commands/__init__.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
from __future__ import absolute_import
from . import _bigquery
__all__ = ['_bigquery']
================================================
FILE: datalab/bigquery/commands/_bigquery.py
================================================
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
"""Google Cloud Platform library - BigQuery IPython Functionality."""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from builtins import zip
from builtins import str
from past.builtins import basestring
try:
import IPython
import IPython.core.display
import IPython.core.magic
except ImportError:
raise Exception('This module can only be loaded in ipython.')
import fnmatch
import json
import re
import datalab.bigquery
import datalab.data
import datalab.utils
import datalab.utils.commands
def _create_create_subparser(parser):
create_parser = parser.subcommand('create', 'Create a dataset or table.')
sub_commands = create_parser.add_subparsers(dest='command')
create_dataset_parser = sub_commands.add_parser('dataset', help='Create a dataset.')
create_dataset_parser.add_argument('-n', '--name', help='The name of the dataset to create.',
required=True)
create_dataset_parser.add_argument('-f', '--friendly', help='The friendly name of the dataset.')
create_table_parser = sub_commands.add_parser('table', help='Create a table.')
create_table_parser.add_argument('-n', '--name', help='The name of the table to create.',
required=True)
create_table_parser.add_argument('-o', '--overwrite', help='Overwrite table if it exists.',
action='store_true')
return create_parser
def _create_delete_subparser(parser):
delete_parser = parser.subcommand('delete', 'Delete a dataset or table.')
sub_commands = delete_parser.add_subparsers(dest='command')
delete_dataset_parser = sub_commands.add_parser('dataset', help='Delete a dataset.')
delete_dataset_parser.add_argument('-n', '--name', help='The name of the dataset to delete.',
required=True)
delete_table_parser = sub_commands.add_parser('table', help='Delete a table.')
delete_table_parser.add_argument('-n', '--name', help='The name of the table to delete.',
required=True)
return delete_parser
def _create_sample_subparser(parser):
sample_parser = parser.subcommand('sample',
'Display a sample of the results of a BigQuery SQL query.\nThe '
'cell can optionally contain arguments for expanding variables '
'in the query,\nif -q/--query was used, or it can contain SQL '
'for a query.')
group = sample_parser.add_mutually_exclusive_group()
group.add_argument('-q', '--query', help='the name of the query to sample')
group.add_argument('-t', '--table', help='the name of the table to sample')
group.add_argument('-v', '--view', help='the name of the view to sample')
sample_parser.add_argument('-d', '--dialect', help='BigQuery SQL dialect',
choices=['legacy', 'standard'])
sample_parser.add_argument('-b', '--billing', type=int, help='BigQuery billing tier')
sample_parser.add_argument('-c', '--count', type=int, default=10,
help='The number of rows to limit to, if sampling')
sample_parser.add_argument('-m', '--method', help='The type of sampling to use',
choices=['limit', 'random', 'hashed', 'sorted'], default='limit')
sample_parser.add_argument('-p', '--percent', type=int, default=1,
help='For random or hashed sampling, what percentage to sample from')
sample_parser.add_argument('-f', '--field',
help='The field to use for sorted or hashed sampling')
sample_parser.add_argument('-o', '--order', choices=['ascending', 'descending'],
default='ascending', help='The sort order to use for sorted sampling')
sample_parser.add_argument('-P', '--profile', action='store_true',
default=False, help='Generate an interactive profile of the data')
sample_parser.add_argument('--verbose',
help='Show the expanded SQL that is being executed',
action='store_true')
return sample_parser
def _create_udf_subparser(parser):
udf_parser = parser.subcommand('udf', 'Create a named Javascript BigQuery UDF')
udf_parser.add_argument('-m', '--module', help='The name for this UDF')
return udf_parser
def _create_dry_run_subparser(parser):
dry_run_parser = parser.subcommand('dryrun',
'Execute a dry run of a BigQuery query and display '
'approximate usage statistics')
dry_run_parser.add_argument('-q', '--query',
help='The name of the query to be dry run')
dry_run_parser.add_argument('-d', '--dialect', help='BigQuery SQL dialect',
choices=['legacy', 'standard'])
dry_run_parser.add_argument('-b', '--billing', type=int, help='BigQuery billing tier')
dry_run_parser.add_argument('-v', '--verbose',
help='Show the expanded SQL that is being executed',
action='store_true')
return dry_run_parser
def _create_execute_subparser(parser):
execute_parser = parser.subcommand('execute',
'Execute a BigQuery SQL query and optionally send the results '
'to a named table.\nThe cell can optionally contain arguments '
'for expanding variables in the query.')
execute_parser.add_argument('-nc', '--nocache', help='Don\'t use previously cached results',
action='store_true')
execute_parser.add_argument('-d', '--dialect', help='BigQuery SQL dialect',
choices=['legacy', 'standard'])
execute_parser.add_argument('-b', '--billing', type=int, help='BigQuery billing tier')
execute_parser.add_argument('-m', '--mode', help='The table creation mode', default='create',
choices=['create', 'append', 'overwrite'])
execute_parser.add_argument('-l', '--large', help='Whether to allow large results',
action='store_true')
execute_parser.add_argument('-q', '--query', help='The name of query to run')
execute_parser.add_argument('-t', '--target', help='target table name')
execute_parser.add_argument('-v', '--verbose',
help='Show the expanded SQL that is being executed',
action='store_true')
return execute_parser
def _create_pipeline_subparser(parser):
pipeline_parser = parser.subcommand('pipeline',
'Define a deployable pipeline based on a BigQuery query.\n'
'The cell can optionally contain arguments for expanding '
'variables in the query.')
pipeline_parser.add_argument('-n', '--name', help='The pipeline name')
pipeline_parser.add_argument('-nc', '--nocache', help='Don\'t use previously cached results',
action='store_true')
pipeline_parser.add_argument('-d', '--dialect', help='BigQuery SQL dialect',
choices=['legacy', 'standard'])
pipeline_parser.add_argument('-b', '--billing', type=int, help='BigQuery billing tier')
pipeline_parser.add_argument('-m', '--mode', help='The table creation mode', default='create',
choices=['create', 'append', 'overwrite'])
pipeline_parser.add_argument('-l', '--large', help='Allow large results', action='store_true')
pipeline_parser.add_argument('-q', '--query', help='The name of query to run', required=True)
pipeline_parser.add_argument('-t', '--target', help='The target table name', nargs='?')
pipeline_parser.add_argument('-v', '--verbose',
help='Show the expanded SQL that is being executed',
action='store_true')
pipeline_parser.add_argument('action', nargs='?', choices=('deploy', 'run', 'dryrun'),
default='dryrun',
help='Whether to deploy the pipeline, execute it immediately in ' +
'the notebook, or validate it with a dry run')
# TODO(gram): we may want to move some command line arguments to the cell body config spec
# eventually.
return pipeline_parser
def _create_table_subparser(parser):
table_parser = parser.subcommand('table', 'View a BigQuery table.')
table_parser.add_argument('-r', '--rows', type=int, default=25,
help='Rows to display per page')
table_parser.add_argument('-c', '--cols',
help='Comma-separated list of column names to restrict to')
table_parser.add_argument('table', help='The name of, or a reference to, the table or view')
return table_parser
def _create_schema_subparser(parser):
schema_parser = parser.subcommand('schema', 'View a BigQuery table or view schema.')
group = schema_parser.add_mutually_exclusive_group()
group.add_argument('-v', '--view', help='the name of the view whose schema should be displayed')
group.add_argument('-t', '--table', help='the name of the table whose schema should be displayed')
return schema_parser
def _create_datasets_subparser(parser):
datasets_parser = parser.subcommand('datasets', 'List the datasets in a BigQuery project.')
datasets_parser.add_argument('-p', '--project',
help='The project whose datasets should be listed')
datasets_parser.add_argument('-f', '--filter',
help='Optional wildcard filter string used to limit the results')
return datasets_parser
def _create_tables_subparser(parser):
tables_parser = parser.subcommand('tables', 'List the tables in a BigQuery project or dataset.')
tables_parser.add_argument('-p', '--project',
help='The project whose tables should be listed')
tables_parser.add_argument('-d', '--dataset',
help='The dataset to restrict to')
tables_parser.add_argument('-f', '--filter',
help='Optional wildcard filter string used to limit the results')
return tables_parser
def _create_extract_subparser(parser):
extract_parser = parser.subcommand('extract', 'Extract BigQuery query results or table to GCS.')
extract_parser.add_argument('-f', '--format', choices=['csv', 'json'], default='csv',
help='The format to use for the export')
extract_parser.add_argument('-c', '--compress', action='store_true',
help='Whether to compress the data')
extract_parser.add_argument('-H', '--header', action='store_true',
help='Whether to include a header line (CSV only)')
extract_parser.add_argument('-d', '--delimiter', default=',',
help='The field delimiter to use (CSV only)')
extract_parser.add_argument('-S', '--source', help='The name of the query or table to extract')
extract_parser.add_argument('-D', '--destination', help='The URL of the destination')
return extract_parser
def _create_load_subparser(parser):
load_parser = parser.subcommand('load', 'Load data from GCS into a BigQuery table.')
load_parser.add_argument('-m', '--mode', help='One of create (default), append or overwrite',
choices=['create', 'append', 'overwrite'], default='create')
load_parser.add_argument('-f', '--format', help='The source format', choices=['json', 'csv'],
default='csv')
load_parser.add_argument('-n', '--skip',
help='The number of initial lines to skip; useful for CSV headers',
type=int, default=0)
load_parser.add_argument('-s', '--strict', help='Whether to reject bad values and jagged lines',
action='store_true')
load_parser.add_argument('-d', '--delimiter', default=',',
help='The inter-field delimiter for CVS (default ,)')
load_parser.add_argument('-q', '--quote', default='"',
help='The quoted field delimiter for CVS (default ")')
load_parser.add_argument('-i', '--infer',
help='Whether to attempt to infer the schema from source; '
'if false the table must already exist',
action='store_true')
load_parser.add_argument('-S', '--source', help='The URL of the GCS source(s)')
load_parser.add_argument('-D', '--destination', help='The destination table name')
return load_parser
def _get_query_argument(args, cell, env):
""" Get a query argument to a cell magic.
The query is specified with args['query']. We look that up and if it is a BQ query
just return it. If it is instead a SqlModule or SqlStatement it may have variable
references. We resolve those using the arg parser for the SqlModule, then override
the resulting defaults with either the Python code in cell, or the dictionary in
overrides. The latter is for if the overrides are specified with YAML or JSON and
eventually we should eliminate code in favor of this.
Args:
args: the dictionary of magic arguments.
cell: the cell contents which can be variable value overrides (if args has a 'query'
value) or inline SQL otherwise.
env: a dictionary that is used for looking up variable values.
Returns:
A Query object.
"""
sql_arg = args.get('query', None)
if sql_arg is None:
# Assume we have inline SQL in the cell
if not isinstance(cell, basestring):
raise Exception('Expected a --query argument or inline SQL')
return datalab.bigquery.Query(cell, values=env)
item = datalab.utils.commands.get_notebook_item(sql_arg)
if isinstance(item, datalab.bigquery.Query): # Queries are already expanded.
return item
# Create an expanded BQ Query.
config = datalab.utils.commands.parse_config(cell, env)
item, env = datalab.data.SqlModule.get_sql_statement_with_environment(item, config)
if cell:
env.update(config) # config is both a fallback and an override.
return datalab.bigquery.Query(item, values=env)
def _sample_cell(args, cell_body):
"""Implements the bigquery sample cell magic for ipython notebooks.
Args:
args: the optional arguments following '%%bigquery sample'.
cell_body: optional contents of the cell interpreted as SQL, YAML or JSON.
Returns:
The results of executing the sampling query, or a profile of the sample data.
"""
env = datalab.utils.commands.notebook_environment()
query = None
table = None
view = None
if args['query']:
query = _get_query_argument(args, cell_body, env)
elif args['table']:
table = _get_table(args['table'])
elif args['view']:
view = datalab.utils.commands.get_notebook_item(args['view'])
if not isinstance(view, datalab.bigquery.View):
raise Exception('%s is not a view' % args['view'])
else:
query = datalab.bigquery.Query(cell_body, values=env)
count = args['count']
method = args['method']
if method == 'random':
sampling = datalab.bigquery.Sampling.random(percent=args['percent'], count=count)
elif method == 'hashed':
sampling = datalab.bigquery.Sampling.hashed(field_name=args['field'], percent=args['percent'],
count=count)
elif method == 'sorted':
ascending = args['order'] == 'ascending'
sampling = datalab.bigquery.Sampling.sorted(args['field'],
ascending=ascending,
count=count)
elif method == 'limit':
sampling = datalab.bigquery.Sampling.default(count=count)
else:
sampling = datalab.bigquery.Sampling.default(count=count)
if query:
results = query.sample(sampling=sampling, dialect=args['dialect'], billing_tier=args['billing'])
elif view:
results = view.sample(sampling=sampling)
else:
results = table.sample(sampling=sampling)
if args['verbose']:
print(results.sql)
if args['profile']:
return datalab.utils.commands.profile_df(results.to_dataframe())
else:
return results
def _create_cell(args, cell_body):
"""Implements the BigQuery cell magic used to create datasets and tables.
The supported syntax is:
%%bigquery create dataset -n|--name <name> [-f|--friendly <friendlyname>]
[<description>]
or:
%%bigquery create table -n|--name <tablename> [--overwrite]
[<YAML or JSON cell_body defining schema to use for tables>]
Args:
args: the argument following '%bigquery create <command>'.
"""
if args['command'] == 'dataset':
try:
datalab.bigquery.Dataset(args['name']).create(friendly_name=args['friendly'],
description=cell_body)
except Exception as e:
print('Failed to create dataset %s: %s' % (args['name'], e))
else:
if cell_body is None:
print('Failed to create %s: no schema specified' % args['name'])
else:
try:
record = datalab.utils.commands.parse_config(cell_body,
datalab.utils.commands.notebook_environment(),
as_dict=False)
schema = datalab.bigquery.Schema(record)
datalab.bigquery.Table(args['name']).create(schema=schema, overwrite=args['overwrite'])
except Exception as e:
print('Failed to create table %s: %s' % (args['name'], e))
def _delete_cell(args, _):
"""Implements the BigQuery cell magic used to delete datasets and tables.
The supported syntax is:
%%bigquery delete dataset -n|--name <name>
or:
%%bigquery delete table -n|--name <name>
Args:
args: the argument following '%bigquery delete <command>'.
"""
# TODO(gram): add support for wildchars and multiple arguments at some point. The latter is
# easy, the former a bit more tricky if non-default projects are involved.
if args['command'] == 'dataset':
try:
datalab.bigquery.Dataset(args['name']).delete()
except Exception as e:
print('Failed to delete dataset %s: %s' % (args['name'], e))
else:
try:
datalab.bigquery.Table(args['name']).delete()
except Exception as e:
print('Failed to delete table %s: %s' % (args['name'], e))
def _dryrun_cell(args, cell_body):
"""Implements the BigQuery cell magic used to dry run BQ queries.
The supported syntax is:
%%bigquery dryrun [-q|--sql <query identifier>]
[<YAML or JSON cell_body or inline SQL>]
Args:
args: the argument following '%bigquery dryrun'.
cell_body: optional contents of the cell interpreted as YAML or JSON.
Returns:
The response wrapped in a DryRunStats object
"""
query = _get_query_argument(args, cell_body, datalab.utils.commands.notebook_environment())
if args['verbose']:
print(query.sql)
result = query.execute_dry_run(dialect=args['dialect'], billing_tier=args['billing'])
return datalab.bigquery._query_stats.QueryStats(total_bytes=result['totalBytesProcessed'],
is_cached=result['cacheHit'])
def _udf_cell(args, js):
"""Implements the bigquery_udf cell magic for ipython notebooks.
The supported syntax is:
%%bigquery udf --module <var>
<js function>
Args:
args: the optional arguments following '%%bigquery udf'.
js: the UDF declaration (inputs and outputs) and implementation in javascript.
Returns:
The results of executing the UDF converted to a dataframe if no variable
was specified. None otherwise.
"""
variable_name = args['module']
if not variable_name:
raise Exception('Declaration must be of the form %%bigquery udf --module <variable name>')
# Parse out the input and output specification
spec_pattern = r'\{\{([^}]+)\}\}'
spec_part_pattern = r'[a-z_][a-z0-9_]*'
specs = re.findall(spec_pattern, js)
if len(specs) < 2:
raise Exception('The JavaScript must declare the input row and output emitter parameters '
'using valid jsdoc format comments.\n'
'The input row param declaration must be typed as {{field:type, field2:type}} '
'and the output emitter param declaration must be typed as '
'function({{field:type, field2:type}}.')
inputs = []
input_spec_parts = re.findall(spec_part_pattern, specs[0], flags=re.IGNORECASE)
if len(input_spec_parts) % 2 != 0:
raise Exception('Invalid input row param declaration. The jsdoc type expression must '
'define an object with field and type pairs.')
for n, t in zip(input_spec_parts[0::2], input_spec_parts[1::2]):
inputs.append((n, t))
outputs = []
output_spec_parts = re.findall(spec_part_pattern, specs[1], flags=re.IGNORECASE)
if len(output_spec_parts) % 2 != 0:
raise Exception('Invalid output emitter param declaration. The jsdoc type expression must '
'define a function accepting an an object with field and type pairs.')
for n, t in zip(output_spec_parts[0::2], output_spec_parts[1::2]):
outputs.append((n, t))
# Look for imports. We use a non-standard @import keyword; we could alternatively use @requires.
# Object names can contain any characters except \r and \n.
import_pattern = r'@import[\s]+(gs://[a-z\d][a-z\d_\.\-]*[a-z\d]/[^\n\r]+)'
imports = re.findall(import_pattern, js)
# Split the cell if necessary. We look for a 'function(' with no name and a header comment
# block with @param and assume this is the primary function, up to a closing '}' at the start
# of the line. The remaining cell content is used as support code.
split_pattern = r'(.*)(/\*.*?@param.*?@param.*?\*/\w*\n\w*function\w*\(.*?^}\n?)(.*)'
parts = re.match(split_pattern, js, re.MULTILINE | re.DOTALL)
support_code = ''
if parts:
gitextract_nogfp4_v/ ├── .build-bot.json ├── .coveragerc ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── datalab/ │ ├── README │ ├── __init__.py │ ├── bigquery/ │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _csv_options.py │ │ ├── _dataset.py │ │ ├── _dialect.py │ │ ├── _federated_table.py │ │ ├── _job.py │ │ ├── _parser.py │ │ ├── _query.py │ │ ├── _query_job.py │ │ ├── _query_results_table.py │ │ ├── _query_stats.py │ │ ├── _sampling.py │ │ ├── _schema.py │ │ ├── _table.py │ │ ├── _udf.py │ │ ├── _utils.py │ │ ├── _view.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _bigquery.py │ ├── context/ │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _context.py │ │ ├── _project.py │ │ ├── _utils.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _projects.py │ ├── data/ │ │ ├── __init__.py │ │ ├── _csv.py │ │ ├── _sql_module.py │ │ ├── _sql_statement.py │ │ ├── _utils.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _sql.py │ ├── kernel/ │ │ └── __init__.py │ ├── notebook/ │ │ ├── __init__.py │ │ └── static/ │ │ ├── bigquery.css │ │ ├── bigquery.ts │ │ ├── charting.css │ │ ├── charting.ts │ │ ├── element.ts │ │ ├── extern/ │ │ │ ├── d3.parcoords.css │ │ │ ├── d3.parcoords.js │ │ │ ├── lantern-browser.html │ │ │ ├── parcoords-LICENSE.txt │ │ │ ├── sylvester-LICENSE.txt │ │ │ └── sylvester.js │ │ ├── job.css │ │ ├── job.ts │ │ ├── parcoords.ts │ │ ├── style.ts │ │ └── visualization.ts │ ├── stackdriver/ │ │ ├── __init__.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ └── _monitoring.py │ │ └── monitoring/ │ │ ├── __init__.py │ │ ├── _group.py │ │ ├── _metric.py │ │ ├── _query.py │ │ ├── _query_metadata.py │ │ ├── _resource.py │ │ └── _utils.py │ ├── storage/ │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _bucket.py │ │ ├── _item.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _storage.py │ └── utils/ │ ├── __init__.py │ ├── _async.py │ ├── _dataflow_job.py │ ├── _gcp_job.py │ ├── _http.py │ ├── _iterator.py │ ├── _job.py │ ├── _json_encoder.py │ ├── _lambda_job.py │ ├── _lru_cache.py │ ├── _utils.py │ └── commands/ │ ├── __init__.py │ ├── _chart.py │ ├── _chart_data.py │ ├── _commands.py │ ├── _csv.py │ ├── _extension.py │ ├── _html.py │ ├── _job.py │ ├── _modules.py │ └── _utils.py ├── docs/ │ ├── .nojekyll │ ├── Makefile │ ├── README │ ├── conf.py │ ├── datalab Commands.rst │ ├── datalab.bigquery.rst │ ├── datalab.context.rst │ ├── datalab.data.rst │ ├── datalab.stackdriver.monitoring.rst │ ├── datalab.storage.rst │ ├── gen-magic-rst.ipy │ ├── google.datalab Commands.rst │ ├── google.datalab.bigquery.rst │ ├── google.datalab.data.rst │ ├── google.datalab.ml.rst │ ├── google.datalab.rst │ ├── google.datalab.stackdriver.monitoring.rst │ ├── google.datalab.storage.rst │ ├── index.rst │ ├── make.bat │ ├── mltoolbox.classification.dnn.rst │ ├── mltoolbox.classification.linear.rst │ ├── mltoolbox.image.classification.rst │ ├── mltoolbox.regression.dnn.rst │ └── mltoolbox.regression.linear.rst ├── externs/ │ └── ts/ │ └── require/ │ └── require.d.ts ├── google/ │ ├── __init__.py │ └── datalab/ │ ├── __init__.py │ ├── _context.py │ ├── _job.py │ ├── bigquery/ │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _csv_options.py │ │ ├── _dataset.py │ │ ├── _external_data_source.py │ │ ├── _job.py │ │ ├── _parser.py │ │ ├── _query.py │ │ ├── _query_job.py │ │ ├── _query_output.py │ │ ├── _query_results_table.py │ │ ├── _query_stats.py │ │ ├── _sampling.py │ │ ├── _schema.py │ │ ├── _table.py │ │ ├── _udf.py │ │ ├── _utils.py │ │ ├── _view.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _bigquery.py │ ├── commands/ │ │ ├── __init__.py │ │ └── _datalab.py │ ├── contrib/ │ │ ├── __init__.py │ │ ├── bigquery/ │ │ │ ├── __init__.py │ │ │ ├── commands/ │ │ │ │ ├── __init__.py │ │ │ │ └── _bigquery.py │ │ │ └── operators/ │ │ │ ├── __init__.py │ │ │ ├── _bq_execute_operator.py │ │ │ ├── _bq_extract_operator.py │ │ │ └── _bq_load_operator.py │ │ ├── mlworkbench/ │ │ │ ├── __init__.py │ │ │ ├── _archive.py │ │ │ ├── _local_predict.py │ │ │ ├── _prediction_explainer.py │ │ │ ├── _shell_process.py │ │ │ └── commands/ │ │ │ ├── __init__.py │ │ │ └── _ml.py │ │ └── pipeline/ │ │ ├── __init__.py │ │ ├── _pipeline.py │ │ ├── airflow/ │ │ │ ├── __init__.py │ │ │ └── _airflow.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ └── _pipeline.py │ │ └── composer/ │ │ ├── __init__.py │ │ ├── _api.py │ │ └── _composer.py │ ├── data/ │ │ ├── __init__.py │ │ └── _csv_file.py │ ├── kernel/ │ │ └── __init__.py │ ├── ml/ │ │ ├── __init__.py │ │ ├── _cloud_models.py │ │ ├── _cloud_training_config.py │ │ ├── _confusion_matrix.py │ │ ├── _dataset.py │ │ ├── _fasets.py │ │ ├── _feature_slice_view.py │ │ ├── _job.py │ │ ├── _metrics.py │ │ ├── _summary.py │ │ ├── _tensorboard.py │ │ └── _util.py │ ├── notebook/ │ │ ├── __init__.py │ │ └── static/ │ │ ├── bigquery.css │ │ ├── bigquery.ts │ │ ├── charting.css │ │ ├── charting.ts │ │ ├── element.ts │ │ ├── extern/ │ │ │ ├── d3.parcoords.css │ │ │ ├── d3.parcoords.js │ │ │ ├── facets-jupyter.html │ │ │ ├── lantern-browser.html │ │ │ ├── parcoords-LICENSE.txt │ │ │ ├── sylvester-LICENSE.txt │ │ │ └── sylvester.js │ │ ├── job.css │ │ ├── job.ts │ │ ├── parcoords.ts │ │ ├── style.ts │ │ └── visualization.ts │ ├── stackdriver/ │ │ ├── __init__.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ └── _monitoring.py │ │ └── monitoring/ │ │ ├── __init__.py │ │ ├── _group.py │ │ ├── _metric.py │ │ ├── _query.py │ │ ├── _query_metadata.py │ │ ├── _resource.py │ │ └── _utils.py │ ├── storage/ │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _bucket.py │ │ ├── _object.py │ │ └── commands/ │ │ ├── __init__.py │ │ └── _storage.py │ └── utils/ │ ├── __init__.py │ ├── _async.py │ ├── _dataflow_job.py │ ├── _gcp_job.py │ ├── _http.py │ ├── _iterator.py │ ├── _json_encoder.py │ ├── _lambda_job.py │ ├── _lru_cache.py │ ├── _utils.py │ ├── commands/ │ │ ├── __init__.py │ │ ├── _chart.py │ │ ├── _chart_data.py │ │ ├── _commands.py │ │ ├── _csv.py │ │ ├── _html.py │ │ ├── _job.py │ │ └── _utils.py │ └── facets/ │ ├── __init__.py │ ├── base_feature_statistics_generator.py │ ├── base_generic_feature_statistics_generator.py │ ├── feature_statistics_generator.py │ ├── feature_statistics_pb2.py │ └── generic_feature_statistics_generator.py ├── install-no-virtualenv.sh ├── install-virtualenv.sh ├── legacy_tests/ │ ├── _util/ │ │ ├── __init__.py │ │ ├── http_tests.py │ │ ├── lru_cache_tests.py │ │ └── util_tests.py │ ├── bigquery/ │ │ ├── __init__.py │ │ ├── api_tests.py │ │ ├── dataset_tests.py │ │ ├── federated_table_tests.py │ │ ├── jobs_tests.py │ │ ├── parser_tests.py │ │ ├── query_tests.py │ │ ├── sampling_tests.py │ │ ├── schema_tests.py │ │ ├── table_tests.py │ │ ├── udf_tests.py │ │ └── view_tests.py │ ├── data/ │ │ ├── __init__.py │ │ └── sql_tests.py │ ├── kernel/ │ │ ├── __init__.py │ │ ├── bigquery_tests.py │ │ ├── chart_data_tests.py │ │ ├── chart_tests.py │ │ ├── commands_tests.py │ │ ├── html_tests.py │ │ ├── module_tests.py │ │ ├── sql_tests.py │ │ ├── storage_tests.py │ │ └── utils_tests.py │ ├── main.py │ ├── stackdriver/ │ │ ├── __init__.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ └── monitoring_tests.py │ │ └── monitoring/ │ │ ├── __init__.py │ │ ├── group_tests.py │ │ ├── metric_tests.py │ │ ├── query_metadata_tests.py │ │ ├── query_tests.py │ │ ├── resource_tests.py │ │ └── utils_tests.py │ └── storage/ │ ├── __init__.py │ ├── api_tests.py │ ├── bucket_tests.py │ └── item_tests.py ├── release.sh ├── setup.cfg ├── setup.py ├── solutionbox/ │ ├── image_classification/ │ │ ├── mltoolbox/ │ │ │ ├── __init__.py │ │ │ └── image/ │ │ │ ├── __init__.py │ │ │ └── classification/ │ │ │ ├── __init__.py │ │ │ ├── _api.py │ │ │ ├── _cloud.py │ │ │ ├── _inceptionlib.py │ │ │ ├── _local.py │ │ │ ├── _model.py │ │ │ ├── _predictor.py │ │ │ ├── _preprocess.py │ │ │ ├── _trainer.py │ │ │ ├── _util.py │ │ │ ├── setup.py │ │ │ └── task.py │ │ └── setup.py │ ├── ml_workbench/ │ │ ├── setup.py │ │ ├── tensorflow/ │ │ │ ├── __init__.py │ │ │ ├── analyze.py │ │ │ ├── setup.py │ │ │ ├── trainer/ │ │ │ │ ├── __init__.py │ │ │ │ ├── feature_analysis.py │ │ │ │ ├── feature_transforms.py │ │ │ │ └── task.py │ │ │ └── transform.py │ │ ├── test_tensorflow/ │ │ │ ├── run_all.sh │ │ │ ├── test_analyze.py │ │ │ ├── test_cloud_workflow.py │ │ │ ├── test_feature_transforms.py │ │ │ ├── test_training.py │ │ │ └── test_transform.py │ │ ├── test_xgboost/ │ │ │ ├── run_all.sh │ │ │ ├── test_analyze.py │ │ │ └── test_transform.py │ │ └── xgboost/ │ │ ├── __init__.py │ │ ├── analyze.py │ │ ├── setup.py │ │ ├── trainer/ │ │ │ ├── __init__.py │ │ │ ├── feature_analysis.py │ │ │ ├── feature_transforms.py │ │ │ └── task.py │ │ └── transform.py │ └── structured_data/ │ ├── build.sh │ ├── mltoolbox/ │ │ ├── __init__.py │ │ ├── _structured_data/ │ │ │ ├── __init__.py │ │ │ ├── __version__.py │ │ │ ├── _package.py │ │ │ ├── master_setup.py │ │ │ ├── prediction/ │ │ │ │ ├── __init__.py │ │ │ │ └── predict.py │ │ │ ├── preprocess/ │ │ │ │ ├── __init__.py │ │ │ │ ├── cloud_preprocess.py │ │ │ │ └── local_preprocess.py │ │ │ └── trainer/ │ │ │ ├── __init__.py │ │ │ ├── task.py │ │ │ └── util.py │ │ ├── classification/ │ │ │ ├── __init__.py │ │ │ ├── dnn/ │ │ │ │ ├── __init__.py │ │ │ │ └── _classification_dnn.py │ │ │ └── linear/ │ │ │ ├── __init__.py │ │ │ └── _classification_linear.py │ │ └── regression/ │ │ ├── __init__.py │ │ ├── dnn/ │ │ │ ├── __init__.py │ │ │ └── _regression_dnn.py │ │ └── linear/ │ │ ├── __init__.py │ │ └── _regression_linear.py │ ├── setup.py │ └── test_mltoolbox/ │ ├── __init__.py │ ├── e2e_functions.py │ ├── test_datalab_e2e.py │ ├── test_package_functions.py │ ├── test_sd_preprocess.py │ └── test_sd_trainer.py ├── tests/ │ ├── _util/ │ │ ├── __init__.py │ │ ├── commands_tests.py │ │ ├── feature_statistics_generator_test.py │ │ ├── generic_feature_statistics_generator_test.py │ │ ├── http_tests.py │ │ ├── lru_cache_tests.py │ │ └── util_tests.py │ ├── bigquery/ │ │ ├── __init__.py │ │ ├── api_tests.py │ │ ├── dataset_tests.py │ │ ├── external_data_source_tests.py │ │ ├── jobs_tests.py │ │ ├── operator_tests.py │ │ ├── parser_tests.py │ │ ├── pipeline_tests.py │ │ ├── query_tests.py │ │ ├── sampling_tests.py │ │ ├── schema_tests.py │ │ ├── table_tests.py │ │ ├── udf_tests.py │ │ └── view_tests.py │ ├── context_tests.py │ ├── integration/ │ │ └── storage_test.py │ ├── kernel/ │ │ ├── __init__.py │ │ ├── bigquery_tests.py │ │ ├── chart_data_tests.py │ │ ├── chart_tests.py │ │ ├── html_tests.py │ │ ├── pipeline_tests.py │ │ ├── storage_tests.py │ │ └── utils_tests.py │ ├── main.py │ ├── ml/ │ │ ├── __init__.py │ │ ├── confusion_matrix_tests.py │ │ ├── dataset_tests.py │ │ ├── facets_tests.py │ │ ├── metrics_tests.py │ │ ├── summary_tests.py │ │ └── tensorboard_tests.py │ ├── ml_workbench/ │ │ ├── __init__.py │ │ └── all_tests.py │ ├── mltoolbox_structured_data/ │ │ ├── __init__.py │ │ ├── dl_interface_tests.py │ │ ├── sd_e2e_tests.py │ │ └── traininglib_tests.py │ ├── mlworkbench_magic/ │ │ ├── __init__.py │ │ ├── archive_tests.py │ │ ├── explainer_tests.py │ │ ├── local_predict_tests.py │ │ ├── ml_tests.py │ │ └── shell_process_tests.py │ ├── pipeline/ │ │ ├── __init__.py │ │ ├── airflow_tests.py │ │ ├── composer_api_tests.py │ │ ├── composer_tests.py │ │ └── pipeline_tests.py │ ├── stackdriver/ │ │ ├── __init__.py │ │ ├── commands/ │ │ │ ├── __init__.py │ │ │ └── monitoring_tests.py │ │ └── monitoring/ │ │ ├── __init__.py │ │ ├── group_tests.py │ │ ├── metric_tests.py │ │ ├── query_metadata_tests.py │ │ ├── query_tests.py │ │ ├── resource_tests.py │ │ └── utils_tests.py │ └── storage/ │ ├── __init__.py │ ├── api_tests.py │ ├── bucket_tests.py │ └── object_tests.py └── tox.ini
Showing preview only (244K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (2933 symbols across 295 files)
FILE: datalab/bigquery/__init__.py
function wait_any (line 37) | def wait_any(jobs, timeout=None):
function wait_all (line 51) | def wait_all(jobs, timeout=None):
FILE: datalab/bigquery/_api.py
class Api (line 23) | class Api(object):
method __init__ (line 36) | def __init__(self, context):
method project_id (line 46) | def project_id(self):
method jobs_insert_load (line 50) | def jobs_insert_load(self, source, table_name, append=False, overwrite...
method jobs_insert_query (line 125) | def jobs_insert_query(self, sql, code=None, imports=None, table_name=N...
method jobs_query_results (line 211) | def jobs_query_results(self, job_id, project_id, page_size, timeout, s...
method jobs_get (line 239) | def jobs_get(self, job_id, project_id=None):
method datasets_insert (line 255) | def datasets_insert(self, dataset_name, friendly_name=None, descriptio...
method datasets_delete (line 281) | def datasets_delete(self, dataset_name, delete_contents):
method datasets_update (line 300) | def datasets_update(self, dataset_name, dataset_info):
method datasets_get (line 311) | def datasets_get(self, dataset_name):
method datasets_list (line 324) | def datasets_list(self, project_id=None, max_results=0, page_token=None):
method tables_get (line 348) | def tables_get(self, table_name):
method tables_list (line 361) | def tables_list(self, dataset_name, max_results=0, page_token=None):
method tables_insert (line 384) | def tables_insert(self, table_name, schema=None, query=None, friendly_...
method tabledata_insert_all (line 422) | def tabledata_insert_all(self, table_name, rows):
method tabledata_list (line 442) | def tabledata_list(self, table_name, start_index=None, max_results=Non...
method table_delete (line 465) | def table_delete(self, table_name):
method table_extract (line 479) | def table_extract(self, table_name, destination, format='CSV', compres...
method table_update (line 521) | def table_update(self, table_name, table_info):
FILE: datalab/bigquery/_csv_options.py
class CSVOptions (line 19) | class CSVOptions(object):
method __init__ (line 21) | def __init__(self, delimiter=',', skip_leading_rows=0, encoding='utf-8...
method delimiter (line 52) | def delimiter(self):
method skip_leading_rows (line 56) | def skip_leading_rows(self):
method encoding (line 60) | def encoding(self):
method quote (line 64) | def quote(self):
method allow_quoted_newlines (line 68) | def allow_quoted_newlines(self):
method allow_jagged_rows (line 72) | def allow_jagged_rows(self):
method _to_query_json (line 75) | def _to_query_json(self):
FILE: datalab/bigquery/_dataset.py
class Dataset (line 27) | class Dataset(object):
method __init__ (line 30) | def __init__(self, name, context=None):
method name (line 54) | def name(self):
method description (line 59) | def description(self):
method friendly_name (line 69) | def friendly_name(self):
method _get_info (line 78) | def _get_info(self):
method exists (line 90) | def exists(self):
method delete (line 101) | def delete(self, delete_contents=False):
method create (line 121) | def create(self, friendly_name=None, description=None):
method update (line 143) | def update(self, friendly_name=None, description=None):
method _retrieve_items (line 166) | def _retrieve_items(self, page_token, item_type):
method _retrieve_tables (line 194) | def _retrieve_tables(self, page_token, _):
method _retrieve_views (line 197) | def _retrieve_views(self, page_token, _):
method tables (line 200) | def tables(self):
method views (line 204) | def views(self):
method __iter__ (line 208) | def __iter__(self):
method __str__ (line 212) | def __str__(self):
method __repr__ (line 220) | def __repr__(self):
class Datasets (line 226) | class Datasets(object):
method __init__ (line 229) | def __init__(self, project_id=None, context=None):
method _retrieve_datasets (line 245) | def _retrieve_datasets(self, page_token, count):
method __iter__ (line 264) | def __iter__(self):
FILE: datalab/bigquery/_dialect.py
class Dialect (line 17) | class Dialect(object):
method __init__ (line 23) | def __init__(self, bq_dialect):
method bq_dialect (line 27) | def bq_dialect(self):
method set_bq_dialect (line 35) | def set_bq_dialect(self, bq_dialect):
method default (line 41) | def default():
FILE: datalab/bigquery/_federated_table.py
class FederatedTable (line 21) | class FederatedTable(object):
method from_storage (line 24) | def from_storage(source, source_format='csv', csv_options=None, ignore...
method __init__ (line 66) | def __init__(self):
method schema (line 80) | def schema(self):
method _to_query_json (line 83) | def _to_query_json(self):
FILE: datalab/bigquery/_job.py
class Job (line 25) | class Job(datalab.utils.GCPJob):
method __init__ (line 29) | def __init__(self, job_id, context):
method _create_api (line 38) | def _create_api(self, context):
method _refresh_state (line 41) | def _refresh_state(self):
method _process_job_status (line 70) | def _process_job_status(self, status):
FILE: datalab/bigquery/_parser.py
class Parser (line 24) | class Parser(object):
method __init__ (line 27) | def __init__(self):
method parse_row (line 31) | def parse_row(schema, data):
method parse_timestamp (line 92) | def parse_timestamp(value):
FILE: datalab/bigquery/_query.py
class Query (line 30) | class Query(object):
method sampling_query (line 37) | def sampling_query(sql, context, fields=None, count=5, sampling=None, ...
method __init__ (line 56) | def __init__(self, sql, context=None, values=None, udfs=None, data_sou...
method _repr_sql_ (line 175) | def _repr_sql_(self):
method __str__ (line 183) | def __str__(self):
method __repr__ (line 191) | def __repr__(self):
method sql (line 200) | def sql(self):
method scripts (line 205) | def scripts(self):
method results (line 209) | def results(self, use_cache=True, dialect=None, billing_tier=None):
method extract (line 231) | def extract(self, storage_uris, format='csv', csv_delimiter=',', csv_h...
method extract_async (line 264) | def extract_async(self, storage_uris, format='csv', csv_delimiter=',',...
method to_dataframe (line 303) | def to_dataframe(self, start_row=0, max_rows=None, use_cache=True, dia...
method to_file (line 325) | def to_file(self, path, format='csv', csv_delimiter=',', csv_header=Tr...
method to_file_async (line 353) | def to_file_async(self, path, format='csv', csv_delimiter=',', csv_hea...
method sample (line 379) | def sample(self, count=5, fields=None, sampling=None, use_cache=True, ...
method execute_dry_run (line 408) | def execute_dry_run(self, dialect=None, billing_tier=None):
method execute_async (line 434) | def execute_async(self, table_name=None, table_mode='create', use_cach...
method execute (line 496) | def execute(self, table_name=None, table_mode='create', use_cache=True...
method to_view (line 531) | def to_view(self, view_name):
FILE: datalab/bigquery/_query_job.py
class QueryJob (line 22) | class QueryJob(_job.Job):
method __init__ (line 25) | def __init__(self, job_id, table_name, sql, context):
method bytes_processed (line 44) | def bytes_processed(self):
method total_rows (line 49) | def total_rows(self):
method cache_hit (line 54) | def cache_hit(self):
method sql (line 59) | def sql(self):
method wait (line 63) | def wait(self, timeout=None):
method results (line 102) | def results(self):
FILE: datalab/bigquery/_query_results_table.py
class QueryResultsTable (line 20) | class QueryResultsTable(_table.Table):
method __init__ (line 26) | def __init__(self, name, context, job, is_temporary=False):
method __repr__ (line 41) | def __repr__(self):
method job (line 50) | def job(self):
method job_id (line 55) | def job_id(self):
method sql (line 60) | def sql(self):
method is_temporary (line 65) | def is_temporary(self):
FILE: datalab/bigquery/_query_stats.py
class QueryStats (line 20) | class QueryStats(object):
method __init__ (line 25) | def __init__(self, total_bytes, is_cached):
method _repr_html_ (line 29) | def _repr_html_(self):
method _size_formatter (line 36) | def _size_formatter(byte_num, suf='B'):
FILE: datalab/bigquery/_sampling.py
class Sampling (line 20) | class Sampling(object):
method __init__ (line 29) | def __init__(self):
method _create_projection (line 33) | def _create_projection(fields):
method default (line 44) | def default(fields=None, count=5):
method sorted (line 57) | def sorted(field_name, ascending=True, fields=None, count=5):
method sampling_query (line 74) | def sampling_query(sql, fields=None, count=5, sampling=None):
method hashed (line 91) | def hashed(field_name, percent, fields=None, count=0):
method random (line 112) | def random(percent, fields=None, count=0):
FILE: datalab/bigquery/_schema.py
class Schema (line 25) | class Schema(list):
class Field (line 33) | class Field(object):
method __init__ (line 49) | def __init__(self, name, data_type, mode='NULLABLE', description=''):
method _repr_sql_ (line 55) | def _repr_sql_(self):
method __eq__ (line 63) | def __eq__(self, other):
method __str__ (line 68) | def __str__(self):
method __repr__ (line 73) | def __repr__(self):
method __getitem__ (line 77) | def __getitem__(self, item):
method _from_dataframe (line 92) | def _from_dataframe(dataframe, default_type='STRING'):
method from_dataframe (line 127) | def from_dataframe(dataframe, default_type='STRING'):
method _get_field_entry (line 145) | def _get_field_entry(name, value):
method _from_dict_record (line 164) | def _from_dict_record(data):
method _from_list_record (line 179) | def _from_list_record(data):
method _from_record (line 192) | def _from_record(data):
method from_record (line 211) | def from_record(source):
method from_data (line 230) | def from_data(source):
method __init__ (line 281) | def __init__(self, definition=None):
method __getitem__ (line 295) | def __getitem__(self, key):
method _add_field (line 303) | def _add_field(self, name, data_type, mode='NULLABLE', description=''):
method find (line 308) | def find(self, name):
method _populate_fields (line 321) | def _populate_fields(self, data, prefix=''):
method __str__ (line 332) | def __str__(self):
method __eq__ (line 338) | def __eq__(self, other):
method __ne__ (line 350) | def __ne__(self, other):
FILE: datalab/bigquery/_table.py
class TableMetadata (line 45) | class TableMetadata(object):
method __init__ (line 48) | def __init__(self, table, info):
method created_on (line 59) | def created_on(self):
method description (line 65) | def description(self):
method expires_on (line 70) | def expires_on(self):
method friendly_name (line 78) | def friendly_name(self):
method modified_on (line 83) | def modified_on(self):
method rows (line 89) | def rows(self):
method size (line 94) | def size(self):
method refresh (line 98) | def refresh(self):
class Table (line 103) | class Table(object):
method __init__ (line 115) | def __init__(self, name, context=None):
method name (line 139) | def name(self):
method job (line 144) | def job(self):
method is_temporary (line 152) | def is_temporary(self):
method _load_info (line 156) | def _load_info(self):
method metadata (line 165) | def metadata(self):
method exists (line 176) | def exists(self):
method is_listable (line 195) | def is_listable(self):
method delete (line 204) | def delete(self):
method create (line 220) | def create(self, schema, overwrite=False):
method sample (line 248) | def sample(self, fields=None, count=5, sampling=None, use_cache=True, ...
method _encode_dict_as_row (line 280) | def _encode_dict_as_row(record, column_name_map):
method insert_data (line 309) | def insert_data(self, data, include_index=False, index_name=None):
method _init_job_from_response (line 419) | def _init_job_from_response(self, response):
method extract_async (line 426) | def extract_async(self, destination, format='csv', csv_delimiter=',', ...
method extract (line 452) | def extract(self, destination, format='csv', csv_delimiter=',', csv_he...
method load_async (line 472) | def load_async(self, source, mode='create', source_format='csv', csv_o...
method load (line 525) | def load(self, source, mode='create', source_format='csv', csv_options...
method _get_row_fetcher (line 555) | def _get_row_fetcher(self, start_row=0, max_rows=None, page_size=_DEFA...
method range (line 614) | def range(self, start_row=0, max_rows=None):
method to_dataframe (line 627) | def to_dataframe(self, start_row=0, max_rows=None):
method to_file (line 655) | def to_file(self, destination, format='csv', csv_delimiter=',', csv_he...
method to_file_async (line 680) | def to_file_async(self, destination, format='csv', csv_delimiter=',', ...
method schema (line 696) | def schema(self):
method update (line 712) | def update(self, friendly_name=None, description=None, expiry=None, sc...
method _repr_sql_ (line 744) | def _repr_sql_(self):
method __repr__ (line 752) | def __repr__(self):
method __str__ (line 757) | def __str__(self):
method length (line 766) | def length(self):
method __iter__ (line 772) | def __iter__(self):
method __getitem__ (line 777) | def __getitem__(self, item):
method _convert_decorator_time (line 821) | def _convert_decorator_time(when):
method snapshot (line 845) | def snapshot(self, at):
method window (line 870) | def window(self, begin, end=None):
method to_query (line 914) | def to_query(self, fields=None):
FILE: datalab/bigquery/_udf.py
class UDF (line 22) | class UDF(object):
method name (line 27) | def name(self):
method imports (line 31) | def imports(self):
method code (line 35) | def code(self):
method __init__ (line 38) | def __init__(self, inputs, outputs, name, implementation, support_code...
method _build_js (line 59) | def _build_js(inputs, outputs, name, implementation, support_code):
FILE: datalab/bigquery/_utils.py
function parse_dataset_name (line 58) | def parse_dataset_name(name, project_id=None):
function parse_table_name (line 105) | def parse_table_name(name, project_id=None, dataset_id=None):
function format_query_errors (line 169) | def format_query_errors(errors):
FILE: datalab/bigquery/_view.py
class View (line 27) | class View(object):
method __init__ (line 35) | def __init__(self, name, context=None):
method __str__ (line 54) | def __str__(self):
method name (line 59) | def name(self):
method description (line 64) | def description(self):
method friendly_name (line 69) | def friendly_name(self):
method query (line 74) | def query(self):
method exists (line 83) | def exists(self):
method delete (line 87) | def delete(self):
method create (line 91) | def create(self, query):
method sample (line 112) | def sample(self, fields=None, count=5, sampling=None, use_cache=True, ...
method schema (line 139) | def schema(self):
method update (line 149) | def update(self, friendly_name=None, description=None, query=None):
method results (line 166) | def results(self, use_cache=True, dialect=None, billing_tier=None):
method execute_async (line 189) | def execute_async(self, table_name=None, table_mode='create', use_cach...
method execute (line 221) | def execute(self, table_name=None, table_mode='create', use_cache=True...
method _repr_sql_ (line 253) | def _repr_sql_(self):
method __repr__ (line 261) | def __repr__(self):
FILE: datalab/bigquery/commands/_bigquery.py
function _create_create_subparser (line 38) | def _create_create_subparser(parser):
function _create_delete_subparser (line 55) | def _create_delete_subparser(parser):
function _create_sample_subparser (line 69) | def _create_sample_subparser(parser):
function _create_udf_subparser (line 100) | def _create_udf_subparser(parser):
function _create_dry_run_subparser (line 106) | def _create_dry_run_subparser(parser):
function _create_execute_subparser (line 121) | def _create_execute_subparser(parser):
function _create_pipeline_subparser (line 143) | def _create_pipeline_subparser(parser):
function _create_table_subparser (line 171) | def _create_table_subparser(parser):
function _create_schema_subparser (line 181) | def _create_schema_subparser(parser):
function _create_datasets_subparser (line 189) | def _create_datasets_subparser(parser):
function _create_tables_subparser (line 198) | def _create_tables_subparser(parser):
function _create_extract_subparser (line 209) | def _create_extract_subparser(parser):
function _create_load_subparser (line 224) | def _create_load_subparser(parser):
function _get_query_argument (line 248) | def _get_query_argument(args, cell, env):
function _sample_cell (line 285) | def _sample_cell(args, cell_body):
function _create_cell (line 342) | def _create_cell(args, cell_body):
function _delete_cell (line 378) | def _delete_cell(args, _):
function _dryrun_cell (line 406) | def _dryrun_cell(args, cell_body):
function _udf_cell (line 428) | def _udf_cell(args, js):
function _execute_cell (line 495) | def _execute_cell(args, cell_body):
function _pipeline_cell (line 516) | def _pipeline_cell(args, cell_body):
function _table_line (line 551) | def _table_line(args):
function _get_schema (line 574) | def _get_schema(name):
function _get_table (line 592) | def _get_table(name):
function _schema_line (line 615) | def _schema_line(args):
function _render_table (line 636) | def _render_table(data, fields=None):
function _render_list (line 641) | def _render_list(data):
function _datasets_line (line 646) | def _datasets_line(args):
function _tables_line (line 663) | def _tables_line(args):
function _extract_line (line 691) | def _extract_line(args):
function _load_cell (line 725) | def _load_cell(args, schema):
function _add_command (line 770) | def _add_command(parser, subparser_fn, handler, cell_required=False, cel...
function _create_bigquery_parser (line 777) | def _create_bigquery_parser():
function bigquery (line 840) | def bigquery(line, cell=None):
function _dispatch_handler (line 863) | def _dispatch_handler(args, cell, parser, handler, cell_required=False, ...
function _table_viewer (line 891) | def _table_viewer(table, rows_per_page=25, fields=None):
function _repr_html_query (line 1007) | def _repr_html_query(query):
function _repr_html_query_results_table (line 1012) | def _repr_html_query_results_table(results):
function _repr_html_table (line 1016) | def _repr_html_table(results):
function _repr_html_table_schema (line 1020) | def _repr_html_table_schema(schema):
function _register_html_formatters (line 1049) | def _register_html_formatters():
FILE: datalab/context/_api.py
class Api (line 21) | class Api(object):
method __init__ (line 28) | def __init__(self, credentials):
method projects_list (line 31) | def projects_list(self, max_results=0, page_token=None):
method project_get (line 41) | def project_get(self, projectId):
FILE: datalab/context/_context.py
class Context (line 24) | class Context(object):
method __init__ (line 30) | def __init__(self, project_id, credentials):
method credentials (line 41) | def credentials(self):
method set_credentials (line 49) | def set_credentials(self, credentials):
method project_id (line 54) | def project_id(self):
method set_project_id (line 65) | def set_project_id(self, project_id):
method is_signed_in (line 77) | def is_signed_in():
method default (line 86) | def default():
FILE: datalab/context/_project.py
class Project (line 35) | class Project(object):
method __init__ (line 38) | def __init__(self, api, id, number, name):
method id (line 45) | def id(self):
method name (line 49) | def name(self):
method number (line 53) | def number(self):
method __str__ (line 56) | def __str__(self):
class Projects (line 60) | class Projects(object):
method __init__ (line 63) | def __init__(self, credentials=None):
method _retrieve_projects (line 73) | def _retrieve_projects(self, page_token, count):
method __iter__ (line 91) | def __iter__(self):
method get_default_id (line 97) | def get_default_id(credentials=None):
method save_default_id (line 110) | def save_default_id(project_id):
FILE: datalab/context/_utils.py
function _in_datalab_docker (line 40) | def _in_datalab_docker():
function get_config_dir (line 44) | def get_config_dir():
function _convert_oauth2client_creds (line 59) | def _convert_oauth2client_creds(credentials):
function get_credentials (line 72) | def get_credentials():
function save_project_id (line 106) | def save_project_id(project_id):
function get_project_id (line 127) | def get_project_id():
FILE: datalab/context/commands/_projects.py
function projects (line 31) | def projects(line, cell=None):
function _list_line (line 46) | def _list_line(args, _):
function _set_line (line 55) | def _set_line(args, _):
FILE: datalab/data/_csv.py
class Csv (line 40) | class Csv(object):
method __init__ (line 43) | def __init__(self, path, delimiter=b','):
method path (line 53) | def path(self):
method _read_gcs_lines (line 57) | def _read_gcs_lines(path, max_lines=None):
method _read_local_lines (line 61) | def _read_local_lines(path, max_lines=None):
method _is_probably_categorical (line 69) | def _is_probably_categorical(self, column):
method browse (line 79) | def browse(self, max_lines=None, headers=None):
method _create_federated_table (line 112) | def _create_federated_table(self, skip_header_rows):
method _get_gcs_csv_row_count (line 123) | def _get_gcs_csv_row_count(self, federated_table):
method sample_to (line 129) | def sample_to(self, count, skip_header_rows, strategy, target):
FILE: datalab/data/_sql_module.py
class SqlModule (line 29) | class SqlModule(object):
method _get_sql_args (line 33) | def _get_sql_args(parser, args=None):
method get_default_query_from_module (line 65) | def get_default_query_from_module(module):
method get_sql_statement_with_environment (line 77) | def get_sql_statement_with_environment(item, args=None):
method expand (line 110) | def expand(sql, args=None):
FILE: datalab/data/_sql_statement.py
class SqlStatement (line 28) | class SqlStatement(object):
method __init__ (line 32) | def __init__(self, sql, module=None):
method __str__ (line 42) | def __str__(self):
method __repr__ (line 50) | def __repr__(self):
method sql (line 59) | def sql(self):
method module (line 64) | def module(self):
method _find_recursive_dependencies (line 69) | def _find_recursive_dependencies(sql, values, code, resolved_vars, res...
method _escape_string (line 123) | def _escape_string(s):
method format (line 127) | def format(sql, args=None):
method _get_tokens (line 196) | def _get_tokens(sql):
method _get_dependencies (line 202) | def _get_dependencies(sql):
FILE: datalab/data/_utils.py
function get_default_query_from_module (line 27) | def get_default_query_from_module(module):
function _next_token (line 41) | def _next_token(sql):
function tokenize (line 140) | def tokenize(sql):
FILE: datalab/data/commands/_sql.py
function _create_sql_parser (line 38) | def _create_sql_parser():
function sql (line 93) | def sql(line, cell=None):
function _date (line 119) | def _date(val, offset=None):
function _resolve_table (line 180) | def _resolve_table(v, format, delta):
function _make_string_formatter (line 189) | def _make_string_formatter(f, offset=None):
function _make_table_formatter (line 196) | def _make_table_formatter(f, offset=None):
function _make_table (line 203) | def _make_table(v):
function _datestring (line 207) | def _datestring(format, offset=''):
function _table (line 211) | def _table(name=None, format=None, offset=''):
function _arguments (line 215) | def _arguments(code, module):
function _split_cell (line 284) | def _split_cell(cell, module):
function sql_cell (line 370) | def sql_cell(args, cell):
FILE: datalab/kernel/__init__.py
function load_ipython_extension (line 56) | def load_ipython_extension(shell):
function unload_ipython_extension (line 140) | def unload_ipython_extension(shell):
FILE: datalab/notebook/__init__.py
function _jupyter_nbextension_paths (line 23) | def _jupyter_nbextension_paths():
FILE: datalab/notebook/static/bigquery.ts
function _toggleNode (line 20) | function _toggleNode(e: any): void {
function _renderSchema (line 29) | function _renderSchema(table: any, schema: any, title: string,
function renderSchema (line 95) | function renderSchema(dom: any, schema: any) {
FILE: datalab/notebook/static/charting.ts
method constructor (line 27) | constructor(protected dom:HTMLElement, protected chartStyle:string) {
method init (line 32) | init(chartModule:any):void {
method addPageChangedHandler (line 42) | addPageChangedHandler(handler:Function):void {
method error (line 45) | error(message:string):void {
class PlotlyDriver (line 49) | class PlotlyDriver extends ChartLibraryDriver {
method constructor (line 52) | constructor(dom:HTMLElement, chartStyle:string) {
method requires (line 56) | requires(url: string, chartStyle:string):Array<string> {
method draw (line 60) | public draw(data:any, options:any):void {
method getStaticImage (line 302) | getStaticImage(callback:Function):void {
method addChartReadyHandler (line 309) | addChartReadyHandler(handler:Function):void {
type IStringMap (line 314) | interface IStringMap {
class GChartsDriver (line 318) | class GChartsDriver extends ChartLibraryDriver {
method constructor (line 361) | constructor(dom:HTMLElement, chartStyle:string) {
method requires (line 365) | requires(url: string, chartStyle:string):Array<string> {
method init (line 373) | init(chartModule:any):void {
method error (line 379) | error(message:string):void {
method draw (line 384) | draw(data:any, options:any):void {
method getStaticImage (line 389) | getStaticImage(callback:Function):void {
method addChartReadyHandler (line 395) | addChartReadyHandler(handler:Function) {
method addPageChangedHandler (line 399) | addPageChangedHandler(handler:Function) {
class Chart (line 406) | class Chart {
method constructor (line 413) | constructor(protected driver:ChartLibraryDriver,
method convertDates (line 441) | public static convertDates(data:any):void {
method extend (line 463) | private static extend(base:any, update:any):void {
method getCell (line 474) | private getCell() {
method getRefreshHandler (line 496) | protected getRefreshHandler(useCache:boolean):Function {
method addControls (line 504) | private addControls():void {
method getControlSettings (line 557) | protected getControlSettings():any {
method getEnvironment (line 602) | private getEnvironment():string {
method refresh (line 613) | protected refresh(useCache:boolean):void {
method handleNewData (line 646) | private handleNewData(env: any, error:any, response: any) {
method removeStaticChart (line 687) | protected removeStaticChart():void {
method addStaticChart (line 743) | private addStaticChart():void {
method handleStaticChart (line 750) | private handleStaticChart(img: string) {
method configureRefresh (line 771) | private configureRefresh(refreshInterval:number):void {
method draw (line 778) | public draw(data:any, options:any):void {
class PagedTable (line 792) | class PagedTable extends Chart {
method constructor (line 796) | constructor(driver:ChartLibraryDriver,
method getControlSettings (line 817) | protected getControlSettings():any {
method draw (line 825) | public draw(data:any, options:any):void {
method handlePageEvent (line 854) | handlePageEvent(page:number):void {
function convertListToDataTable (line 863) | function convertListToDataTable(data:any):any {
function _render (line 890) | function _render(driver:ChartLibraryDriver,
function render (line 920) | function render(driverName:string,
FILE: datalab/notebook/static/element.ts
function resolve (line 25) | function resolve(cbInfo: any): void {
function domReadyCallback (line 29) | function domReadyCallback(): void {
function load (line 39) | function load(name: any, req: any, loadCallback: any, config: any): void {
FILE: datalab/notebook/static/extern/d3.parcoords.js
function getset (line 120) | function getset(obj,state,events) {
function extend (line 135) | function extend(target, source) {
function without (line 142) | function without(arr, item) {
function compute_cluster_centroids (line 335) | function compute_cluster_centroids(d) {
function compute_centroids (line 369) | function compute_centroids(row) {
function compute_control_points (line 402) | function compute_control_points(centroids) {
function single_curve (line 446) | function single_curve(d, ctx) {
function color_path (line 463) | function color_path(d, i, ctx) {
function paths (line 475) | function paths(data, ctx) {
function single_path (line 488) | function single_path(d, ctx) {
function path_foreground (line 498) | function path_foreground(d, i) {
function path_highlight (line 502) | function path_highlight(d, i) {
function flipAxisAndUpdatePCP (line 509) | function flipAxisAndUpdatePCP(dimension, i) {
function rotateLabels (line 522) | function rotateLabels() {
function brushUpdated (line 714) | function brushUpdated(newSelection) {
function brushPredicate (line 720) | function brushPredicate(predicate) {
function is_brushed (line 776) | function is_brushed(p) {
function selected (line 781) | function selected() {
function brushExtents (line 823) | function brushExtents() {
function brushFor (line 836) | function brushFor(axis) {
function brushReset (line 853) | function brushReset(dimension) {
function install (line 867) | function install() {
function drawStrum (line 904) | function drawStrum(strum, activePoint) {
function dimensionsForPoint (line 956) | function dimensionsForPoint(p) {
function onDragStart (line 984) | function onDragStart() {
function onDrag (line 1011) | function onDrag() {
function containmentTest (line 1023) | function containmentTest(strum, width) {
function selected (line 1046) | function selected() {
function removeStrum (line 1080) | function removeStrum() {
function onDragEnd (line 1090) | function onDragEnd() {
function brushReset (line 1109) | function brushReset(strums) {
function install (line 1123) | function install() {
function position (line 1281) | function position(d) {
function doFrame (line 1310) | function doFrame() {
FILE: datalab/notebook/static/extern/sylvester.js
function Vector (line 28) | function Vector() {}
function Matrix (line 330) | function Matrix() {}
function Line (line 826) | function Line() {}
function Plane (line 1029) | function Plane() {}
FILE: datalab/notebook/static/job.ts
function refresh (line 22) | function refresh(dom: any, job_name: any, job_type: any, interval: any,
function render (line 48) | function render(dom: any, events: any, job_name: string, job_type: string,
FILE: datalab/notebook/static/parcoords.ts
function getCentroids (line 19) | function getCentroids(data: any, graph: any): any {
function getActiveData (line 34) | function getActiveData(graph: any): any{
function findAxes (line 39) | function findAxes(testPt: any, cenPts: any): number {
function isOnLine (line 50) | function isOnLine(startPt: any, endPt: any, testPt: any, tol: number){
function getClickedLines (line 64) | function getClickedLines(mouseClick: any, graph: any): any {
function highlightLineOnClick (line 86) | function highlightLineOnClick(mouseClick: any, graph: any) {
function plot (line 98) | function plot(d3: any, color_domain: number[], maximize: boolean, data: ...
FILE: datalab/notebook/static/style.ts
function addStyleSheet (line 29) | function addStyleSheet(url: string): void {
function domReadyCallback (line 40) | function domReadyCallback(): void {
function load (line 50) | function load(url: string, req: any, loadCallback: any, config: any): vo...
FILE: datalab/notebook/static/visualization.ts
function loadGoogleApiLoader (line 32) | function loadGoogleApiLoader(callback: any): void {
function invokeVisualizationCallback (line 45) | function invokeVisualizationCallback(cb: any) {
function loadVisualizationPackages (line 49) | function loadVisualizationPackages(names: any, callbacks: any): void {
function load (line 67) | function load(name: any, req: any, callback: any, config: any) {
FILE: datalab/stackdriver/commands/_monitoring.py
function monitoring (line 27) | def monitoring(line, cell=None):
function _list_metric_descriptors (line 76) | def _list_metric_descriptors(args, _):
function _list_resource_descriptors (line 85) | def _list_resource_descriptors(args, _):
function _list_groups (line 94) | def _list_groups(args, _):
function _render_dataframe (line 103) | def _render_dataframe(dataframe):
FILE: datalab/stackdriver/monitoring/_group.py
class Groups (line 29) | class Groups(object):
method __init__ (line 35) | def __init__(self, context=None):
method list (line 45) | def list(self, pattern='*'):
method as_dataframe (line 63) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: datalab/stackdriver/monitoring/_metric.py
class MetricDescriptors (line 26) | class MetricDescriptors(object):
method __init__ (line 32) | def __init__(self, filter_string=None, type_prefix=None, context=None):
method list (line 47) | def list(self, pattern='*'):
method as_dataframe (line 64) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: datalab/stackdriver/monitoring/_query.py
class Query (line 23) | class Query(google.cloud.monitoring_v3.query.Query):
method __init__ (line 26) | def __init__(self,
method metadata (line 66) | def metadata(self):
FILE: datalab/stackdriver/monitoring/_query_metadata.py
class QueryMetadata (line 25) | class QueryMetadata(object):
method __init__ (line 28) | def __init__(self, query):
method __iter__ (line 40) | def __iter__(self):
method metric_type (line 45) | def metric_type(self):
method resource_types (line 50) | def resource_types(self):
method as_dataframe (line 54) | def as_dataframe(self, max_rows=None):
FILE: datalab/stackdriver/monitoring/_resource.py
class ResourceDescriptors (line 24) | class ResourceDescriptors(object):
method __init__ (line 29) | def __init__(self, filter_string=None, context=None):
method list (line 41) | def list(self, pattern='*'):
method as_dataframe (line 57) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: datalab/stackdriver/monitoring/_utils.py
class _MonitoringClient (line 27) | class _MonitoringClient(object):
method __init__ (line 28) | def __init__(self, context):
method list_metric_descriptors (line 40) | def list_metric_descriptors(self, filter_string=None, type_prefix=None):
method list_resource_descriptors (line 54) | def list_resource_descriptors(self, filter_string=None):
method list_groups (line 59) | def list_groups(self):
function make_client (line 64) | def make_client(context=None):
FILE: datalab/storage/_api.py
class Api (line 27) | class Api(object):
method __init__ (line 40) | def __init__(self, context):
method project_id (line 50) | def project_id(self):
method buckets_insert (line 54) | def buckets_insert(self, bucket, project_id=None):
method buckets_delete (line 71) | def buckets_delete(self, bucket):
method buckets_get (line 83) | def buckets_get(self, bucket, projection='noAcl'):
method buckets_list (line 98) | def buckets_list(self, projection='noAcl', max_results=0, page_token=N...
method object_download (line 123) | def object_download(self, bucket, key, start_offset=0, byte_count=None):
method object_upload (line 147) | def object_upload(self, bucket, key, content, content_type):
method objects_copy (line 165) | def objects_copy(self, source_bucket, source_key, target_bucket, targe...
method objects_delete (line 182) | def objects_delete(self, bucket, key):
method objects_get (line 195) | def objects_get(self, bucket, key, projection='noAcl'):
method objects_list (line 214) | def objects_list(self, bucket, prefix=None, delimiter=None, projection...
method objects_patch (line 249) | def objects_patch(self, bucket, key, info):
method _escape_key (line 265) | def _escape_key(key):
method verify_permitted_to_read (line 270) | def verify_permitted_to_read(gs_path):
FILE: datalab/storage/_bucket.py
function parse_name (line 34) | def parse_name(name):
class BucketMetadata (line 59) | class BucketMetadata(object):
method __init__ (line 62) | def __init__(self, info):
method created_on (line 71) | def created_on(self):
method etag (line 77) | def etag(self):
method name (line 82) | def name(self):
class Bucket (line 87) | class Bucket(object):
method __init__ (line 90) | def __init__(self, name, info=None, context=None):
method name (line 108) | def name(self):
method __repr__ (line 112) | def __repr__(self):
method metadata (line 118) | def metadata(self):
method item (line 134) | def item(self, key):
method items (line 146) | def items(self, prefix=None, delimiter=None):
method exists (line 160) | def exists(self):
method create (line 167) | def create(self, project_id=None):
method delete (line 186) | def delete(self):
class Buckets (line 199) | class Buckets(object):
method __init__ (line 202) | def __init__(self, project_id=None, context=None):
method contains (line 218) | def contains(self, name):
method create (line 238) | def create(self, name):
method _retrieve_buckets (line 250) | def _retrieve_buckets(self, page_token, _):
method __iter__ (line 266) | def __iter__(self):
FILE: datalab/storage/_item.py
class ItemMetadata (line 29) | class ItemMetadata(object):
method __init__ (line 32) | def __init__(self, info):
method content_type (line 41) | def content_type(self):
method etag (line 46) | def etag(self):
method name (line 51) | def name(self):
method size (line 56) | def size(self):
method updated_on (line 61) | def updated_on(self):
class Item (line 67) | class Item(object):
method __init__ (line 70) | def __init__(self, bucket, key, info=None, context=None):
method from_url (line 90) | def from_url(url):
method key (line 96) | def key(self):
method uri (line 101) | def uri(self):
method __repr__ (line 106) | def __repr__(self):
method copy_to (line 111) | def copy_to(self, new_key, bucket=None):
method exists (line 130) | def exists(self):
method delete (line 139) | def delete(self):
method metadata (line 152) | def metadata(self):
method read_from (line 167) | def read_from(self, start_offset=0, byte_count=None):
method read_lines (line 184) | def read_lines(self, max_lines=None):
method write_to (line 212) | def write_to(self, content, content_type):
class Items (line 227) | class Items(object):
method __init__ (line 230) | def __init__(self, bucket, prefix, delimiter, context=None):
method contains (line 252) | def contains(self, key):
method _retrieve_items (line 272) | def _retrieve_items(self, page_token, _):
method __iter__ (line 290) | def __iter__(self):
FILE: datalab/storage/commands/_storage.py
function _extract_storage_api_response_error (line 34) | def _extract_storage_api_response_error(message):
function storage (line 55) | def storage(line, cell=None):
function _parser_exit (line 137) | def _parser_exit(status=0, message=None):
function _expand_list (line 144) | def _expand_list(names):
function _storage_copy (line 199) | def _storage_copy(args, _):
function _storage_create (line 229) | def _storage_create(args, _):
function _storage_delete (line 247) | def _storage_delete(args, _):
function _storage_list_buckets (line 276) | def _storage_list_buckets(project, pattern):
function _storage_get_keys (line 284) | def _storage_get_keys(bucket, pattern):
function _storage_get_key_names (line 289) | def _storage_get_key_names(bucket, pattern):
function _storage_list_keys (line 294) | def _storage_list_keys(bucket, pattern):
function _storage_list (line 304) | def _storage_list(args, _):
function _get_item_contents (line 344) | def _get_item_contents(source_name):
function _storage_read (line 356) | def _storage_read(args, _):
function _storage_view (line 362) | def _storage_view(args, _):
function _storage_write (line 377) | def _storage_write(args, _):
FILE: datalab/utils/_async.py
class async_ (line 26) | class async_(with_metaclass(abc.ABCMeta, object)):
method __init__ (line 33) | def __init__(self, function):
method _preprocess_args (line 39) | def _preprocess_args(*args):
method _preprocess_kwargs (line 44) | def _preprocess_kwargs(**kwargs):
method _call (line 50) | def _call(self, *args, **kwargs):
method __call__ (line 53) | def __call__(self, *args, **kwargs):
class async_function (line 58) | class async_function(async_):
method _call (line 64) | def _call(self, *args, **kwargs):
class async_method (line 69) | class async_method(async_):
method _call (line 75) | def _call(self, *args, **kwargs):
method __get__ (line 80) | def __get__(self, instance, owner):
FILE: datalab/utils/_dataflow_job.py
class DataflowJob (line 19) | class DataflowJob(_job.Job):
method __init__ (line 23) | def __init__(self, runner_results):
method _refresh_state (line 32) | def _refresh_state(self):
FILE: datalab/utils/_gcp_job.py
class GCPJob (line 21) | class GCPJob(_job.Job):
method __init__ (line 25) | def __init__(self, job_id, context):
method _create_api (line 38) | def _create_api(self, context):
method __repr__ (line 41) | def __repr__(self):
FILE: datalab/utils/_http.py
class RequestException (line 39) | class RequestException(Exception):
method __init__ (line 41) | def __init__(self, status, content):
method __str__ (line 56) | def __str__(self):
class Http (line 60) | class Http(object):
method __init__ (line 74) | def __init__(self):
method request (line 78) | def request(url, args=None, data=None, headers=None, method=None,
FILE: datalab/utils/_iterator.py
class Iterator (line 19) | class Iterator(object):
method __init__ (line 22) | def __init__(self, retriever):
method __iter__ (line 33) | def __iter__(self):
method reset (line 46) | def reset(self):
FILE: datalab/utils/_job.py
class JobError (line 26) | class JobError(Exception):
method __init__ (line 29) | def __init__(self, location, message, reason):
method __str__ (line 34) | def __str__(self):
class Job (line 38) | class Job(object):
method __init__ (line 50) | def __init__(self, job_id=None, future=None):
method __str__ (line 66) | def __str__(self):
method id (line 70) | def id(self):
method is_complete (line 79) | def is_complete(self):
method failed (line 89) | def failed(self):
method fatal_error (line 100) | def fatal_error(self):
method errors (line 110) | def errors(self):
method result (line 119) | def result(self):
method start_time_utc (line 135) | def start_time_utc(self):
method end_time_utc (line 140) | def end_time_utc(self):
method total_time (line 145) | def total_time(self):
method _refresh_state (line 151) | def _refresh_state(self):
method _timeout (line 171) | def _timeout(self):
method wait (line 175) | def wait(self, timeout=None):
method state (line 202) | def state(self):
method __repr__ (line 217) | def __repr__(self):
method _wait (line 222) | def _wait(jobs, timeout, return_when):
method wait_any (line 257) | def wait_any(jobs, timeout=None):
method wait_all (line 270) | def wait_all(jobs, timeout=None):
FILE: datalab/utils/_json_encoder.py
class JSONEncoder (line 21) | class JSONEncoder(json.JSONEncoder):
method default (line 24) | def default(self, obj):
FILE: datalab/utils/_lambda_job.py
class LambdaJob (line 21) | class LambdaJob(_job.Job):
method __init__ (line 25) | def __init__(self, fn, job_id, *args, **kwargs):
method __repr__ (line 35) | def __repr__(self):
FILE: datalab/utils/_lru_cache.py
class LRUCache (line 23) | class LRUCache(object):
method __init__ (line 26) | def __init__(self, cache_size):
method __getitem__ (line 37) | def __getitem__(self, key):
method __delitem__ (line 59) | def __delitem__(self, key):
method __setitem__ (line 69) | def __setitem__(self, key, value):
method __contains__ (line 96) | def __contains__(self, key):
method get (line 99) | def get(self, key, value):
FILE: datalab/utils/_utils.py
function print_exception_with_last_stack (line 31) | def print_exception_with_last_stack(e):
function get_item (line 41) | def get_item(env, name, default=None):
function compare_datetimes (line 63) | def compare_datetimes(d1, d2):
function pick_unused_port (line 86) | def pick_unused_port():
function is_http_running_on (line 99) | def is_http_running_on(port):
function gcs_copy_file (line 116) | def gcs_copy_file(source, dest):
FILE: datalab/utils/commands/_chart.py
function chart (line 29) | def chart(line, cell=None):
function _chart_cell (line 52) | def _chart_cell(args, cell):
FILE: datalab/utils/commands/_chart_data.py
function _get_chart_data (line 36) | def _get_chart_data(line, cell_body=''):
FILE: datalab/utils/commands/_commands.py
class CommandParser (line 27) | class CommandParser(argparse.ArgumentParser):
method __init__ (line 30) | def __init__(self, *args, **kwargs):
method create (line 36) | def create(name):
method exit (line 40) | def exit(self, status=0, message=None):
method format_usage (line 44) | def format_usage(self):
method create_args (line 49) | def create_args(line, namespace):
method parse (line 66) | def parse(self, line, namespace=None):
method subcommand (line 78) | def subcommand(self, name, help):
FILE: datalab/utils/commands/_csv.py
function csv (line 34) | def csv(line, cell=None):
function _view (line 51) | def _view(args, cell):
FILE: datalab/utils/commands/_extension.py
function extension (line 29) | def extension(line, cell=None):
function _extension (line 40) | def _extension(args, cell):
FILE: datalab/utils/commands/_html.py
class Html (line 24) | class Html(object):
method next_id (line 33) | def next_id():
method __init__ (line 38) | def __init__(self, markup=None):
method add_class (line 49) | def add_class(self, class_name):
method add_dependency (line 54) | def add_dependency(self, path, name):
method add_script (line 59) | def add_script(self, script):
method _repr_html_ (line 64) | def _repr_html_(self):
class HtmlBuilder (line 87) | class HtmlBuilder(object):
method __init__ (line 91) | def __init__(self):
method _render_objects (line 96) | def _render_objects(self, items, attributes=None, datatype='object'):
method _render_text (line 150) | def _render_text(self, text, preformatted=False):
method _render_list (line 160) | def _render_list(self, items, empty='<pre><empty></pre>'):
method _to_html (line 177) | def _to_html(self):
method _format (line 186) | def _format(value, nbsp=False):
method render_text (line 195) | def render_text(text, preformatted=False):
method render_table (line 209) | def render_table(data, headers=None):
method render_chart_data (line 221) | def render_chart_data(data):
method render_list (line 232) | def render_list(data):
FILE: datalab/utils/commands/_job.py
function html_job_status (line 34) | def html_job_status(job_name, job_type, refresh_interval, html_on_runnin...
function _get_job_status (line 61) | def _get_job_status(line):
FILE: datalab/utils/commands/_modules.py
function pymodule (line 31) | def pymodule(line, cell=None):
function _pymodule_cell (line 41) | def _pymodule_cell(args, cell):
function _create_python_module (line 54) | def _create_python_module(name, code):
FILE: datalab/utils/commands/_utils.py
function notebook_environment (line 47) | def notebook_environment():
function get_notebook_item (line 53) | def get_notebook_item(name):
function render_list (line 59) | def render_list(data):
function render_dictionary (line 63) | def render_dictionary(data, headers=None):
function render_text (line 73) | def render_text(text, preformatted=False):
function get_field_list (line 83) | def get_field_list(fields, schema):
function _get_cols (line 99) | def _get_cols(fields, schema):
function _get_data_from_empty_list (line 128) | def _get_data_from_empty_list(source, fields='*', first_row=0, count=-1,...
function _get_data_from_list_of_dicts (line 134) | def _get_data_from_list_of_dicts(source, fields='*', first_row=0, count=...
function _get_data_from_list_of_lists (line 144) | def _get_data_from_list_of_lists(source, fields='*', first_row=0, count=...
function _get_data_from_dataframe (line 155) | def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, ...
function _get_data_from_table (line 176) | def _get_data_from_table(source, fields='*', first_row=0, count=-1, sche...
function get_data (line 188) | def get_data(source, fields='*', env=None, first_row=0, count=-1, schema...
function handle_magic_line (line 247) | def handle_magic_line(line, cell, parser, namespace=None):
function expand_var (line 260) | def expand_var(v, env):
function replace_vars (line 284) | def replace_vars(config, env):
function parse_config (line 313) | def parse_config(config, env, as_dict=True):
function validate_config (line 336) | def validate_config(config, required_keys, optional_keys=None):
function validate_config_must_have (line 360) | def validate_config_must_have(config, required_keys):
function validate_config_has_one_of (line 375) | def validate_config_has_one_of(config, one_of_keys):
function validate_config_value (line 393) | def validate_config_value(value, possible_values):
function get_data_source_index (line 415) | def get_data_source_index(name):
function validate_gcs_path (line 421) | def validate_gcs_path(path, require_object):
function parse_control_options (line 438) | def parse_control_options(controls, variable_defaults=None):
function chart_html (line 553) | def chart_html(driver_name, chart_type, source, chart_options=None, fiel...
function profile_df (line 675) | def profile_df(df):
FILE: externs/ts/require/require.d.ts
type RequireError (line 41) | interface RequireError extends Error {
type RequireShim (line 59) | interface RequireShim {
type RequireConfig (line 82) | interface RequireConfig {
type RequireModule (line 216) | interface RequireModule {
type RequireMap (line 228) | interface RequireMap {
type Require (line 261) | interface Require {
type RequireDefine (line 332) | interface RequireDefine {
FILE: google/datalab/_context.py
class Context (line 23) | class Context(object):
method __init__ (line 29) | def __init__(self, project_id, credentials, config=None):
method credentials (line 42) | def credentials(self):
method set_credentials (line 50) | def set_credentials(self, credentials):
method project_id (line 55) | def project_id(self):
method set_project_id (line 66) | def set_project_id(self, project_id):
method config (line 73) | def config(self):
method set_config (line 81) | def set_config(self, config):
method _is_signed_in (line 86) | def _is_signed_in():
method _get_default_config (line 95) | def _get_default_config():
method default (line 102) | def default():
FILE: google/datalab/_job.py
class JobError (line 26) | class JobError(Exception):
method __init__ (line 29) | def __init__(self, location, message, reason):
method __str__ (line 34) | def __str__(self):
class Job (line 38) | class Job(object):
method __init__ (line 50) | def __init__(self, job_id=None, future=None):
method __str__ (line 66) | def __str__(self):
method id (line 70) | def id(self):
method is_complete (line 79) | def is_complete(self):
method failed (line 89) | def failed(self):
method fatal_error (line 100) | def fatal_error(self):
method errors (line 110) | def errors(self):
method result (line 119) | def result(self):
method start_time_utc (line 135) | def start_time_utc(self):
method end_time_utc (line 140) | def end_time_utc(self):
method total_time (line 145) | def total_time(self):
method _refresh_state (line 151) | def _refresh_state(self):
method _timeout (line 171) | def _timeout(self):
method wait (line 175) | def wait(self, timeout=None):
method state (line 202) | def state(self):
method __repr__ (line 217) | def __repr__(self):
method _wait (line 222) | def _wait(jobs, timeout, return_when):
FILE: google/datalab/bigquery/_api.py
class Api (line 23) | class Api(object):
method __init__ (line 36) | def __init__(self, context):
method project_id (line 45) | def project_id(self):
method credentials (line 50) | def credentials(self):
method bigquery_billing_tier (line 55) | def bigquery_billing_tier(self):
method jobs_insert_load (line 59) | def jobs_insert_load(self, source, table_name, append=False, overwrite...
method jobs_insert_query (line 134) | def jobs_insert_query(self, sql, table_name=None, append=False,
method jobs_query_results (line 204) | def jobs_query_results(self, job_id, project_id, page_size, timeout, s...
method jobs_get (line 232) | def jobs_get(self, job_id, project_id=None):
method datasets_insert (line 248) | def datasets_insert(self, dataset_name, friendly_name=None, descriptio...
method datasets_delete (line 274) | def datasets_delete(self, dataset_name, delete_contents):
method datasets_update (line 293) | def datasets_update(self, dataset_name, dataset_info):
method datasets_get (line 304) | def datasets_get(self, dataset_name):
method datasets_list (line 317) | def datasets_list(self, project_id=None, max_results=0, page_token=None):
method tables_get (line 341) | def tables_get(self, table_name):
method tables_list (line 354) | def tables_list(self, dataset_name, max_results=0, page_token=None):
method tables_insert (line 377) | def tables_insert(self, table_name, schema=None, query=None, friendly_...
method tabledata_insert_all (line 415) | def tabledata_insert_all(self, table_name, rows):
method tabledata_list (line 435) | def tabledata_list(self, table_name, start_index=None, max_results=Non...
method table_delete (line 458) | def table_delete(self, table_name):
method table_extract (line 473) | def table_extract(self, table_name, destination, format='CSV', compres...
method table_update (line 515) | def table_update(self, table_name, table_info):
FILE: google/datalab/bigquery/_csv_options.py
class CSVOptions (line 19) | class CSVOptions(object):
method __init__ (line 21) | def __init__(self, delimiter=',', skip_leading_rows=0, encoding='utf-8...
method delimiter (line 52) | def delimiter(self):
method skip_leading_rows (line 56) | def skip_leading_rows(self):
method encoding (line 60) | def encoding(self):
method quote (line 64) | def quote(self):
method allow_quoted_newlines (line 68) | def allow_quoted_newlines(self):
method allow_jagged_rows (line 72) | def allow_jagged_rows(self):
method _to_query_json (line 75) | def _to_query_json(self):
FILE: google/datalab/bigquery/_dataset.py
class Dataset (line 27) | class Dataset(object):
method __init__ (line 30) | def __init__(self, name, context=None):
method name (line 54) | def name(self):
method description (line 59) | def description(self):
method friendly_name (line 69) | def friendly_name(self):
method _get_info (line 78) | def _get_info(self):
method exists (line 90) | def exists(self):
method delete (line 101) | def delete(self, delete_contents=False):
method create (line 121) | def create(self, friendly_name=None, description=None):
method update (line 143) | def update(self, friendly_name=None, description=None):
method _retrieve_items (line 166) | def _retrieve_items(self, page_token, item_type):
method _retrieve_tables (line 194) | def _retrieve_tables(self, page_token, _):
method _retrieve_views (line 197) | def _retrieve_views(self, page_token, _):
method tables (line 200) | def tables(self):
method views (line 204) | def views(self):
method __iter__ (line 208) | def __iter__(self):
method __str__ (line 212) | def __str__(self):
method __repr__ (line 220) | def __repr__(self):
class Datasets (line 226) | class Datasets(object):
method __init__ (line 229) | def __init__(self, context=None):
method _retrieve_datasets (line 244) | def _retrieve_datasets(self, page_token, _):
method __iter__ (line 264) | def __iter__(self):
FILE: google/datalab/bigquery/_external_data_source.py
class ExternalDataSource (line 21) | class ExternalDataSource(object):
method __init__ (line 23) | def __init__(self, source, source_format='csv', csv_options=None, igno...
method schema (line 64) | def schema(self):
method __repr__ (line 67) | def __repr__(self):
method _to_query_json (line 70) | def _to_query_json(self):
FILE: google/datalab/bigquery/_job.py
class Job (line 27) | class Job(GCPJob):
method __init__ (line 31) | def __init__(self, job_id, context):
method _create_api (line 40) | def _create_api(self, context):
method _refresh_state (line 43) | def _refresh_state(self):
method _process_job_status (line 72) | def _process_job_status(self, status):
FILE: google/datalab/bigquery/_parser.py
class Parser (line 24) | class Parser(object):
method __init__ (line 27) | def __init__(self):
method parse_row (line 31) | def parse_row(schema, data):
method parse_timestamp (line 92) | def parse_timestamp(value):
FILE: google/datalab/bigquery/_query.py
class Query (line 32) | class Query(object):
method __init__ (line 38) | def __init__(self, sql, env=None, udfs=None, data_sources=None, subque...
method from_view (line 93) | def from_view(view):
method from_table (line 105) | def from_table(table, fields=None):
method _expanded_sql (line 122) | def _expanded_sql(self, sampling=None):
method _repr_sql_ (line 169) | def _repr_sql_(self):
method __repr__ (line 177) | def __repr__(self):
method sql (line 186) | def sql(self):
method udfs (line 191) | def udfs(self):
method subqueries (line 196) | def subqueries(self):
method data_sources (line 201) | def data_sources(self):
method dry_run (line 205) | def dry_run(self, context=None, query_params=None):
method execute_async (line 230) | def execute_async(self, output_options=None, sampling=None, context=No...
method execute (line 324) | def execute(self, output_options=None, sampling=None, context=None, qu...
method get_query_parameters (line 342) | def get_query_parameters(config_parameters, date_time=datetime.datetim...
method resolve_parameters (line 371) | def resolve_parameters(value, parameters, date_time=datetime.datetime....
method _resolve_parameters (line 394) | def _resolve_parameters(operator_param_value, merged_parameters):
method _airflow_macro_formats (line 422) | def _airflow_macro_formats(date_time, macros, types_and_values):
method merge_parameters (line 478) | def merge_parameters(parameters, date_time, macros, types_and_values):
FILE: google/datalab/bigquery/_query_job.py
class QueryJob (line 22) | class QueryJob(_job.Job):
method __init__ (line 25) | def __init__(self, job_id, table_name, sql, context):
method bytes_processed (line 44) | def bytes_processed(self):
method total_rows (line 49) | def total_rows(self):
method cache_hit (line 54) | def cache_hit(self):
method sql (line 59) | def sql(self):
method wait (line 63) | def wait(self, timeout=None):
method result (line 101) | def result(self):
FILE: google/datalab/bigquery/_query_output.py
class QueryOutput (line 16) | class QueryOutput(object):
method table (line 19) | def table(name=None, mode='create', use_cache=True, priority='interact...
method file (line 46) | def file(path, format='csv', csv_delimiter=',', csv_header=True, compr...
method dataframe (line 76) | def dataframe(start_row=0, max_rows=None, use_cache=True):
method __init__ (line 91) | def __init__(self):
method type (line 108) | def type(self):
method table_name (line 112) | def table_name(self):
method table_mode (line 116) | def table_mode(self):
method use_cache (line 120) | def use_cache(self):
method priority (line 124) | def priority(self):
method allow_large_results (line 128) | def allow_large_results(self):
method file_path (line 132) | def file_path(self):
method file_format (line 136) | def file_format(self):
method csv_delimiter (line 140) | def csv_delimiter(self):
method csv_header (line 144) | def csv_header(self):
method compress_file (line 148) | def compress_file(self):
method dataframe_start_row (line 152) | def dataframe_start_row(self):
method dataframe_max_rows (line 156) | def dataframe_max_rows(self):
FILE: google/datalab/bigquery/_query_results_table.py
class QueryResultsTable (line 20) | class QueryResultsTable(_table.Table):
method __init__ (line 26) | def __init__(self, name, context, job, is_temporary=False):
method __repr__ (line 41) | def __repr__(self):
method insert (line 49) | def insert(self, *args, **kwargs):
method job (line 53) | def job(self):
method job_id (line 58) | def job_id(self):
method sql (line 63) | def sql(self):
method is_temporary (line 68) | def is_temporary(self):
FILE: google/datalab/bigquery/_query_stats.py
class QueryStats (line 20) | class QueryStats(object):
method __init__ (line 25) | def __init__(self, total_bytes, is_cached):
method _repr_html_ (line 29) | def _repr_html_(self):
method _size_formatter (line 36) | def _size_formatter(byte_num, suf='B'):
FILE: google/datalab/bigquery/_sampling.py
class Sampling (line 20) | class Sampling(object):
method __init__ (line 29) | def __init__(self):
method _create_projection (line 33) | def _create_projection(fields):
method default (line 44) | def default(fields=None, count=5):
method sorted (line 57) | def sorted(field_name, ascending=True, fields=None, count=5):
method hashed (line 76) | def hashed(field_name, percent, fields=None, count=0):
method random (line 100) | def random(percent, fields=None, count=0):
method _auto (line 122) | def _auto(method, fields, count, percent, key_field, ascending):
FILE: google/datalab/bigquery/_schema.py
class SchemaField (line 26) | class SchemaField(object):
method __init__ (line 39) | def __init__(self, name, type, mode='NULLABLE', description=''):
method _repr_sql_ (line 45) | def _repr_sql_(self):
method __eq__ (line 53) | def __eq__(self, other):
method __repr__ (line 58) | def __repr__(self):
method __getitem__ (line 62) | def __getitem__(self, item):
class Schema (line 77) | class Schema(list):
method _from_dataframe (line 86) | def _from_dataframe(dataframe, default_type='STRING'):
method _get_field_entry (line 120) | def _get_field_entry(name, value):
method _from_dict_record (line 143) | def _from_dict_record(data):
method _from_list_record (line 158) | def _from_list_record(data):
method _from_record (line 171) | def _from_record(data):
method from_record (line 190) | def from_record(source):
method from_data (line 209) | def from_data(source):
method __init__ (line 259) | def __init__(self, definition=None):
method __getitem__ (line 273) | def __getitem__(self, key):
method _add_field (line 281) | def _add_field(self, name, type, mode='NULLABLE', description=''):
method find (line 286) | def find(self, name):
method _populate_fields (line 299) | def _populate_fields(self, data, prefix=''):
method __repr__ (line 310) | def __repr__(self):
method __eq__ (line 314) | def __eq__(self, other):
method __ne__ (line 326) | def __ne__(self, other):
FILE: google/datalab/bigquery/_table.py
class TableMetadata (line 42) | class TableMetadata(object):
method __init__ (line 45) | def __init__(self, table, info):
method created_on (line 56) | def created_on(self):
method description (line 62) | def description(self):
method expires_on (line 67) | def expires_on(self):
method friendly_name (line 75) | def friendly_name(self):
method modified_on (line 80) | def modified_on(self):
method rows (line 86) | def rows(self):
method size (line 91) | def size(self):
method refresh (line 95) | def refresh(self):
class Table (line 100) | class Table(object):
method __init__ (line 115) | def __init__(self, name, context=None):
method name (line 139) | def name(self):
method full_name (line 144) | def full_name(self):
method job (line 149) | def job(self):
method is_temporary (line 157) | def is_temporary(self):
method _load_info (line 161) | def _load_info(self):
method metadata (line 170) | def metadata(self):
method exists (line 181) | def exists(self):
method is_listable (line 200) | def is_listable(self):
method delete (line 209) | def delete(self):
method create (line 225) | def create(self, schema, overwrite=False):
method _encode_dict_as_row (line 254) | def _encode_dict_as_row(record, column_name_map):
method insert (line 283) | def insert(self, data, include_index=False, index_name=None):
method _init_job_from_response (line 393) | def _init_job_from_response(self, response):
method extract_async (line 400) | def extract_async(self, destination, format='csv', csv_delimiter=None,...
method extract (line 428) | def extract(self, destination, format='csv', csv_delimiter=None, csv_h...
method load_async (line 448) | def load_async(self, source, mode='create', source_format='csv', csv_o...
method load (line 501) | def load(self, source, mode='create', source_format='csv', csv_options...
method _get_row_fetcher (line 531) | def _get_row_fetcher(self, start_row=0, max_rows=None, page_size=_DEFA...
method range (line 590) | def range(self, start_row=0, max_rows=None):
method to_dataframe (line 603) | def to_dataframe(self, start_row=0, max_rows=None):
method to_file (line 637) | def to_file(self, destination, format='csv', csv_delimiter=',', csv_he...
method schema (line 662) | def schema(self):
method update (line 678) | def update(self, friendly_name=None, description=None, expiry=None, sc...
method _repr_sql_ (line 710) | def _repr_sql_(self):
method __repr__ (line 718) | def __repr__(self):
method length (line 724) | def length(self):
method __iter__ (line 730) | def __iter__(self):
method __getitem__ (line 735) | def __getitem__(self, item):
method _convert_decorator_time (line 779) | def _convert_decorator_time(when):
method snapshot (line 803) | def snapshot(self, at):
method window (line 828) | def window(self, begin, end=None):
FILE: google/datalab/bigquery/_udf.py
class UDF (line 20) | class UDF(object):
method name (line 25) | def name(self):
method imports (line 29) | def imports(self):
method code (line 33) | def code(self):
method __init__ (line 36) | def __init__(self, name, code, return_type, params=None, language='js'...
method _expanded_sql (line 65) | def _expanded_sql(self):
method _repr_sql_ (line 76) | def _repr_sql_(self):
method __repr__ (line 79) | def __repr__(self):
method _build_udf (line 83) | def _build_udf(name, code, return_type, params, language, imports):
FILE: google/datalab/bigquery/_utils.py
function parse_dataset_name (line 58) | def parse_dataset_name(name, project_id=None):
function parse_table_name (line 105) | def parse_table_name(name, project_id=None, dataset_id=None):
function format_query_errors (line 169) | def format_query_errors(errors):
FILE: google/datalab/bigquery/_view.py
class View (line 27) | class View(object):
method __init__ (line 35) | def __init__(self, name, context=None):
method name (line 55) | def name(self):
method description (line 60) | def description(self):
method friendly_name (line 65) | def friendly_name(self):
method query (line 70) | def query(self):
method exists (line 79) | def exists(self):
method delete (line 83) | def delete(self):
method create (line 87) | def create(self, query):
method schema (line 109) | def schema(self):
method update (line 119) | def update(self, friendly_name=None, description=None, query=None):
method _repr_sql_ (line 136) | def _repr_sql_(self):
method __repr__ (line 144) | def __repr__(self):
FILE: google/datalab/bigquery/commands/_bigquery.py
class BigQuerySchema (line 42) | class BigQuerySchema(object):
function _create_dataset_subparser (line 109) | def _create_dataset_subparser(parser):
function _create_table_subparser (line 134) | def _create_table_subparser(parser):
function _create_sample_subparser (line 167) | def _create_sample_subparser(parser):
function _create_udf_subparser (line 199) | def _create_udf_subparser(parser):
function _create_datasource_subparser (line 207) | def _create_datasource_subparser(parser):
function _create_dryrun_subparser (line 224) | def _create_dryrun_subparser(parser):
function _create_query_subparser (line 234) | def _create_query_subparser(parser):
function _create_execute_subparser (line 252) | def _create_execute_subparser(parser):
function _create_extract_subparser (line 278) | def _create_extract_subparser(parser):
function _create_load_subparser (line 302) | def _create_load_subparser(parser):
function _get_query_argument (line 325) | def _get_query_argument(args, cell, env):
function get_query_parameters (line 355) | def get_query_parameters(args, cell_body, date_time=datetime.datetime.no...
function _sample_cell (line 385) | def _sample_cell(args, cell_body):
function _dryrun_cell (line 456) | def _dryrun_cell(args, cell_body):
function _udf_cell (line 480) | def _udf_cell(args, cell_body):
function _datasource_cell (line 519) | def _datasource_cell(args, cell_body):
function _query_cell (line 548) | def _query_cell(args, cell_body):
function _execute_cell (line 576) | def _execute_cell(args, cell_body):
function _get_table (line 622) | def _get_table(name):
function _render_list (line 645) | def _render_list(data):
function _dataset_line (line 650) | def _dataset_line(args):
function _table_cell (line 683) | def _table_cell(args, cell_body):
function _extract_cell (line 757) | def _extract_cell(args, cell_body):
function _load_cell (line 805) | def _load_cell(args, cell_body):
function _create_pipeline_subparser (line 858) | def _create_pipeline_subparser(parser):
function _pipeline_cell (line 913) | def _pipeline_cell(args, cell_body):
function _add_command (line 965) | def _add_command(parser, subparser_fn, handler, cell_required=False, cel...
function _create_bigquery_parser (line 972) | def _create_bigquery_parser():
function bq (line 1028) | def bq(line, cell=None):
function _dispatch_handler (line 1046) | def _dispatch_handler(args, cell, parser, handler, cell_required=False, ...
function _table_viewer (line 1074) | def _table_viewer(table, rows_per_page=25, fields=None):
function _repr_html_query (line 1189) | def _repr_html_query(query):
function _repr_html_query_results_table (line 1194) | def _repr_html_query_results_table(results):
function _repr_html_table (line 1198) | def _repr_html_table(results):
function _repr_html_table_schema (line 1202) | def _repr_html_table_schema(schema):
function _register_html_formatters (line 1230) | def _register_html_formatters():
FILE: google/datalab/commands/_datalab.py
function datalab (line 30) | def datalab(line, cell=None):
function _config_list_fn (line 84) | def _config_list_fn(args, cell):
function _config_set_fn (line 89) | def _config_set_fn(args, cell):
function _project_get_fn (line 97) | def _project_get_fn(args, cell):
function _project_set_fn (line 102) | def _project_set_fn(args, cell):
FILE: google/datalab/contrib/bigquery/commands/_bigquery.py
function get_airflow_spec_from_config (line 21) | def get_airflow_spec_from_config(name, bq_pipeline_config):
function _get_pipeline_spec_from_config (line 28) | def _get_pipeline_spec_from_config(bq_pipeline_config):
function _get_load_parameters (line 76) | def _get_load_parameters(bq_pipeline_input_config, bq_pipeline_transform...
function _get_execute_parameters (line 122) | def _get_execute_parameters(load_task_id, bq_pipeline_input_config,
function _get_extract_parameters (line 193) | def _get_extract_parameters(execute_task_id, bq_pipeline_input_config,
FILE: google/datalab/contrib/bigquery/operators/_bq_execute_operator.py
class ExecuteOperator (line 18) | class ExecuteOperator(BaseOperator):
method __init__ (line 23) | def __init__(self, sql, parameters=None, table=None, mode=None, data_s...
method execute (line 37) | def execute(self, context):
FILE: google/datalab/contrib/bigquery/operators/_bq_extract_operator.py
class ExtractOperator (line 18) | class ExtractOperator(BaseOperator):
method __init__ (line 23) | def __init__(self, path, table, format='csv', csv_options=None, *args,...
method execute (line 30) | def execute(self, context):
FILE: google/datalab/contrib/bigquery/operators/_bq_load_operator.py
class LoadOperator (line 18) | class LoadOperator(BaseOperator):
method __init__ (line 32) | def __init__(self, table, path, mode='append', format='csv', schema=No...
method execute (line 43) | def execute(self, context):
FILE: google/datalab/contrib/mlworkbench/_archive.py
function extract_archive (line 27) | def extract_archive(archive_path, dest):
FILE: google/datalab/contrib/mlworkbench/_local_predict.py
function _tf_load_model (line 39) | def _tf_load_model(sess, model_dir):
function _tf_predict (line 56) | def _tf_predict(model_dir, input_csvlines):
function _download_images (line 90) | def _download_images(data, img_cols):
function _get_predicton_csv_lines (line 111) | def _get_predicton_csv_lines(data, headers, images):
function _get_display_data_with_images (line 138) | def _get_display_data_with_images(data, images):
function get_model_schema_and_features (line 160) | def get_model_schema_and_features(model_dir):
function get_prediction_results (line 175) | def get_prediction_results(model_dir_or_id, data, headers, img_cols=None,
function get_probs_for_labels (line 242) | def get_probs_for_labels(labels, prediction_results):
function _batch_csv_reader (line 300) | def _batch_csv_reader(csv_file, n):
function _get_output_schema (line 306) | def _get_output_schema(session, output_alias_map):
function _format_results (line 321) | def _format_results(output_format, output_schema, batched_results):
function local_batch_predict (line 359) | def local_batch_predict(model_dir, csv_file_pattern, output_dir, output_...
FILE: google/datalab/contrib/mlworkbench/_prediction_explainer.py
class PredictionExplainer (line 32) | class PredictionExplainer(object):
method __init__ (line 35) | def __init__(self, model_dir):
method _make_text_predict_fn (line 56) | def _make_text_predict_fn(self, labels, instance, column_to_explain):
method _make_image_predict_fn (line 73) | def _make_image_predict_fn(self, labels, instance, column_to_explain):
method _get_unique_categories (line 92) | def _get_unique_categories(self, df):
method _preprocess_data_for_tabular_explain (line 103) | def _preprocess_data_for_tabular_explain(self, df, categories):
method _make_tabular_predict_fn (line 130) | def _make_tabular_predict_fn(self, labels, instance, categories):
method explain_tabular (line 157) | def explain_tabular(self, trainset, labels, instance, num_features=5, ...
method explain_text (line 201) | def explain_text(self, labels, instance, column_name=None, num_feature...
method explain_image (line 246) | def explain_image(self, labels, instance, column_name=None, num_featur...
method _image_gradients (line 301) | def _image_gradients(self, input_csvlines, label, image_column_name):
method probe_image (line 334) | def probe_image(self, labels, instance, column_name=None, num_scaled_i...
FILE: google/datalab/contrib/mlworkbench/_shell_process.py
function _wait_and_kill (line 27) | def _wait_and_kill(pid_to_wait, pids_to_kill):
function run_and_monitor (line 43) | def run_and_monitor(args, pid_to_wait, std_out_filter_fn=None, cwd=None):
FILE: google/datalab/contrib/mlworkbench/commands/_ml.py
function ml (line 57) | def ml(line, cell=None):
function _abs_path (line 576) | def _abs_path(path):
function _create_json_file (line 596) | def _create_json_file(tmpdir, data, filename):
function _show_job_link (line 603) | def _show_job_link(job):
function get_dataset_from_arg (line 615) | def get_dataset_from_arg(dataset_arg):
function _analyze (line 626) | def _analyze(args, cell):
function _transform (line 669) | def _transform(args, cell):
function _train (line 730) | def _train(args, cell):
function _predict (line 798) | def _predict(args, cell):
function _batch_predict (line 833) | def _batch_predict(args, cell):
class _TextLimeExplainerInstance (line 873) | class _TextLimeExplainerInstance(object):
method __init__ (line 875) | def __init__(self, explainer, labels, args):
method visualize (line 884) | def visualize(self, label_index):
class _ImageLimeExplainerInstance (line 898) | class _ImageLimeExplainerInstance(object):
method __init__ (line 900) | def __init__(self, explainer, labels, args):
method visualize (line 911) | def visualize(self, label_index):
class _ImageIgExplainerInstance (line 924) | class _ImageIgExplainerInstance(object):
method __init__ (line 926) | def __init__(self, explainer, labels, args):
method visualize (line 933) | def visualize(self, label_index):
class _TabularLimeExplainerInstance (line 943) | class _TabularLimeExplainerInstance(object):
method __init__ (line 945) | def __init__(self, explainer, labels, args):
method visualize (line 962) | def visualize(self, label_index):
function _explain (line 979) | def _explain(args, cell):
function _tensorboard_start (line 1013) | def _tensorboard_start(args, cell):
function _tensorboard_stop (line 1017) | def _tensorboard_stop(args, cell):
function _tensorboard_list (line 1021) | def _tensorboard_list(args, cell):
function _get_evaluation_csv_schema (line 1025) | def _get_evaluation_csv_schema(csv_file):
function _evaluate_cm (line 1033) | def _evaluate_cm(args, cell):
function _create_metrics (line 1052) | def _create_metrics(args):
function _evaluate_accuracy (line 1068) | def _evaluate_accuracy(args, cell):
function _evaluate_regression (line 1073) | def _evaluate_regression(args, cell):
function _evaluate_pr (line 1099) | def _evaluate_pr(args, cell):
function _evaluate_roc (line 1117) | def _evaluate_roc(args, cell):
function _model_list (line 1135) | def _model_list(args, cell):
function _model_delete (line 1157) | def _model_delete(args, cell):
function _model_deploy (line 1169) | def _model_deploy(args, cell):
function _dataset_create (line 1193) | def _dataset_create(args, cell):
function _dataset_explore (line 1221) | def _dataset_explore(args, cell):
FILE: google/datalab/contrib/pipeline/_pipeline.py
class PipelineGenerator (line 26) | class PipelineGenerator(object):
method generate_airflow_spec (line 48) | def generate_airflow_spec(name, pipeline_spec):
method _get_default_args (line 71) | def _get_default_args(schedule_config, emails):
method _get_datetime_expr_str (line 104) | def _get_datetime_expr_str(datetime_obj):
method _get_operator_definition (line 120) | def _get_operator_definition(task_id, task_details, parameters):
method _get_param_format_string (line 152) | def _get_param_format_string(param_value):
method _get_dag_definition (line 159) | def _get_dag_definition(name, schedule_interval, catchup=False):
method _get_dependency_definition (line 167) | def _get_dependency_definition(task_id, dependencies):
method _get_operator_class_name (line 178) | def _get_operator_class_name(task_detail_type):
method _get_operator_param_name_and_values (line 201) | def _get_operator_param_name_and_values(operator_class_name, task_deta...
method _get_bq_execute_params (line 239) | def _get_bq_execute_params(operator_task_details):
method _get_bq_extract_params (line 256) | def _get_bq_extract_params(operator_task_details):
method _get_bq_load_params (line 281) | def _get_bq_load_params(operator_task_details):
method _get_bucket_and_source_object (line 313) | def _get_bucket_and_source_object(gcs_path):
FILE: google/datalab/contrib/pipeline/airflow/_airflow.py
class Airflow (line 16) | class Airflow(object):
method __init__ (line 23) | def __init__(self, gcs_dag_bucket, gcs_dag_file_path=None):
method deploy (line 33) | def deploy(self, name, dag_string):
FILE: google/datalab/contrib/pipeline/commands/_pipeline.py
function _create_cell (line 21) | def _create_cell(args, cell_body):
function _create_create_subparser (line 43) | def _create_create_subparser(parser):
function _add_command (line 57) | def _add_command(parser, subparser_fn, handler, cell_required=False,
function _create_pipeline_parser (line 66) | def _create_pipeline_parser():
function pipeline (line 90) | def pipeline(line, cell=None):
function _dispatch_handler (line 108) | def _dispatch_handler(args, cell, parser, handler, cell_required=False,
FILE: google/datalab/contrib/pipeline/composer/_api.py
class Api (line 17) | class Api(object):
method get_environment_details (line 24) | def get_environment_details(zone, environment):
FILE: google/datalab/contrib/pipeline/composer/_composer.py
class Composer (line 18) | class Composer(object):
method __init__ (line 27) | def __init__(self, zone, environment):
method deploy (line 38) | def deploy(self, name, dag_string):
method gcs_dag_location (line 47) | def gcs_dag_location(self):
FILE: google/datalab/data/_csv_file.py
class CsvFile (line 40) | class CsvFile(object):
method __init__ (line 43) | def __init__(self, path, delimiter=b','):
method path (line 53) | def path(self):
method _read_gcs_lines (line 57) | def _read_gcs_lines(path, max_lines=None):
method _read_local_lines (line 61) | def _read_local_lines(path, max_lines=None):
method _is_probably_categorical (line 69) | def _is_probably_categorical(self, column):
method browse (line 79) | def browse(self, max_lines=None, headers=None):
method _create_external_data_source (line 112) | def _create_external_data_source(self, skip_header_rows):
method _get_gcs_csv_row_count (line 123) | def _get_gcs_csv_row_count(self, external_data_source):
method sample_to (line 129) | def sample_to(self, count, skip_header_rows, strategy, target):
FILE: google/datalab/kernel/__init__.py
function load_ipython_extension (line 44) | def load_ipython_extension(shell):
function unload_ipython_extension (line 125) | def unload_ipython_extension(shell):
FILE: google/datalab/ml/_cloud_models.py
class Models (line 24) | class Models(object):
method __init__ (line 27) | def __init__(self, project_id=None):
method _retrieve_models (line 39) | def _retrieve_models(self, page_token, _):
method get_iterator (line 48) | def get_iterator(self):
method get_model_details (line 53) | def get_model_details(self, model_name):
method create (line 66) | def create(self, model_name):
method delete (line 82) | def delete(self, model_name):
method list (line 97) | def list(self, count=10):
method describe (line 119) | def describe(self, model_name):
class ModelVersions (line 129) | class ModelVersions(object):
method __init__ (line 132) | def __init__(self, model_name, project_id=None):
method _retrieve_versions (line 150) | def _retrieve_versions(self, page_token, _):
method get_iterator (line 160) | def get_iterator(self):
method get_version_details (line 166) | def get_version_details(self, version_name):
method deploy (line 176) | def deploy(self, version_name, path, runtime_version=None):
method delete (line 224) | def delete(self, version_name):
method predict (line 236) | def predict(self, version_name, data):
method describe (line 263) | def describe(self, version_name):
method list (line 273) | def list(self):
FILE: google/datalab/ml/_cloud_training_config.py
class CloudTrainingConfig (line 24) | class CloudTrainingConfig(_CloudTrainingConfig):
FILE: google/datalab/ml/_confusion_matrix.py
class ConfusionMatrix (line 27) | class ConfusionMatrix(object):
method __init__ (line 30) | def __init__(self, cm, labels):
method from_csv (line 41) | def from_csv(input_csv, headers=None, schema_file=None):
method from_bigquery (line 80) | def from_bigquery(sql):
method to_dataframe (line 115) | def to_dataframe(self):
method plot (line 129) | def plot(self, figsize=None, rotation=45):
FILE: google/datalab/ml/_dataset.py
class CsvDataSet (line 31) | class CsvDataSet(object):
method __init__ (line 34) | def __init__(self, file_pattern, schema=None, schema_file=None):
method input_files (line 77) | def input_files(self):
method files (line 82) | def files(self):
method schema (line 91) | def schema(self):
method size (line 95) | def size(self):
method sample (line 105) | def sample(self, n):
class BigQueryDataSet (line 148) | class BigQueryDataSet(object):
method __init__ (line 151) | def __init__(self, sql=None, table=None):
method query (line 168) | def query(self):
method table (line 172) | def table(self):
method _get_source (line 175) | def _get_source(self):
method schema (line 181) | def schema(self):
method size (line 188) | def size(self):
method sample (line 196) | def sample(self, n):
class TransformedDataSet (line 221) | class TransformedDataSet(object):
method __init__ (line 224) | def __init__(self, file_pattern):
method input_files (line 238) | def input_files(self):
method files (line 243) | def files(self):
method size (line 252) | def size(self):
FILE: google/datalab/ml/_fasets.py
class FacetsOverview (line 24) | class FacetsOverview(object):
method _remove_nonascii (line 27) | def _remove_nonascii(self, df):
method plot (line 38) | def plot(self, data):
class FacetsDiveview (line 65) | class FacetsDiveview(object):
method plot (line 68) | def plot(self, data, height=1000, render_large_data=False):
FILE: google/datalab/ml/_feature_slice_view.py
class FeatureSliceView (line 21) | class FeatureSliceView(object):
method _get_lantern_format (line 24) | def _get_lantern_format(self, df):
method plot (line 46) | def plot(self, data):
FILE: google/datalab/ml/_job.py
class Job (line 23) | class Job(datalab.Job):
method __init__ (line 26) | def __init__(self, name, context=None):
method _refresh_state (line 44) | def _refresh_state(self):
method info (line 52) | def info(self):
method describe (line 56) | def describe(self):
method submit_training (line 62) | def submit_training(job_request, job_id=None):
method submit_batch_prediction (line 119) | def submit_batch_prediction(job_request, job_id=None):
class Jobs (line 154) | class Jobs(object):
method __init__ (line 157) | def __init__(self, filter=None):
method _retrieve_jobs (line 170) | def _retrieve_jobs(self, page_token, _):
method get_iterator (line 179) | def get_iterator(self):
method list (line 184) | def list(self, count=10):
FILE: google/datalab/ml/_metrics.py
class Metrics (line 25) | class Metrics(object):
method __init__ (line 28) | def __init__(self, input_csv_pattern=None, headers=None, bigquery=None):
method from_csv (line 56) | def from_csv(input_csv_pattern, headers=None, schema_file=None):
method from_bigquery (line 84) | def from_bigquery(sql):
method _get_data_from_csv_files (line 106) | def _get_data_from_csv_files(self):
method _get_data_from_bigquery (line 116) | def _get_data_from_bigquery(self, queries):
method accuracy (line 125) | def accuracy(self):
method roc (line 180) | def roc(self, num_thresholds, target_class, probability_column=None):
method precision_recall (line 282) | def precision_recall(self, num_thresholds, target_class, probability_c...
method rmse (line 374) | def rmse(self):
method mae (line 406) | def mae(self):
method percentile_nearest (line 437) | def percentile_nearest(self, percentile):
FILE: google/datalab/ml/_summary.py
class Summary (line 29) | class Summary(object):
method __init__ (line 32) | def __init__(self, paths):
method _glob_events_files (line 44) | def _glob_events_files(self, paths, recursive):
method list_events (line 64) | def list_events(self):
method get_events (line 89) | def get_events(self, event_names):
method plot (line 141) | def plot(self, event_names, x_axis='step'):
FILE: google/datalab/ml/_tensorboard.py
class TensorBoard (line 29) | class TensorBoard(object):
method list (line 33) | def list():
method start (line 50) | def start(logdir):
method stop (line 82) | def stop(pid):
FILE: google/datalab/ml/_util.py
function wait_for_long_running_operation (line 30) | def wait_for_long_running_operation(operation_full_name):
function package_and_copy (line 45) | def package_and_copy(package_root_dir, setup_py, output_tar_path):
function read_file_to_string (line 91) | def read_file_to_string(path):
function open_local_or_gcs (line 97) | def open_local_or_gcs(path, mode):
function glob_files (line 102) | def glob_files(path):
FILE: google/datalab/notebook/__init__.py
function _jupyter_nbextension_paths (line 23) | def _jupyter_nbextension_paths():
FILE: google/datalab/notebook/static/bigquery.ts
function _toggleNode (line 20) | function _toggleNode(e: any): void {
function _renderSchema (line 29) | function _renderSchema(table: any, schema: any, title: string,
function renderSchema (line 95) | function renderSchema(dom: any, schema: any) {
FILE: google/datalab/notebook/static/charting.ts
method constructor (line 27) | constructor(protected dom:HTMLElement, protected chartStyle:string) {
method init (line 32) | init(chartModule:any):void {
method addPageChangedHandler (line 42) | addPageChangedHandler(handler:Function):void {
method error (line 45) | error(message:string):void {
class PlotlyDriver (line 49) | class PlotlyDriver extends ChartLibraryDriver {
method constructor (line 52) | constructor(dom:HTMLElement, chartStyle:string) {
method requires (line 56) | requires(url: string, chartStyle:string):Array<string> {
method draw (line 60) | public draw(data:any, options:any):void {
method getStaticImage (line 302) | getStaticImage(callback:Function):void {
method addChartReadyHandler (line 309) | addChartReadyHandler(handler:Function):void {
type IStringMap (line 314) | interface IStringMap {
class GChartsDriver (line 318) | class GChartsDriver extends ChartLibraryDriver {
method constructor (line 361) | constructor(dom:HTMLElement, chartStyle:string) {
method requires (line 365) | requires(url: string, chartStyle:string):Array<string> {
method init (line 373) | init(chartModule:any):void {
method error (line 379) | error(message:string):void {
method draw (line 384) | draw(data:any, options:any):void {
method getStaticImage (line 389) | getStaticImage(callback:Function):void {
method addChartReadyHandler (line 395) | addChartReadyHandler(handler:Function) {
method addPageChangedHandler (line 399) | addPageChangedHandler(handler:Function) {
class Chart (line 406) | class Chart {
method constructor (line 413) | constructor(protected driver:ChartLibraryDriver,
method convertDates (line 441) | public convertDates(data:any):void {
method extend (line 524) | private static extend(base:any, update:any):void {
method getCell (line 535) | private getCell() {
method getRefreshHandler (line 557) | protected getRefreshHandler(useCache:boolean):Function {
method addControls (line 565) | private addControls():void {
method getControlSettings (line 618) | protected getControlSettings():any {
method getEnvironment (line 663) | private getEnvironment():string {
method refresh (line 674) | protected refresh(useCache:boolean):void {
method handleNewData (line 707) | private handleNewData(env: any, error:any, response: any) {
method removeStaticChart (line 748) | protected removeStaticChart():void {
method addStaticChart (line 804) | private addStaticChart():void {
method handleStaticChart (line 811) | private handleStaticChart(img: string) {
method configureRefresh (line 832) | private configureRefresh(refreshInterval:number):void {
method draw (line 839) | public draw(data:any, options:any):void {
class PagedTable (line 853) | class PagedTable extends Chart {
method constructor (line 857) | constructor(driver:ChartLibraryDriver,
method getControlSettings (line 878) | protected getControlSettings():any {
method draw (line 886) | public draw(data:any, options:any):void {
method handlePageEvent (line 915) | handlePageEvent(page:number):void {
function convertListToDataTable (line 924) | function convertListToDataTable(data:any):any {
function _render (line 951) | function _render(driver:ChartLibraryDriver,
function render (line 981) | function render(driverName:string,
FILE: google/datalab/notebook/static/element.ts
function resolve (line 25) | function resolve(cbInfo: any): void {
function domReadyCallback (line 29) | function domReadyCallback(): void {
function load (line 39) | function load(name: any, req: any, loadCallback: any, config: any): void {
FILE: google/datalab/notebook/static/extern/d3.parcoords.js
function getset (line 120) | function getset(obj,state,events) {
function extend (line 135) | function extend(target, source) {
function without (line 142) | function without(arr, item) {
function compute_cluster_centroids (line 335) | function compute_cluster_centroids(d) {
function compute_centroids (line 369) | function compute_centroids(row) {
function compute_control_points (line 402) | function compute_control_points(centroids) {
function single_curve (line 446) | function single_curve(d, ctx) {
function color_path (line 463) | function color_path(d, i, ctx) {
function paths (line 475) | function paths(data, ctx) {
function single_path (line 488) | function single_path(d, ctx) {
function path_foreground (line 498) | function path_foreground(d, i) {
function path_highlight (line 502) | function path_highlight(d, i) {
function flipAxisAndUpdatePCP (line 509) | function flipAxisAndUpdatePCP(dimension, i) {
function rotateLabels (line 522) | function rotateLabels() {
function brushUpdated (line 714) | function brushUpdated(newSelection) {
function brushPredicate (line 720) | function brushPredicate(predicate) {
function is_brushed (line 776) | function is_brushed(p) {
function selected (line 781) | function selected() {
function brushExtents (line 823) | function brushExtents() {
function brushFor (line 836) | function brushFor(axis) {
function brushReset (line 853) | function brushReset(dimension) {
function install (line 867) | function install() {
function drawStrum (line 904) | function drawStrum(strum, activePoint) {
function dimensionsForPoint (line 956) | function dimensionsForPoint(p) {
function onDragStart (line 984) | function onDragStart() {
function onDrag (line 1011) | function onDrag() {
function containmentTest (line 1023) | function containmentTest(strum, width) {
function selected (line 1046) | function selected() {
function removeStrum (line 1080) | function removeStrum() {
function onDragEnd (line 1090) | function onDragEnd() {
function brushReset (line 1109) | function brushReset(strums) {
function install (line 1123) | function install() {
function position (line 1281) | function position(d) {
function doFrame (line 1310) | function doFrame() {
FILE: google/datalab/notebook/static/extern/sylvester.js
function Vector (line 28) | function Vector() {}
function Matrix (line 330) | function Matrix() {}
function Line (line 826) | function Line() {}
function Plane (line 1029) | function Plane() {}
FILE: google/datalab/notebook/static/job.ts
function refresh (line 22) | function refresh(dom: any, job_name: any, job_type: any, interval: any,
function render (line 48) | function render(dom: any, events: any, job_name: string, job_type: string,
FILE: google/datalab/notebook/static/parcoords.ts
function getCentroids (line 19) | function getCentroids(data: any, graph: any): any {
function getActiveData (line 34) | function getActiveData(graph: any): any{
function findAxes (line 39) | function findAxes(testPt: any, cenPts: any): number {
function isOnLine (line 50) | function isOnLine(startPt: any, endPt: any, testPt: any, tol: number){
function getClickedLines (line 64) | function getClickedLines(mouseClick: any, graph: any): any {
function highlightLineOnClick (line 86) | function highlightLineOnClick(mouseClick: any, graph: any) {
function plot (line 98) | function plot(d3: any, color_domain: number[], maximize: boolean, data: ...
FILE: google/datalab/notebook/static/style.ts
function addStyleSheet (line 29) | function addStyleSheet(url: string): void {
function domReadyCallback (line 40) | function domReadyCallback(): void {
function load (line 50) | function load(url: string, req: any, loadCallback: any, config: any): vo...
FILE: google/datalab/notebook/static/visualization.ts
function loadGoogleApiLoader (line 32) | function loadGoogleApiLoader(callback: any): void {
function invokeVisualizationCallback (line 45) | function invokeVisualizationCallback(cb: any) {
function loadVisualizationPackages (line 49) | function loadVisualizationPackages(names: any, callbacks: any): void {
function load (line 67) | function load(name: any, req: any, callback: any, config: any) {
FILE: google/datalab/stackdriver/commands/_monitoring.py
function sd (line 28) | def sd(line, cell=None):
function _create_monitoring_subparser (line 45) | def _create_monitoring_subparser(parser):
function _monitoring_metrics_list (line 84) | def _monitoring_metrics_list(args, _):
function _monitoring_resource_types_list (line 93) | def _monitoring_resource_types_list(args, _):
function _monitoring_groups_list (line 102) | def _monitoring_groups_list(args, _):
function _render_dataframe (line 111) | def _render_dataframe(dataframe):
function _make_context (line 119) | def _make_context(project_id):
FILE: google/datalab/stackdriver/monitoring/_group.py
class Groups (line 29) | class Groups(object):
method __init__ (line 35) | def __init__(self, context=None):
method list (line 45) | def list(self, pattern='*'):
method as_dataframe (line 63) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: google/datalab/stackdriver/monitoring/_metric.py
class MetricDescriptors (line 26) | class MetricDescriptors(object):
method __init__ (line 32) | def __init__(self, filter_string=None, type_prefix=None, context=None):
method list (line 47) | def list(self, pattern='*'):
method as_dataframe (line 64) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: google/datalab/stackdriver/monitoring/_query.py
class Query (line 23) | class Query(google.cloud.monitoring_v3.query.Query):
method __init__ (line 26) | def __init__(self,
method metadata (line 66) | def metadata(self):
FILE: google/datalab/stackdriver/monitoring/_query_metadata.py
class QueryMetadata (line 25) | class QueryMetadata(object):
method __init__ (line 28) | def __init__(self, query):
method __iter__ (line 40) | def __iter__(self):
method metric_type (line 45) | def metric_type(self):
method resource_types (line 50) | def resource_types(self):
method as_dataframe (line 54) | def as_dataframe(self, max_rows=None):
FILE: google/datalab/stackdriver/monitoring/_resource.py
class ResourceDescriptors (line 24) | class ResourceDescriptors(object):
method __init__ (line 29) | def __init__(self, filter_string=None, context=None):
method list (line 41) | def list(self, pattern='*'):
method as_dataframe (line 57) | def as_dataframe(self, pattern='*', max_rows=None):
FILE: google/datalab/stackdriver/monitoring/_utils.py
class _MonitoringClient (line 27) | class _MonitoringClient(object):
method __init__ (line 28) | def __init__(self, context):
method list_metric_descriptors (line 40) | def list_metric_descriptors(self, filter_string=None, type_prefix=None):
method list_resource_descriptors (line 54) | def list_resource_descriptors(self, filter_string=None):
method list_groups (line 59) | def list_groups(self):
function make_client (line 64) | def make_client(context=None):
FILE: google/datalab/storage/_api.py
class Api (line 27) | class Api(object):
method __init__ (line 40) | def __init__(self, context):
method project_id (line 50) | def project_id(self):
method buckets_insert (line 54) | def buckets_insert(self, bucket, project_id=None):
method buckets_delete (line 72) | def buckets_delete(self, bucket):
method buckets_get (line 84) | def buckets_get(self, bucket, projection='noAcl'):
method buckets_list (line 99) | def buckets_list(self, projection='noAcl', max_results=0, page_token=N...
method object_download (line 124) | def object_download(self, bucket, key, start_offset=0, byte_count=None):
method object_upload (line 148) | def object_upload(self, bucket, key, content, content_type):
method objects_copy (line 166) | def objects_copy(self, source_bucket, source_key, target_bucket, targe...
method objects_delete (line 183) | def objects_delete(self, bucket, key):
method objects_get (line 196) | def objects_get(self, bucket, key, projection='noAcl'):
method objects_list (line 215) | def objects_list(self, bucket, prefix=None, delimiter=None, projection...
method objects_patch (line 250) | def objects_patch(self, bucket, key, info):
method _escape_key (line 267) | def _escape_key(key):
method verify_permitted_to_read (line 272) | def verify_permitted_to_read(gs_path):
FILE: google/datalab/storage/_bucket.py
function parse_name (line 34) | def parse_name(name):
class BucketMetadata (line 59) | class BucketMetadata(object):
method __init__ (line 62) | def __init__(self, info):
method created_on (line 71) | def created_on(self):
method etag (line 77) | def etag(self):
method name (line 82) | def name(self):
class Bucket (line 87) | class Bucket(object):
method __init__ (line 90) | def __init__(self, name, info=None, context=None):
method name (line 108) | def name(self):
method __repr__ (line 112) | def __repr__(self):
method metadata (line 118) | def metadata(self):
method object (line 134) | def object(self, key):
method objects (line 146) | def objects(self, prefix=None, delimiter=None):
method exists (line 160) | def exists(self):
method create (line 167) | def create(self, context=None):
method delete (line 185) | def delete(self):
class Buckets (line 198) | class Buckets(object):
method __init__ (line 201) | def __init__(self, context=None):
method contains (line 215) | def contains(self, name):
method _retrieve_buckets (line 235) | def _retrieve_buckets(self, page_token, _):
method __iter__ (line 251) | def __iter__(self):
FILE: google/datalab/storage/_object.py
class ObjectMetadata (line 37) | class ObjectMetadata(object):
method __init__ (line 40) | def __init__(self, info):
method content_type (line 49) | def content_type(self):
method etag (line 54) | def etag(self):
method name (line 59) | def name(self):
method size (line 64) | def size(self):
method updated_on (line 69) | def updated_on(self):
class Object (line 75) | class Object(object):
method __init__ (line 78) | def __init__(self, bucket, key, info=None, context=None):
method from_url (line 98) | def from_url(url):
method key (line 104) | def key(self):
method uri (line 109) | def uri(self):
method __repr__ (line 114) | def __repr__(self):
method copy_to (line 119) | def copy_to(self, new_key, bucket=None):
method exists (line 138) | def exists(self):
method delete (line 147) | def delete(self, wait_for_deletion=True):
method metadata (line 175) | def metadata(self):
method read_stream (line 190) | def read_stream(self, start_offset=0, byte_count=None):
method download (line 207) | def download(self):
method read_lines (line 217) | def read_lines(self, max_lines=None):
method write_stream (line 245) | def write_stream(self, content, content_type):
method upload (line 259) | def upload(self, content):
class Objects (line 270) | class Objects(object):
method __init__ (line 273) | def __init__(self, bucket, prefix, delimiter, context=None):
method contains (line 295) | def contains(self, key):
method _retrieve_objects (line 315) | def _retrieve_objects(self, page_token, _):
method __iter__ (line 334) | def __iter__(self):
FILE: google/datalab/storage/commands/_storage.py
function _extract_gcs_api_response_error (line 34) | def _extract_gcs_api_response_error(message):
function gcs (line 55) | def gcs(line, cell=None):
function _parser_exit (line 134) | def _parser_exit(status=0, message=None):
function _expand_list (line 141) | def _expand_list(names):
function _gcs_copy (line 196) | def _gcs_copy(args, _):
function _gcs_create (line 226) | def _gcs_create(args, _):
function _gcs_delete (line 243) | def _gcs_delete(args, _):
function _make_context (line 272) | def _make_context(project_id=None):
function _gcs_list_buckets (line 278) | def _gcs_list_buckets(project, pattern):
function _gcs_get_keys (line 286) | def _gcs_get_keys(bucket, pattern):
function _gcs_get_key_names (line 291) | def _gcs_get_key_names(bucket, pattern):
function _gcs_list_keys (line 296) | def _gcs_list_keys(bucket, pattern):
function _gcs_list (line 306) | def _gcs_list(args, _):
function _get_object_contents (line 348) | def _get_object_contents(source_name):
function _gcs_read (line 360) | def _gcs_read(args, _):
function _gcs_view (line 366) | def _gcs_view(args, _):
function _gcs_write (line 383) | def _gcs_write(args, _):
FILE: google/datalab/utils/_async.py
class async_ (line 26) | class async_(with_metaclass(abc.ABCMeta, object)):
method __init__ (line 33) | def __init__(self, function):
method _preprocess_args (line 39) | def _preprocess_args(*args):
method _preprocess_kwargs (line 44) | def _preprocess_kwargs(**kwargs):
method _call (line 50) | def _call(self, *args, **kwargs):
method __call__ (line 53) | def __call__(self, *args, **kwargs):
class async_function (line 58) | class async_function(async_):
method _call (line 64) | def _call(self, *args, **kwargs):
class async_method (line 69) | class async_method(async_):
method _call (line 75) | def _call(self, *args, **kwargs):
method __get__ (line 80) | def __get__(self, instance, owner):
FILE: google/datalab/utils/_dataflow_job.py
class DataflowJob (line 19) | class DataflowJob(_job.Job):
method __init__ (line 23) | def __init__(self, runner_results):
method _refresh_state (line 32) | def _refresh_state(self):
FILE: google/datalab/utils/_gcp_job.py
class GCPJob (line 21) | class GCPJob(_job.Job):
method __init__ (line 25) | def __init__(self, job_id, context):
method _create_api (line 38) | def _create_api(self, context):
method __repr__ (line 41) | def __repr__(self):
FILE: google/datalab/utils/_http.py
class RequestException (line 39) | class RequestException(Exception):
method __init__ (line 41) | def __init__(self, status, content):
method __str__ (line 59) | def __str__(self):
class Http (line 63) | class Http(object):
method __init__ (line 77) | def __init__(self):
method request (line 81) | def request(url, args=None, data=None, headers=None, method=None,
FILE: google/datalab/utils/_iterator.py
class Iterator (line 19) | class Iterator(object):
method __init__ (line 22) | def __init__(self, retriever):
method __iter__ (line 33) | def __iter__(self):
method reset (line 45) | def reset(self):
FILE: google/datalab/utils/_json_encoder.py
class JSONEncoder (line 21) | class JSONEncoder(json.JSONEncoder):
method default (line 24) | def default(self, obj):
FILE: google/datalab/utils/_lambda_job.py
class LambdaJob (line 21) | class LambdaJob(_job.Job):
method __init__ (line 25) | def __init__(self, fn, job_id, *args, **kwargs):
method __repr__ (line 35) | def __repr__(self):
FILE: google/datalab/utils/_lru_cache.py
class LRUCache (line 23) | class LRUCache(object):
method __init__ (line 26) | def __init__(self, cache_size):
method __getitem__ (line 37) | def __getitem__(self, key):
method __delitem__ (line 59) | def __delitem__(self, key):
method __setitem__ (line 69) | def __setitem__(self, key, value):
method __contains__ (line 96) | def __contains__(self, key):
method get (line 99) | def get(self, key, value):
FILE: google/datalab/utils/_utils.py
function print_exception_with_last_stack (line 41) | def print_exception_with_last_stack(e):
function get_item (line 51) | def get_item(env, name, default=None):
function compare_datetimes (line 75) | def compare_datetimes(d1, d2):
function pick_unused_port (line 98) | def pick_unused_port():
function is_http_running_on (line 111) | def is_http_running_on(port):
function gcs_copy_file (line 128) | def gcs_copy_file(source, dest):
function _in_datalab_docker (line 150) | def _in_datalab_docker():
function get_config_dir (line 154) | def get_config_dir():
function _convert_oauth2client_creds (line 169) | def _convert_oauth2client_creds(credentials):
function get_credentials (line 182) | def get_credentials():
function save_project_id (line 222) | def save_project_id(project_id):
function get_default_project_id (line 243) | def get_default_project_id():
function _construct_context_for_args (line 276) | def _construct_context_for_args(args):
function python_portable_string (line 300) | def python_portable_string(string, encoding='utf-8'):
FILE: google/datalab/utils/commands/_chart.py
function chart (line 29) | def chart(line, cell=None):
function _chart_cell (line 52) | def _chart_cell(args, cell):
FILE: google/datalab/utils/commands/_chart_data.py
function _get_chart_data (line 35) | def _get_chart_data(line, cell_body=''):
FILE: google/datalab/utils/commands/_commands.py
class CommandParser (line 30) | class CommandParser(argparse.ArgumentParser):
method __init__ (line 33) | def __init__(self, *args, **kwargs):
method create (line 44) | def create(name):
method exit (line 48) | def exit(self, status=0, message=None):
method format_help (line 56) | def format_help(self):
method format_usage (line 79) | def format_usage(self):
method create_args (line 84) | def create_args(line, namespace):
method _get_subparsers (line 101) | def _get_subparsers(self):
method _get_subparser_line_args (line 115) | def _get_subparser_line_args(self, subparser_prog):
method _get_subparser_cell_args (line 132) | def _get_subparser_cell_args(self, subparser_prog):
method add_cell_argument (line 142) | def add_cell_argument(self, name, help, required=False):
method parse (line 157) | def parse(self, line, cell, namespace=None):
method subcommand (line 259) | def subcommand(self, name, help, **kwargs):
FILE: google/datalab/utils/commands/_csv.py
function csv (line 34) | def csv(line, cell=None):
function _view (line 51) | def _view(args, cell):
FILE: google/datalab/utils/commands/_html.py
class Html (line 24) | class Html(object):
method next_id (line 33) | def next_id():
method __init__ (line 38) | def __init__(self, markup=None):
method add_class (line 49) | def add_class(self, class_name):
method add_dependency (line 54) | def add_dependency(self, path, name):
method add_script (line 59) | def add_script(self, script):
method _repr_html_ (line 64) | def _repr_html_(self):
class HtmlBuilder (line 87) | class HtmlBuilder(object):
method __init__ (line 91) | def __init__(self):
method _render_objects (line 96) | def _render_objects(self, items, attributes=None, datatype='object'):
method _render_text (line 151) | def _render_text(self, text, preformatted=False):
method _render_list (line 161) | def _render_list(self, items, empty='<pre><empty></pre>'):
method _to_html (line 178) | def _to_html(self):
method _format (line 187) | def _format(value, nbsp=False):
method render_text (line 196) | def render_text(text, preformatted=False):
method render_table (line 210) | def render_table(data, headers=None):
method render_chart_data (line 222) | def render_chart_data(data):
method render_list (line 233) | def render_list(data):
FILE: google/datalab/utils/commands/_job.py
function html_job_status (line 34) | def html_job_status(job_name, job_type, refresh_interval, html_on_runnin...
function _get_job_status (line 61) | def _get_job_status(line):
FILE: google/datalab/utils/commands/_utils.py
function notebook_environment (line 44) | def notebook_environment():
function get_notebook_item (line 50) | def get_notebook_item(name):
function render_list (line 56) | def render_list(data):
function render_dictionary (line 60) | def render_dictionary(data, headers=None):
function render_text (line 70) | def render_text(text, preformatted=False):
function get_field_list (line 80) | def get_field_list(fields, schema):
function _get_cols (line 109) | def _get_cols(fields, schema):
function _get_data_from_empty_list (line 138) | def _get_data_from_empty_list(source, fields='*', first_row=0, count=-1,...
function _get_data_from_list_of_dicts (line 144) | def _get_data_from_list_of_dicts(source, fields='*', first_row=0, count=...
function _get_data_from_list_of_lists (line 154) | def _get_data_from_list_of_lists(source, fields='*', first_row=0, count=...
function _get_data_from_dataframe (line 165) | def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, ...
function _get_data_from_table (line 186) | def _get_data_from_table(source, fields='*', first_row=0, count=-1, sche...
function get_data (line 198) | def get_data(source, fields='*', env=None, first_row=0, count=-1, schema...
function handle_magic_line (line 250) | def handle_magic_line(line, cell, parser, namespace=None):
function expand_var (line 265) | def expand_var(v, env):
function replace_vars (line 289) | def replace_vars(config, env):
function parse_config (line 318) | def parse_config(config, env, as_dict=True):
function parse_config_for_selected_keys (line 341) | def parse_config_for_selected_keys(content, keys):
function validate_config (line 396) | def validate_config(config, required_keys, optional_keys=None):
function validate_config_must_have (line 421) | def validate_config_must_have(config, required_keys):
function validate_config_has_one_of (line 436) | def validate_config_has_one_of(config, one_of_keys):
function validate_config_value (line 454) | def validate_config_value(value, possible_values):
function get_data_source_index (line 476) | def get_data_source_index(name):
function validate_gcs_path (line 482) | def validate_gcs_path(path, require_object):
function parse_control_options (line 499) | def parse_control_options(controls, variable_defaults=None):
function chart_html (line 614) | def chart_html(driver_name, chart_type, source, chart_options=None, fiel...
function profile_df (line 730) | def profile_df(df):
FILE: google/datalab/utils/facets/base_feature_statistics_generator.py
class BaseFeatureStatisticsGenerator (line 32) | class BaseFeatureStatisticsGenerator(BaseGenericFeatureStatisticsGenerat...
method __init__ (line 35) | def __init__(self, fs_proto, datasets_proto, histogram_proto):
method ProtoFromTfRecordFiles (line 39) | def ProtoFromTfRecordFiles(self,
method _ParseExample (line 74) | def _ParseExample(self, example_features, example_feature_lists, entries,
method _GetEntries (line 162) | def _GetEntries(self,
method _GetTfRecordEntries (line 202) | def _GetTfRecordEntries(self, path, max_entries, is_sequence,
FILE: google/datalab/utils/facets/base_generic_feature_statistics_generator.py
class BaseGenericFeatureStatisticsGenerator (line 27) | class BaseGenericFeatureStatisticsGenerator(object):
method __init__ (line 30) | def __init__(self, fs_proto, datasets_proto, histogram_proto):
method ProtoFromDataFrames (line 35) | def ProtoFromDataFrames(self, dataframes):
method DtypeToType (line 58) | def DtypeToType(self, dtype):
method DtypeToNumberConverter (line 69) | def DtypeToNumberConverter(self, dtype):
method NdarrayToEntry (line 96) | def NdarrayToEntry(self, x):
method GetDatasetsProto (line 139) | def GetDatasetsProto(self, datasets, features=None):
method _PopulateQuantilesHistogram (line 288) | def _PopulateQuantilesHistogram(self, hist, nums):
FILE: google/datalab/utils/facets/feature_statistics_generator.py
class FeatureStatisticsGenerator (line 25) | class FeatureStatisticsGenerator(BaseFeatureStatisticsGenerator):
method __init__ (line 28) | def __init__(self):
function ProtoFromTfRecordFiles (line 34) | def ProtoFromTfRecordFiles(files,
FILE: google/datalab/utils/facets/generic_feature_statistics_generator.py
class GenericFeatureStatisticsGenerator (line 25) | class GenericFeatureStatisticsGenerator(BaseGenericFeatureStatisticsGene...
method __init__ (line 28) | def __init__(self):
function ProtoFromDataFrames (line 34) | def ProtoFromDataFrames(dataframes):
FILE: legacy_tests/_util/http_tests.py
class TestCases (line 23) | class TestCases(unittest.TestCase):
method test_get_request_is_invoked (line 27) | def test_get_request_is_invoked(self, mock_request, mock_response):
method test_post_request_is_invoked (line 36) | def test_post_request_is_invoked(self, mock_request, mock_response):
method test_explicit_post_request_is_invoked (line 44) | def test_explicit_post_request_is_invoked(self, mock_request, mock_res...
method test_query_string_format (line 52) | def test_query_string_format(self, mock_request, mock_response):
method test_formats_json_request (line 63) | def test_formats_json_request(self, mock_request, mock_response):
method test_supports_custom_content (line 75) | def test_supports_custom_content(self, mock_request, mock_response):
method test_parses_json_response (line 87) | def test_parses_json_response(self, mock_request, mock_response):
method test_raises_http_error (line 95) | def test_raises_http_error(self, mock_request, mock_response):
method _setup_mocks (line 106) | def _setup_mocks(mock_request, mock_response, content, status=200):
FILE: legacy_tests/_util/lru_cache_tests.py
class TestCases (line 20) | class TestCases(unittest.TestCase):
method test_cache_no_entry (line 22) | def test_cache_no_entry(self):
method test_cache_lookup (line 27) | def test_cache_lookup(self):
method test_cache_overflow (line 35) | def test_cache_overflow(self):
FILE: legacy_tests/_util/util_tests.py
class TestCases (line 21) | class TestCases(unittest.TestCase):
method _get_data (line 24) | def _get_data():
method test_no_entry (line 37) | def test_no_entry(self):
method test_entry (line 44) | def test_entry(self):
FILE: legacy_tests/bigquery/api_tests.py
class TestCases (line 25) | class TestCases(unittest.TestCase):
method validate (line 27) | def validate(self, mock_http_request, expected_url, expected_args=None...
method test_jobs_insert_load (line 50) | def test_jobs_insert_load(self, mock_http_request):
method test_jobs_insert_query (line 107) | def test_jobs_insert_query(self, mock_http_request):
method test_jobs_query_results (line 160) | def test_jobs_query_results(self, mock_http_request):
method test_jobs_get (line 168) | def test_jobs_get(self, mock_http_request):
method test_datasets_insert (line 175) | def test_datasets_insert(self, mock_http_request):
method test_datasets_delete (line 201) | def test_datasets_delete(self, mock_http_request):
method test_datasets_update (line 213) | def test_datasets_update(self, mock_http_request):
method test_datasets_get (line 220) | def test_datasets_get(self, mock_http_request):
method test_datasets_list (line 226) | def test_datasets_list(self, mock_http_request):
method test_tables_get (line 239) | def test_tables_get(self, mock_http_request):
method test_tables_list (line 246) | def test_tables_list(self, mock_http_request):
method test_tables_insert (line 259) | def test_tables_insert(self, mock_http_request):
method test_tabledata_insertAll (line 295) | def test_tabledata_insertAll(self, mock_http_request):
method test_tabledata_list (line 307) | def test_tabledata_list(self, mock_http_request):
method test_table_delete (line 324) | def test_table_delete(self, mock_http_request):
method test_table_extract (line 332) | def test_table_extract(self, mock_http_request):
method test_table_update (line 380) | def test_table_update(self, mock_http_request):
method _create_api (line 388) | def _create_api():
method _create_context (line 393) | def _create_context():
FILE: legacy_tests/bigquery/dataset_tests.py
class TestCases (line 25) | class TestCases(unittest.TestCase):
method _check_name_parts (line 27) | def _check_name_parts(self, dataset):
method test_parse_full_name (line 34) | def test_parse_full_name(self):
method test_parse_local_name (line 38) | def test_parse_local_name(self):
method test_parse_dict_full_name (line 42) | def test_parse_dict_full_name(self):
method test_parse_dict_local_name (line 46) | def test_parse_dict_local_name(self):
method test_parse_named_tuple_name (line 50) | def test_parse_named_tuple_name(self):
method test_parse_tuple_full_name (line 54) | def test_parse_tuple_full_name(self):
method test_parse_tuple_local (line 58) | def test_parse_tuple_local(self):
method test_parse_array_full_name (line 62) | def test_parse_array_full_name(self):
method test_parse_array_local (line 66) | def test_parse_array_local(self):
method test_parse_invalid_name (line 70) | def test_parse_invalid_name(self):
method test_dataset_exists (line 75) | def test_dataset_exists(self, mock_api_datasets_get):
method test_datasets_create_fails (line 85) | def test_datasets_create_fails(self, mock_api_datasets_get, mock_api_d...
method test_datasets_create_succeeds (line 95) | def test_datasets_create_succeeds(self, mock_api_datasets_get, mock_ap...
method test_datasets_create_redundant (line 103) | def test_datasets_create_redundant(self, mock_api_datasets_get, mock_a...
method test_datasets_delete_succeeds (line 111) | def test_datasets_delete_succeeds(self, mock_api_datasets_delete, mock...
method test_datasets_delete_fails (line 119) | def test_datasets_delete_fails(self, mock_api_datasets_delete, mock_ap...
method test_tables_list (line 127) | def test_tables_list(self, mock_api_tables_list):
method test_datasets_list (line 148) | def test_datasets_list(self, mock_api_datasets_list, mock_dataset_get_...
method test_datasets_update (line 165) | def test_datasets_update(self, mock_api_datasets_update, mock_api_data...
method _create_context (line 188) | def _create_context():
method _create_dataset (line 194) | def _create_dataset(name, metadata=None):
FILE: legacy_tests/bigquery/federated_table_tests.py
class TestCases (line 25) | class TestCases(unittest.TestCase):
method _request_result (line 31) | def _request_result():
method _get_data (line 49) | def _get_data():
method _get_table_definition (line 61) | def _get_table_definition(uris, skip_rows=0):
method _get_expected_request_data (line 87) | def _get_expected_request_data(sql, table_definitions):
method test_external_table_query (line 105) | def test_external_table_query(self, mock_http_request):
method test_external_table_query2 (line 128) | def test_external_table_query2(self, mock_http_request):
method test_external_tables_query (line 149) | def test_external_tables_query(self, mock_http_request):
method _create_context (line 176) | def _create_context():
FILE: legacy_tests/bigquery/jobs_tests.py
class TestCases (line 23) | class TestCases(unittest.TestCase):
method _make_job (line 26) | def _make_job(id):
method test_job_complete (line 30) | def test_job_complete(self, mock_api_jobs_get):
method test_job_fatal_error (line 40) | def test_job_fatal_error(self, mock_api_jobs_get):
method test_job_errors (line 61) | def test_job_errors(self, mock_api_jobs_get):
method _create_context (line 91) | def _create_context():
method _create_api (line 97) | def _create_api():
FILE: legacy_tests/bigquery/parser_tests.py
class TestCases (line 19) | class TestCases(unittest.TestCase):
method test_repeating_data (line 21) | def test_repeating_data(self):
method test_non_nested_data (line 28) | def test_non_nested_data(self):
method test_parse_nested_data (line 242) | def test_parse_nested_data(self):
FILE: legacy_tests/bigquery/query_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_single_result_query (line 31) | def test_single_result_query(self, mock_api_tables_get, mock_api_jobs_...
method test_empty_result_query (line 55) | def test_empty_result_query(self, mock_api_tables_get, mock_api_jobs_get,
method test_incomplete_result_query (line 71) | def test_incomplete_result_query(self,
method test_malformed_response_raises_exception (line 88) | def test_malformed_response_raises_exception(self, mock_api_insert_que...
method test_udf_expansion (line 97) | def test_udf_expansion(self):
method _create_query (line 109) | def _create_query(sql=None):
method _create_context (line 115) | def _create_context():
method _create_insert_done_result (line 121) | def _create_insert_done_result():
method _create_single_row_result (line 140) | def _create_single_row_result():
method _create_empty_result (line 150) | def _create_empty_result():
method _create_incomplete_result (line 157) | def _create_incomplete_result():
method _create_page_result (line 176) | def _create_page_result(page_token=None):
method _create_tables_get_result (line 187) | def _create_tables_get_result(num_rows=1, schema=None):
FILE: legacy_tests/bigquery/sampling_tests.py
class TestCases (line 19) | class TestCases(unittest.TestCase):
method test_default (line 23) | def test_default(self):
method test_default_custom_count (line 27) | def test_default_custom_count(self):
method test_default_custom_fields (line 31) | def test_default_custom_fields(self):
method test_default_all_fields (line 35) | def test_default_all_fields(self):
method test_hashed (line 39) | def test_hashed(self):
method test_hashed_and_limited (line 43) | def test_hashed_and_limited(self):
method test_hashed_with_fields (line 48) | def test_hashed_with_fields(self):
method test_sorted_ascending (line 52) | def test_sorted_ascending(self):
method test_sorted_descending (line 56) | def test_sorted_descending(self):
method test_sorted_with_fields (line 60) | def test_sorted_with_fields(self):
method _apply_sampling (line 64) | def _apply_sampling(self, sampling, expected_query):
FILE: legacy_tests/bigquery/schema_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_schema_from_dataframe (line 26) | def test_schema_from_dataframe(self):
method test_schema_from_data (line 31) | def test_schema_from_data(self):
method _create_data_frame (line 104) | def _create_data_frame():
method _create_inferred_schema (line 119) | def _create_inferred_schema(extra_field=None):
FILE: legacy_tests/bigquery/table_tests.py
class TestCases (line 29) | class TestCases(unittest.TestCase):
method _check_name_parts (line 31) | def _check_name_parts(self, table):
method test_api_paths (line 40) | def test_api_paths(self):
method test_parse_full_name (line 49) | def test_parse_full_name(self):
method test_parse_local_name (line 53) | def test_parse_local_name(self):
method test_parse_dict_full_name (line 57) | def test_parse_dict_full_name(self):
method test_parse_dict_local_name (line 62) | def test_parse_dict_local_name(self):
method test_parse_named_tuple_name (line 66) | def test_parse_named_tuple_name(self):
method test_parse_tuple_full_name (line 71) | def test_parse_tuple_full_name(self):
method test_parse_tuple_local (line 75) | def test_parse_tuple_local(self):
method test_parse_array_full_name (line 79) | def test_parse_array_full_name(self):
method test_parse_array_local (line 83) | def test_parse_array_local(self):
method test_parse_invalid_name (line 87) | def test_parse_invalid_name(self):
method test_table_metadata (line 92) | def test_table_metadata(self, mock_api_tables_get):
method test_table_schema (line 108) | def test_table_schema(self, mock_api_tables):
method test_table_schema_nested (line 118) | def test_table_schema_nested(self, mock_api_tables):
method test_malformed_response_raises_exception (line 134) | def test_malformed_response_raises_exception(self, mock_api_tables_get):
method test_dataset_list (line 145) | def test_dataset_list(self, mock_api_datasets_get, mock_api_tables_list):
method test_table_list (line 160) | def test_table_list(self, mock_api_datasets_get, mock_api_tables_list):
method test_view_list (line 175) | def test_view_list(self, mock_api_datasets_get, mock_api_tables_list):
method test_table_list_empty (line 189) | def test_table_list_empty(self, mock_api_datasets_get, mock_api_tables...
method test_table_exists (line 202) | def test_table_exists(self, mock_api_tables_get):
method test_tables_create (line 213) | def test_tables_create(self,
method test_insert_data_no_table (line 237) | def test_insert_data_no_table(self,
method test_insert_data_missing_field (line 267) | def test_insert_data_missing_field(self,
method test_insert_data_mismatched_schema (line 300) | def test_insert_data_mismatched_schema(self,
method test_insert_data_dataframe (line 335) | def test_insert_data_dataframe(self,
method test_insert_data_dictlist (line 371) | def test_insert_data_dictlist(self,
method test_insert_data_dictlist_index (line 411) | def test_insert_data_dictlist_index(self,
method test_insert_data_dictlist_named_index (line 451) | def test_insert_data_dictlist_named_index(self,
method test_table_load (line 487) | def test_table_load(self, mock_api_jobs_get, mock_api_jobs_insert_load...
method test_table_extract (line 502) | def test_table_extract(self, mock_api_tables_get, mock_api_jobs_get, m...
method test_table_to_dataframe (line 515) | def test_table_to_dataframe(self, mock_api_tables_get, mock_api_tabled...
method test_encode_dict_as_row (line 534) | def test_encode_dict_as_row(self):
method test_decorators (line 539) | def test_decorators(self):
method test_window_decorators (line 593) | def test_window_decorators(self):
method test_table_update (line 620) | def test_table_update(self, mock_api_table_update, mock_api_tables_get):
method test_table_to_query (line 640) | def test_table_to_query(self):
method _create_context (line 650) | def _create_context():
method _create_table (line 656) | def _create_table(name):
method _create_table_info_result (line 660) | def _create_table_info_result(ts=None):
method _create_table_info_nested_schema_result (line 682) | def _create_table_info_nested_schema_result(ts=None):
method _create_dataset (line 709) | def _create_dataset(dataset_id):
method _create_table_list_result (line 713) | def _create_table_list_result():
method _create_table_list_empty_result (line 732) | def _create_table_list_empty_result():
method _create_data_frame (line 738) | def _create_data_frame():
method _create_inferred_schema (line 753) | def _create_inferred_schema(extra_field=None):
method _create_table_with_schema (line 764) | def _create_table_with_schema(schema, name='test:testds.testTable0'):
class _uuid (line 767) | class _uuid(object):
method hex (line 769) | def hex(self):
method _create_uuid (line 773) | def _create_uuid():
FILE: legacy_tests/bigquery/udf_tests.py
class TestCases (line 23) | class TestCases(unittest.TestCase):
method test_sql_building (line 25) | def test_sql_building(self):
method _create_udf (line 41) | def _create_udf():
method _create_context (line 49) | def _create_context():
FILE: legacy_tests/bigquery/view_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_view_repr_sql (line 26) | def test_view_repr_sql(self):
method test_view_create (line 35) | def test_view_create(self,
method test_view_result (line 60) | def test_view_result(self, mock_api_tables_get, mock_api_jobs_get, moc...
method test_view_update (line 84) | def test_view_update(self, mock_context_default, mock_api_table_update,
method _create_tables_insert_success_result (line 113) | def _create_tables_insert_success_result():
method _create_insert_done_result (line 117) | def _create_insert_done_result():
method _create_tables_get_result (line 136) | def _create_tables_get_result(num_rows=1, schema=None):
method _create_single_row_result (line 147) | def _create_single_row_result():
method _create_context (line 157) | def _create_context():
FILE: legacy_tests/data/sql_tests.py
class TestCases (line 21) | class TestCases(unittest.TestCase):
method test_sql_tokenizer (line 23) | def test_sql_tokenizer(self):
method test_zero_placeholders (line 73) | def test_zero_placeholders(self):
method test_single_placeholder (line 81) | def test_single_placeholder(self):
method test_multiple_placeholders (line 89) | def test_multiple_placeholders(self):
method test_escaped_placeholder (line 99) | def test_escaped_placeholder(self):
method test_string_escaping (line 107) | def test_string_escaping(self):
method test_all_combinations (line 115) | def test_all_combinations(self):
method test_missing_args (line 135) | def test_missing_args(self):
method test_invalid_args (line 145) | def test_invalid_args(self):
method test_nested_queries (line 155) | def test_nested_queries(self):
method test_shared_nested_queries (line 182) | def test_shared_nested_queries(self):
method test_circular_references (line 190) | def test_circular_references(self):
method test_module_reference (line 208) | def test_module_reference(self):
method test_get_sql_statement_with_environment (line 223) | def test_get_sql_statement_with_environment(self):
method test_get_query_from_module (line 227) | def test_get_query_from_module(self):
method test_get_sql_args (line 231) | def test_get_sql_args(self):
FILE: legacy_tests/kernel/bigquery_tests.py
function noop_decorator (line 24) | def noop_decorator(func):
class TestCases (line 40) | class TestCases(unittest.TestCase):
method test_udf_cell (line 44) | def test_udf_cell(self, mock_default_context, mock_notebook_environment):
method _create_context (line 69) | def _create_context():
method test_sample_cell (line 74) | def test_sample_cell(self):
method test_get_schema (line 78) | def test_get_schema(self):
method test_get_table (line 82) | def test_get_table(self):
method test_table_viewer (line 86) | def test_table_viewer(self):
FILE: legacy_tests/kernel/chart_data_tests.py
function noop_decorator (line 26) | def noop_decorator(func):
class TestCases (line 37) | class TestCases(unittest.TestCase):
method test_get_chart_data (line 40) | def test_get_chart_data(self, mock_get_item):
FILE: legacy_tests/kernel/chart_tests.py
function noop_decorator (line 24) | def noop_decorator(func):
class TestCases (line 35) | class TestCases(unittest.TestCase):
method test_chart_cell (line 37) | def test_chart_cell(self):
method test_chart_magic (line 50) | def test_chart_magic(self):
FILE: legacy_tests/kernel/commands_tests.py
function noop_decorator (line 23) | def noop_decorator(func):
class TestCases (line 33) | class TestCases(unittest.TestCase):
method test_create_args (line 35) | def test_create_args(self):
FILE: legacy_tests/kernel/html_tests.py
function noop_decorator (line 23) | def noop_decorator(func):
class TestCases (line 33) | class TestCases(unittest.TestCase):
method test_render_table (line 35) | def test_render_table(self):
method test_render_text (line 39) | def test_render_text(self):
FILE: legacy_tests/kernel/module_tests.py
function noop_decorator (line 26) | def noop_decorator(func):
class TestCases (line 36) | class TestCases(unittest.TestCase):
method test_create_python_module (line 38) | def test_create_python_module(self):
method test_pymodule (line 43) | def test_pymodule(self):
method test_pymodule_magic (line 49) | def test_pymodule_magic(self, mock_pymodule_cell):
FILE: legacy_tests/kernel/sql_tests.py
function noop_decorator (line 31) | def noop_decorator(func):
class TestCases (line 41) | class TestCases(unittest.TestCase):
method test_split_cell (line 46) | def test_split_cell(self):
method test_arguments (line 117) | def test_arguments(self, mock_default_context):
method test_date (line 162) | def test_date(self):
method test_sql_cell (line 166) | def test_sql_cell(self):
method _create_context (line 171) | def _create_context():
FILE: legacy_tests/kernel/storage_tests.py
function noop_decorator (line 28) | def noop_decorator(func):
class TestCases (line 38) | class TestCases(unittest.TestCase):
method test_expand_list (line 44) | def test_expand_list(self, mock_context_default, mock_api_objects_get,...
method test_storage_copy (line 96) | def test_storage_copy(self, mock_context_default, mock_api_objects_get...
method test_storage_copy_magic (line 123) | def test_storage_copy_magic(self, mock_storage_copy):
method test_storage_create (line 134) | def test_storage_create(self, mock_context_default, mock_api_buckets_i...
method test_storage_delete (line 163) | def test_storage_delete(self, mock_context_default, mock_api_bucket_de...
method test_storage_view (line 191) | def test_storage_view(self, mock_context_default):
method test_storage_write (line 197) | def test_storage_write(self, mock_context_default):
method _create_context (line 203) | def _create_context():
method _mock_bucket_items_return (line 209) | def _mock_bucket_items_return(context):
method _mock_api_objects_get (line 229) | def _mock_api_objects_get():
method _mock_api_buckets_get (line 241) | def _mock_api_buckets_get():
FILE: legacy_tests/kernel/utils_tests.py
class TestCases (line 39) | class TestCases(unittest.TestCase):
method _get_expected_cols (line 42) | def _get_expected_cols():
method _timestamp (line 54) | def _timestamp(d):
method _get_raw_rows (line 58) | def _get_raw_rows():
method _get_expected_rows (line 88) | def _get_expected_rows():
method _get_test_data_as_list_of_dicts (line 112) | def _get_test_data_as_list_of_dicts():
method test_get_data_from_list_of_dicts (line 133) | def test_get_data_from_list_of_dicts(self):
method test_get_data_from_list_of_lists (line 140) | def test_get_data_from_list_of_lists(self):
method test_get_data_from_dataframe (line 155) | def test_get_data_from_dataframe(self):
method test_get_data_from_table (line 166) | def test_get_data_from_table(self, mock_context_default, mock_api_tabl...
method test_get_data_from_empty_list (line 200) | def test_get_data_from_empty_list(self):
method test_get_data_from_malformed_list (line 203) | def test_get_data_from_malformed_list(self):
method _test_get_data (line 209) | def _test_get_data(self, test_data, cols, rows, expected_count, fn):
method _create_api (line 260) | def _create_api():
method _create_context (line 265) | def _create_context():
FILE: legacy_tests/stackdriver/commands/monitoring_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_list_metric_descriptors (line 28) | def test_list_metric_descriptors(self, mock_metric_descriptors, mock_r...
method test_list_resource_descriptors (line 46) | def test_list_resource_descriptors(self, mock_resource_descriptors, mo...
method test_list_groups (line 63) | def test_list_groups(self, mock_groups, mock_render_dataframe):
FILE: legacy_tests/stackdriver/monitoring/group_tests.py
class TestCases (line 35) | class TestCases(unittest.TestCase):
method setUp (line 37) | def setUp(self):
method test_constructor_minimal (line 42) | def test_constructor_minimal(self, mock_context_default):
method test_constructor_maximal (line 52) | def test_constructor_maximal(self):
method test_list (line 60) | def test_list(self, mock_api_list_groups):
method test_list_w_pattern_match (line 72) | def test_list_w_pattern_match(self, mock_api_list_groups):
method test_list_caching (line 83) | def test_list_caching(self, mock_gcloud_list_groups):
method test_as_dataframe (line 94) | def test_as_dataframe(self, mock_gcloud_list_groups):
method test_as_dataframe_w_all_args (line 113) | def test_as_dataframe_w_all_args(self, mock_gcloud_list_groups):
method _create_context (line 125) | def _create_context(project_id):
method _list_groups_get_result (line 130) | def _list_groups_get_result(context):
FILE: legacy_tests/stackdriver/monitoring/metric_tests.py
class TestCases (line 40) | class TestCases(unittest.TestCase):
method setUp (line 42) | def setUp(self):
method test_constructor_minimal (line 47) | def test_constructor_minimal(self, mock_context_default):
method test_constructor_maximal (line 58) | def test_constructor_maximal(self):
method test_list (line 71) | def test_list(self, mock_gcloud_list_descriptors):
method test_list_w_api_filter (line 84) | def test_list_w_api_filter(self, mock_gcloud_list_descriptors):
method test_list_w_pattern_match (line 103) | def test_list_w_pattern_match(self, mock_gcloud_list_descriptors):
method test_list_caching (line 115) | def test_list_caching(self, mock_gcloud_list_descriptors):
method test_as_dataframe (line 127) | def test_as_dataframe(self, mock_datalab_list_descriptors):
method test_as_dataframe_w_all_args (line 148) | def test_as_dataframe_w_all_args(self, mock_datalab_list_descriptors):
method _create_context (line 160) | def _create_context(project_id):
method _list_metrics_get_result (line 165) | def _list_metrics_get_result(context):
FILE: legacy_tests/stackdriver/monitoring/query_metadata_tests.py
class TestCases (line 35) | class TestCases(unittest.TestCase):
method setUp (line 37) | def setUp(self):
method test_constructor (line 43) | def test_constructor(self, mock_query_iter):
method test_iteration (line 55) | def test_iteration(self, mock_query_iter):
method test_as_dataframe (line 66) | def test_as_dataframe(self, mock_query_iter):
method test_as_dataframe_w_max_rows (line 96) | def test_as_dataframe_w_max_rows(self, mock_query_iter):
method test_as_dataframe_w_no_data (line 125) | def test_as_dataframe_w_no_data(self, mock_query_iter):
method _query_iter_get_result (line 134) | def _query_iter_get_result():
FILE: legacy_tests/stackdriver/monitoring/query_tests.py
class TestCases (line 33) | class TestCases(unittest.TestCase):
method setUp (line 35) | def setUp(self):
method test_constructor_minimal (line 40) | def test_constructor_minimal(self, mock_context_default):
method test_constructor_maximal (line 55) | def test_constructor_maximal(self):
method test_metadata (line 76) | def test_metadata(self, mock_query_iter):
FILE: legacy_tests/stackdriver/monitoring/resource_tests.py
class TestCases (line 35) | class TestCases(unittest.TestCase):
method setUp (line 37) | def setUp(self):
method test_constructor_minimal (line 42) | def test_constructor_minimal(self, mock_context_default):
method test_constructor_maximal (line 49) | def test_constructor_maximal(self):
method test_list (line 59) | def test_list(self, mock_api_list_descriptors):
method test_list_w_api_filter (line 71) | def test_list_w_api_filter(self, mock_api_list_descriptors):
method test_list_w_pattern_match (line 85) | def test_list_w_pattern_match(self, mock_api_list_descriptors):
method test_list_caching (line 96) | def test_list_caching(self, mock_gcloud_list_descriptors):
method test_as_dataframe (line 108) | def test_as_dataframe(self, mock_datalab_list_descriptors):
method test_as_dataframe_w_all_args (line 129) | def test_as_dataframe_w_all_args(self, mock_datalab_list_descriptors):
method _create_context (line 142) | def _create_context(project_id):
method _list_resources_get_result (line 147) | def _list_resources_get_result():
FILE: legacy_tests/stackdriver/monitoring/utils_tests.py
class TestCases (line 22) | class TestCases(unittest.TestCase):
method test_make_client (line 24) | def test_make_client(self):
method test_make_client_w_defaults (line 31) | def test_make_client_w_defaults(self, mock_context_default):
method _create_context (line 39) | def _create_context():
FILE: legacy_tests/storage/api_tests.py
class TestCases (line 23) | class TestCases(unittest.TestCase):
method validate (line 25) | def validate(self, mock_http_request, expected_url, expected_args=None...
method test_buckets_insert (line 40) | def test_buckets_insert(self, mock_http_request):
method test_buckets_delete (line 52) | def test_buckets_delete(self, mock_http_request):
method test_buckets_get (line 59) | def test_buckets_get(self, mock_http_request):
method test_buckets_list (line 69) | def test_buckets_list(self, mock_http_request):
method test_object_download (line 81) | def test_object_download(self, mock_http_request):
method test_object_upload (line 88) | def test_object_upload(self, mock_http_request):
method test_objects_copy (line 96) | def test_objects_copy(self, mock_http_request):
method test_objects_delete (line 104) | def test_objects_delete(self, mock_http_request):
method test_objects_get (line 111) | def test_objects_get(self, mock_http_request):
method test_objects_list (line 122) | def test_objects_list(self, mock_http_request):
method test_objects_patch (line 135) | def test_objects_patch(self, mock_http_request):
method _create_api (line 142) | def _create_api():
method _create_context (line 147) | def _create_context():
FILE: legacy_tests/storage/bucket_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_bucket_existence (line 27) | def test_bucket_existence(self, mock_api_buckets):
method test_bucket_metadata (line 37) | def test_bucket_metadata(self, mock_api_buckets):
method _create_bucket (line 46) | def _create_bucket(name='test_bucket'):
method _create_context (line 50) | def _create_context():
method _create_buckets_get_result (line 56) | def _create_buckets_get_result():
FILE: legacy_tests/storage/item_tests.py
class TestCases (line 24) | class TestCases(unittest.TestCase):
method test_item_existence (line 28) | def test_item_existence(self, mock_api_objects_get, mock_api_objects_l...
method test_item_metadata (line 39) | def test_item_metadata(self, mock_api_objects):
method test_enumerate_items_empty (line 50) | def test_enumerate_items_empty(self, mock_api_objects):
method test_enumerate_items_single (line 59) | def test_enumerate_items_single(self, mock_api_objects):
method test_enumerate_items_multi_page (line 69) | def test_enumerate_items_multi_page(self, mock_api_objects):
method _create_bucket (line 83) | def _create_bucket(name='test_bucket'):
method _create_context (line 87) | def _create_context():
method _create_objects_get_result (line 93) | def _create_objects_get_result():
method _create_enumeration_empty_result (line 97) | def _create_enumeration_empty_result():
method _create_enumeration_single_result (line 101) | def _create_enumeration_single_result():
method _create_enumeration_multipage_result1 (line 109) | def _create_enumeration_multipage_result1():
method _create_enumeration_multipage_result2 (line 118) | def _create_enumeration_multipage_result2():
FILE: solutionbox/image_classification/mltoolbox/image/classification/_api.py
function preprocess_async (line 25) | def preprocess_async(train_dataset, output_dir, eval_dataset=None, check...
function preprocess (line 55) | def preprocess(train_dataset, output_dir, eval_dataset=None, checkpoint=...
function train_async (line 66) | def train_async(input_dir, batch_size, max_steps, output_dir, checkpoint...
function train (line 89) | def train(input_dir, batch_size, max_steps, output_dir, checkpoint=None,...
function predict (line 100) | def predict(model, image_files, resize=False, show_image=True, cloud=None):
function batch_predict_async (line 126) | def batch_predict_async(dataset, model_dir, output_csv=None, output_bq_t...
function batch_predict (line 159) | def batch_predict(dataset, model_dir, output_csv=None, output_bq_table=N...
FILE: solutionbox/image_classification/mltoolbox/image/classification/_cloud.py
class Cloud (line 36) | class Cloud(object):
method preprocess (line 40) | def preprocess(train_dataset, output_dir, eval_dataset, checkpoint, pi...
method train (line 99) | def train(input_dir, batch_size, max_steps, output_dir, checkpoint, cl...
method predict (line 135) | def predict(model_id, image_files, resize, show_image):
method batch_predict (line 169) | def batch_predict(dataset, model_dir, output_csv, output_bq_table, pip...
FILE: solutionbox/image_classification/mltoolbox/image/classification/_inceptionlib.py
function trunc_normal (line 28) | def trunc_normal(stddev):
function inception_v3_base (line 32) | def inception_v3_base(inputs,
function inception_v3 (line 442) | def inception_v3(inputs,
function _reduced_kernel_size_for_small_input (line 556) | def _reduced_kernel_size_for_small_input(input_tensor, kernel_size):
function inception_v3_arg_scope (line 587) | def inception_v3_arg_scope(weight_decay=0.00004,
FILE: solutionbox/image_classification/mltoolbox/image/classification/_local.py
class Local (line 28) | class Local(object):
method preprocess (line 32) | def preprocess(train_dataset, output_dir, eval_dataset, checkpoint):
method train (line 54) | def train(input_dir, batch_size, max_steps, output_dir, checkpoint):
method predict (line 70) | def predict(model_dir, image_files, resize, show_image):
method batch_predict (line 82) | def batch_predict(dataset, model_dir, output_csv, output_bq_table):
FILE: solutionbox/image_classification/mltoolbox/image/classification/_model.py
class GraphMod (line 43) | class GraphMod(Enum):
class GraphReferences (line 49) | class GraphReferences(object):
method __init__ (line 52) | def __init__(self):
class Model (line 62) | class Model(object):
method __init__ (line 65) | def __init__(self, labels, dropout, inception_checkpoint_file):
method add_final_training_ops (line 71) | def add_final_training_ops(self,
method build_inception_graph (line 114) | def build_inception_graph(self):
method build_graph (line 154) | def build_graph(self, data_paths, batch_size, graph_mod):
method build_train_graph (line 239) | def build_train_graph(self, data_paths, batch_size):
method build_eval_graph (line 242) | def build_eval_graph(self, data_paths, batch_size):
method restore_from_checkpoint (line 245) | def restore_from_checkpoint(self, session, inception_checkpoint_file,
method build_prediction_graph (line 286) | def build_prediction_graph(self):
method export (line 319) | def export(self, last_checkpoint, output_dir):
method format_metric_values (line 348) | def format_metric_values(self, metric_values):
method format_prediction_values (line 362) | def format_prediction_values(self, prediction):
function loss (line 367) | def loss(logits, labels):
function training (line 382) | def training(loss_op):
FILE: solutionbox/image_classification/mltoolbox/image/classification/_predictor.py
function _load_tf_model (line 29) | def _load_tf_model(model_dir):
function _tf_predict (line 44) | def _tf_predict(model_dir, images):
function predict (line 58) | def predict(model_dir, images):
class EmitAsBatchDoFn (line 69) | class EmitAsBatchDoFn(beam.DoFn):
method __init__ (line 72) | def __init__(self, batch_size):
method process (line 76) | def process(self, element):
method finish_bundle (line 83) | def finish_bundle(self, context=None):
class UnbatchDoFn (line 88) | class UnbatchDoFn(beam.DoFn):
method process (line 91) | def process(self, element):
class LoadImagesDoFn (line 96) | class LoadImagesDoFn(beam.DoFn):
method process (line 99) | def process(self, element):
class PredictBatchDoFn (line 109) | class PredictBatchDoFn(beam.DoFn):
method __init__ (line 112) | def __init__(self, model_dir):
method start_bundle (line 118) | def start_bundle(self, context=None):
method finish_bundle (line 121) | def finish_bundle(self, context=None):
method process (line 125) | def process(self, element):
class ProcessResultsDoFn (line 145) | class ProcessResultsDoFn(beam.DoFn):
method process (line 150) | def process(self, element):
class MakeCsvLineDoFn (line 171) | class MakeCsvLineDoFn(beam.DoFn):
method process (line 174) | def process(self, element):
function configure_pipeline (line 187) | def configure_pipeline(p, dataset, model_dir, output_csv, output_bq_table):
FILE: solutionbox/image_classification/mltoolbox/image/classification/_preprocess.py
class ExtractLabelIdsDoFn (line 42) | class ExtractLabelIdsDoFn(beam.DoFn):
method start_bundle (line 46) | def start_bundle(self, context=None):
method process (line 49) | def process(self, element, all_labels):
class ReadImageAndConvertToJpegDoFn (line 78) | class ReadImageAndConvertToJpegDoFn(beam.DoFn):
method process (line 85) | def process(self, element):
class EmbeddingsGraph (line 107) | class EmbeddingsGraph(object):
method __init__ (line 116) | def __init__(self, tf_session, checkpoint_path):
method build_graph (line 126) | def build_graph(self):
method restore_from_checkpoint (line 169) | def restore_from_checkpoint(self, checkpoint_path):
method calculate_embedding (line 188) | def calculate_embedding(self, batch_image_bytes):
class TFExampleFromImageDoFn (line 201) | class TFExampleFromImageDoFn(beam.DoFn):
method __init__ (line 215) | def __init__(self, checkpoint_path):
method start_bundle (line 221) | def start_bundle(self, context=None):
method finish_bundle (line 233) | def finish_bundle(self, context=None):
method process (line 237) | def process(self, element):
class TrainEvalSplitPartitionFn (line 271) | class TrainEvalSplitPartitionFn(beam.PartitionFn):
method partition_for (line 273) | def partition_for(self, element, num_partitions):
class ExampleProtoCoder (line 278) | class ExampleProtoCoder(beam.coders.Coder):
method encode (line 281) | def encode(self, example_proto):
method decode (line 284) | def decode(self, serialized_str):
class SaveFeatures (line 291) | class SaveFeatures(beam.PTransform):
method __init__ (line 295) | def __init__(self, file_path_prefix):
method expand (line 299) | def expand(self, features):
function _labels_pipeline (line 307) | def _labels_pipeline(sources):
function _transformation_pipeline (line 316) | def _transformation_pipeline(source, checkpoint, labels, mode):
function configure_pipeline (line 327) | def configure_pipeline(p, dataset_train, dataset_eval, checkpoint_path, ...
FILE: solutionbox/image_classification/mltoolbox/image/classification/_trainer.py
function start_server (line 27) | def start_server(cluster, task):
class Evaluator (line 41) | class Evaluator(object):
method __init__ (line 44) | def __init__(self, model, data_paths, batch_size, output_path, dataset...
method _data_size (line 56) | def _data_size(self, data_paths):
method evaluate (line 65) | def evaluate(self, num_eval_batches=None):
class Trainer (line 104) | class Trainer(object):
method __init__ (line 107) | def __init__(self, input_dir, batch_size, max_steps, output_path, mode...
method run_training (line 120) | def run_training(self):
method log (line 232) | def log(self, session):
method eval (line 246) | def eval(self, session):
method save_summaries (line 268) | def save_summaries(self, session):
FILE: solutionbox/image_classification/mltoolbox/image/classification/_util.py
function is_in_IPython (line 30) | def is_in_IPython():
function default_project (line 38) | def default_project():
function _get_latest_data_dir (line 43) | def _get_latest_data_dir(input_dir):
function get_train_eval_files (line 53) | def get_train_eval_files(input_dir):
function get_labels (line 63) | def get_labels(input_dir):
function read_examples (line 72) | def read_examples(input_files, batch_size, shuffle, num_epochs=None):
function override_if_not_in_args (line 120) | def override_if_not_in_args(flag, argument, args):
function loss (line 126) | def loss(loss_value):
function accuracy (line 136) | def accuracy(logits, labels):
function check_dataset (line 150) | def check_dataset(dataset, mode):
function get_sources_from_dataset (line 165) | def get_sources_from_dataset(p, dataset, mode):
function decode_and_resize (line 192) | def decode_and_resize(image_str_tensor):
function resize_image (line 211) | def resize_image(image_str_tensor):
function load_images (line 219) | def load_images(image_files, resize=True):
function process_prediction_results (line 242) | def process_prediction_results(results, show_image):
function repackage_to_staging (line 257) | def repackage_to_staging(output_path):
FILE: solutionbox/image_classification/mltoolbox/image/classification/task.py
function main (line 33) | def main(_):
FILE: solutionbox/ml_workbench/tensorflow/analyze.py
function parse_arguments (line 32) | def parse_arguments(argv):
function run_cloud_analysis (line 184) | def run_cloud_analysis(output_dir, csv_file_pattern, bigquery_table, sch...
function main (line 321) | def main(argv=None):
FILE: solutionbox/ml_workbench/tensorflow/trainer/feature_analysis.py
function check_schema_transforms_match (line 32) | def check_schema_transforms_match(schema, inverted_features):
function save_schema_features (line 103) | def save_schema_features(schema, features, output):
function expand_defaults (line 114) | def expand_defaults(schema, features):
function invert_features (line 172) | def invert_features(features):
function run_local_analysis (line 187) | def run_local_analysis(output_dir, csv_file_pattern, schema, features):
FILE: solutionbox/ml_workbench/tensorflow/trainer/feature_transforms.py
function _scale (line 84) | def _scale(x, min_x_value, max_x_value, output_min, output_max):
function _string_to_int (line 111) | def _string_to_int(x, vocab):
function _tfidf (line 140) | def _tfidf(x, reduced_term_freq, vocab_size, corpus_size):
function _bag_of_words (line 203) | def _bag_of_words(x):
function _make_image_to_vec_tito (line 224) | def _make_image_to_vec_tito(feature_name, tmp_dir=None, checkpoint=None):
function make_preprocessing_fn (line 344) | def make_preprocessing_fn(output_dir, features, keep_target):
function get_transformed_feature_info (line 435) | def get_transformed_feature_info(features, schema):
function csv_header_and_defaults (line 503) | def csv_header_and_defaults(features, schema, stats, keep_target):
function build_csv_serving_tensors_for_transform_step (line 534) | def build_csv_serving_tensors_for_transform_step(analysis_path,
function build_csv_serving_tensors_for_training_step (line 570) | def build_csv_serving_tensors_for_training_step(analysis_path,
function build_csv_transforming_training_input_fn (line 598) | def build_csv_transforming_training_input_fn(schema,
function build_tfexample_transfored_training_input_fn (line 701) | def build_tfexample_transfored_training_input_fn(schema,
function image_feature_engineering (line 809) | def image_feature_engineering(features, feature_tensors_dict):
function get_target_name (line 829) | def get_target_name(features):
function read_vocab_file (line 837) | def read_vocab_file(file_path):
FILE: solutionbox/ml_workbench/tensorflow/trainer/task.py
class DatalabParser (line 61) | class DatalabParser():
method __init__ (line 73) | def __init__(self, epilog=None, datalab_epilog=None):
method make_datalab_help_action (line 107) | def make_datalab_help_action(self):
method add_argument (line 132) | def add_argument(self, name, **kwargs):
method parse_known_args (line 151) | def parse_known_args(self, args=None):
function parse_arguments (line 155) | def parse_arguments(argv):
function is_linear_model (line 264) | def is_linear_model(model_type):
function is_dnn_model (line 268) | def is_dnn_model(model_type):
function is_regression_model (line 272) | def is_regression_model(model_type):
function is_classification_model (line 276) | def is_classification_model(model_type):
function build_feature_columns (line 280) | def build_feature_columns(features, stats, model_type):
function recursive_copy (line 359) | def recursive_copy(src_dir, dest_dir):
function make_prediction_output_tensors (line 377) | def make_prediction_output_tensors(args, features, input_ops, model_fn_ops,
function make_export_strategy (line 462) | def make_export_strategy(
function get_estimator (line 585) | def get_estimator(args, output_dir, features, stats, target_vocab_size):
function read_vocab (line 642) | def read_vocab(args, column_name):
function get_key_names (line 662) | def get_key_names(features):
function read_json_file (line 670) | def read_json_file(file_path):
function get_experiment_fn (line 676) | def get_experiment_fn(args):
function local_analysis (line 817) | def local_analysis(args):
function set_logging_level (line 844) | def set_logging_level(args):
function main (line 857) | def main(argv=None):
FILE: solutionbox/ml_workbench/tensorflow/transform.py
function parse_arguments (line 33) | def parse_arguments(argv):
function shuffle (line 171) | def shuffle(pcoll): # pylint: disable=invalid-name
function image_transform_columns (line 179) | def image_transform_columns(features):
function prepare_image_transforms (line 197) | def prepare_image_transforms(element, image_columns):
class EmitAsBatchDoFn (line 241) | class EmitAsBatchDoFn(beam.DoFn):
method __init__ (line 244) | def __init__(self, batch_size):
method process (line 253) | def process(self, element):
method finish_bundle (line 260) | def finish_bundle(self, element=None):
class TransformFeaturesDoFn (line 268) | class TransformFeaturesDoFn(beam.DoFn):
method __init__ (line 271) | def __init__(self, analysis_output_dir, features, schema, stats):
method start_bundle (line 278) | def start_bundle(self, element=None):
method finish_bundle (line 301) | def finish_bundle(self, element=None):
method process (line 304) | def process(self, element):
function decode_csv (line 355) | def decode_csv(csv_string, column_names):
function encode_csv (line 373) | def encode_csv(data_dict, column_names):
function serialize_example (line 392) | def serialize_example(transformed_json_data, info_dict):
function preprocess (line 431) | def preprocess(pipeline, args):
function main (line 509) | def main(argv=None):
FILE: solutionbox/ml_workbench/test_tensorflow/test_analyze.py
class TestConfigFiles (line 37) | class TestConfigFiles(unittest.TestCase):
method test_expand_defaults_do_nothing (line 40) | def test_expand_defaults_do_nothing(self):
method test_expand_defaults_unknown_schema_type (line 54) | def test_expand_defaults_unknown_schema_type(self):
method test_expand_defaults (line 63) | def test_expand_defaults(self):
method test_check_schema_transforms_match (line 85) | def test_check_schema_transforms_match(self):
class TestLocalAnalyze (line 132) | class TestLocalAnalyze(unittest.TestCase):
method test_numerics (line 135) | def test_numerics(self):
method test_categorical (line 169) | def test_categorical(self):
method test_text (line 219) | def test_text(self):
class TestCloudAnalyzeFromBQTable (line 277) | class TestCloudAnalyzeFromBQTable(unittest.TestCase):
method test_numerics (line 285) | def test_numerics(self):
class TestCloudAnalyzeFromCSVFiles (line 338) | class TestCloudAnalyzeFromCSVFiles(unittest.TestCase):
method setUpClass (line 342) | def setUpClass(cls):
method tearDownClass (line 348) | def tearDownClass(cls):
method test_numerics (line 354) | def test_numerics(self):
method test_categorical (line 392) | def test_categorical(self):
method test_text (line 445) | def test_text(self):
class TestOneSourceColumnManyFeatures (line 499) | class TestOneSourceColumnManyFeatures(unittest.TestCase):
method test_multiple_usage (line 502) | def test_multiple_usage(self):
FILE: solutionbox/ml_workbench/test_tensorflow/test_cloud_workflow.py
class TestCloudServices (line 23) | class TestCloudServices(unittest.TestCase):
method __init__ (line 46) | def __init__(self, *args, **kwargs):
method setUp (line 57) | def setUp(self):
method tearDown (line 80) | def tearDown(self):
method _make_image_files (line 86) | def _make_image_files(self):
method _make_csv_data (line 107) | def _make_csv_data(self, filename, num_rows, keep_target=True, embedde...
method _get_default_project_id (line 161) | def _get_default_project_id(self):
method _run_analyze (line 166) | def _run_analyze(self):
method _run_transform (line 205) | def _run_transform(self):
method _run_training_transform (line 247) | def _run_training_transform(self):
method _run_batch_prediction (line 281) | def _run_batch_prediction(self):
method test_cloud_workflow (line 306) | def test_cloud_workflow(self):
FILE: solutionbox/ml_workbench/test_tensorflow/test_feature_transforms.py
class TestGraphBuilding (line 35) | class TestGraphBuilding(unittest.TestCase):
method _run_graph (line 38) | def _run_graph(self, analysis_path, features, schema, stats, predict_d...
method test_make_transform_graph_numerics (line 60) | def test_make_transform_graph_numerics(self):
method test_make_transform_graph_category (line 97) | def test_make_transform_graph_category(self):
method test_make_transform_graph_text_tfidf (line 131) | def test_make_transform_graph_text_tfidf(self):
method test_make_transform_graph_text_multi_hot (line 201) | def test_make_transform_graph_text_multi_hot(self):
method test_make_transform_graph_text_bag_of_words (line 248) | def test_make_transform_graph_text_bag_of_words(self):
method test_make_transform_graph_images (line 311) | def test_make_transform_graph_images(self):
FILE: solutionbox/ml_workbench/test_tensorflow/test_training.py
function run_exported_model (line 26) | def run_exported_model(model_path, csv_data):
class TestSpecialCharacters (line 59) | class TestSpecialCharacters(unittest.TestCase):
method testCommaQuote (line 62) | def testCommaQuote(self):
class TestClassificationTopN (line 170) | class TestClassificationTopN(unittest.TestCase):
method testTopNZero (line 173) | def testTopNZero(self):
class TestMultipleFeatures (line 228) | class TestMultipleFeatures(unittest.TestCase):
method testMultipleColumnsRaw (line 231) | def testMultipleColumnsRaw(self):
method testMultipleColumnsTransformed (line 288) | def testMultipleColumnsTransformed(self):
class TestOptionalKeys (line 369) | class TestOptionalKeys(unittest.TestCase):
method testNoKeys (line 370) | def testNoKeys(self):
method testManyKeys (line 418) | def testManyKeys(self):
class TestTrainer (line 473) | class TestTrainer(unittest.TestCase):
method __init__ (line 480) | def __init__(self, *args, **kwargs):
method setUp (line 494) | def setUp(self):
method tearDown (line 511) | def tearDown(self):
method make_image_files (line 515) | def make_image_files(self):
method make_csv_data (line 527) | def make_csv_data(self, filename, num_rows, problem_type, keep_target=...
method _create_schema_features (line 642) | def _create_schema_features(self, problem_type, with_image=False):
method _run_analyze (line 688) | def _run_analyze(self, problem_type, with_image=False):
method _run_transform (line 699) | def _run_transform(self):
method _run_training_transform (line 719) | def _run_training_transform(self, problem_type, model_type, extra_args...
method _run_training_raw (line 741) | def _run_training_raw(self, problem_type, model_type, extra_args=[]):
method _run_training_with_analysis (line 764) | def _run_training_with_analysis(self, problem_type, model_type, extra_...
method _check_model (line 788) | def _check_model(self, problem_type, model_type, with_image=False):
method testClassificationLinear (line 864) | def testClassificationLinear(self):
method testRegressionLinear (line 878) | def testRegressionLinear(self):
method testRegressionDNN (line 892) | def testRegressionDNN(self):
method testClassificationDNNWithImage (line 908) | def testClassificationDNNWithImage(self):
method testTrainingWithAnalysis (line 925) | def testTrainingWithAnalysis(self):
FILE: solutionbox/ml_workbench/test_tensorflow/test_transform.py
class TestTransformRawData (line 31) | class TestTransformRawData(unittest.TestCase):
method setUpClass (line 35) | def setUpClass(cls):
method tearDownClass (line 95) | def tearDownClass(cls):
method test_local_csv_transform (line 98) | def test_local_csv_transform(self):
method test_local_bigquery_transform (line 144) | def test_local_bigquery_transform(self):
FILE: solutionbox/ml_workbench/test_xgboost/test_analyze.py
class TestConfigFiles (line 26) | class TestConfigFiles(unittest.TestCase):
method test_expand_defaults_do_nothing (line 29) | def test_expand_defaults_do_nothing(self):
method test_expand_defaults_unknown_schema_type (line 43) | def test_expand_defaults_unknown_schema_type(self):
method test_expand_defaults (line 52) | def test_expand_defaults(self):
class TestLocalAnalyze (line 75) | class TestLocalAnalyze(unittest.TestCase):
method test_numerics (line 78) | def test_numerics(self):
method test_categorical (line 112) | def test_categorical(self):
method test_text (line 148) | def test_text(self):
FILE: solutionbox/ml_workbench/test_xgboost/test_transform.py
class TestTransformRawData (line 21) | class TestTransformRawData(unittest.TestCase):
method setUpClass (line 25) | def setUpClass(cls):
method tearDownClass (line 85) | def tearDownClass(cls):
method test_local_csv_transform (line 89) | def test_local_csv_transform(self):
FILE: solutionbox/ml_workbench/xgboost/analyze.py
function parse_arguments (line 32) | def parse_arguments(argv):
function run_cloud_analysis (line 173) | def run_cloud_analysis(output_dir, csv_file_pattern, bigquery_table, sch...
function main (line 310) | def main(argv=None):
FILE: solutionbox/ml_workbench/xgboost/trainer/feature_analysis.py
function check_schema_transforms_match (line 32) | def check_schema_transforms_match(schema, inverted_features):
function save_schema_features (line 103) | def save_schema_features(schema, features, output):
function expand_defaults (line 114) | def expand_defaults(schema, features):
function invert_features (line 172) | def invert_features(features):
function run_local_analysis (line 185) | def run_local_analysis(output_dir, csv_file_pattern, schema, features):
FILE: solutionbox/ml_workbench/xgboost/trainer/feature_transforms.py
function _scale (line 78) | def _scale(x, min_x_value, max_x_value, output_min, output_max):
function _string_to_int (line 105) | def _string_to_int(x, vocab):
function _make_image_to_vec_tito (line 132) | def _make_image_to_vec_tito(feature_name, tmp_dir=None, checkpoint=None):
function make_preprocessing_fn (line 252) | def make_preprocessing_fn(output_dir, features, keep_target):
function csv_header_and_defaults (line 317) | def csv_header_and_defaults(features, schema, stats, keep_target):
function build_csv_serving_tensors_for_transform_step (line 348) | def build_csv_serving_tensors_for_transform_step(analysis_path,
function get_target_name (line 384) | def get_target_name(features):
function read_vocab_file (line 392) | def read_vocab_file(file_path):
function get_transformed_feature_indices (line 415) | def get_transformed_feature_indices(features, stats):
function create_feature_map (line 447) | def create_feature_map(features, feature_indices, output_dir):
FILE: solutionbox/ml_workbench/xgboost/trainer/task.py
class DatalabParser (line 61) | class DatalabParser():
method __init__ (line 73) | def __init__(self, epilog=None, datalab_epilog=None):
method make_datalab_help_action (line 107) | def make_datalab_help_action(self):
method add_argument (line 132) | def add_argument(self, name, **kwargs):
method parse_known_args (line 151) | def parse_known_args(self, args=None):
function parse_arguments (line 155) | def parse_arguments(argv):
function is_linear_model (line 264) | def is_linear_model(model_type):
function is_dnn_model (line 268) | def is_dnn_model(model_type):
function is_regression_model (line 272) | def is_regression_model(model_type):
function is_classification_model (line 276) | def is_classification_model(model_type):
function build_feature_columns (line 280) | def build_feature_columns(features, stats, model_type):
function recursive_copy (line 359) | def recursive_copy(src_dir, dest_dir):
function make_prediction_output_tensors (line 377) | def make_prediction_output_tensors(args, features, input_ops, model_fn_ops,
function make_export_strategy (line 462) | def make_export_strategy(
function get_estimator (line 585) | def get_estimator(args, output_dir, features, stats, target_vocab_size):
function read_vocab (line 642) | def read_vocab(args, column_name):
function get_key_names (line 662) | def get_key_names(features):
function read_json_file (line 670) | def read_json_file(file_path):
function get_experiment_fn (line 676) | def get_experiment_fn(args):
function local_analysis (line 817) | def local_analysis(args):
function set_logging_level (line 844) | def set_logging_level(args):
function main (line 857) | def main(argv=None):
FILE: solutionbox/ml_workbench/xgboost/transform.py
function parse_arguments (line 33) | def parse_arguments(argv):
function shuffle (line 171) | def shuffle(pcoll): # pylint: disable=invalid-name
function image_transform_columns (line 179) | def image_transform_columns(features):
function prepare_image_transforms (line 197) | def prepare_image_transforms(element, image_columns):
class EmitAsBatchDoFn (line 241) | class EmitAsBatchDoFn(beam.DoFn):
method __init__ (line 244) | def __init__(self, batch_size):
method process (line 253) | def process(self, element):
method finish_bundle (line 260) | def finish_bundle(self, element=None):
class TransformFeaturesDoFn (line 268) | class TransformFeaturesDoFn(beam.DoFn):
method __init__ (line 271) | def __init__(self, analysis_output_dir, features, schema, stats):
method start_bundle (line 278) | def start_bundle(self, element=None):
method finish_bundle (line 301) | def finish_bundle(self, element=None):
method process (line 304) | def process(self, element):
function decode_csv (line 355) | def decode_csv(csv_string, column_names):
function encode_csv (line 373) | def encode_csv(data_dict, column_names):
function serialize_example (line 392) | def serialize_example(transformed_json_data, features, feature_indices, ...
function preprocess (line 424) | def preprocess(pipeline, args):
function main (line 514) | def main(argv=None):
FILE: solutionbox/structured_data/mltoolbox/_structured_data/_package.py
class FileNotFoundError (line 58) | class FileNotFoundError(IOError):
function _default_project (line 62) | def _default_project():
function _assert_gcs_files (line 67) | def _assert_gcs_files(files):
function _package_to_staging (line 87) | def _package_to_staging(staging_package_url):
function _wait_and_kill (line 108) | def _wait_and_kill(pid_to_wait, pids_to_kill):
function analyze (line 136) | def analyze(output_dir, dataset, cloud=False, project_id=None):
function analyze_async (line 147) | def analyze_async(output_dir, dataset, cloud=False, project_id=None):
function _analyze (line 171) | def _analyze(output_dir, dataset, cloud=False, project_id=None):
function train_async (line 216) | def train_async(train_dataset,
function local_train (line 363) | def local_train(train_dataset,
function cloud_train (line 456) | def cloud_train(train_dataset,
function predict (line 550) | def predict(data, training_dir=None, model_name=None, model_version=None...
function local_predict (line 595) | def local_predict(training_dir, data):
function cloud_predict (line 671) | def cloud_predict(model_name, model_version, data):
function batch_predict (line 722) | def batch_predict(training_dir, prediction_input_file, output_dir,
function batch_predict_async (line 742) | def batch_predict_async(training_dir, prediction_input_file, output_dir,
function local_batch_predict (line 780) | def local_batch_predict(training_dir, prediction_input_file, output_dir,...
function cloud_batch_predict (line 810) | def cloud_batch_predict(training_dir, prediction_input_file, output_dir,...
FILE: solutionbox/structured_data/mltoolbox/_structured_data/master_setup.py
function get_version (line 21) | def get_version():
FILE: solutionbox/structured_data/mltoolbox/_structured_data/prediction/predict.py
function parse_arguments (line 30) | def parse_arguments(argv):
class EmitAsBatchDoFn (line 112) | class EmitAsBatchDoFn(beam.DoFn):
method __init__ (line 115) | def __init__(self, batch_size):
method process (line 124) | def process(self, element):
method finish_bundle (line 131) | def finish_bundle(self, element=None):
class RunGraphDoFn (line 136) | class RunGraphDoFn(beam.DoFn):
method __init__ (line 139) | def __init__(self, trained_model_dir):
method start_bundle (line 143) | def start_bundle(self, element=None):
method finish_bundle (line 159) | def finish_bundle(self, element=None):
method process (line 165) | def process(self, element):
class RawJsonCoder (line 221) | class RawJsonCoder(beam.coders.Coder):
method encode (line 224) | def encode(self, obj):
class CSVCoder (line 237) | class CSVCoder(beam.coders.Coder):
method __init__ (line 240) | def __init__(self, header):
method make_header_string (line 248) | def make_header_string(self):
method encode (line 251) | def encode(self, tf_graph_predictions):
class FormatAndSave (line 267) | class FormatAndSave(beam.PTransform):
method __init__ (line 269) | def __init__(self, args):
method apply (line 304) | def apply(self, datasets):
method expand (line 307) | def expand(self, datasets):
function make_prediction_pipeline (line 349) | def make_prediction_pipeline(pipeline, args):
function main (line 376) | def main(argv=None):
FILE: solutionbox/structured_data/mltoolbox/_structured_data/preprocess/cloud_preprocess.py
function parse_arguments (line 33) | def parse_arguments(argv):
function parse_table_name (line 91) | def parse_table_name(bigquery_table):
function run_numerical_analysis (line 111) | def run_numerical_analysis(table, schema_list, args):
function run_categorical_analysis (line 159) | def run_categorical_analysis(table, schema_list, args):
function run_analysis (line 222) | def run_analysis(args):
function main (line 259) | def main(argv=None):
FILE: solutionbox/structured_data/mltoolbox/_structured_data/preprocess/local_preprocess.py
function parse_arguments (line 37) | def parse_arguments(argv):
function run_numerical_categorical_analysis (line 69) | def run_numerical_categorical_analysis(args, schema_list):
function run_analysis (line 147) | def run_analysis(args):
function main (line 162) | def main(argv=None):
FILE: solutionbox/structured_data/mltoolbox/_structured_data/trainer/task.py
function get_reader_input_fn (line 30) | def get_reader_input_fn(train_config, preprocess_output_dir, model_type,
function get_experiment_fn (line 60) | def get_experiment_fn(args):
function parse_arguments (line 135) | def parse_arguments(argv):
function main (line 231) | def main(argv=None):
FILE: solutionbox/structured_data/mltoolbox/_structured_data/trainer/util.py
class NotFittedError (line 61) | class NotFittedError(ValueError):
function _recursive_copy (line 69) | def _recursive_copy(src_dir, dest_dir):
function serving_from_csv_input (line 90) | def serving_from_csv_input(train_config, args, keep_target):
function make_output_tensors (line 118) | def make_output_tensors(train_config, args, input_ops, model_fn_ops, kee...
function make_export_strategy (line 185) | def make_export_strategy(train_config, args, keep_target, assets_extra=N...
function parse_example_tensor (line 281) | def parse_example_tensor(examples, train_config, keep_target):
function read_examples (line 315) | def read_examples(input_files, batch_size, shuffle, num_epochs=None):
function get_estimator (line 367) | def get_estimator(output_dir, train_config, args):
function preprocess_input (line 448) | def preprocess_input(features, target, train_config, preprocess_output_dir,
function _scale_tensor (line 545) | def _scale_tensor(tensor, range_min, range_max, scale_min, scale_max):
function _tflearn_features (line 570) | def _tflearn_features(train_config, args):
function get_vocabulary (line 655) | def get_vocabulary(preprocess_output_dir, name):
function merge_metadata (line 680) | def merge_metadata(preprocess_output_dir, transforms_file):
function validate_metadata (line 811) | def validate_metadata(train_config):
function is_linear_model (line 841) | def is_linear_model(model_type):
function is_dnn_model (line 845) | def is_dnn_model(model_type):
function is_regression_model (line 849) | def is_regression_model(model_type):
function is_classification_model (line 853) | def is_classification_model(model_type):
function python_portable_string (line 859) | def python_portable_string(string, encoding='utf-8'):
FILE: solutionbox/structured_data/mltoolbox/classification/dnn/_classification_dnn.py
function train (line 4) | def train(train_dataset,
function train_async (line 44) | def train_async(train_dataset,
FILE: solutionbox/structured_data/mltoolbox/classification/linear/_classification_linear.py
function train (line 4) | def train(train_dataset,
function train_async (line 42) | def train_async(train_dataset,
FILE: solutionbox/structured_data/mltoolbox/regression/dnn/_regression_dnn.py
function train (line 4) | def train(train_dataset,
function train_async (line 42) | def train_async(train_dataset,
FILE: solutionbox/structured_data/mltoolbox/regression/linear/_regression_linear.py
function train (line 4) | def train(train_dataset,
function train_async (line 40) | def train_async(train_dataset,
FILE: solutionbox/structured_data/setup.py
function get_version (line 21) | def get_version():
FILE: solutionbox/structured_data/test_mltoolbox/e2e_functions.py
function make_csv_data (line 24) | def make_csv_data(filename, num_rows, problem_type, keep_target=True):
function make_preprocess_schema (line 83) | def make_preprocess_schema(filename, problem_type):
function run_preprocess (line 138) | def run_preprocess(output_dir, csv_filename, schema_filename, logger):
function run_training (line 160) | def run_training(
FILE: solutionbox/structured_data/test_mltoolbox/test_datalab_e2e.py
class TestLinearRegression (line 38) | class TestLinearRegression(unittest.TestCase):
method __init__ (line 46) | def __init__(self, *args, **kwargs):
method _make_test_files (line 55) | def _make_test_files(self):
method _run_analyze (line 102) | def _run_analyze(self):
method _run_train (line 114) | def _run_train(self):
method _run_predict (line 133) | def _run_predict(self):
method _run_batch_prediction (line 142) | def _run_batch_prediction(self, output_dir, use_target):
method _cleanup (line 176) | def _cleanup(self):
method test_e2e (line 179) | def test_e2e(self):
FILE: solutionbox/structured_data/test_mltoolbox/test_package_functions.py
class TestAnalyze (line 36) | class TestAnalyze(unittest.TestCase):
method test_not_csvdataset (line 38) | def test_not_csvdataset(self):
method test_csvdataset_one_file (line 44) | def test_csvdataset_one_file(self):
method test_projectid (line 55) | def test_projectid(self):
method test_cloud_with_local_output_folder (line 66) | def test_cloud_with_local_output_folder(self):
method test_cloud_but_local_files (line 76) | def test_cloud_but_local_files(self):
method test_unsupported_schema (line 86) | def test_unsupported_schema(self):
class TestFunctionSignature (line 119) | class TestFunctionSignature(unittest.TestCase):
method _argspec (line 121) | def _argspec(self, fn_obj):
method test_same_analysis (line 127) | def test_same_analysis(self):
method test_same_analysis_async (line 134) | def test_same_analysis_async(self):
method test_analysis_argspec (line 141) | def test_analysis_argspec(self):
FILE: solutionbox/structured_data/test_mltoolbox/test_sd_preprocess.py
class TestPreprocess (line 30) | class TestPreprocess(unittest.TestCase):
method __init__ (line 35) | def __init__(self, *args, **kwargs):
method setUp (line 44) | def setUp(self):
method tearDown (line 52) | def tearDown(self):
method _make_test_data (line 56) | def _make_test_data(self, problem_type):
method _test_preprocess (line 65) | def _test_preprocess(self, problem_type):
method testRegression (line 105) | def testRegression(self):
method testClassification (line 108) | def testClassification(self):
FILE: solutionbox/structured_data/test_mltoolbox/test_sd_trainer.py
class TestTrainer (line 29) | class TestTrainer(unittest.TestCase):
method __init__ (line 36) | def __init__(self, *args, **kwargs):
method setUp (line 50) | def setUp(self):
method tearDown (line 66) | def tearDown(self):
method _run_training (line 70) | def _run_training(self, problem_type, model_type, transforms, extra_ar...
method _check_training_screen_output (line 110) | def _check_training_screen_output(self, accuracy=None, loss=None):
method _check_train_files (line 171) | def _check_train_files(self):
method _check_savedmodel (line 175) | def _check_savedmodel(self, model_folder):
method testRegressionDnn (line 185) | def testRegressionDnn(self):
method testRegressionLinear (line 206) | def testRegressionLinear(self):
method testClassificationDnn (line 225) | def testClassificationDnn(self):
method testClassificationLinear (line 246) | def testClassificationLinear(self):
FILE: tests/_util/commands_tests.py
class TestCases (line 27) | class TestCases(unittest.TestCase):
method redirect_stderr (line 32) | def redirect_stderr(target):
method test_subcommand_line (line 38) | def test_subcommand_line(self):
method test_subcommand_line_cell (line 66) | def test_subcommand_line_cell(self):
method test_subcommand_var_replacement (line 119) | def test_subcommand_var_replacement(self):
method test_subcommand_help (line 146) | def test_subcommand_help(self):
FILE: tests/_util/feature_statistics_generator_test.py
class FeatureStatisticsGeneratorTest (line 21) | class FeatureStatisticsGeneratorTest(googletest.TestCase):
method setUp (line 23) | def setUp(self):
method testParseExampleInt (line 26) | def testParseExampleInt(self):
method testParseExampleMissingValueList (line 47) | def testParseExampleMissingValueList(self):
method _check_sequence_example_entries (line 71) | def _check_sequence_example_entries(self,
method testParseExampleSequenceContext (line 89) | def testParseExampleSequenceContext(self):
method testParseExampleSequenceFeatureList (line 104) | def testParseExampleSequenceFeatureList(self):
method testParseExampleSequenceFeatureListMultipleEntriesInner (line 118) | def testParseExampleSequenceFeatureListMultipleEntriesInner(self):
method testParseExampleSequenceFeatureListMultipleEntriesOuter (line 134) | def testParseExampleSequenceFeatureListMultipleEntriesOuter(self):
method testVaryingCountsAndMissing (line 150) | def testVaryingCountsAndMissing(self):
method testParseExampleStringsAndFloats (line 174) | def testParseExampleStringsAndFloats(self):
method testParseExamplesTypeMismatch (line 197) | def testParseExamplesTypeMismatch(self):
method testGetDatasetsProtoFromEntriesLists (line 212) | def testGetDatasetsProtoFromEntriesLists(self):
method testGetProtoNums (line 234) | def testGetProtoNums(self):
method testQuantiles (line 306) | def testQuantiles(self):
method testInfinityAndNan (line 337) | def testInfinityAndNan(self):
method testInfinitysOnly (line 383) | def testInfinitysOnly(self):
method testGetProtoStrings (line 411) | def testGetProtoStrings(self):
method testGetProtoMultipleDatasets (line 476) | def testGetProtoMultipleDatasets(self):
method testGetEntriesNoFiles (line 520) | def testGetEntriesNoFiles(self):
method get_example_iter (line 527) | def get_example_iter():
method testGetEntries_one (line 539) | def testGetEntries_one(self):
method testGetEntries_oneFile (line 545) | def testGetEntries_oneFile(self):
method testGetEntries_twoFiles (line 550) | def testGetEntries_twoFiles(self):
method testGetEntries_stopInSecondFile (line 556) | def testGetEntries_stopInSecondFile(self):
FILE: tests/_util/generic_feature_statistics_generator_test.py
class GenericFeatureStatisticsGeneratorTest (line 22) | class GenericFeatureStatisticsGeneratorTest(googletest.TestCase):
method setUp (line 24) | def setUp(self):
method testProtoFromDataFrames (line 27) | def testProtoFromDataFrames(self):
method testNdarrayToEntry (line 54) | def testNdarrayToEntry(self):
method testNdarrayToEntryTimeTypes (line 64) | def testNdarrayToEntryTimeTypes(self):
method testDTypeToType (line 78) | def testDTypeToType(self):
method testGetDatasetsProtoFromEntriesLists (line 96) | def testGetDatasetsProtoFromEntriesLists(self):
method testGetDatasetsProtoSequenceExampleHistogram (line 128) | def testGetDatasetsProtoSequenceExampleHistogram(self):
method testGetDatasetsProtoWithAllowlist (line 151) | def testGetDatasetsProtoWithAllowlist(self):
FILE: tests/_util/http_tests.py
class TestCases (line 23) | class TestCases(unittest.TestCase):
method test_get_request_is_invoked (line 27) | def test_get_request_is_invoked(self, mock_request, mock_response):
method test_post_request_is_invoked (line 36) | def test_post_request_is_invoked(self, mock_request, mock_response):
method test_explicit_post_request_is_invoked (line 44) | def test_explicit_post_request_is_invoked(self, mock_request, mock_res...
method test_query_string_format (line 52) | def test_query_string_format(self, mock_request, mock_response):
method test_formats_json_request (line 63) | def test_formats_json_request(self, mock_request, mock_response):
method test_supports_custom_content (line 75) | def test_supports_custom_content(self, mock_request, mock_response):
method test_parses_json_response (line 87) | def test_parses_json_response(self, mock_request, mock_response):
method test_raises_http_error_json (line 95) | def test_raises_http_error_json(self, mock_request, mock_response):
method test_raises_http_error_str (line 108) | def test_raises_http_error_str(self, mock_request, mock_response):
method test_raises_http_error_bytes (line 120) | def test_raises_http_error_bytes(self, mock_request, mock_response):
method _setup_mocks (line 131) | def _setup_mocks(mock_request, mock_response, content, status=200):
FILE: tests/_util/lru_cache_tests.py
class TestCases (line 20) | class TestCases(unittest.TestCase):
method test_cache_no_entry (line 22) | def test_cache_no_entry(self):
method test_cache_lookup (line 27) | def test_cache_lookup(self):
method test_cache_overflow (line 35) | def test_cache_overflow(self):
FILE: tests/_util/util_tests.py
class TestCases (line 28) | class TestCases(unittest.TestCase):
method _get_data (line 31) | def _get_data():
method test_no_entry (line 44) | def test_no_entry(self):
method test_entry (line 52) | def test_entry(self):
method test_compare_datetimes (line 60) | def test_compare_datetimes(self):
method test_compare_datetimes_tz (line 67) | def test_compare_datetimes_tz(self):
method test_get_config_dir (line 78) | def test_get_config_dir(self, mock_expand_user):
method test_get_config_dir_win (line 87) | def test_get_config_dir_win(self, mock_path_join):
method test_get_credentials_from_file (line 99) | def test_get_credentials_from_file(self, mock_path_exists, mock_google...
method test_save_project_id (line 147) | def test_save_project_id(self, mock_path_exists, mock_subprocess_call):
method test_get_default_project_id (line 167) | def test_get_default_project_id(self, mock_path_exists, mock_subproces...
method test_iterator (line 191) | def test_iterator(self):
FILE: tests/bigquery/api_tests.py
class TestCases (line 25) | class TestCases(unittest.TestCase):
method validate (line 27) | def validate(self, mock_http_request, expected_url, expected_args=None...
method test_jobs_insert_load (line 50) | def test_jobs_insert_load(self, mock_http_request):
method test_jobs_insert_query (line 107) | def test_jobs_insert_query(self, mock_http_request):
method test_jobs_query_results (line 158) | def test_jobs_query_results(self, mock_http_request):
method test_jobs_get (line 166) | def test_jobs_get(self, mock_http_request):
method test_datasets_insert (line 173) | def test_datasets_insert(self, mock_http_request):
method test_datasets_delete (line 200) | def test_datasets_delete(self, mock_http_request):
method test_datasets_update (line 212) | def test_datasets_update(self, mock_http_request):
method test_datasets_get (line 219) | def test_datasets_get(self, mock_http_request):
method test_datasets_list (line 225) | def test_datasets_list(self, mock_http_request):
method test_tables_get (line 238) | def test_tables_get(self, mock_http_request):
method test_tables_list (line 245) | def test_tables_list(self, mock_http_request):
method test_tables_insert (line 258) | def test_tables_insert(self, mock_http_request):
method test_tabledata_insertAll (line 294) | def test_tabledata_insertAll(self, mock_http_request):
method test_tabledata_list (line 306) | def test_tabledata_list(self, mock_http_request):
method test_table_delete (line 323) | def test_table_delete(self, mock_http_request):
method test_table_extract (line 331) | def test_table_extract(self, mock_http_request):
method test_table_update (line 379) | def test_table_update(self, mock_http_request):
method _create_api (line 387) | def _create_api(context=None):
method _create_context (line 393) | def _create_context():
FILE: tests/bigquery/dataset_tests.py
class TestCases (line 25) | class TestCases(unittest.TestCase):
method _check_name_parts (line 27) | def _check_name_parts(self, dataset):
method test_parse_full_name (line 34) | def test_parse_full_name(self):
method test_parse_local_name (line 38) | def test_parse_local_name(self):
method test_parse_dict_full_name (line 42) | def test_parse_dict_full_name(self):
method test_parse_dict_local_name (line 46) | def test_parse_dict_local_name(self):
method test_parse_named_tuple_name (line 50) | def test_parse_named_tuple_name(self):
method test_parse_tuple_full_name (line 55) | def test_parse_tuple_full_name(self):
method test_parse_tuple_local (line 59) | def test_parse_tuple_local(self):
method test_parse_array_full_name (line 63) | def test_parse_array_full_name(self):
method test_parse_array_local (line 67) | def test_parse_array_local(self):
method test_parse_invalid_name (line 71) | def test_parse_invalid_name(self):
method test_dataset_exists (line 76) | def test_dataset_exists(self, mock_api_datasets_get):
method test_datasets_create_fails (line 86) | def test_datasets_create_fails(self, mock_api_datasets_get, mock_api_d...
method test_datasets_create_succeeds (line 96) | def test_datasets_create_succeeds(self, mock_api_da
Condensed preview — 438 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (6,795K chars).
[
{
"path": ".coveragerc",
"chars": 96,
"preview": "# .coveragerc to control coverage.py\n[run]\n\n[report]\ninclude = */site-packages/google/datalab/*\n"
},
{
"path": ".gitignore",
"chars": 193,
"preview": "*.pyc\n*.pyi\n*.map\n*.egg-info\n*.iml\n.idea\n.DS_Store\nMANIFEST\nbuild\n.coverage\ndist\ndatalab.magics.rst\ndatalab/notebook/sta"
},
{
"path": ".travis.yml",
"chars": 528,
"preview": "language: python\ndist: trusty\nsudo: false\n\nmatrix:\n include:\n - python: 2.7\n env: TOX_ENV=py27\n - python: 3.5\n "
},
{
"path": "CONTRIBUTING.md",
"chars": 1947,
"preview": "Want to contribute? Great! First, read this page (including the small print at the end).\n\n### Before you contribute\nBefo"
},
{
"path": "LICENSE.txt",
"chars": 11359,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 654,
"preview": "# Google Cloud DataLab\n\nDatalab is deprecated. [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench) "
},
{
"path": "datalab/README",
"chars": 599,
"preview": "Everything under datalab namespace is actively maintained but no new features are being added. Please use corresponding "
},
{
"path": "datalab/__init__.py",
"chars": 588,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/__init__.py",
"chars": 2325,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_api.py",
"chars": 21200,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_csv_options.py",
"chars": 3259,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_dataset.py",
"chars": 8970,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_dialect.py",
"chars": 1547,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_federated_table.py",
"chars": 4086,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_job.py",
"chars": 2983,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_parser.py",
"chars": 3103,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_query.py",
"chars": 25430,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_query_job.py",
"chars": 3883,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_query_results_table.py",
"chars": 2402,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_query_stats.py",
"chars": 1643,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_sampling.py",
"chars": 5151,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_schema.py",
"chars": 13893,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_table.py",
"chars": 35768,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_udf.py",
"chars": 3301,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_utils.py",
"chars": 6273,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/_view.py",
"chars": 11620,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/commands/__init__.py",
"chars": 679,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/bigquery/commands/_bigquery.py",
"chars": 43171,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/context/__init__.py",
"chars": 786,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/context/_api.py",
"chars": 1553,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/context/_context.py",
"chars": 3372,
"preview": "# Copyright 2014 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "datalab/context/_project.py",
"chars": 3439,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/context/_utils.py",
"chars": 5412,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "datalab/context/commands/__init__.py",
"chars": 679,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/context/commands/_projects.py",
"chars": 2323,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/__init__.py",
"chars": 915,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/_csv.py",
"chars": 7063,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/_sql_module.py",
"chars": 4625,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/_sql_statement.py",
"chars": 7788,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/_utils.py",
"chars": 5038,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/data/commands/__init__.py",
"chars": 668,
"preview": "from __future__ import absolute_import\n# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache L"
},
{
"path": "datalab/data/commands/_sql.py",
"chars": 15107,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/kernel/__init__.py",
"chars": 5640,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/notebook/__init__.py",
"chars": 898,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/notebook/static/bigquery.css",
"chars": 1009,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/bigquery.ts",
"chars": 3799,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/charting.css",
"chars": 2083,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/charting.ts",
"chars": 32558,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/element.ts",
"chars": 1698,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/extern/d3.parcoords.css",
"chars": 1142,
"preview": ".parcoords > svg, .parcoords > canvas {\n /*font: 14px sans-serif;*/\n position: absolute;\n}\n.parcoords > canvas {\n poi"
},
{
"path": "datalab/notebook/static/extern/d3.parcoords.js",
"chars": 39110,
"preview": "d3.parcoords = function(config) {\n var __ = {\n data: [],\n highlighted: [],\n dimensions: [],\n dimensionTitle"
},
{
"path": "datalab/notebook/static/extern/lantern-browser.html",
"chars": 1042271,
"preview": "<html><head><!--\n@license\nCopyright (c) 2016 The Polymer Project Authors. All rights reserved.\nThis code may only be use"
},
{
"path": "datalab/notebook/static/extern/parcoords-LICENSE.txt",
"chars": 1412,
"preview": "Copyright (c) 2012, Kai Chang\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nm"
},
{
"path": "datalab/notebook/static/extern/sylvester-LICENSE.txt",
"chars": 1080,
"preview": "(The MIT License)\n\nCopyright (c) 2007-2015 James Coglan\n\nPermission is hereby granted, free of charge, to any person obt"
},
{
"path": "datalab/notebook/static/extern/sylvester.js",
"chars": 43637,
"preview": "// === Sylvester ===\n// Vector and Matrix mathematics modules for JavaScript\n// Copyright (c) 2007 James Coglan\n// \n// P"
},
{
"path": "datalab/notebook/static/job.css",
"chars": 706,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/job.ts",
"chars": 2386,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/parcoords.ts",
"chars": 5337,
"preview": "/*\n * Copyright 2016 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/style.ts",
"chars": 2428,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/notebook/static/visualization.ts",
"chars": 2646,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "datalab/stackdriver/__init__.py",
"chars": 656,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/commands/__init__.py",
"chars": 683,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/commands/_monitoring.py",
"chars": 4071,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/__init__.py",
"chars": 1114,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_group.py",
"chars": 2940,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_metric.py",
"chars": 3345,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_query.py",
"chars": 2969,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_query_metadata.py",
"chars": 3384,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_resource.py",
"chars": 2837,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/stackdriver/monitoring/_utils.py",
"chars": 2412,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/__init__.py",
"chars": 814,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/_api.py",
"chars": 11362,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/_bucket.py",
"chars": 7868,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/_item.py",
"chars": 9068,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/commands/__init__.py",
"chars": 677,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/storage/commands/_storage.py",
"chars": 15320,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/__init__.py",
"chars": 1464,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_async.py",
"chars": 3389,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_dataflow_job.py",
"chars": 1506,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_gcp_job.py",
"chars": 1521,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_http.py",
"chars": 5804,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_iterator.py",
"chars": 1641,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_job.py",
"chars": 8370,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_json_encoder.py",
"chars": 1166,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_lambda_job.py",
"chars": 1522,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_lru_cache.py",
"chars": 3090,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/_utils.py",
"chars": 3247,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/__init__.py",
"chars": 973,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_chart.py",
"chars": 2792,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_chart_data.py",
"chars": 2567,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_commands.py",
"chars": 2839,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_csv.py",
"chars": 2397,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_extension.py",
"chars": 1692,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_html.py",
"chars": 7741,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_job.py",
"chars": 2819,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_modules.py",
"chars": 2055,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "datalab/utils/commands/_utils.py",
"chars": 25707,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "docs/.nojekyll",
"chars": 0,
"preview": ""
},
{
"path": "docs/Makefile",
"chars": 8416,
"preview": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD "
},
{
"path": "docs/README",
"chars": 294,
"preview": "To use, install the prerequisites and the pydatalab module:\n\n pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleo"
},
{
"path": "docs/conf.py",
"chars": 11389,
"preview": "# -*- coding: utf-8 -*-\n#\n# api documentation build configuration file, created by\n# sphinx-quickstart on Tue Nov 3 12:"
},
{
"path": "docs/datalab Commands.rst",
"chars": 6866,
"preview": "datalab Commands\n=======================\n\n.. attribute:: %bigquery\n.. parsed-literal::\n\n usage: bigquery [-h]\n "
},
{
"path": "docs/datalab.bigquery.rst",
"chars": 1096,
"preview": "datalab.bigquery Module\n=======================\n\n.. automodule:: datalab.bigquery\n :members:\n :undoc-members:\n "
},
{
"path": "docs/datalab.context.rst",
"chars": 210,
"preview": "datalab.context Module\n======================\n\n.. autoclass:: datalab.context.Context\n :members:\n\n.. autoclass:: data"
},
{
"path": "docs/datalab.data.rst",
"chars": 284,
"preview": "datalab.data Module\n===================\n\n.. automodule:: datalab.data\n :members:\n :undoc-members:\n :show-inheri"
},
{
"path": "docs/datalab.stackdriver.monitoring.rst",
"chars": 446,
"preview": "datalab.stackdriver.monitoring Module\n=====================================\n\n.. autoclass:: datalab.stackdriver.monitori"
},
{
"path": "docs/datalab.storage.rst",
"chars": 347,
"preview": "datalab.storage Module\n======================\n\n.. automodule:: datalab.storage\n :members:\n :undoc-members:\n :sh"
},
{
"path": "docs/gen-magic-rst.ipy",
"chars": 1758,
"preview": "import subprocess, pkgutil, importlib, sys\nfrom cStringIO import StringIO\n\n# import submodules\ndatalab_submodules = ['da"
},
{
"path": "docs/google.datalab Commands.rst",
"chars": 6615,
"preview": "google.datalab Commands\n=======================\n\n.. attribute:: %bq\n.. parsed-literal::\n\n usage: %bq [-h]\n "
},
{
"path": "docs/google.datalab.bigquery.rst",
"chars": 1251,
"preview": "google.datalab.bigquery Module\n==============================\n\n.. automodule:: google.datalab.bigquery\n :members:\n "
},
{
"path": "docs/google.datalab.data.rst",
"chars": 207,
"preview": "google.datalab.data Module\n==========================\n\n.. automodule:: google.datalab.data\n :members:\n :undoc-memb"
},
{
"path": "docs/google.datalab.ml.rst",
"chars": 801,
"preview": "google.datalab.ml Module\n========================\n\n.. automodule:: google.datalab.ml\n :members:\n :undoc-members:\n "
},
{
"path": "docs/google.datalab.rst",
"chars": 236,
"preview": "google.datalab Module\n=====================\n\n.. automodule:: google.datalab\n :members:\n :undoc-members:\n :show-"
},
{
"path": "docs/google.datalab.stackdriver.monitoring.rst",
"chars": 495,
"preview": "google.datalab.stackdriver.monitoring Module\n============================================\n\n.. autoclass:: google.datalab"
},
{
"path": "docs/google.datalab.storage.rst",
"chars": 400,
"preview": "google.datalab.storage Module\n=============================\n\n.. automodule:: google.datalab.storage\n :members:\n :u"
},
{
"path": "docs/index.rst",
"chars": 949,
"preview": "Welcome to Cloud Datalab's documentation\n========================================\n\ngoogle.datalab namespace\n############"
},
{
"path": "docs/make.bat",
"chars": 7238,
"preview": "@ECHO OFF\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\n"
},
{
"path": "docs/mltoolbox.classification.dnn.rst",
"chars": 580,
"preview": "mltoolbox.classification.dnn\n============================\n\n.. automodule:: mltoolbox.classification.dnn\n :members:\n "
},
{
"path": "docs/mltoolbox.classification.linear.rst",
"chars": 610,
"preview": "mltoolbox.classification.linear\n===============================\n\n.. automodule:: mltoolbox.classification.linear\n :me"
},
{
"path": "docs/mltoolbox.image.classification.rst",
"chars": 606,
"preview": "mltoolbox.image.classification\n==============================\n\n.. automodule:: mltoolbox.image.classification\n :membe"
},
{
"path": "docs/mltoolbox.regression.dnn.rst",
"chars": 540,
"preview": "mltoolbox.regression.dnn\n========================\n\n.. automodule:: mltoolbox.regression.dnn\n :members:\n :undoc-mem"
},
{
"path": "docs/mltoolbox.regression.linear.rst",
"chars": 570,
"preview": "mltoolbox.regression.linear\n===========================\n\n.. automodule:: mltoolbox.regression.linear\n :members:\n :"
},
{
"path": "externs/ts/require/require.d.ts",
"chars": 9366,
"preview": "// Type definitions for RequireJS 2.1.20\n// Project: http://requirejs.org/\n// Definitions by: Josh Baldwin <https://gith"
},
{
"path": "google/__init__.py",
"chars": 588,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/__init__.py",
"chars": 884,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/_context.py",
"chars": 3907,
"preview": "# Copyright 2014 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/_job.py",
"chars": 7503,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/__init__.py",
"chars": 1437,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_api.py",
"chars": 20890,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_csv_options.py",
"chars": 3259,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_dataset.py",
"chars": 8953,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_external_data_source.py",
"chars": 3665,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_job.py",
"chars": 3022,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_parser.py",
"chars": 3103,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_query.py",
"chars": 21547,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_query_job.py",
"chars": 3870,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_query_output.py",
"chars": 5455,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_query_results_table.py",
"chars": 2501,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_query_stats.py",
"chars": 1643,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_sampling.py",
"chars": 6074,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_schema.py",
"chars": 12451,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_table.py",
"chars": 33007,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_udf.py",
"chars": 4191,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_utils.py",
"chars": 6273,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/_view.py",
"chars": 5053,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/commands/__init__.py",
"chars": 679,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/bigquery/commands/_bigquery.py",
"chars": 51028,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/commands/__init__.py",
"chars": 729,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/commands/_datalab.py",
"chars": 3565,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/__init__.py",
"chars": 588,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/__init__.py",
"chars": 588,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/commands/__init__.py",
"chars": 648,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/commands/_bigquery.py",
"chars": 10304,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/operators/__init__.py",
"chars": 588,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/operators/_bq_execute_operator.py",
"chars": 3158,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/operators/_bq_extract_operator.py",
"chars": 1839,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/bigquery/operators/_bq_load_operator.py",
"chars": 2667,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/mlworkbench/__init__.py",
"chars": 898,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/mlworkbench/_archive.py",
"chars": 2088,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/mlworkbench/_local_predict.py",
"chars": 14116,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/mlworkbench/_prediction_explainer.py",
"chars": 18756,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/mlworkbench/_shell_process.py",
"chars": 2694,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/mlworkbench/commands/__init__.py",
"chars": 667,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/mlworkbench/commands/_ml.py",
"chars": 53995,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/contrib/pipeline/__init__.py",
"chars": 588,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/_pipeline.py",
"chars": 15536,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/airflow/__init__.py",
"chars": 626,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/airflow/_airflow.py",
"chars": 1619,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/commands/__init__.py",
"chars": 588,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/commands/_pipeline.py",
"chars": 4985,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/composer/__init__.py",
"chars": 628,
"preview": "# Copyright 2018 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/composer/_api.py",
"chars": 1562,
"preview": "# Copyright 2018 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/contrib/pipeline/composer/_composer.py",
"chars": 2541,
"preview": "# Copyright 2018 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/data/__init__.py",
"chars": 782,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/data/_csv_file.py",
"chars": 7141,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/kernel/__init__.py",
"chars": 4980,
"preview": "# Copyright 2015 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/__init__.py",
"chars": 1147,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_cloud_models.py",
"chars": 11760,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_cloud_training_config.py",
"chars": 2614,
"preview": "# Copyright 2017 Google Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/ml/_confusion_matrix.py",
"chars": 5872,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_dataset.py",
"chars": 8647,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_fasets.py",
"chars": 3449,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_feature_slice_view.py",
"chars": 3621,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_job.py",
"chars": 7143,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_metrics.py",
"chars": 16353,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_summary.py",
"chars": 5983,
"preview": "# Copyright 2017 Google Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/ml/_tensorboard.py",
"chars": 2946,
"preview": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/ml/_util.py",
"chars": 3427,
"preview": "# Copyright 2017 Google Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# "
},
{
"path": "google/datalab/notebook/__init__.py",
"chars": 898,
"preview": "# Copyright 2016 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); yo"
},
{
"path": "google/datalab/notebook/static/bigquery.css",
"chars": 1009,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "google/datalab/notebook/static/bigquery.ts",
"chars": 3802,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "google/datalab/notebook/static/charting.css",
"chars": 2083,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "google/datalab/notebook/static/charting.ts",
"chars": 36217,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "google/datalab/notebook/static/element.ts",
"chars": 1701,
"preview": "/*\n * Copyright 2015 Google Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "google/datalab/notebook/static/extern/d3.parcoords.css",
"chars": 1142,
"preview": ".parcoords > svg, .parcoords > canvas {\n /*font: 14px sans-serif;*/\n position: absolute;\n}\n.parcoords > canvas {\n poi"
},
{
"path": "google/datalab/notebook/static/extern/d3.parcoords.js",
"chars": 39110,
"preview": "d3.parcoords = function(config) {\n var __ = {\n data: [],\n highlighted: [],\n dimensions: [],\n dimensionTitle"
},
{
"path": "google/datalab/notebook/static/extern/facets-jupyter.html",
"chars": 1728783,
"preview": "<!doctype html><script>define=undefined</script><!--\n@license\nCopyright 2017 Google Inc.\n\nLicensed under the Apache Lice"
},
{
"path": "google/datalab/notebook/static/extern/lantern-browser.html",
"chars": 1042271,
"preview": "<html><head><!--\n@license\nCopyright (c) 2016 The Polymer Project Authors. All rights reserved.\nThis code may only be use"
},
{
"path": "google/datalab/notebook/static/extern/parcoords-LICENSE.txt",
"chars": 1412,
"preview": "Copyright (c) 2012, Kai Chang\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nm"
},
{
"path": "google/datalab/notebook/static/extern/sylvester-LICENSE.txt",
"chars": 1080,
"preview": "(The MIT License)\n\nCopyright (c) 2007-2015 James Coglan\n\nPermission is hereby granted, free of charge, to any person obt"
}
]
// ... and 238 more files (download for full content)
About this extraction
This page contains the full source code of the googledatalab/pydatalab GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 438 files (6.3 MB), approximately 1.7M tokens, and a symbol index with 2933 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.