Copy disabled (too large)
Download .txt
Showing preview only (18,987K chars total). Download the full file to get everything.
Repository: kubeflow/community
Branch: master
Commit: 7fc0641842fb
Files: 500
Total size: 31.4 MB
Directory structure:
gitextract_nd5su600/
├── .github/
│ ├── issue_label_bot.yaml
│ └── workflows/
│ └── stale.yml
├── .gitignore
├── ADOPTERS.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── INCLUSIVITY.md
├── KUBEFLOW-GENERAL-TECHNICAL-REVIEW.md
├── KUBEFLOW-OUTREACH-COMMITTEE.md
├── KUBEFLOW-STEERING-COMMITTEE.md
├── LICENSE
├── MAINTAINERS.md
├── Makefile
├── OWNERS
├── OWNERS_ALIASES
├── README.md
├── calendar/
│ ├── Dockerfile
│ ├── Makefile
│ ├── OWNERS
│ ├── README.md
│ ├── calendar.yaml
│ ├── calendar_import.py
│ ├── latest_image.yaml
│ └── manifests/
│ ├── deployment.yaml
│ ├── kustomization.yaml
│ └── service-account.yaml
├── dco-signoff-hook/
│ ├── README.md
│ └── prepare-commit-msg
├── devstats/
│ ├── Dockerfile.devstats
│ ├── Makefile
│ ├── README.md
│ ├── config/
│ │ ├── INSTALL_UBUNTU18.md
│ │ ├── README.md
│ │ ├── SETUP_OTHER_PROJECT.md
│ │ ├── copy_devstats_binaries.sh
│ │ ├── cron/
│ │ │ └── net_tcp_config.sh
│ │ ├── crontab
│ │ ├── deploy.sh
│ │ ├── devel/
│ │ │ ├── add_single_metric.sh
│ │ │ ├── add_single_metric_all.sh
│ │ │ ├── create_databases.sh
│ │ │ ├── create_grafana.sh
│ │ │ ├── create_psql_user.sh
│ │ │ ├── deploy_all.sh
│ │ │ ├── deploy_proj.sh
│ │ │ ├── drop_psql_db.sh
│ │ │ ├── drop_ts_tables.sh
│ │ │ ├── drop_tsdb_affs_tables.sh
│ │ │ ├── get_all_sqlite_jsons.sh
│ │ │ ├── get_from_sqlite.sh
│ │ │ ├── get_icon_type.sh
│ │ │ ├── grafana_start.sh
│ │ │ ├── grafana_stop.sh
│ │ │ ├── import_jsons_to_sqlite.sh
│ │ │ ├── init_database.sh
│ │ │ ├── mass_replace.sh
│ │ │ ├── net_tcp_config.sh
│ │ │ ├── psql_user_grants.sh
│ │ │ ├── put_all_charts.sh
│ │ │ ├── put_all_charts_cleanup.sh
│ │ │ ├── restore_db.sh
│ │ │ ├── ro_user_grants.sh
│ │ │ ├── sync_lock.sh
│ │ │ ├── sync_unlock.sh
│ │ │ ├── test_metrics.yaml
│ │ │ ├── test_tags.yaml
│ │ │ ├── update_from_sqlite.sh
│ │ │ └── vars_all.sh
│ │ ├── devstats.sh
│ │ ├── docs/
│ │ │ ├── annotations.md
│ │ │ ├── dashboards/
│ │ │ │ ├── dashboards.md
│ │ │ │ ├── dashboards_devel.md
│ │ │ │ ├── kubernetes/
│ │ │ │ │ ├── blocked_prs.md
│ │ │ │ │ ├── bot_commands.md
│ │ │ │ │ ├── companies_table.md
│ │ │ │ │ ├── company_stats.md
│ │ │ │ │ ├── contributing_companies.md
│ │ │ │ │ ├── developer_stats.md
│ │ │ │ │ ├── episodic_issues.md
│ │ │ │ │ ├── episodic_prs.md
│ │ │ │ │ ├── gh_stats_commits.md
│ │ │ │ │ ├── gh_stats_iclosed.md
│ │ │ │ │ ├── gh_stats_icommenters.md
│ │ │ │ │ ├── gh_stats_icomments.md
│ │ │ │ │ ├── gh_stats_iopened.md
│ │ │ │ │ ├── gh_stats_prclosed.md
│ │ │ │ │ ├── gh_stats_prcommenters.md
│ │ │ │ │ ├── gh_stats_prcomments.md
│ │ │ │ │ ├── gh_stats_prmerged.md
│ │ │ │ │ ├── gh_stats_propened.md
│ │ │ │ │ ├── gh_stats_reviewers.md
│ │ │ │ │ ├── ghstats_devel.md
│ │ │ │ │ ├── issues_age.md
│ │ │ │ │ ├── issues_opened_closed.md
│ │ │ │ │ ├── pr_approval.md
│ │ │ │ │ ├── pr_approve_to_merge.md
│ │ │ │ │ ├── pr_authors.md
│ │ │ │ │ ├── pr_comments.md
│ │ │ │ │ ├── pr_labels.md
│ │ │ │ │ ├── pr_reviews_by_contributor.md
│ │ │ │ │ ├── pr_time_to_engagement.md
│ │ │ │ │ ├── pr_workload.md
│ │ │ │ │ ├── project_stats.md
│ │ │ │ │ ├── prs_age.md
│ │ │ │ │ ├── sig_mentions.md
│ │ │ │ │ ├── sig_mentions_devel.md
│ │ │ │ │ ├── sig_milestones.md
│ │ │ │ │ └── stars_and_forks.md
│ │ │ │ └── shared/
│ │ │ │ ├── activity.md
│ │ │ │ ├── commits.md
│ │ │ │ ├── community_stats.md
│ │ │ │ ├── companies_stats.md
│ │ │ │ ├── companies_summary.md
│ │ │ │ ├── company_prs.md
│ │ │ │ ├── contributing_companies.md
│ │ │ │ ├── countries_stats.md
│ │ │ │ ├── dashboards.md
│ │ │ │ ├── developers_summary.md
│ │ │ │ ├── gender_stats.md
│ │ │ │ ├── github_events_docs_html.md
│ │ │ │ ├── issues.md
│ │ │ │ ├── issues_age.md
│ │ │ │ ├── new_and_episodic_issues.md
│ │ │ │ ├── new_and_episodic_prs.md
│ │ │ │ ├── new_contributors.md
│ │ │ │ ├── new_prs.md
│ │ │ │ ├── non_author_activity.md
│ │ │ │ ├── opened_to_merged.md
│ │ │ │ ├── pr_authors.md
│ │ │ │ ├── pr_comments.md
│ │ │ │ ├── pr_companies.md
│ │ │ │ ├── project_health.md
│ │ │ │ ├── projects_health.md
│ │ │ │ ├── projects_stats.md
│ │ │ │ ├── prs_age.md
│ │ │ │ ├── prs_approval.md
│ │ │ │ ├── prs_authors_chart.md
│ │ │ │ ├── prs_merged.md
│ │ │ │ ├── repo_commenters.md
│ │ │ │ ├── repo_comments.md
│ │ │ │ ├── stub.md
│ │ │ │ ├── time_metrics.md
│ │ │ │ ├── top_commenters.md
│ │ │ │ ├── tzs_stats.md
│ │ │ │ ├── user_reviews.md
│ │ │ │ └── users_stats.md
│ │ │ ├── excluding_bots.md
│ │ │ ├── github.md
│ │ │ ├── periods.md
│ │ │ ├── presentation/
│ │ │ │ ├── DevStatsPresentation.odp
│ │ │ │ └── DevStatsPresentation.pptx
│ │ │ ├── repository_aliases.md
│ │ │ ├── repository_groups.md
│ │ │ ├── tables/
│ │ │ │ ├── const_table.md
│ │ │ │ ├── gha_actors.md
│ │ │ │ ├── gha_comments.md
│ │ │ │ ├── gha_commits.md
│ │ │ │ ├── gha_commits_files.md
│ │ │ │ ├── gha_events.md
│ │ │ │ ├── gha_events_commits_files.md
│ │ │ │ ├── gha_issues.md
│ │ │ │ ├── gha_issues_events_labels.md
│ │ │ │ ├── gha_issues_labels.md
│ │ │ │ ├── gha_issues_pull_requests.md
│ │ │ │ ├── gha_labels.md
│ │ │ │ ├── gha_milestones.md
│ │ │ │ ├── gha_orgs.md
│ │ │ │ ├── gha_payloads.md
│ │ │ │ ├── gha_postprocess_scripts.md
│ │ │ │ ├── gha_pull_requests.md
│ │ │ │ ├── gha_repos.md
│ │ │ │ ├── gha_skip_commits.md
│ │ │ │ ├── gha_texts.md
│ │ │ │ ├── gha_vars.md
│ │ │ │ └── variable_table.md
│ │ │ ├── tags.md
│ │ │ └── vars.md
│ │ ├── find.sh
│ │ ├── git/
│ │ │ ├── git_files.sh
│ │ │ ├── git_reset_pull.sh
│ │ │ └── git_tags.sh
│ │ ├── github_users.json
│ │ ├── grafana/
│ │ │ ├── dashboards/
│ │ │ │ └── kubeflow/
│ │ │ │ ├── activity-repository-groups.json
│ │ │ │ ├── commits-repository-groups.json
│ │ │ │ ├── community-stats.json
│ │ │ │ ├── companies-stats.json
│ │ │ │ ├── companies-summary.json
│ │ │ │ ├── company-prs-in-repository-groups.json
│ │ │ │ ├── contributing-companies.json
│ │ │ │ ├── countries-stats.json
│ │ │ │ ├── dashboards.json
│ │ │ │ ├── developers-summary.json
│ │ │ │ ├── first-non-author-activity.json
│ │ │ │ ├── gender-stats.json
│ │ │ │ ├── github-events.json
│ │ │ │ ├── issues-age.json
│ │ │ │ ├── issues-repository-group.json
│ │ │ │ ├── new-and-episodic-issue-creators.json
│ │ │ │ ├── new-and-episodic-pr-contributors.json
│ │ │ │ ├── new-contributors-table.json
│ │ │ │ ├── new-prs.json
│ │ │ │ ├── opened-to-merged.json
│ │ │ │ ├── pr-comments.json
│ │ │ │ ├── project-statistics.json
│ │ │ │ ├── prs-age.json
│ │ │ │ ├── prs-approval.json
│ │ │ │ ├── prs-authors-companies-histogram.json
│ │ │ │ ├── prs-authors-histogram.json
│ │ │ │ ├── prs-authors.json
│ │ │ │ ├── prs-merged-repository-groups.json
│ │ │ │ ├── repository-commenters.json
│ │ │ │ ├── repository-comments.json
│ │ │ │ ├── time-metrics.json
│ │ │ │ ├── timezones-stats.json
│ │ │ │ ├── top-commenters.json
│ │ │ │ ├── user-reviews.json
│ │ │ │ └── users-stats.json
│ │ │ ├── etc/
│ │ │ │ └── grafana.ini.example
│ │ │ ├── kubeflow/
│ │ │ │ ├── change_title_and_icons.sh
│ │ │ │ └── grafana_start.sh
│ │ │ └── provisioning/
│ │ │ ├── dashboards/
│ │ │ │ └── providers.yaml
│ │ │ └── datasources/
│ │ │ └── datasource.yaml
│ │ ├── grafana.sh
│ │ ├── hide/
│ │ │ └── hide.csv
│ │ ├── kubeflow/
│ │ │ └── psql.sh
│ │ ├── metrics/
│ │ │ ├── kubeflow/
│ │ │ │ ├── sync_vars.yaml
│ │ │ │ └── vars.yaml
│ │ │ └── shared/
│ │ │ ├── activity_repo_groups.sql
│ │ │ ├── all_prs_merged.sql
│ │ │ ├── columns.yaml
│ │ │ ├── columns_affs.yaml
│ │ │ ├── commits_repo_groups.sql
│ │ │ ├── committers_countries.sql
│ │ │ ├── committers_countries_cum.sql
│ │ │ ├── committers_gender.sql
│ │ │ ├── committers_gender_cum.sql
│ │ │ ├── committers_tz.sql
│ │ │ ├── companies_tags.sql
│ │ │ ├── company_activity.sql
│ │ │ ├── company_activity_commits.sql
│ │ │ ├── company_prs.sql
│ │ │ ├── countries.sql
│ │ │ ├── countries_cum.sql
│ │ │ ├── countries_tags.sql
│ │ │ ├── cumulative_periods.sql
│ │ │ ├── episodic_contributors.sql
│ │ │ ├── episodic_issues.sql
│ │ │ ├── es_periods_tags.sql
│ │ │ ├── event_types.sql
│ │ │ ├── event_types_tags.sql
│ │ │ ├── events.sql
│ │ │ ├── first_non_author_activity.sql
│ │ │ ├── gender.sql
│ │ │ ├── gender_cum.sql
│ │ │ ├── hist_commenters.sql
│ │ │ ├── hist_pr_authors.sql
│ │ │ ├── hist_pr_companies.sql
│ │ │ ├── issues_age.sql
│ │ │ ├── issues_closed.sql
│ │ │ ├── issues_opened.sql
│ │ │ ├── labels_priorities_tags_with_all.sql
│ │ │ ├── metrics.yaml
│ │ │ ├── metrics_affs.yaml
│ │ │ ├── new_contributors.sql
│ │ │ ├── new_contributors_data.sql
│ │ │ ├── new_issues.sql
│ │ │ ├── new_prs.sql
│ │ │ ├── num_stats.sql
│ │ │ ├── opened_to_merged.sql
│ │ │ ├── pr_comments.sql
│ │ │ ├── project_company_stats.sql
│ │ │ ├── project_developer_stats.sql
│ │ │ ├── project_stats.sql
│ │ │ ├── projects_health.sql
│ │ │ ├── prs_age.sql
│ │ │ ├── prs_authors.sql
│ │ │ ├── prs_merged_groups.sql
│ │ │ ├── prs_state.sql
│ │ │ ├── repo_commenters.sql
│ │ │ ├── repo_comments.sql
│ │ │ ├── repo_groups_tags.sql
│ │ │ ├── repo_groups_tags_with_all.sql
│ │ │ ├── reviewers_tags.sql
│ │ │ ├── reviews_per_user.sql
│ │ │ ├── tags.yaml
│ │ │ ├── tags_affs.yaml
│ │ │ ├── time_metrics.sql
│ │ │ ├── top_repo_aliases_tags.sql
│ │ │ ├── top_repos_tags_with_all.sql
│ │ │ ├── tz.sql
│ │ │ ├── tz_offset_tags.sql
│ │ │ ├── user_activity.sql
│ │ │ ├── user_activity_commits.sql
│ │ │ ├── users_tags.sql
│ │ │ └── watchers.sql
│ │ ├── partials/
│ │ │ ├── projects.html
│ │ │ └── projects_health.html
│ │ ├── projects.yaml
│ │ ├── run.sh
│ │ ├── scripts/
│ │ │ ├── clean_affiliations.sql
│ │ │ └── kubeflow/
│ │ │ └── repo_groups.sql
│ │ ├── shared/
│ │ │ ├── get_repos.sh
│ │ │ ├── import_affs.sh
│ │ │ ├── reinit.sh
│ │ │ ├── setup_repo_groups.sh
│ │ │ ├── setup_scripts.sh
│ │ │ ├── sync.sh
│ │ │ └── tags.sh
│ │ ├── sqlite/
│ │ │ └── touch
│ │ └── util_sql/
│ │ ├── country_codes.sql
│ │ ├── create_events_commits.sql
│ │ ├── default_postprocess_scripts.sql
│ │ ├── devstats_log_table.sql
│ │ ├── drop_psql_user.sql
│ │ ├── drop_ro_user.sql
│ │ ├── exclude_bots.sql
│ │ ├── list_unprocessed_commits.sql
│ │ ├── postprocess_issues_prs.sql
│ │ ├── postprocess_labels.sql
│ │ ├── postprocess_repo_groups_from_repos.sql
│ │ ├── postprocess_texts.sql
│ │ ├── repo_groups_postprocess_script_from_repos.sql
│ │ └── update_country_names.sql
│ ├── data/
│ │ └── github_users.json
│ ├── deploy-config/
│ │ ├── devstats.jinja
│ │ ├── devstats.yaml
│ │ └── gcfs.yaml
│ ├── k8s_manifests/
│ │ ├── cli_home_pvc.yaml
│ │ └── nfs_pvc.yaml
│ ├── ks-app/
│ │ ├── .ksonnet/
│ │ │ └── registries/
│ │ │ ├── incubator/
│ │ │ │ └── 40285d8a14f1ac5787e405e1023cf0c07f6aa28c.yaml
│ │ │ └── kubeflow/
│ │ │ └── e2c2c7c49e578472ba57f91f3b2a5f4d1a2d0289.yaml
│ │ ├── app.yaml
│ │ ├── components/
│ │ │ ├── backfill.jsonnet
│ │ │ ├── cert-manager.jsonnet
│ │ │ ├── devstats-parts.libsonnet
│ │ │ ├── devstats.jsonnet
│ │ │ ├── params.libsonnet
│ │ │ └── syncronjob.jsonnet
│ │ ├── environments/
│ │ │ ├── base.libsonnet
│ │ │ └── devstats2/
│ │ │ ├── globals.libsonnet
│ │ │ ├── main.jsonnet
│ │ │ └── params.libsonnet
│ │ ├── lib/
│ │ │ └── ksonnet-lib/
│ │ │ ├── v1.11.6/
│ │ │ │ ├── k.libsonnet
│ │ │ │ ├── k8s.libsonnet
│ │ │ │ └── swagger.json
│ │ │ └── v1.7.0/
│ │ │ ├── k.libsonnet
│ │ │ ├── k8s.libsonnet
│ │ │ └── swagger.json
│ │ └── vendor/
│ │ └── kubeflow/
│ │ └── core@e2c2c7c49e578472ba57f91f3b2a5f4d1a2d0289/
│ │ ├── README.md
│ │ ├── all.libsonnet
│ │ ├── ambassador.libsonnet
│ │ ├── centraldashboard.libsonnet
│ │ ├── cert-manager.libsonnet
│ │ ├── cloud-endpoints.libsonnet
│ │ ├── iap.libsonnet
│ │ ├── jupyterhub.libsonnet
│ │ ├── kubeform_spawner.py
│ │ ├── nfs.libsonnet
│ │ ├── parts.yaml
│ │ ├── prototypes/
│ │ │ ├── all.jsonnet
│ │ │ ├── cert-manager.jsonnet
│ │ │ ├── cloud-endpoints.jsonnet
│ │ │ ├── iap-ingress.jsonnet
│ │ │ └── tensorboard.jsonnet
│ │ ├── spartakus.libsonnet
│ │ ├── tensorboard.libsonnet
│ │ ├── tests/
│ │ │ ├── ambassador_test.jsonnet
│ │ │ ├── iap_test.jsonnet
│ │ │ ├── jupyterhub_test.jsonnet
│ │ │ ├── nfs_test.jsonnet
│ │ │ ├── spartakus_test.jsonnet
│ │ │ ├── tf-job_test.jsonnet
│ │ │ └── util_test.jsonnet
│ │ ├── tf-job.libsonnet
│ │ ├── util.libsonnet
│ │ ├── version-info.json
│ │ └── version.libsonnet
│ ├── modify_dashboards.sh
│ ├── postgre-docker-entrypoint.sh
│ ├── print_imports.sh
│ └── scripts/
│ ├── generate_actors.sh
│ ├── modify_devstats_scripts.sh
│ ├── setup_nfs.sh
│ └── setup_postgres_home.sh
├── elections/
│ ├── eligible-candidates-and-voters-2023-ksc.md
│ ├── eligible-candidates-and-voters-2024-ksc.md
│ ├── eligible-candidates-and-voters-2025-ksc.md
│ ├── kubeflow-steering-committee-elections-2023.md
│ ├── kubeflow-steering-committee-elections-2024.md
│ └── kubeflow-steering-committee-elections-2025.md
├── generator/
│ ├── README.md
│ ├── aliases.tmpl
│ ├── app.go
│ ├── header.tmpl
│ ├── list.tmpl
│ ├── sig_readme.tmpl
│ ├── ug_readme.tmpl
│ └── wg_readme.tmpl
├── go.mod
├── go.sum
├── guidelines/
│ ├── README.md
│ ├── application_requirements.md
│ ├── auth.md
│ └── creating_dockerfiles.md
├── how-to/
│ ├── README.md
│ ├── community_meetings.md
│ ├── github_admin.md
│ ├── join_kubeflow_ecosystem.md
│ ├── kubeflow_assets.md
│ └── sharing_docs.md
├── issue-labels.md
├── ksc/
│ ├── README.md
│ └── decision-log.md
├── labels-owners.yaml
├── member_organizations.yaml
├── proposal-workflow.md
├── proposals/
│ ├── 139-mpi-operator/
│ │ └── README.md
│ ├── 141-chainer-operator/
│ │ └── README.md
│ ├── 263-pvc-template/
│ │ └── README.md
│ ├── 2655-kubeflow-data-cache/
│ │ └── README.md
│ ├── 280-issue-triage/
│ │ └── README.md
│ ├── 30-tf-operator-v1alpha2/
│ │ └── tf-operator-design-v1alpha2.md
│ ├── 33-pytorch-operator/
│ │ └── README.md
│ ├── 335-fate-operator/
│ │ └── README.md
│ ├── 41-caffe2-operator/
│ │ └── README.md
│ ├── 434-kubeflow-distribution/
│ │ └── README.md
│ ├── 502-paddle-operator/
│ │ └── README.md
│ ├── 524-kubeflow-conformance-program/
│ │ └── README.md
│ ├── 525-kfserving-transition/
│ │ └── README.md
│ ├── 585-kubeflow-governance/
│ │ ├── CONFORMANCE-COMMITTEE.md
│ │ ├── GOVERNANCE.md
│ │ └── TECH-OVERSIGHT-COMMITTEE.md
│ ├── 645-kubeflow-steering-committee-election/
│ │ └── README.md
│ ├── 648-spark-operator/
│ │ └── README.md
│ ├── 748-expand-kubeflow-ecosystem/
│ │ └── README.md
│ ├── 819-kubeflow-sdk/
│ │ └── README.md
│ ├── 839-outreach-committee/
│ │ └── README.md
│ ├── 841-release-process/
│ │ └── README.md
│ ├── 867-kubeflow-documentation-ai/
│ │ └── README.md
│ ├── 872-spark-history-server-mcp/
│ │ └── README.md
│ ├── 897-experiment-tracking/
│ │ └── README.md
│ ├── 907-model-registry-renaming/
│ │ └── README.md
│ ├── 913-components-repo/
│ │ └── README.md
│ ├── 936-kubeflow-mcp-server/
│ │ ├── DESIGN.md
│ │ └── README.md
│ ├── NNNN-template/
│ │ └── README.md
│ ├── README.md
│ ├── gsoc/
│ │ ├── README.md
│ │ └── gsoc-proposal-template.md
│ ├── kale-donation.md
│ └── model-registry-proposal.md
├── psg/
│ └── ROADMAP.md
├── repository-setup.md
├── scripts/
│ ├── OWNERS
│ ├── company_stats.ipynb
│ ├── github_stats.ipynb
│ ├── kf1.0_stats.ipynb
│ ├── open_pr_stats.ipynb
│ ├── print_companies.py
│ ├── project_stats.ipynb
│ ├── project_stats.py
│ ├── prs_by_company.py
│ └── requirements.txt
├── security/
│ ├── README.md
│ └── self-assessment.md
├── sig-feature-store/
│ ├── README.md
│ └── charter.md
├── sig-onprem/
│ ├── README.md
│ └── charter.md
├── slack/
│ ├── OWNERS
│ ├── README.md
│ └── slack_channels.md
├── wg-automl/
│ ├── README.md
│ └── charter.md
├── wg-data/
│ ├── README.md
│ └── charter.md
├── wg-deployment/
│ ├── README.md
│ └── charter.md
├── wg-list.md
├── wg-manifests/
│ ├── README.md
│ └── charter.md
├── wg-ml-experience/
│ ├── README.md
│ └── charter.md
├── wg-notebooks/
│ ├── README.md
│ └── charter.md
├── wg-pipelines/
│ ├── README.md
│ └── charter.md
├── wg-serving/
│ ├── README.md
│ └── charter.md
├── wg-training/
│ ├── README.md
│ └── charter.md
├── wgs/
│ ├── community-membership.md
│ ├── overview.md
│ ├── templates/
│ │ └── wg-charter-template.md
│ ├── wg-charter.md
│ ├── wg-foo/
│ │ └── OWNERS
│ ├── wg-governance-requirements.md
│ ├── wg-governance.md
│ └── wg-lifecycle.md
└── wgs.yaml
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/issue_label_bot.yaml
================================================
label-alias:
bug: 'kind/bug'
feature_request: 'kind/feature'
feature: 'kind/feature'
question: 'kind/question'
================================================
FILE: .github/workflows/stale.yml
================================================
# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
#
# You can adjust the behavior by modifying this file.
# For more information, see:
# https://github.com/actions/stale
name: Mark stale issues and pull requests
on:
schedule:
- cron: '0 0 * * *' # Run every day at midnight
jobs:
stale:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/stale@v5
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
days-before-stale: 90
days-before-close: 21
# The message that will be added as a comment to the issues
# when the stale workflow marks it automatically as stale with a label.
stale-issue-message: >
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
# The message that will be added as a comment to the issues
# when the stale workflow closes it automatically after being stale for too long.
close-issue-message: >
This issue has been automatically closed because it has not had recent activity. Please comment "/reopen" to reopen it.
stale-issue-label: lifecycle/stale
# Exclude them from being marked as stale
exempt-issue-labels: lifecycle/frozen,enhancement,good first issue
# The message that will be added as a comment to the pull requests
# when the stale workflow marks it automatically as stale with a label.
stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions. \n"
# The message that will be added as a comment to the pull requests
# when the stale workflow closes it automatically after being stale for too long.
close-pr-message: "This pull request has been automatically closed because it has not had recent activity.You can reopen the PR if you want. \n"
stale-pr-label: lifecycle/stale
# Exclude them from being marked as stale
exempt-pr-labels: lifecycle/frozen,enhancement,good first issue
# The issues or the pull requests with a milestone will not be marked as stale automatically
exempt-all-milestones: true
# Learn more about operations: https://github.com/actions/stale#operations-per-run.
operations-per-run: 250
================================================
FILE: .gitignore
================================================
flymd.*
**.pyc
**.swp
**/.ipynb_checkpoints
**/.cache
.DS_Store
# IntelliJ
.idea/
.vscode/
================================================
FILE: ADOPTERS.md
================================================
# Adopters of Kubeflow
Our adopter files list organizations that have adopted Kubeflow in some way.
Sharing your adoption helps build project momentum, and this benefits everyone.
It's a small contribution with a big impact!
We invite you to add your organization to these lists by raising a pull request on the relevant GitHub repository.
## Adopters of Kubeflow Projects
For adopters of specific Kubeflow Projects, please refer to the respective component's repository.
| Component | Repository | Adopter List |
|-------------------------|-------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|
| Kubeflow Katib | [`kubeflow/katib`](https://github.com/kubeflow/katib) | [`ADOPTERS.md`](https://github.com/kubeflow/katib/blob/master/ADOPTERS.md) |
| Kubeflow MPI Operator | [`kubeflow/mpi-operator`](https://github.com/kubeflow/mpi-operator) | [`ADOPTERS.md`](https://github.com/kubeflow/mpi-operator/blob/master/ADOPTERS.md) |
| Kubeflow Hub | [`kubeflow/hub`](https://github.com/kubeflow/hub) | [`ADOPTERS.md`](https://github.com/kubeflow/hub/blob/master/ADOPTERS.md) |
| Kubeflow Notebooks | [`kubeflow/notebooks`](https://github.com/kubeflow/notebooks) | [`ADOPTERS.md`](https://github.com/kubeflow/notebooks/blob/main/ADOPTERS.md) |
| Kubeflow Pipelines | [`kubeflow/pipelines`](https://github.com/kubeflow/pipelines) | [`ADOPTERS.md`](https://github.com/kubeflow/pipelines/blob/master/ADOPTERS.md) |
| Kubeflow Spark Operator | [`kubeflow/spark-operator`](https://github.com/kubeflow/spark-operator) | [`ADOPTERS.md`](https://github.com/kubeflow/spark-operator/blob/master/ADOPTERS.md) |
| Kubeflow Trainer | [`kubeflow/trainer`](https://github.com/kubeflow/trainer) | [`ADOPTERS.md`](https://github.com/kubeflow/trainer/blob/master/ADOPTERS.md) |
| KServe | [`kserve/kserve`](https://github.com/kserve/kserve) | [`ADOPTERS.md`](https://github.com/kserve/website/blob/main/docs/community/adopters.md) |
## Adopters of Kubeflow Platform
For organizations that have adopted Kubeflow Platform, please add your organization to the following list.
<br>
Please keep the list in alphabetical order.
| Organization | Contact | Description of Use |
| --- | --- | --- |
| [AT&T](https://www.att.com) | [@ma-armenta](https://github.com/ma-armenta) | Kubeflow is used as the ML Platform for research and PoCs. |
| [CERN](https://www.home.cern) | [@gigabyte132](https://github.com/gigabyte132) | Kubeflow is used as our main ML platform. |
| [DHL Data & Analytics](https://www.linkedin.com/company/dhl-data-analytics/) | [@juliusvonkohout](https://github.com/juliusvonkohout) | Kubeflow is used as our main ML platform. |
| [Jio Platforms](https://www.jio.com) | [@abdullahpathan22](https://github.com/abdullahpathan22) | Kubeflow is used as our ML platform for running AI workloads on Kubernetes for JioTV+. |
| [Slalom](https://www.slalom.com/us/en/services/artificial-intelligence) | [@robcube](https://github.com/robcube) | Kubeflow is used to prototype gesture or hand-based recognition. |
| [Telia](https://www.teliacompany.com/) | [@akeed](https://github.com/akeed), [@sarahberenji](https://github.com/sarahberenji), [@tarekabouzeid](https://github.com/tarekabouzeid) | Kubeflow is used as our main ML platform for on-premises analytics. |
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Kubeflow Community Code of Conduct
Kubeflow follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting
the [Kubeflow Steering Committee](./proposals/STEERING-COMMITTEE.md).
================================================
FILE: CONTRIBUTING.md
================================================
# Kubeflow Contributor Guide
Welcome to the Kubeflow project! We'd love to accept your patches and
contributions to this project. Please read the
[contributor's guide in our docs](https://www.kubeflow.org/docs/about/contributing/).
The contributor's guide:
* Shows you where to find the Developer Certificate of Origin (DCO) that you need
to agree to
* Helps you get started with your first contribution to Kubeflow
* Describes the pull request and review workflow in detail, including the
OWNERS files and automated workflow tool
================================================
FILE: INCLUSIVITY.md
================================================
# Kubeflow Inclusivity
This document is a guide to inclusivity for leaders involved in the Kubeflow
project. It describes conscious efforts and behavior patterns that contribute
to an inclusive environment where Kubeflow members feel safe to collaborate.
The Kubeflow community is a global group of contributors with a wide variety of
backgrounds. As such, leaders are expected to take purposeful steps to ensure
sustainability of the collaborative space. This is essential for project health
and future growth. We expect all community members, and especially leaders, to practice and grow in the areas covered in this document.
## Carve out space
Carve out well-defined spaces for contribution. Encourage members of the
community to engage in these spaces. Reach out to specific individuals to let
them know you think they would be a good fit.
## Get out of the way
Get out of the way when someone steps up. Give them ownership and set
expectations for delivery and accountability. Follow up on those expectations.
If you have concerns or need more information, increase the frequency of
communication rather than taking over or overstepping.
## Make opportunities
Seek out situations that provide opportunities for members of the community.
Examples of this include connecting event organizers with potential speakers,
introducing leaders to individual contributors, and inviting others to
collaborate. Consciously drive the creation of opportunities in areas that
community members want to grow in.
## Ask members where they want to grow
Find out which areas community members want to grow in. This could be in the
form of 1:1 conversations, small groups, or weekly meetings. Ask how you can
help.
Rather than making assumptions and assigning tasks, ask people where they want
to contribute and help them figure out how to make the most impact. Stretch them
just enough that they can see progress and sustained growth.
## Empower members to say no
Make it clear that members are empowered to turn down opportunities. Encourage
them to define their own boundaries and give them space to assert those
boundaries. Communicate that it is their responsibility to balance their
commitments and that they will be supported in doing so. Before presenting a
specific opportunity to an individual, provide a disclaimer that it is
perfectly acceptable to say no.
## Encourage members to ask for what they want
Encourage members of the community to make requests. That could be for
improvements to the product, community, or their own personal growth. Respond
to these requests with kindness and fairness.
Ask for volunteers and make time for the community to bring up topics they care
about.
## Explicitly call out challenges
Name specific challenges that affect members of the community and state your
position on how to resolve them. Make statements such as, "I understand how
difficult it must be to X," and "I wish you didn't have to face such blatant
challenges doing Y." Offer advice on how to deal with them or just be there to
commiserate.
Simply acknowledging the struggle is an act of empathy that makes it easier to
face these challenges. This is a means of lightening the load on
underrepresented groups by not requiring them to shoulder these burdens
silently.
## Give clear, specific, and actionable feedback
Be proactive about providing feedback, but ask first and be kind. Include
concrete steps that can be taken to improve the outcome and steer clear of
criticism involving something that cannot be reasonably changed.
## Treat everyone with respect
Set an example and uphold that standard. Do not tolerate double standards or
casual deprecation, even in jest. Ensure that community members understand the
group is open to everyone.
## Follow up on complaints
When you observe a
[code of conduct](https://github.com/kubeflow/community/blob/master/CODE_OF_CONDUCT.md)
violation or become aware of one, follow through on enforcing community
standards. Do this with care, showing respect and kindness for everyone
involved. These instances have a broader impact than just the involved parties,
since they set downstream expectations for the entire community.
This is a responsibility that the Kubeflow project does not take lightly, since
it directly impacts the ability of members to feel safe in the community.
## Indicators of success
It can be difficult to assess whether these efforts are effective. In many ways,
success can be invisible since it involves the prevention of conflict. A few
indicators are:
* Diverse membership across various dimensions (geographic, corporate, level of
experience, etc.)
* Presence of members from frequently marginalized groups
* Continued engagement by long-term members
* Sentiment within the community that ideas are heard and contributions valued
* Accountability of leaders by members
## Attribution
The origins of this document are an enumeration of efforts by project cofounder
[David Aronchick](https://github.com/aronchick). This was not a solo effort and
included support from [Jeremy Lewi](https://github.com/jlewi), [Michelle Casbon](https://github.com/texasmichelle),
[Edd Wilder-James](https://github.com/ewilderj), and other members of the team.
================================================
FILE: KUBEFLOW-GENERAL-TECHNICAL-REVIEW.md
================================================
# General Technical Review - Kubeflow / Graduation
- **Project:** Kubeflow
- **Project Version:** Every Kubeflow sub-project has its own version.
- **Website:** https://www.kubeflow.org/
- **Date Updated**: 2025-09-05
- **Template Version**: v1.0
- **Description**:
Kubeflow is the foundation of tools for AI Platforms on Kubernetes.
AI platform teams can build on top of Kubeflow by using each project independently or deploying the
entire AI reference platform to meet their specific needs. The Kubeflow AI reference platform is
composable, modular, portable, and scalable, backed by an ecosystem of Kubernetes-native projects
that cover every stage of the
[AI lifecycle.](https://www.kubeflow.org/docs/started/architecture/#kubeflow-projects-in-the-ai-lifecycle)
Whether you’re an AI practitioner, a platform administrator, or a team of developers,
Kubeflow offers modular, scalable, and extensible tools to support your AI use cases.
## What are Kubeflow Projects
Kubeflow is composed of multiple open source projects that address different aspects of the AI
lifecycle. These projects are designed to be usable both independently and as part of the Kubeflow
AI reference platform. This provides flexibility for users who may not need the full end-to-end
AI platform capabilities but want to leverage specific functionalities, such as model training or
model serving.
### Kubeflow Projects in scope for CNCF Graduation
- Kubeflow Spark Operator
- Kubeflow Notebooks
- Kubeflow Trainer
- Kubeflow Katib
- Kubeflow Model Registry
- Kubeflow Pipelines
## What is the Kubeflow AI Reference Platform
The Kubeflow AI reference platform refers to the full suite of Kubeflow projects bundled together
with additional integration and management tools. Kubeflow AI reference platform deploys the
comprehensive toolkit for the entire AI lifecycle. The Kubeflow AI reference platform can be
installed via [Packaged Distributions](https://www.kubeflow.org/docs/started/installing-kubeflow/#packaged-distributions)
or [Kubeflow Manifests](https://www.kubeflow.org/docs/started/installing-kubeflow/#kubeflow-manifests).
## Day 0 - Planning Phase
### Scope
#### Describe the roadmap process
Kubeflow projects are managed by community members that are part of working groups.
Each working group defines and agrees on the features for each release. The release cadence for
each working group varies according to community agreement among the working groups.
Then, all Kubeflow projects have individual roadmap files in the Git repository defining each
release and available to the public. This ensures we have a standard structure for each proposed
feature, auditing, versioning, and transparency, since it is recorded along with the history in the Git repo.
For more information, check ROADMAP for each Kubeflow Project:
- [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator/blob/master/ROADMAP.md)
- [Kubeflow Trainer](https://github.com/kubeflow/trainer/blob/master/ROADMAP.md)
- [Kubeflow Katib](https://github.com/kubeflow/katib/blob/master/ROADMAP.md)
- [Kubeflow Hub](https://github.com/kubeflow/hub/blob/main/ROADMAP.md)
- [Kubeflow Pipelines](https://github.com/kubeflow/pipelines/blob/master/ROADMAP.md)
Community-wide changes are proposed as [Kubeflow Enhancement proposals (KEPs)](https://github.com/kubeflow/community/tree/master/proposals)
in the `kubeflow/community` repository or in the [Kubeflow sub-projects KEPs](https://github.com/kubeflow/trainer/tree/master/docs/proposals).
#### Describe the target persona or user(s) for the project
1. Users: Data Scientists, ML Engineer, AI Practitioners, Data Engineers, AI Practitioners.
2. Operators: MLOps Engineers, AIOps engineers, Platform Engineers, AI Platform Engineers.
3. Vendors: Vendors and projects building Kubernetes based AI Platform products.
#### Explain the primary use case for the project. What additional use cases are supported by the project
The goal of Kubeflow is to run Cloud Native AI workloads for every stage in AI lifecycle. By
using Kubeflow projects users can develop and deploy AI applications.
The primary use-cases include:
- Large-scale data processing and feature engineering.
- Distributed pre-training of foundation models.
- Post-training and fine-tuning of LLMs.
- Hyperparameter optimization and model tuning.
- LLM inference and multi-host serving.
Additional Use Cases:
- End-to-End GenAI pipeline building.
- Interactive AI development.
- Multiple users / projects with hard multi-tenancy on the same cluster.
#### Explain which use cases have been identified as unsupported by the project
As Kubeflow is composed of multiple projects, each working group makes its own determinations as t
what will be excluded from them. However we have an overarching theme and governance structure
(Steering Committee) that has identified the following areas as not being a priority for all projects:
- The projects are deployed in any Kubernetes (each release will specify tested versions),
regardless of the underlying infrastructure, independently through Kubernetes manifests leveraging
Kustomize and/or Helm Charts. However, the project doesn’t provide an implementation to be deployed
on infrastructure besides Kubernetes. - We do not officially enforce a deployment method or distribution.
- Kubeflow doesn’t provide a GitOps implementation, however Kubeflow manifests can be integrated
into a GitOps solution. For example, Platform Engineers can create an ArgoCD Application (CRD)
to install and configure Kubeflow projects. by providing Kubeflow individual project manifests,
for example Pipelines. The GitOps application will read from the Kubeflow Pipelines manifest and
Argo CD will deploy the configurations in the target cluster.
#### Describe the intended types of organizations who would benefit from adopting this project
Kubeflow is intended to be used by any organization which needs to run AI workloads on Kubernetes,
in any of the AI lifecycle stages an organization might choose to use just one or more projects,
such as Pipelines or Training. Organizations can also use all the projects from Kubeflow to increase
the user experience to build AI workloads. Additionally, organizations can develop their own
customizations on top of the Kubeflow platform or choose to build distributions to help other
organizations adopt a customized platform based on Kubeflow projects.
As Kubeflow maintains flexibility, organizations can choose their own path according to their needs.
We encourage adopters to be part of the adopters list in [GitHub](https://github.com/kubeflow/community/blob/master/ADOPTERS.md).
Examples of organizations that use Kubeflow:
- AWS
- Red Hat
- Capital One
- CERN
- Google
- Alibaba Cloud
- Bloomberg
- IBM
- Cisco
- Huawei
- Microsoft
- Tencent
- DHL Data & Analytics
- Telia
- Roblox
- Toyota
- PepsiCo
- Volvo
- and others…
#### Describe any completed end user research and link to any reports.
We regularly undertake user research about Kubeflow and its users. Research could be done by
working groups, during events, or by conducting at least one study annually. We ensure these surveys
are visible to the public and shared with the community.
Here are some results from previous years.
- [Kubeflow 2025 Survey](https://docs.google.com/forms/d/11cSe5vmGLrGekJISHBMfjVh_97WFGuhcvGnd0l5aNLg/edit#responses)
- [2025:UX designers supporting Model Registry conducted a series of user sessions to understand preferred interaction patterns (link)](https://docs.google.com/forms/d/11cSe5vmGLrGekJISHBMfjVh_97WFGuhcvGnd0l5aNLg/edit#responses)
- [Kubeflow Survey 2024](https://github.com/kubeflow/community/issues/708)
- [Kubeflow Survey 2023](https://blog.kubeflow.org/kubeflow-user-survey-2023/)
- [Kubeflow Survey 2022](https://blog.kubeflow.org/kubeflow-user-survey-2022/)
### Usability
#### How should your target personas interact with your project
AI Practitioner - Kubeflow SDK, Kubeflow UIs
Platform Admins - Operator guides, installing and configuring any Kubeflow projects via Helm charts
or Kustomize manifests predefined and available in the Kubeflow documentation from the command line.
- [Getting Started guide](https://www.kubeflow.org/docs/started/introduction/).
- [How to install Kubeflow projects](https://www.kubeflow.org/docs/started/installing-kubeflow/).
- [How to contribute to Kubeflow](https://www.kubeflow.org/docs/about/contributing/).
#### Describe the user experience (UX) and user interface (UI) of the project
Kubeflow user experience in each project is a collection of projects, the user experience for the
projects are each with their own [interfaces, APIs and SDKs](https://www.kubeflow.org/docs/started/architecture/#kubeflow-interfaces).
##### Describing User Experience through SDK
We are working on a unified [Kubeflow SDK](https://github.com/kubeflow/sdk) that gives
AI practitioners Python-native experience to interact with Kubeflow APIs.
Through Kubeflow SDK, users will be able to interact with the different projects, increasing user experience and reducing complexity.
<div style="text-align: center;">
<img
src="https://raw.githubusercontent.com/kubeflow/sdk/main/docs/images/persona_diagram.svg"
width="600"
title="Kubeflow SDK Personas"
alt="Kubeflow SDK Personas"
/>
</div>
Example to interact with Kubeflow Trainer using Kubeflow SDK:
```py
from kubeflow.trainer import TrainerClient, BuiltinTrainer, TorchTuneConfig
# Fine-Tune Llama 3.2
job_id = TrainerClient().train(
runtime=TrainerClient().get_runtime(name="torchtune-llama3.2-1b"),
trainer=BuiltinTrainer(
config=TorchTuneConfig(
resources_per_node={
"memory": "200G",
"gpu": 1,
},
)
),
)
# Wait for TrainJob to complete
TrainerClient().wait_for_job_status(job_id)
# Print TrainJob logs
print("\n".join(TrainerClient().get_job_logs(name=job_id)))
```
Example to interact with Kubeflow Katib using Kubeflow SDK
```py
from kubeflow.optimizer import OptimizerClient, search
from kubeflow.trainer import TrainerClient, BuiltinTrainer, TorchTuneConfig
OptimizerClient().optimize(
objective="loss",
mode="min",
num_trials=5,
trainer=BuiltinTrainer(
config=TorchTuneConfig(
resources_per_node={
"memory": "200G",
"gpu": 1,
},
lr=search("0.1", "0.2", "lognormal"),
num_epochs=search("4", "8", "uniform"),
)
),
runtime=TrainerClient().get_runtime(name="torchtune-llama3.2-1b"),
)
```
Find more details and examples about [Kubeflow SDK](https://github.com/kubeflow/sdk).
##### Describing User Experience through APIs
- Kubeflow Pipelines
REST API is available under the `/pipeline/` HTTP path. For example, if you host Kubeflow at
https://kubeflow.example.com, the API will be available at https://kubeflow.example.com/pipeline/.
The API is documented using swagger and examples about its usage
[can be found here](https://www.kubeflow.org/docs/components/pipelines/reference/api/kubeflow-pipeline-api-spec/)
- Kubeflow Model Registry
Model Registry REST API is available under the `/api/model_registry/` HTTP path. More information
[can be found here](https://www.kubeflow.org/docs/components/hub/reference/rest-api/)
The Model Catalog Service provides a read-only discovery service for ML models across multiple
catalog sources. It acts as a federated metadata aggregation layer, allowing users to search and
discover models from various external catalogs through a unified REST API.
##### Describing User Interfaces through web UI
[Kubeflow Central Dashboard](https://www.kubeflow.org/docs/components/central-dash/overview/)
acts as a hub for the AI platform and tools by exposing the UIs of components running in the cluster.
<div style="text-align: center;">
<img
src="https://github.com/kubeflow/website/blob/498691a0eae4bef5ae5ae7935dd367a547274f85/content/en/docs/images/dashboard/homepage.png?raw=true"
width="600"
title="Kubeflow Dashboard"
alt="Kubeflow Dashboard"
/>
</div>
End users can access the Central Dashboard installed version on their organization according to
the user and access control setup previously by Platform Admin. Users can access Kubeflow projects
such as Kubeflow Pipelines (KFP) to manage experiments, pipeline definitions, runs, and recurrent
runs. KFP Artifacts to track artifacts produced by pipelines stored in MLMD. KFP Executions to
track executions of pipeline components stored in MLMD. Kubeflow Katib experiments to manage
AutoML experiments. Kubeflow Notebooks to manage Kubeflow Notebooks. Kubeflow TensorBoards to
manage TensorBoard instances. Kubeflow Volumes to manage Kubernetes PVC Volumes. Contributors page
to manage contributors of profiles (namespaces) that you own.
More information can be found [in the Kubeflow Dashboard docs](https://www.kubeflow.org/docs/components/central-dash/overview/).
#### Describe how this project integrates with other projects in a production environment
All Kubeflow projects are Kubernetes native and so fit into the wider ecosystem of Kubernetes
based tools. Kubeflow projects extensively use tools from the cloud native ecosystem,
including, Argo Workflow, Istio, Helm, Knative, KServe, JobSet, Kueue, and other projects.
Specific components integrate with popular AI frameworks where applicable (e.g. Kubeflow Trainer
integrates with PyTorch and other model frameworks)
The following diagram gives an overview of the
[Kubeflow Ecosystem](https://www.kubeflow.org/docs/started/architecture/#kubeflow-ecosystem)
and how it relates to the wider Kubernetes AI landscapes.
<div style="text-align: center;">
<img
src="https://github.com/kubeflow/website/blob/498691a0eae4bef5ae5ae7935dd367a547274f85/content/en/docs/started/images/kubeflow-architecture.drawio.svg?raw=true"
width="600"
title="Kubeflow Ecosystem"
alt="Kubeflow Ecosystem"
/>
</div>
Specifically for a production environment, Kubeflow can integrate and leverage the following
resources and projects:
- Cloud Providers and underlying infrastructure
- Kubeflow can leverage the underlying infrastructure provided through Kubernetes, such as access
hardware accelerators from Intel, Nvidia, and AMD through Kubernetes Operators.
- Organizations can run Kubeflow projects on any Kubernetes platform and use cloud provider
services, such as scalability across different nodes, taking advantage of projects such as Open Cluster Management.
- Configuration as code
- GitOps approach can be used to set up Kubeflow setup and configuration, for example, using
Argo CD to manage configuration as code in multiple clusters.
- Security
- Security is becoming a key aspect, Kubeflow projects implement security best practices in
Kubernetes including Pod Security Standards, Network policies, RBAC. Additionally, they can
leverage projects such as the External Secrets Operator.
- Data Scientists can also use Sigstore projects to sign and validate their models to avoid type
of attacks such as tampering with Models to ensure the integrity and authenticity of models
persists during the Model Development Lifecycle.
- Simplify User experience
- Organizations can simplify Data scientists' experience by scaling Platform Engineers by
providing software templates through Backstage to access environments quickly, leveraging
Kubeflow projects such as Notebooks.
### Design
#### Explain the design principles and best practices the project is following
- Loosely Coupled and Distributable Services
- Built as modular, independent microservices that can scale and evolve.
- Services communicate through well defined APIs, enabling flexible orchestration and integration.
- Kubernetes native with Declarative APIs and Automation
- Uses Kubernetes CRD for each project.
- Integrate natively with Kubernetes core APIs like Jobs, Pods, Deployment.
- Supports automated deployment, orchestration, and CI/CD workflows.
- Enables GitOps management for reproducible and auditable operations.
- Portability and Platform-Agnosticism
- Designed to run on any Kubernetes cluster, including onprem, edge, public cloud or hybrid environments.
- Avoids vendor lock-in while ensuring consistent behavior across different infrastructures.
- Observability
- Provides metrics, logs, and traces to monitor system performance and workflow execution.
- Enables debugging, performance analysis, and monitoring of workloads and processes.
- Resilience and Availability
- Leverages platform features such as health checks, automatic restarts, and scaling to maintain high availability.
- Designed to tolerate failures and minimizing disruption to workloads.
- Security and Multi-Tenancy
- Implements access control, isolation, and resource quotas to separate workloads and users.
- Supports secure communication and policy enforcement to maintain a safe multi-tenant environment.
- Extensibility and Interoperability
- Provides APIs and runtime contracts to allow integration of new tools, frameworks, or workflows.
- Supports extension and customization without breaking existing functionality.
Kubeflow overall has strong [contributing guidelines](https://www.kubeflow.org/docs/about/contributing/)
that inform our development and design. Each project also provides new contributors specific
guidance on their GitHub repositories:
- [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator/blob/master/CONTRIBUTING.md)
- [Kubeflow Notebooks](https://github.com/kubeflow/notebooks/blob/main/CONTRIBUTING.md)
- [Kubeflow Trainer](https://github.com/kubeflow/trainer/blob/master/CONTRIBUTING.md)
- [Kubeflow Katib](https://github.com/kubeflow/katib/blob/master/CONTRIBUTING.md)
- [Kubeflow Model Registry](https://github.com/kubeflow/model-registry/blob/main/CONTRIBUTING.md)
- [Kubeflow Pipelines](https://github.com/kubeflow/pipelines/blob/master/CONTRIBUTING.md)
For the new features every Kubeflow project follows
[the KEP guidelines](https://github.com/kubeflow/community/tree/master/proposals#kep-format-to-propose-and-document-enhancements-to-kubeflow)
#### Outline or link to the project’s architecture requirements
We do follow the same practices as Kubernetes for APIs, with alpha, beta, and stable status of APIs.
Details [are described here](https://www.kubeflow.org/docs/started/support/#component-status).
#### Define any specific service dependencies the project relies on in the cluster
The following projects are required to install Kubeflow projects:
- Istio, Knative, Cert-manager
- Detailed information [can be found in the Kubeflow manifests](https://github.com/kubeflow/manifests#kubeflow-components-versions).
Specific projects have other dependencies:
- Kubeflow Pipelines: Argo Workflows>=v3.1
- Kubeflow Trainer: JobSet>=v0.8.0
- Kubeflow Katib: MySQL>=v8.0
- Kubeflow Model Registry: MySQL>=v8.0
#### Describe how the project implements Identity and Access Management
Kubeflow projects use a pluggable system of IAM that connects back to Kubernetes IAM in most cases.
As a reference implementation, the optional Kubeflow Manifests/Dashboard components use the
following IAM systems:
- [OAuth2 Proxy](https://github.com/kubeflow/manifests#oauth2-proxy)
- [Kubeflow manifest with Dex](https://github.com/kubeflow/manifests#dex)
General security Talks about Kubeflow including an architectural introduction:
- [KubeCon / Kubeflow Summit 2024 Lightning Talk](https://youtu.be/f7C3v0gJRVI).
- [KubeCon 2023 Hardening Kubeflow Security for Enterprise Environments](https://youtu.be/wQaOa4Kjxs0).
- [Kubeflow Summit 2023 Security Working Group Update](https://youtu.be/XKHVt2yxQFo).
- [Blog Hardening Kubeflow Security for Enterprise Environments](https://blogs.vmware.com/opensource/2023/06/20/hardening-kubeflow-security-for-enterprise-environments-2/).
#### Describe how the project has addressed sovereignty
Kubeflow’s projects can be self-hosted on any Kubernetes cluster, including air-gapped environments,
which is critical for those organizations using disconnected environments. Additionally,
Kubeflow projects support multi-tenancy, which allows organizations to isolate workloads in a
Kubernetes cluster, which can be used with Network Policies to restrict isolation from different
components by ports or application name. Kubeflow projects support custom service accounts,
allowing organizations to control how Kubeflow projects interact with the cluster by leveraging RBAC.
Further, Platform Administrators can decide which users in Kubernetes will have access to the
Kubeflow projects by creating different users and RBAC policies. Kubeflow Notebooks also supports
profiles, which allows simplifying the permission management.
In terms of data management, data sovereignty is managed by those who deploy or package the projects.
#### Describe any compliance requirements addressed by the project
Kubeflow projects are extensible which allows users to fit their internal compliance requirements.
As a result, specific common compliance frameworks (SOC-2, GDPR, etc.) are the responsibility of
end users and vendors. However, we aim to provide a strong foundation through reference architectures
similar things from which to build on.
#### Describe the project’s High Availability requirements
The end users can adjust the replicas. Kubeflow project controllers support leader election,
[for example Kubeflow Trainer](https://github.com/kubeflow/trainer/blob/master/cmd/trainer-controller-manager/main.go#L80).
#### Describe the project’s resource requirements, including CPU, Network and Memory
The following table shows the resource requirements for each Kubeflow project, calculated as the
maximum of actual usage and configured requests for CPU/memory, plus storage requirements from PVCs:
The maximum looks hefty so rather consider in general the maximum (manifest requests, average usage).
| Component | CPU (cores) | Memory (Mi) | Storage (GB) |
| ----------------------- | ----------- | ----------- | ------------ |
| Cert Manager | 3m | 130Mi | 0GB |
| Dex + OAuth2-Proxy | 3m | 28Mi | 0GB |
| Istio | 2300m | 3502Mi | 0GB |
| Kubeflow KServe | 600m | 1200Mi | 0GB |
| Kubeflow Katib | 9m | 471Mi | 13GB |
| Kubeflow Core | 35m | 841Mi | 0GB |
| Metadata | 78m | 687Mi | 30GB |
| Kubeflow Model Registry | 510m | 2112Mi | 0GB |
| Other | 20m | 354Mi | 6GB |
| Kubeflow Pipelines | 970m | 3552Mi | 90GB |
| Kubeflow Spark Operator | 4m | 41Mi | 0GB |
| Kubeflow Trainer | 3m | 25Mi | 0GB |
| **Total** | **4535m** | **12943Mi** | **139GB** |
#### Describe the project’s storage requirements, including its use of ephemeral and/or persistent storage
Each project can configure storage in different ways. This is also true of the manifests which
configure storage for a collection of projects.
This [can be seen here](https://github.com/kubeflow/manifests/pull/3091).
#### Please outline the project’s API Design
Various Kubeflow projects offer APIs and Python SDKs. See the following sets of reference documentation:
- [Pipelines reference docs](https://www.kubeflow.org/docs/components/pipelines/reference/)
for the Kubeflow Pipelines API and SDK, including the Kubeflow Pipelines domain-specific language (DSL).
- [Kubeflow SDK](https://github.com/kubeflow/sdk)
to interact with Kubeflow APIs with Python-native interface.
See also [Kubeflow APIs and SDKs](https://www.kubeflow.org/docs/started/architecture/#kubeflow-apis-and-sdks).
Kubeflow CRDs are following Kubernetes best practices for
[the API changes](https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-changes).
#### Describe the project’s API topology and conventions
[Kubeflow APIs and SDKs documentation](https://www.kubeflow.org/docs/started/architecture/#kubeflow-apis-and-sdks).
#### Describe the project defaults
Some defaults can be seen [in the Kubeflow manifests](https://github.com/kubeflow/manifests#installation).
Additionally, API defaults are shown in the API docs, such as:
[Kubeflow Trainer](https://github.com/kubeflow/trainer/blob/master/pkg/apis/trainer/v1alpha1/trainjob_types.go#L128)
or [Kubeflow Katib](https://github.com/kubeflow/katib/blob/master/pkg/apis/controller/experiments/v1beta1/experiment_types.go#L46).
#### Outline any additional configurations from default to make reasonable use of the project
[Kubeflow manifests guides](https://github.com/kubeflow/manifests/blob/master/README.md).
Some public and listed distributions have their own ways
[to install Kubeflow AI platform](https://www.kubeflow.org/docs/started/installing-kubeflow/#packaged-distributions)
#### Describe any new or changed API types and calls—including to cloud providers—that will result from this project being enabled and used
Deploying Kubeflow manifests and individual projects results in exposing new APIs and the possibility
to call configured 3rd party APIs if integrations exist. These all have to be explicitly set by the users.
#### Describe compatibility of any new or changed APIs with API servers, including the Kubernetes API server
Kubeflow follows the same practice for [API compatibility as Kubernetes](https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-changes).
#### Describe versioning of any new or changed APIs, including how breaking changes are handled
Many Kubeflow projects use Kubernetes CRDs, and for these resources follows
[the same deprecation policy as Kubernetes](https://kubernetes.io/docs/reference/using-api/deprecation-policy/).
#### Describe the project’s release processes, including major, minor and patch releases
Every Kubeflow project follow its own release lifecycle, for example
[Kubeflow Trainer](https://github.com/kubeflow/trainer/tree/master/docs/release)
or [Kubeflow Katib](https://github.com/kubeflow/katib/tree/master/docs/release).
However, community also maintain [the Kubeflow AI reference platform](https://www.kubeflow.org/docs/kubeflow-platform/releases/)
releases which install all Kubeflow projects together for end-to-end AI platform:
### Installation
#### Describe how the project is installed and initialized
[Kubeflow Installation guide](https://www.kubeflow.org/docs/started/installing-kubeflow/).
Kubeflow projects can be installed as a standalone applications or together using
the Kubeflow Manifests or Kubeflow Distributions (public or private).
#### How does an adopter test and validate the installation
Distributions can verify the installation [by following this guide](https://github.com/kubeflow/manifests?tab=readme-ov-file#installation)
or [executing this test suites](https://github.com/kubeflow/manifests/blob/master/.github/workflows/full_kubeflow_integration_test.yaml).
Kubeflow projects docs also explain how platform admins should validate the installation of individual
applications, [for example Kubeflow Trainer](https://www.kubeflow.org/docs/components/trainer/operator-guides/installation/#installing-the-kubeflow-trainer-controller-manager)
### Security
#### Provide a link to the project’s cloud native [security self assessment](https://tag-security.cncf.io/community/assessments/).
- [Kubeflow Security Self Assessment](https://github.com/kubeflow/community/blob/master/security/self-assessment.md)
#### How are you satisfying the tenets of cloud native security projects
1. Make security a design requirement.
Kubeflow projects are built with security as a fundamental concern. Kubeflow projects follow
Kubernetes and cloud native security best practices. By leveraging Kubernetes Custom Resources
Definition (CRDs), Role-Based Access Control (RBAC), network policies, and pod security standards,
Kubeflow projects seamlessly integrate into cloud native secure environments. Additionally,
Kubeflow projects re-use functionality from other cloud native tools like Istio, Cert-Manager,
Kubernetes and inherit its security best practices.
Default configuration for Kubeflow projects follow best practices such as rootless containers,
limited privileges, etc.
2. Applying secure configuration has the best user experience
Default configuration for Kubeflow projects follow best practices such as rootless containers,
limited privileges, etc. Users can migrate workloads to more secure configurations without breaking
changes, leveraging Kubernetes’ declarative model.
3. Selecting insecure configuration is a conscious decision
Insecure options require explicit user setup and configurations.
#### Describe how each of the [cloud native principles](https://github.com/cncf/toc/blob/main/DEFINITION.md) apply to your project
Kubeflow uses cloud native principles by building on Kubernetes and other cloud native technologies
while extending them in our composable projects.
Kubeflow projects use containers as a fundamental unit of task/deployment. For example,
every TrainJob uses separate containers to train models, allowing users to package their training
code along with its dependencies in a containerized environment
Kubeflow projects are designed to run natively on Kubernetes and define various CRDs for each
AI workload. For example, TrainJob CR for model training or Notebook CR for interactive development.
It uses native Kubernetes primitives like Deployments, Services, Job, JobSet, and CRDs to
orchestrate AI workloads.
Kubeflow follows a loosely-coupled microservices architecture model, where each project can
be deployed as a standalone service and be integrated into desired AI platform on Kubernetes.
Kubeflow APIs are declarative and follow Kubernetes best practices. For instance,
SparkApplication to manage Spark jobs or TrainJob to manage distributed training jobs.
#### How do you recommend users alter security defaults in order to "loosen" the security of the project
Users can install Kubeflow Pipelines without multi-user isolation if they want to loosen the
security of deployment. Users suggested to follow the Kubeflow AI reference platform deployment
mode to install Kubeflow Pipelines control plane
[in multi-tenant model](https://www.kubeflow.org/docs/components/pipelines/operator-guides/multi-user/).
It provides isolation between Pipelines and Runs for users.
#### Security Hygiene
##### Describe the frameworks, practices and procedures the project uses to maintain the basic health and security of the project
- **Robust CI/CD infrastructure:** Kubeflow projects have automated unit, integration, and end-to-end
tests that are integrated into the CI pipelines to ensure code stability and correctness of PRs.
- **Dependency management:** Kubeflow projects leverage lock files like `go.mod` and `pyproject.toml`,
and progressing towards Software Bill of Materials (SBOM) generation.
- **Release stability:** Kubeflow projects maintain release branches and release tags which allow
maintainers create patch releases to address bug fixes and security vulnerabilities. Also,
projects maintain changelogs and roadmaps to ensure traceability and transparency for all changes.
- **Static and Dynamic Code Analysis:** Kubeflow projects CI integrates code linting, static,
and dynamic code analysis tools to enforce code quality and detect potential issues before they
reach production.
- **Secure Defaults:** Defaults manifests and Helm charts are configured with security best practices,
like pod security standards, rootless containers, and enforces least privileges.
- **Code Review Process:** Kubeflow projects’ maintainers follow a strong review process with peer
review and explicit approval before PRs being merged into the main branch.
- **Open Governance:** Kubeflow follows open governance with regular public meetings and
communication channels to keep users aware of development and roadmap.
- **Secure Image Build:** Image builds are validated through CI/CD checks to ensure they are
reproducible, security, and consistent.
Some examples can be found here:
- [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator/tree/master/.github/workflows)
- [Kubeflow Katib](https://github.com/kubeflow/katib/tree/master/.github/workflows)
- [Kubeflow Trainer](https://github.com/kubeflow/trainer/tree/master/.github/workflows)
- [Kubeflow Model Registry](https://github.com/kubeflow/model-registry/tree/main/.github/workflows)
- [Kubeflow Pipelines](https://github.com/kubeflow/pipelines/tree/master/.github/workflows)
- [Kubeflow Manifests](https://github.com/kubeflow/manifests/blob/master/.github/workflows/full_kubeflow_integration_test.yaml)
##### Describe how the project has evaluated which features will be a security risk to users if they are not maintained by the project
Features are discussed and reviewed in the community, with security implications considered during
design and code review processes. The project tracks the security health of dependencies and evaluates
the impact of vulnerabilities or unmaintained projects. The project identifies potential risks by
analyzing the attack surface, especially for features that interact with user-supplied code,
custom containers, user supplied models or external data sources.
#### Cloud Native Threat Modeling
##### Explain the least minimal privileges required by the project and reasons for additional privileges
Each Kubeflow project's control plane needs to watch for its CRDs. For example, Kubeflow Trainer:
TrainJob and ClusterTrainingRuntime, Spark Operator for SparkApplications, Katib for Experiments, etc.
Controllers have minimal privileges to orchestrate its CRDs and corresponding resources that need to be created
Kubeflow control plane is installed cluster-wide, and it requires RBAC to manage namespace-scoped resources.
The default controllers run with PodSecurityStandards restricted and rootless containers,
minimizing the risk of privilege escalation.
Users who interact with Kubeflow SDK get namespace-scoped permission to interact with required
resources. For example, users’ role for
[Kubeflow Trainer can be found here](https://github.com/kubeflow/trainer/blob/master/manifests/overlays/kubeflow-platform/kubeflow-trainer-roles.yaml#L17).
All roles are aggregated in the `kubeflow-edit` cluster role after installing Kubeflow Manifests.
##### Describe how the project is handling certificate rotation and mitigates any issues with certificates
Kubeflow projects use [cert-manager to generate and rotate certificates](https://github.com/kubeflow/manifests/tree/master/common/cert-manager).
Those certificates are used for validation and mutation webhooks across Kubeflow projects. Cert-manager
handles the issuance, renewal, and rotation of these certificates automatically without downtime.
##### Describe how the project is following and implementing [secure software supply chain best practices](https://project.linuxfoundation.org/hubfs/CNCF_SSCP_v1.pdf)
- **Automated CI/CD infrastructure:** Kubeflow projects have automated unit, integration, and
end-to-end tests that are integrated into the CI pipelines to ensure code stability and reduce
risk of introducing vulnerabilities.
- **Vulnerability Scanning:** Kubeflow uses tools for vulnerability scanning like Dependabot,
Trivy to identify CVEs and address known CVEs in dependencies.
- **Dependency management:** Kubeflow projects leverage lock files like `go.mod` and `pyproject.toml`,
and progressing towards Software Bill of Materials (SBOM) generation.
- **Code Review Process:** Kubeflow projects’ maintainers follow a strong review process with peer
review and explicit approval before PRs being merged into the main branch.
- **DCO Check:** Committers are required to sign and comply with the Developer Certificate of Origin
(DCO) to affirm the legitimacy and authorship of their contributions.
- **Branch Protection:** The project enforces branch protection rules to prevent unauthorized changes,
enforce status checks, require pull request reviews, and control who can push to protected branches.
- **Prevent Secrets in Source Code:** Proactively prevents committing secrets to the source code
by using tools and GitHub workflows that detect sensitive data in PRs.
- **License compliance:** Kubeflow uses
[the FOSSA product](https://app.fossa.com/reports/bb8e2d41-254a-4af3-9044-e7f484c34dd1)
provided by the CNCF to scan for license compliance on a regular and ongoing basis.
## Day 1 - Installation and Deployment Phase
### Project Installation and Configuration
#### Describe what project installation and configuration look like
Each project has its [own standalone installation guide](https://www.kubeflow.org/docs/started/installing-kubeflow/#standalone-kubeflow-components).
Kubeflow projects are packaged by [distributions and vendors](https://www.kubeflow.org/docs/started/installing-kubeflow/#kubeflow-platform)
into platform products which can be used to install our tools.
We provide optional manifests to deploy all Kubeflow projects as
[Kubeflow AI reference platform](https://github.com/kubeflow/manifests/blob/master/README.md).
### Project Enablement and Rollback
#### How can this project be enabled or disabled in a live cluster? Please describe any downtime required of the control plane or nodes
Users can set the replica count to 0 in the Kubeflow projects deployment. Existing AI workloads
should not be impacted since it won’t be reconciled by controllers.
Updating [the Kubeflow AI Reference Platform](https://github.com/kubeflow/manifests/blob/master/README.md#upgrading-and-extending).
The installation guide described [when control plane is ready](https://www.kubeflow.org/docs/components/trainer/operator-guides/installation/#installing-the-kubeflow-trainer-controller-manager).
#### Describe how enabling the project changes any default behavior of the cluster or running workloads
Enable projects by scaling replica count back to 1. All running AI workloads will be reconciled again
by controllers, and they perform the appropriate updates to the CRDs.
Istio, Knative, cert-manager might interfere with existing installations.
#### Describe how the project tests enablement and disablement
Conformance program [is work in progress](https://github.com/kubeflow/kubeflow/tree/master/conformance/1.7)
to ensure tests across all Kubeflow projects.
#### How does the project clean up any resources created, including CRDs
The Kubeflow projects control plane can be deleted by cleanup the appropriate resources,
for example:
```sh
kubectl delete -k https://github.com/kubeflow/trainer.git/manifests/overlays/manager?ref=v2.0.0
```
The command will cleanup all CRDs and control plane deployment. Since Kubeflow CRDs maintain
`ownerReference` for associated resources, all resources will be removed after deleting the
actual CRDs. For example:
```sh
kubectl delete trainjob --all --all-namespaces
```
Check [this guide to cleanup](https://github.com/kubeflow/manifests/blob/master/README.md#upgrading-and-extending)
the Kubeflow AI reference platform.
### Rollout, Upgrade and Rollback Planning
#### How does the project intend to provide and maintain compatibility with infrastructure and orchestration management tools like Kubernetes and with what frequency
Kubeflow projects publish its supported Kubernetes version for every release. The supported versions
are evaluated and upgraded on every release. We support
[Kubernetes 1.31+ and test on 1.32+](https://github.com/kubeflow/manifests/blob/master/tests/install_KinD_create_KinD_cluster_install_kustomize.sh)
#### How the project handles rollback procedures
Some projects support leader election and HA to make sure that long-running workloads are complete.
Users can scale up or down replicas of controllers during rollback.
#### How can a rollout or rollback fail? Describe any impact to already running workloads
Newer revision will not be active for many reasons including insufficient cluster resources for the
newer spec, invalid configuration spec. Traffic will not be switched to the new unless the newer
revision is ready to accept traffic. Hence, the already running workloads will not be affected
in any case if rollout fails.
#### Describe any specific metrics that should inform a rollback
Kubeflow projects CRDs expose status that inform about activity of AI workloads. Additionally,
controllers are exposed various Prometheus metrics to indicate workload status.
#### Explain how upgrades and rollbacks were tested and how the upgrade->downgrade->upgrade path was tested
Currently, it’s being manually tested by users, but automated tests are work in progress.
#### Explain how the project informs users of deprecations and removals of features and APIs
All API changes are backward compatible and changes are announced in the release notes and
changelogs. If some APIs are deprecated, newer versions of APIs are introduced with user awareness.
Kubeflow CRDs follow Kubernetes best practices for API compatibility.
For example [Kubeflow Trainer breaking changes](https://github.com/kubeflow/trainer/blob/master/CHANGELOG.md#breaking-changes).
#### Explain how the project permits utilization of alpha and beta capabilities as part of a rollout
At the moment, functionality is logged explicitly if a feature is alpha or beta. We do not have
feature gates. Additionally, we are working on conversion webhook that helps users with updated
API versions of CRDs.
## Day 2 - Day-to-Day Operations Phase
### Scalability/Reliability
#### Describe how the project increases the size or count of existing API objects
In general, the API object count complexity is linear in the number of users and workloads. Users
can list deployed Kubeflow CRDs across all namespaces.
For example, to list all TrainJob from Kubeflow Trainer run:
```sh
kubectl get trainjob --all-namespaces
```
The core controllers and resources are constant (beyond replicating specific controllers),
see [the full kustomize build here](https://github.com/kubeflow/manifests?tab=readme-ov-file#install-with-a-single-command).
#### Describe how the project defines Service Level Objectives (SLOs) and Service Level Indicators (SLIs)
Kubeflow currently doesn’t provide the SLOs and SLIs, however we follow
[the Kubernetes SLOs](https://github.com/kubernetes/community/blob/master/sig-scalability/slos/slos.md)
#### Describe any operations that will increase in time covered by existing SLIs/SLOs
As above.
#### Describe the increase in resource usage in any components as a result of enabling this project, to include CPU, Memory, Storage, Throughput
Resources requirements for Kubeflow projects [are set here](https://github.com/kubeflow/manifests/pull/3091#issuecomment-3016609243).
#### Describe which conditions enabling / using this project would result in resource exhaustion of some node resources
There are some specific issues for each project, we should list some of them (e.g. pipelines
controller can have open file handler issues). Since many workloads are GPU-intensive,
Kubernetes platform admins need to ensure that nodes have enough capacity to run AI workloads with
Kubeflow projects.
#### Describe the load testing that has been performed on the project and the results
Some intensive load testing has been performed by Kubeflow users, for example some LLM foundation
models have been trained using Kubeflow Trainer across a few hundreds GPUs. At KubeCon some users
share their experience of running more than 10,000 GPUs with Kubeflow Training Operator.
#### Describe the recommended limits of users, requests, system resources, etc. and how they were obtained
Users on the order of hundreds are known to work. Scaling beyond hundreds of users may require an
increase in the requests/limits replicas of some deployments, or scaling dependencies like databases.
#### Describe which resilience pattern the project uses and how, including the circuit breaker pattern
Kubeflow uses the Kubernetes resilience pattern to manage controllers with HA. If one replica fails, another part of the control plane takes responsibility to orchestrate workloads.
### Observability Requirements
#### Describe the signals the project is using or producing, including logs, metrics, profiles and traces. Please include supported formats, recommended configurations and data storage
Kubeflow controllers expose Prometheus metrics to report workload status. Controllers also expose
logs and status for platform admins to ensure stability.
#### Describe how the project captures audit logging
Platform admins can leverage [Kubernetes audit](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/)
for Kubeflow projects.
#### Describe any dashboards the project uses or implements as well as any dashboard requirements
[Kubeflow Dashboard requirements](https://www.kubeflow.org/docs/components/central-dash/overview/).
#### Describe how the project surfaces project resource requirements for adopters to monitor cloud and infrastructure costs, e.g. FinOps That must happen on the Kubernetes namespace level
Users are recommended to use third-party tools like Kubecost to measure cloud and infrastructure
cost of running Kubeflow projects.
#### Which parameters is the project covering to ensure the health of the application/service and its workloads
Most project use Kubernetes [Liveness and Readiness probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/).
For example [Kubeflow Trainer controller manager](https://github.com/kubeflow/trainer/blob/master/manifests/base/manager/manager.yaml#L30-L43).
#### How can an operator determine if the project is in use by workloads
- Check the Pods in `kubeflow-profil`e labeled namespaces.
- Check the CRDs in user’s namespaces
- Check the Kubeflow Dashboard resources.
#### How can someone using this project know that it is working for his instance
- [Kubeflow Manifests guide](https://github.com/kubeflow/manifests#installation).
- Run the [Kubeflow manifest test suite](https://github.com/kubeflow/manifests/blob/master/.github/workflows/full_kubeflow_integration_test.yaml)
- Run the upstream examples, for example [Kubeflow Trainer PyTorch training](https://github.com/kubeflow/trainer/blob/master/examples/pytorch/question-answering/fine-tune-distilbert.ipynb).
### Dependencies
#### Describe the specific running services the project depends on in the cluster
Most projects depend on Cert-Manager>=v1.16, and Istio>=v1.26
Specific projects have other dependencies:
- Kubeflow Pipelines: Argo Workflows>=v3.1
- Kubeflow Trainer: JobSet>=v0.8.0
- Kubeflow Katib: MySQL>=v8.0
- Kubeflow Model Registry: MySQL>=v8.0
For details, please take a look at General security Talks about Kubeflow including
[an architectural introduction](https://github.com/kubeflow/manifests#kubeflow-components-versions).
#### Describe the project’s dependency lifecycle policy
We follow Kubernetes deprecation policy for supported versions. For example, we support the
3-4 latest versions of Kubernetes to deploy Kubeflow projects.
#### How does the project incorporate and consider source composition analysis as part of its development and security hygiene? Describe how this source composition analysis (SCA) is tracked
Various static and dynamic code analysis tools are enforced as described above.
#### Describe how the project implements changes based on source composition analysis (SCA) and the timescale
Various static and dynamic code analysis tools are enforced as described above.
### Troubleshooting
#### How does this project recover if a key component or feature becomes unavailable
Specifics depend on the Kubeflow Project (KFP, Katib, etc.). Kubeflow projects are cloud and
Kubernetes native apps, so fault tolerance is strongly tied to the health of the underlying cluster.
In practice user workloads sometimes have a retry policy and the Kubeflow core services should
just recover automatically.
More details are described in the previous sections.
#### Describe the known failure modes
Each Kubeflow project handles failure modes differently beyond native Kubernetes fault tolerance.
Many of them are configured at the application level in user code.
### Security
#### How is the project executing access control
As described above each Kubeflow projects require limited RBAC to manage its CRDs
#### Cloud Native Threat Modeling
Described in the section above.
#### How does the project ensure its security reporting and response team is representative of its community diversity (organizational and individual)
Every Kubeflow project follows security guidelines.
- Kubeflow Spark Operator [security policy](https://github.com/kubeflow/spark-operator/blob/master/SECURITY.md).
- Kubeflow Notebooks [security policy](https://github.com/kubeflow/notebooks/blob/master/SECURITY.md).
- Kubeflow Trainer [security policy](https://github.com/kubeflow/trainer/blob/master/SECURITY.md).
- Kubeflow Katib [security policy](https://github.com/kubeflow/katib/blob/master/SECURITY.md).
- Kubeflow Model Registry [security policy](https://github.com/kubeflow/model-registry/blob/main/SECURITY.md).
- Kubeflow Pipelines [security policy](https://github.com/kubeflow/pipelines/blob/master/SECURITY.md).
#### How does the project invite and rotate security reporting team members
Active project maintainers are responsible to ensure that security reports are addressed. Each
project has its own security reporting and disclosure policy. For projects which use GitHub’s security
disclosure system, access control is managed by having write access to the relevant github repository.
================================================
FILE: KUBEFLOW-OUTREACH-COMMITTEE.md
================================================
# Kubeflow Outreach Committee (KOC)
The Kubeflow Outreach Committee (KOC) is a committee dedicated to fostering growth, engagement, and community outreach for the Kubeflow project. It will focus on activities that promote Kubeflow to new and existing users, contributors, and stakeholders while building an inclusive, vibrant, and diverse community within the broader AI/ML ecosystem.
The governance of the Kubeflow Outreach Committee will evolve as the community grows, ensuring that outreach activities remain relevant and impactful.
### Charter
1. Promote the vision, values, and mission of the Kubeflow project in collaboration with the KSC and WG Leads.
2. Lead efforts to grow and engage the AI/ML community, including organizing events, producing educational materials, creating and sharing content on social media, and advocating for Kubeflow across various tech communities.
3. Help grow the community by bringing new contributors and users to Kubeflow, particularly through accessible resources and community-building initiatives.
4. Collaborate with other committees and working groups to align outreach efforts with the broader goals of the Kubeflow project.
5. Encourage contributions to the Kubeflow Website and Blog, including project documentation, examples, and other educational materials.
### Limitations
Because the actions of the KOC may be perceived to represent the views of the wider Kubeflow project, the KSC places the following limits and requirements on the KOC.
1. The KOC may not endorse, recommend, or promote any product, service, or project except that which is directly provided by Kubeflow without approval by the KSC for each specific occasion.
2. All material published and events organized by the KOC must be exclusively under the "Kubeflow Community" brand, and not that of other organizations or individuals without approval by the KSC for each specific occasion.
3. The KOC may not enter into any partnerships or create co-branded material (including with other open-source projects) without approval by the KSC for each specific occasion.
4. The KOC may not authorize the usage of any trademarks, any such requests must be referred to the KSC.
5. The KOC may not enter into any contracts or other legal obligations, any such requests must be referred to the KSC.
6. The KOC must not organize any events or gatherings without approval by the KSC for each specific occasion.
In all cases, approval by the KSC refers to the normal decision process outlined in the KSC charter.
### Committee Structure
- **Chairs**:
- The Kubeflow Outreach Committee will be co-chaired by established Kubeflow community members, with these Chairs being responsible for guiding the committee’s work.
- KOC Chairs are appointed and removed by a decision of the KSC.
- Importantly, chairs are the only role which may participate in decision votes.
- **Leads**:
- KOC Leads will help KOC Chairs execute on their outreach strategy. For example, post to Kubeflow social media accounts, review blog posts, publish YouTube videos, help with events, or other items that fall under the KOC charter.
- Leads of the KOC are nominated by the KOC Chairs, and confirmed by a decision of the KSC. The KOC Chairs may remove a KOC Lead at their discretion.
- KOC Leads must inform the KOC Chairs of any significant actions they take.
Members of the KOC, both Chairs and Leads, must adhere to the CNCF Code of Conduct. The removal of any members for violating the CNCF Code of Conduct is also at the KSC's discretion through its normal decision process.
## Committee members
The KOC may have at most three (3) Chairs and five (5) Leads. Seats on the KOC are held by an individual, not by their employer.
### Limitations on Company Representation
No more than one Chair seat may be held by employees of the same organization (or conglomerate, in the case of companies owning each other).
Separately, no more than two Lead roles may be held by employees of the same organization (or conglomerate, in the case of companies owning each other). Additionally, no more than one Lead and one Chair can be held by employees of the same organization (or conglomerate, in the case of companies owning each other).
If employers change because of job changes, acquisitions, or other events, in a way that would be in violation of the proceeding limits, sufficient members of the committee must resign their positions until the requirements are satisfied. If it is impossible to find sufficient members to resign, all employees of that organization will be removed and the vacancies will be filled using the normal process.
In the event of a question of company membership (for example evaluating independence of corporate subsidiaries) a majority of all non-involved KSC members will decide.
### Committee Meetings
The KOC will meet regularly (at least monthly, or as needed), and these meetings will be open to the public. Specific sensitive matters, such as code of conduct violations or confidential discussions, may be handled separately.
Meeting notes will be publicly available to the community, except when privacy is required for specific discussions.
### Decision Process
Decisions requiring a vote include:
- Nominating a new KOC Lead for KSC approval
- Removing a KOC Lead
- Granting or removing access to any social media account or other elevated permission
- Proposing a new event or gathering to the KSC for approval
- Proposing a new partnership or co-branded publication to the KSC for approval
- Any other decisions that at least half of the chairs (rounded down) present decide requires a vote
Decisions are made in meetings when a quorum of the chairs are present and may pass with at least half the chairs (rounded up) supporting it.
Quorum is considered reached when at least half of the chairs (rounded up) are present.
Chairs of the KOC may abstain from a vote. Abstaining chairs will only be considered as contributing to quorum, in the event that a vote is called in a meeting.
## Committee members
Seats on the Outreach Committee are held by an individual, not by their employer.
The current membership of the committee is (listed alphabetically by first name):
| Name | Organization | GitHub |
| ------------------- | --------------- | --------------------------------------------------------- |
| Stefano Fioravanzo | Canonical | [StefanoFioravanzo](https://github.com/StefanoFioravanzo/) |
| Valentina Rodriguez Sosa | Red Hat | [varodrig](https://github.com/varodrig/) |
| Tarek Abouzeid | Telia | [tarekabouzeid](https://github.com/tarekabouzeid/) |
## Lead members
| Name | Organization | GitHub | Role |
| ------------------- | ------------- | --------------------------------------------------- | ------------------ |
| Yash Pal | | [yashpal2104](https://github.com/yashpal2104) | Social Media Lead |
### Getting in Touch
To engage with the Outreach Committee, you can:
1. Email the committee at `outreach@kubeflow.org` for private discussions.
2. Open an issue in the `kubeflow/community` repository and tag `@kubeflow/outreach-committee` for attention.
### Changes to the Charter
Changes to the KOC charter can be proposed by any community member via a GitHub PR. Amendments will be subject to approval by a standard decision of the KSC. Proposals will be available for at least one week for community comments before a vote occurs.
================================================
FILE: KUBEFLOW-STEERING-COMMITTEE.md
================================================
# Kubeflow Steering Committee
The Kubeflow Steering Committee (KSC) is the governing body of the Kubeflow project, providing decision-making and oversight pertaining to the Kubeflow project policies, sub-organizations, and financial planning, and defines the project values and structure.
The governance of Kubeflow is an open, living document, and will continue to evolve as the community and project change.
### Charter
1. Define, evolve, and promote the vision, values, and mission of the Kubeflow project.
1. Define and evolve project and group governance structures and policies, including election rules, working group policies, and project roles.
1. Steward, control access, delegate access, and establish processes regarding all Kubeflow project resources and have the final say in the disposition of those resources.
1. Define and evolve the scope of the Kubeflow community, including acceptance of new projects into Kubeflow.
1. Define Kubeflow trademark policy and conformance criteria.
1. Receive and handle reports about code of conduct violations and maintain confidentiality.
1. Act as the final escalation point and arbitrator for any disputes, issues, clarifications, or escalations within the project scope.
## Committee Meetings
KSC currently meets at least bi-weekly, or as-needed. Meetings are open to the public and held online, unless they pertain to sensitive or privileged matters. Examples of such matters are:
- Privacy related issues
- Private emails to the committee
- Code of conduct violations
- Certain Escalations
- Disputes between members
- Security reports
Meeting notes are available to members of the kubeflow-discuss mailing list, unless community member privacy requires otherwise. Public meetings will be recorded and the recordings made available publicly.
Questions and proposals for changes to governance are posted as issues in the kubeflow/community repo, and the KSC invites your feedback there. See [Getting in touch](#getting-in-touch) for other options.
## Committee Structure
KSC is composed of 5 (five) members. They are elected according to [the election policy](proposals/645-kubeflow-steering-committee-election/README.md).
Seats on the Steering Committee are held by an individual, not by their employer.
### Limitations on Company Representation
No more than one (1) seats may be held by employees of the same organization (or conglomerate, in the
case of companies owning each other). If the results of an election result in greater than one employee
of the same organization, the lowest vote getters in the current election from any particular
employer will be removed until representation on the committee is down to one.
In the staggered election schedule, if a particular organization already has one seat among the rotation
not affected by the election, no candidates from that organization will be selected by the election.
If the organization wants to change its representation in KSC, one member from that organization
needs to stand down from KSC, which will trigger a "resignation" event as explained below.
There is no guarantee that vacancy created will be filled by the organization's candidate.
If employers change because of job changes, acquisitions, or other events, in a way that would yield
more than one seat being held by employees of the same organization, sufficient members of the committee
must resign until only one employee of the same employer are left. If it is impossible to find
sufficient members to resign, all employees of that organization will be removed and new special
elections held. In the event of a question of company membership (for example evaluating independence
of corporate subsidiaries) a majority of all non-involved Steering Committee members will decide.
### Current Committee Members
The current membership of the committee is (listed alphabetically by first name):
| Name | Organization | GitHub | Term Start | Term End |
| ------------------- | ------------ | ---------------------------------------------------------------- | ---------- | ---------- |
| Andrey Velichkevich | Apple | [andreyvelich](https://github.com/andreyvelich/) | 02/01/2026 | 02/01/2028 |
| Chase Christensen | Wiz | [chasecadet](https://github.com/chasecadet/) | 02/01/2026 | 02/01/2028 |
| Francisco Arceo | Red Hat | [franciscojavierarceo](https://github.com/franciscojavierarceo/) | 02/01/2025 | 02/01/2027 |
| Julius von Kohout | DHL | [juliusvonkohout](https://github.com/juliusvonkohout/) | 02/01/2025 | 02/01/2027 |
| Mathew Wicks | NVIDIA | [thesuperzapper](https://github.com/thesuperzapper/) | 02/01/2026 | 02/01/2028 |
### Emeritus Committee Members
The list of emeritus members that previously served on KSC:
| Name | Organization | GitHub | Term Start | Term End |
| ------------ | ------------ | -------------------------------------------------- | ---------- | ---------- |
| Johnu George | Nutanix | [johnugeorge](https://github.com/johnugeorge/) | 02/01/2024 | 02/01/2026 |
| Josh Bottum | Independent | [jbottum](https://github.com/jbottum/) | 02/01/2024 | 02/01/2025 |
| James Wu | Google | [james-jwu](https://github.com/james-jwu/) | 02/01/2024 | 02/01/2025 |
| Yuan Tang | Red Hat | [terrytangyuan](https://github.com/terrytangyuan/) | 02/01/2024 | 02/01/2026 |
## Ownership Transfer
KSC members hold administrative ownership of Kubeflow assets. When new members of the KSC are elected,
a GitHub issue must be created to facilitate the transfer to the incoming members.
GitHub issue name:
```
Transfer Ownership to KSC 2025
```
GitHub issue content:
- [ ] Update Kubeflow Steering Committee document with the new members and emeritus members.
- [ ] Archive the current Slack channel (e.g. `#archived-ksc-2024`) and create the new Slack channel (e.g. `kubeflow-steering-committee`).
- [ ] Schedule weekly calls with the new members.
- [ ] Update [admins for Kubeflow GitHub org](https://github.com/kubeflow/internal-acls/blob/master/github-orgs/kubeflow/org.yaml#L7).
- [ ] Update the [`kubeflow-steering-committee` GitHub team](https://github.com/kubeflow/internal-acls/blob/master/github-orgs/kubeflow/org.yaml).
- [ ] Update approvers for the following OWNERS files (e.g the past members should be moved to `emeritus_approvers`):
- `kubeflow/kubeflow` [OWNERS file](https://github.com/kubeflow/kubeflow/blob/master/OWNERS).
- `kubeflow/community` [OWNERS file](https://github.com/kubeflow/community/blob/master/OWNERS).
- `kubeflow/internal-acls` [OWNERS file](https://github.com/kubeflow/internal-acls/blob/master/OWNERS).
- `kubeflow/website` [OWNERS file](https://github.com/kubeflow/website/blob/master/OWNERS).
- `kubeflow/blog` [OWNERS file](https://github.com/kubeflow/blog/blob/master/OWNERS).
- [ ] Kubeflow GCP projects under `kubeflow.org` organization for ACLs and DNS management.
- Access for `kf-admin-cluster` GKE cluster in `kubeflow-admin` GCP project for the GitHub ACLs sync.
- Access for `kubeflow-dns` GCP project for the DNS management.
- [ ] Access for Kubeflow GKE cluster `kf-ci-v1` in `kubeflow-ci` GCP project (No Organization)
where Prow is running.
- [ ] Kubeflow [Google Group](https://groups.google.com/g/kubeflow-discuss).
- [ ] Update members for [KSC Google Group](https://groups.google.com/a/kubeflow.org/g/ksc).
- [ ] Access to Kubeflow `1password` account.
- [ ] Kubeflow social media resources.
- Kubeflow [LinkedIn](https://www.linkedin.com/company/kubeflow/)
- Kubeflow [X](https://x.com/kubeflow).
- Kubeflow [Bluesky](https://bsky.app/profile/kubefloworg.bsky.social).
- [Kubeflow Community](https://www.youtube.com/@KubeflowCommunity) YouTube channel.
- [Kubeflow](https://www.youtube.com/@Kubeflow) YouTube channel.
## Decision process
The steering committee desires to always reach consensus.
### Normal decision process
Decisions requiring a vote include:
- Issuing written policy
- Amending existing written policy
- Accepting, or removing a Kubeflow component
- Creating, removing, or modifying a working group
- All spending, hiring, and contracting decisions
- Official responses to publicly raised issues
- Any other decisions that at least half of the members (rounded down) present decide require a vote
Decisions are made in meetings when a quorum of the members are present and may pass with at least half the members (rounded up) of the committee supporting it.
Quorum is considered reached when at least half of the members (rounded up) are present.
Members of KSC may abstain from a vote. Abstaining members will only be considered as contributing to quorum, in the event that a vote is called in a meeting.
### Special decision process
Issues that impacts the KSC governance requires a special decision process. Issues include:
- Changes to the KSC charter
- KSC voting rules
- Election rules
The issue may pass with 70% of the members (rounded up) of the committee supporting it.
One organization may cast 1 vote. Votes cast by members from the same organization are equally weighted. Example:
- If KSC is made up of employees from organizations A, A, B, C, D, each vote from organization A is weighted by a factor of 0.5. The total number of votes is 4, and 3 votes (70% rounded up) is required to pass a proposal. This rule is designed to remove organization A's ability to defeat a proposal that is supported by all other KSC members.
- Similarly, if KSC is made up of employees from organizations A, A, B, B, C, the total number of votes is 3, and 2.5 votes is required to pass a proposal.
### Results
The results of the decision process are recorded and made publicly available, unless they pertain to sensitive or privileged matters. The results will include:
- Description of the issue
- Names of members who supported, opposed, and abstained from the vote.
## Getting in touch
There are two ways to raise issues to the steering committee for decision:
1. Emailing the steering committee at `ksc@kubeflow.org`. This is a private discussion list to which all members of the committee have access.
1. Open an issue on a kubeflow/community repository and indicate that you would like attention from the steering committee using GitHub tag `@kubeflow/kubeflow-steering-committee`.
## Changes to the charter
Changes to the KSC charter may be proposed via a PR on the charter itself by a Kubeflow community member. Amendments are accepted following the [Special Decision Process](#special-decision-process) detailed above.
Proposals and amendments to the charter are available for at least a period of one week for comments and questions before a vote will occur.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MAINTAINERS.md
================================================
# Maintainers
This document lists the maintainers under [the Kubeflow governance model](https://www.kubeflow.org/docs/about/governance/).
## Kubeflow Steering Committee
The members can be found in [the Kubeflow Steering Committee charter](KUBEFLOW-STEERING-COMMITTEE.md).
## Kubeflow Outreach Committee
The members can be found in [the Kubeflow Outreach Committee charter](KUBEFLOW-OUTREACH-COMMITTEE.md).
## Kubeflow Working Groups
The members can be found in [the Kubeflow Working Group list](wgs.yaml).
## Kubeflow Projects Approvers and Reviewers
The Kubeflow projects approvers and reviewers can be found in the corresponding OWNERS files:
- [Kubeflow Katib](https://github.com/kubeflow/katib/blob/master/OWNERS)
- [Kubeflow Manifests](https://github.com/kubeflow/manifests/blob/master/OWNERS)
- [Kubeflow Hub](https://github.com/kubeflow/hub/blob/main/OWNERS)
- [Kubeflow Notebooks](https://github.com/kubeflow/notebooks/blob/main/OWNERS)
- [Kubeflow Pipelines](https://github.com/kubeflow/pipelines/blob/master/OWNERS)
- [Kubeflow SDK](https://github.com/kubeflow/sdk/blob/main/OWNERS)
- [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator/blob/master/OWNERS)
- [Kubeflow Trainer](https://github.com/kubeflow/trainer/blob/master/OWNERS)
================================================
FILE: Makefile
================================================
IMAGE_NAME=golang:1.14
export GO111MODULE=on
export GOPROXY?=https://proxy.golang.org
default: \
generate \
reset-docs:
git checkout HEAD -- ./wgs/wgs.yaml ./wg-*/README.md
generate:
go run ./generator/app.go
test:
go test -v ./generator/...
.PHONY: default reset-docs generate verify test
================================================
FILE: OWNERS
================================================
approvers:
- andreyvelich
- chasecadet
- franciscojavierarceo
- juliusvonkohout
- thesuperzapper
emeritus_approvers:
- james-jwu
- jbottum
- johnugeorge
- terrytangyuan
================================================
FILE: OWNERS_ALIASES
================================================
aliases:
sig-feature-store-leads:
- kevinstumpf
- pyalex
- woop
sig-onprem-leads:
- jtfogarty
- mameshini
wg-automl-leads:
- andreyvelich
- gaocegege
- johnugeorge
wg-data-leads:
- ChenYi015
- andreyvelich
- franciscojavierarceo
- rareddy
- tarilabs
wg-deployment-leads:
- PatrickXYS
- animeshsingh
- mameshini
- vpavlin
- yanniszark
wg-manifests-leads:
- juliusvonkohout
- kimwnasptd
wg-ml-experience-leads:
- StefanoFioravanzo
- andreyvelich
- ederign
wg-notebooks-leads:
- StefanoFioravanzo
- elikatsis
- kimwnasptd
- thesuperzapper
- yanniszark
wg-pipelines-leads:
- HumairAK
- chensun
- droctothorpe
- james-jwu
- mprahl
- zazulam
wg-serving-leads:
- animeshsingh
- cliveseldon
- ellistarn
- iamlovingit
- rakelkar
- yuzisun
wg-training-leads:
- Jeffwan
- andreyvelich
- gaocegege
- johnugeorge
- tenzen-y
- terrytangyuan
## BEGIN CUSTOM CONTENT
## END CUSTOM CONTENT
================================================
FILE: README.md
================================================
# Kubeflow Community
Welcome to Kubeflow! We're glad to have you here.
## Join us in discussions, forums, and meetings
Visit the
[community page in our docs](https://www.kubeflow.org/docs/about/community/)
to see where you can chat with the community and share your ideas.
## Code of conduct
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of
experience, education, socio-economic status, nationality, personal appearance,
race, religion, or sexual identity and orientation.
The Kubeflow community is guided by our [Code of
Conduct](https://github.com/kubeflow/community/blob/master/CODE_OF_CONDUCT.md),
which we encourage everybody to read before participating.
## Contributing
The [contributor's guide](https://www.kubeflow.org/docs/about/contributing/)
provides a starting point for contributing ideas and bug fixes to the Kubeflow
project.
## Kubeflow Brand Guidelines
The [Kubeflow Brand Guidelines](https://www.linuxfoundation.org/legal/trademark-usage) are intended to provide the community guidance on how to reference the Kubeflow trademarks.
Please make sure your usage of Kubeflow trademarks and logos is in compliance with the guidelines.
If you have any questions about the guidelines or would like to get feedback on a particular use case,
please reach out to ksc@kubeflow.org.
## About this repository
* [proposals](https://github.com/kubeflow/community/tree/master/proposals): Kubeflow design proposals
* [how-to](https://github.com/kubeflow/community/tree/master/how-to): for documenting community and other project processes
## Legal
The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/trademark-usage/).
================================================
FILE: calendar/Dockerfile
================================================
FROM google/cloud-sdk
RUN apt-get update -y && \
apt-get install -y python3-distutils
RUN python3.7 -m pip install \
fire==0.3.1 \
google-api-python-client==1.12.4 \
google-auth-oauthlib==0.4.1 \
google-cloud-secret-manager==2.0.0 \
oauth2client==4.1.3 \
python-dateutil==2.8.1 \
pyyaml==5.3.1 \
six==1.15.0
RUN mkdir -p /opt/kubeflow
COPY calendar_import.py /opt/kubeflow
================================================
FILE: calendar/Makefile
================================================
GIT_VERSION:=$(shell git describe --dirty --always)
PROJECT:=kf-infra-gitops
IMAGE:=gcr.io/$(PROJECT)/calendar-sync:$(GIT_VERSION)
# Echo commands
SHELL = sh -xv
LATESTIMAGES=latest_image.yaml
CONTEXT:=kf-autobot
build-submit:
gcloud --project=$(PROJECT) builds submit --machine-type=n1-highcpu-32 --tag=$(IMAGE) ./
build-output:
yq w -i latest_image.yaml $(PROJECT).calendar.image \
$(IMAGE)@$(shell gcloud container images describe $(IMAGE) | yq r - image_summary.digest)
set-image:
cd manifests && kustomize edit set image calendar=$(shell yq r $(LATESTIMAGES) $(PROJECT).calendar.image)
update-image: build-submit build-output set-image
deploy:
kustomize build manifests | kubectl --context=$(CONTEXT) apply -f -
# Build the image using GCB
# We can't use skaffold due to: https://github.com/GoogleContainerTools/skaffold/issues/3468
build: build-submit build-output
================================================
FILE: calendar/OWNERS
================================================
approvers:
- james-jwu
- jtfogarty
- nicholas-abad
- yuzliu
- zijianjoy
- chensun
================================================
FILE: calendar/README.md
================================================
# Kubeflow community calendar
This is location for [Kubeflow community calendar](https://calendar.google.com/calendar/embed?src=kubeflow.org_7l5vnbn8suj2se10sen81d9428@group.calendar.google.com).
You can find list for all meetings in [`calendar.yaml`](calendar.yaml) file.
## Add new meeting to the calendar
To add new meeting to the Kubeflow calendar follow these steps:
1. Add new record to [`calendar.yaml`](calendar.yaml) with meeting parameters.
Information about each field you can find in the file.
1. After updating the file, you have to ask anyone from [OWNERS](OWNERS) file
to manually run [`calendar_import.py`](../scripts/calendar_import.py) to update
Kubeflow calendar.
1. Then you should be able to see your new meeting in the calendar.
## Automating calendar_import.py
`calendar_import.py` is designed to run automatically using a robot account.
* We deploy it on Kubernetes
* We run [git-sync](https://github.com/kubernetes/git-sync) in a side car to synchronize the repo to a volume mount
* When calendar_import.py detects a change it runs the sync
### Where it Runs
* Project: kf-infra-gitops
* Cluster: kf-org-admin
* Namespace: kf-autobot
* Project configs [kubeflow/community-infra/tree/master/prod/namespaces/kf-infra-gitops](https://github.com/kubeflow/community-infra/tree/master/prod/namespaces/kf-infra-gitops)
### Credentials and Permissions
* The script runs using the gsuite account `autobot@kubeflow.org`
* Password and recovery codes are stored in secret manager
* **secret**: [projects/kf-infra-gitops/secret/kf-autobot-kubeflow-org-password](https://console.cloud.google.com/security/secret-manager/secret/autobot-kubeflow-org-password?project=kf-infra-gitops)
```
gcloud --project=kf-infra-gitops secrets versions access latest --secret="autobot-kubeflow-org-password"
```
* An OAuth2 refresh token is stored in secret manager to allow the script to run without human intervention
* **secret** [projects/kf-infra-gitops/secret/autobot-at-kubeflow-oauth](https://console.cloud.google.com/security/secret-manager/secret/autobot-at-kubeflow-oauth?project=kf-infra-gitops)
* When `calendar_import.py` runs it uses a GSA to read the OAuth2 refresh token from secret manager and then uses it
to authenticate as `autorobt@kubeflow.org` to the calendar API
* To update the refresh token run
```
python calendar_import.py mint-credentials
```
* This will direct you through the OAuth2 Web Login flow
* You will need to login as `autobot@kubeflow.org` using the password/recovery codes stored in secret manager
* The script will save the refresh token to secret manager
* The person running the script needs to be able to modify the secret
* We can't use a Google Service Account (GSA) to directly authenticate to the calendar API
because the calendar API requires [Domain Delegation](https://developers.google.com/identity/protocols/oauth2/service-account)
* Without domain wide configuration the GSA can add events to the calendar but not invite attendees to the meeting
* Domain wide configuration is restricted to the calendar scope to minimize the damage this can do
* Domain wide delegation makes it very difficult to reason about the blast radius as the GSA could impersonate
any user
* The only restriction is via OAuth scopes
* By impersonating `autobot@kubeflow.org` we know automation is limited to the privileges assigned to that account.
## Before running `calendar_import.py`
1. You need to be calendar admin in kubeflow.org.
1. You need to join the [kubeflow-discuss google group](https://groups.google.com/g/kubeflow-discuss), because
> Tip: If you have "View members" access to a group and create a group event, each member receives an invitation email. If you do not have “View members” access, the group receives an invitation. Each user must accept the invite for the event to display appear on their calendar.
Note if you are using a user@kubeflow.org email account, you should join kubeflow-discuss with this account.
1. You need a Google Cloud project (not sure whether it must be in kubeflow.org).
1. Go to https://console.cloud.google.com/apis/credentials, create an OAuth 2.0 Client ID choosing `Desktop app` type.
1. Visit the newly created OAuth 2.0 Client ID and click "Download JSON" button on top.
1. Move the json file to `~/secrest/kf-calendar.oauth_client.json`.
1. `pip install -r scripts/requirements.txt`
================================================
FILE: calendar/calendar.yaml
================================================
# This file controls the meetings on the "Kubeflow Community Calendar":
# https://calendar.google.com/calendar/embed?src=kubeflow.org_7l5vnbn8suj2se10sen81d9428@group.calendar.google.com
# ==================================================================================================
# Each list element is a map with the following keys:
# id: a unique id for the meeting (TIP: increment the last id by 1)
# name: name of the meeting
# date: date of the first meeting "MM/dd/YYYY"
# time: start and end time of the meeting "hour:minuteAM/PM-hour:minuteAM/PM"
# timezone: timezone in IANA Time Zone Database format (default: 'America/Los_Angeles')
# frequency: how often the meeting takes place (optional)
# until: last date to repeat until "MM/dd/YYYY" (optional)
# attendees: list of attendees (see: https://developers.google.com/calendar/v3/reference/events/insert#python)
# video: a zoom or Hangouts meeting link
# description: the description of the meeting
# organizer: the github username of the meeting organizer
# ==================================================================================================
# !!!!!!!!!!!!!!! IMPORTANT !!!!!!!!!!!!!!!
# DO NOT REMOVE recurring meetings, set their `until` date in the past, and move them under "legacy meetings".
# Otherwise, the meeting will remain in the calendar forever.
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
####################################################################################################
# Current meetings
####################################################################################################
# - id: kf032
# name: Kubeflow Community Call
# date: 04/15/2025
# time: 8:00AM-8:55AM
# frequency: bi-weekly
# video: https://zoom.us/j/99152427566?pwd=S0djc1FPcXkyNUdneHg3UFR3VTcyQT09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & agenda: https://bit.ly/kf-meeting-notes
# Join with Zoom: https://zoom.us/j/99152427566?pwd=S0djc1FPcXkyNUdneHg3UFR3VTcyQT09
# Meeting ID: 991 5242 7566
# Passcode: 645928
# Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
# International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
# Meeting Host Link: https://zoom.us/s/99152427566
# organizer: autobot@kubeflow.org
# - id: kf043
# name: Kubeflow CNCF Graduation Call
# date: 04/22/2025
# time: 8:00AM-8:55AM
# frequency: bi-weekly
# video: https://zoom.us/j/99152427566?pwd=S0djc1FPcXkyNUdneHg3UFR3VTcyQT09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & agenda: https://bit.ly/kf-meeting-notes
# Join with Zoom: https://zoom.us/j/99152427566?pwd=S0djc1FPcXkyNUdneHg3UFR3VTcyQT09
# Meeting ID: 991 5242 7566
# Passcode: 645928
# Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
# International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
# Meeting Host Link: https://zoom.us/s/99152427566
# organizer: autobot@kubeflow.org
# - id: kf026
# name: Kubeflow Pipelines Community Meeting (PST AM)
# date: 08/19/2020
# time: 10:00am-10:40am
# frequency: bi-weekly
# video: https://zoom.us/j/92607298595?pwd=VlKLUbiguGkbT9oKbaoDmCxrhbRop7.1
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# - email: kubeflow-pipelines@google.com
# description:
# - |
# Notes & Agenda: http://bit.ly/kfp-meeting-notes
# Join Zoom Meeting
# https://zoom.us/j/92607298595?pwd=VlKLUbiguGkbT9oKbaoDmCxrhbRop7.1
# Meeting ID: 926 0729 8595
# Passcode: 876287
# organizer: chensun
# - id: kf035
# name: Kubeflow Release Team Meeting (CET, US friendly)
# date: 05/15/2023
# time: 6:00PM-7:00PM
# timezone: Europe/Madrid
# frequency: bi-weekly
# video: https://meet.google.com/ezk-fmxo-fvu
# until: 06/24/2025
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & agenda: https://bit.ly/kf-release-team-notes
# Join with Google Meet: https://meet.google.com/ezk-fmxo-fvu
# Time zone: Europe/Madrid
# Video call link: https://meet.google.com/ezk-fmxo-fvu
# Or dial: (ES) +34 877 99 40 20 PIN: 889 929 201#
# More phone numbers: https://tel.meet/ezk-fmxo-fvu?pin=9345506892998
# organizer: dnplas
# - id: kf036
# name: Kubeflow Platform Meeting (Manifests + Security WG)
# date: 06/01/2023
# time: 6:00PM-7:00PM
# timezone: Europe/Athens
# frequency: bi-weekly
# video: https://bit.ly/kf-wg-manifests-meet
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & Agenda: https://bit.ly/kf-wg-manifests-notes
# Meeting Link: https://bit.ly/kf-wg-manifests-meet
# Recordings: https://bit.ly/kf-wg-manifests-drive
# organizer: juliusvonkohout
# - id: kf037
# name: Kubeflow Notebooks 2.0 + WG Meeting (US + EMEA)
# date: 06/08/2023
# time: 8:00AM-8:55AM
# frequency: weekly
# video: https://zoom.us/j/96967996819?pwd=aFNkWnBTVW1TWW1iellneFYrRXJPdz09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & Agenda: https://bit.ly/kf-notebooks-wg-notes
# Join with Zoom: https://zoom.us/j/96967996819?pwd=aFNkWnBTVW1TWW1iellneFYrRXJPdz09
# Meeting ID: 969 6799 6819
# Passcode: 326059
# Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
# International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
# organizer: kimwnasptd
# - id: kf041
# name: KF Model Registry community meeting (US/EMEA)
# date: 02/19/2024
# time: 7:00PM-8:00PM
# timezone: Europe/Madrid
# # meeting to happen after KF Release meeting (id: kf035), hence align by using same TZ
# frequency: bi-weekly
# video: https://meet.google.com/pni-ywgg-gtt
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Bi-weekly recurring meeting for KF Model Registry community call
# Meeting notes: https://docs.google.com/document/d/1DmMhcae081SItH19gSqBpFtPfbkr9dFhSMCgs-JKzNo/edit?usp=sharing
# organizer: tarilabs
# - id: kf042
# name: Kubeflow Spark Operator Meeting
# date: 03/07/2025
# time: 8:00AM-9:00AM
# frequency: bi-weekly
# video: https://zoom.us/j/93870602975?pwd=NWFNT2xrZU03alVTTXFBTEsvdDdMQT09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Monthly recurring meeting for Kubeflow Spark Operator.
# Notes & Agenda: https://docs.google.com/document/d/1AnG6ptKLBY7O6ddyNm4SVsEbfu6jiyVyN3hDDgDUnxQ/edit?usp=sharing
# Join with Zoom: https://zoom.us/j/93870602975?pwd=NWFNT2xrZU03alVTTXFBTEsvdDdMQT09
# Meeting ID: 938 7060 2975
# Passcode: 307820
# Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
# International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
# organizer: vara-bonthu
# - id: kf038
# name: Kubeflow AutoML and Training WG Meeting (Asia & Europe friendly)
# date: 09/20/2023
# time: 2:00PM-3:00PM
# timezone: Etc/UTC
# frequency: every-4-weeks
# video: https://us04web.zoom.us/j/8394435453?pwd=SHo4QXJhdHltb043NGxnMkZmU1l0UT09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & Agenda: https://bit.ly/2PWVCkV
# Join Zoom Meeting https://us04web.zoom.us/j/8394435453?pwd=SHo4QXJhdHltb043NGxnMkZmU1l0UT09
# Meeting ID: 839 443 5453
# Passcode: R3ScM7
# organizer: andreyvelich
# - id: kf039
# name: Kubeflow AutoML and Training WG Meeting (US friendly)
# date: 10/04/2023
# time: 5:00PM-6:00PM
# timezone: Etc/UTC
# frequency: every-4-weeks
# video: https://us04web.zoom.us/j/8394435453?pwd=SHo4QXJhdHltb043NGxnMkZmU1l0UT09
# attendees:
# - email: kubeflow-discuss@googlegroups.com
# description:
# - |
# Notes & Agenda: https://bit.ly/2PWVCkV
# Join Zoom Meeting https://us04web.zoom.us/j/8394435453?pwd=SHo4QXJhdHltb043NGxnMkZmU1l0UT09
# Meeting ID: 839 443 5453
# Passcode: R3ScM7
# organizer: andreyvelich
####################################################################################################
# Legacy meetings
####################################################################################################
- id: kf001
name: Kubeflow Community Call (US West/APAC)
date: 08/11/2020
time: 5:30PM-6:25PM
frequency: bi-weekly
video: https://zoom.us/j/83583392870?pwd=Q3lYQlAxbVYrdVZQNlp5cktIY2JmUT09
until: 09/20/2022
attendees:
- email: kubeflow-discuss@googlegroups.com
description:
- |
Notes & agenda: https://bit.ly/kf-meeting-notes
Join with Zoom: https://zoom.us/j/83583392870?pwd=Q3lYQlAxbVYrdVZQNlp5cktIY2JmUT09
Meeting ID: 835 8339 2870
Passcode: 499845
Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
Meeting Host Link: https://zoom.us/s/83583392870
organizer: autobot@kubeflow.org
- id: kf033
name: Kubeflow Community Call (Europe/APAC)
date: 10/04/2022
time: 8:00AM-8:55AM
timezone: Europe/London
frequency: bi-weekly
video: https://zoom.us/j/89752932714?pwd=bWRRcEY5MUM3S3U1TmY5NGZaL1ByUT09
until: 12/13/2022
attendees:
- email: kubeflow-discuss@googlegroups.com
description:
- |
Notes & agenda: https://bit.ly/kf-meeting-notes
Join with Zoom: https://zoom.us/j/89752932714?pwd=bWRRcEY5MUM3S3U1TmY5NGZaL1ByUT09
Meeting ID: 897 5293 2714
Passcode: 055458
Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
Meeting Host Link: https://zoom.us/s/89752932714
organizer: autobot@kubeflow.org
- id: kf005
name: Kubeflow Product Management Working Group
date: 08/20/2019
time: 10:00am-11:00am
frequency: bi-weekly
until: 07/14/2020
video: https://zoom.us/j/512655157
description:
- |
Meeting Logs: https://bit.ly/kf-pm-meeting-notes
organizer: theadactyl
- id: kf006
name: Kubeflow Outreach
date: 08/27/2019
time: 10:00am-11:00am
frequency: bi-weekly
until: 07/14/2020
video: https://zoom.us/j/512655157
description:
- |
Meeting Logs: http://bit.ly/kf-outreach-meeting-notes
organizer: theadactyl
- id: kf007
name: Google OSS Hands-on Workshop B - Kubeflow (MLOPS) @ Kubecon
date: 11/18/2019
time: 1:00pm-3:00pm
description:
- |
Kubecon schedule link: https://sched.co/W536
Requires additional registration & Fee
organizer: jlewi
- id: kf008
name: Enabling Kubeflow with Enterprise-Grade Auth for On-Prem Deployments - Yannis Zarkadas, Arrikto & Krishna Durai, Cisco
date: 11/19/2019
time: 11:50am-12:25pm
description:
- |
Room 16AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaY4
organizer: jlewi
- id: kf009
name: "Introducing KFServing: Serverless Model Serving on Kubernetes - Ellis Bigelow, Google & Dan Sun, Bloomberg"
date: 11/19/2019
time: 2:25pm-3:00pm
description:
- |
Room 15AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaZo
organizer: jlewi
- id: kf010
name: Towards Continuous Computer Vision Model Improvement with Kubeflow - Derek Hao Hu & Yanjia Li, Snap Inc.
date: 11/19/2019
time: 3:20pm-3:55pm
description:
- |
Room 31ABC - San Diego Convention Center Upper Level
https://sched.co/Uae4
organizer: jlewi
- id: kf011
name: Measuring and Optimizing Kubeflow Clusters at Lyft - Konstantin Gizdarski, Lyft & Richard Liu, Google
date: 11/19/2019
time: 4:25pm-5:00pm
description:
- |
Room 6C - San Diego Convention Center Upper Level
https://sched.co/UabJ
organizer: jlewi
- id: kf012
name: "KubeFlow’s Serverless Component: 10x Faster, a 1/10 of the Effort - Orit Nissan-Messing, Iguazio"
date: 11/19/2019
time: 4:25pm-5:00pm
description:
- |
Room 6F - San Diego Convention Center Upper Level
https://sched.co/UaaL
organizer: jlewi
- id: kf013
name: Advanced Model Inferencing Leveraging KNative, Istio and Kubeflow Serving - Animesh Singh, IBM & Clive Cox, Seldon
date: 11/20/2019
time: 10:55am-11:30am
description:
- |
Room 17AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaVw
organizer: jlewi
- id: kf014
name: Building and Managing a Centralized Kubeflow Platform at Spotify - Keshi Dai & Ryan Clough, Spotify
date: 11/20/2019
time: 11:50am-12:25pm
description:
- |
Room 29ABCD - San Diego Convention Center Upper Level
https://sched.co/UaWi
organizer: jlewi
- id: kf015
name: "Panel: Enterprise-grade, On-prem Kubeflow in the Financial Sector - Laura Schornack, JPMorgan Chase; Jeff Fogarty, US Bank; Josh Bottum, Arrikto; & Thea Lamkin, Google"
date: 11/20/2019
time: 2:25pm-3:00pm
description:
- |
Room 14AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaYA
organizer: jlewi
- id: kf016
name: "Kubeflow: Multi-Tenant, Self-Serve, Accelerated Platform for Practitioners - Kam Kasravi, Intel & Kunming Qu, Google"
date: 11/20/2019
time: 3:20pm-3:55pm
description:
- |
Room 17AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaaI
organizer: jlewi
- id: kf017
name: "Realizing End to End Reproducible Machine Learning on Kubernetes - Suneeta Mall, Nearmap"
date: 11/20/2019
time: 4:25pm-5:00pm
description:
- |
Room 16AB - San Diego Convention Center Mezzanine Level
https://sched.co/UacQ
organizer: jlewi
- id: kf018
name: "Tutorial: From Notebook to Kubeflow Pipelines: An End-to-End Data Science Workflow - Michelle Casbon, Google, Stefano Fioravanzo, Fondazione Bruno Kessler, & Ilias Katsakioris, Arrikto (Limited Available Seating; First-Come, First-Served Basis)"
date: 11/21/2019
time: 2:25pm-3:55pm
description:
- |
Room 28ABCDE - San Diego Convention Center Upper Level
https://sched.co/UaaL
organizer: jlewi
- id: kf019
name: "Building a Medical AI with Kubernetes and Kubeflow - Jeremie Vallee, Babylon Health"
date: 11/21/2019
time: 3:20pm-3:55pm
description:
- |
Room 11AB - San Diego Convention Center Upper Level
https://sched.co/UaWf
organizer: jlewi
- id: kf020
name: "KubeDirector - Deploying Complex Stateful Applications on Kubernetes - Joel Baxter & Thomas Phelan, Hewlett Packard Enterprise"
date: 11/21/2019
time: 4:25pm-5:00pm
description:
- |
Ballroom Sec 20CD - San Diego Convention Center Upper Level
https://sched.co/UaaF
organizer: jlewi
- id: kf021
name: "GPU as a Service Over K8s: Drive Productivity and Increase Utilization - Yaron Haviv, Iguazio"
date: 11/21/2019
time: 4:25pm-5:00pm
description:
- |
Room 17AB - San Diego Convention Center Mezzanine Level
https://sched.co/UaYt
organizer: jlewi
- id: kf022
name: "Supercharge Kubeflow Performance on GPU Clusters - Meenakshi Kaushik & Neelima Mukiri, Cisco"
date: 11/21/2019
time: 5:20pm-5:55pm
description:
- |
Room 11AB - San Diego Convention Center Upper Level
https://sched.co/Uada
organizer: jlewi
- id: kf023
name: "Book Signing for Oreilly's Kubeflow Operations Guide"
date: 11/20/2019
time: 10:30am-11:30am
description:
- |
San Diego Convention Center – Sails Pavilion – O'Reilly Booth–S25
organizer: jpatanooga
- id: kf029
name: Kubeflow Feature Store SIG Meeting (US/Asia friendly)
date: 11/25/2020
time: 6:00pm-6:30pm
frequency: monthly
until: 07/01/2023
attendees:
- email: kubeflow-discuss@googlegroups.com
description:
- |
Notes & Agenda: https://docs.google.com/document/d/1GHi-NFHmDA2TnH1pDrs4O7OyIQdZRtKatGAdedxte4w
Zoom: Provided in meeting notes
organizer: woop
- id: kf034
name: Kubeflow Security Team Call (US West/APAC)
date: 12/19/2023
time: 7:30AM-8:00AM
frequency: bi-weekly
video: https://us06web.zoom.us/j/87118537300?pwd=NG5ibWN0N2YxUGR5Y2NXSXRiN0FGZz09
until: 06/04/2024
attendees:
- email: kubeflow-discuss@googlegroups.com
- email: akgraner@gmail.com
- email: juliusvonkohout@gmail.com
- email: kimonas.sotirchos@canonical.com
- email: joshbottum@gmail.com
description:
- |
Notes & agenda: https://docs.google.com/document/d/1xGkg9GuO2OjvYhdONJFbSrpF66UKhtYonczttJoTv3s/edit?usp=sharing
Join with Zoom: https://us06web.zoom.us/j/87118537300?pwd=NG5ibWN0N2YxUGR5Y2NXSXRiN0FGZz09
Meeting ID: 871 1853 7300
Passcode: 273734
If you want to be manually added to the invite link, please reach out to Amber Graner (akgraner)
organizer: akgraner
- id: kf025
name: Kubeflow Pipelines Community Meeting (PST PM)
date: 08/05/2020
time: 5:30pm-6:10pm
frequency: every-4-weeks
video: https://meet.google.com/jkr-dupp-wwm
until: 07/03/2024
attendees:
- email: kubeflow-discuss@googlegroups.com
- email: kubeflow-pipelines@google.com
description:
- |
Notes & Agenda: http://bit.ly/kfp-meeting-notes
Join at: https://meet.google.com/jkr-dupp-wwm
organizer: chensun
- id: kf040
name: Kubeflow Notebooks WG Meeting (US + APAC)
date: 01/25/2024
time: 4:00PM-4:55PM
frequency: bi-weekly
video: https://zoom.us/j/95919259449?pwd=VGRNTm05VzRnZlIwN3lJRklsZmdqZz09
until: 09/13/2024
attendees:
- email: kubeflow-discuss@googlegroups.com
description:
- |
Notes & Agenda: https://bit.ly/kf-notebooks-wg-notes
Join with Zoom: https://zoom.us/j/95919259449?pwd=VGRNTm05VzRnZlIwN3lJRklsZmdqZz09
Meeting ID: 959 1925 9449
Passcode: 066005
Join with Phone (USA): +1 669 900 6833 or +1 646 558 8656
International numbers: https://zoom.us/zoomconference?m=Os1EjlUlpb2_XUMaQ6dX1azqMK5CkfWH
organizer: thesuperzapper
================================================
FILE: calendar/calendar_import.py
================================================
"""
Imports meetings in 'caldendar/calendar.yaml' to the Kubeflow Community Calendar
For modifications please refer to the Google Calendar Python API:
https://developers.google.com/resources/api-libraries/documentation/calendar/v3/python/latest/calendar_v3.events.html#insert
Requires the following packages: oauth2client, pyyaml, google-api-python-client
Uses a service account -- the calendar must be shared with the service account
email and given permission: "Make changes to events"
* Using a service account requires domain wide delegation in order to
add attendees to meetings
References:
https://developers.google.com/calendar/quickstart/python
* Shows the web app flow
https://developers.google.com/calendar/auth
"""
from datetime import datetime
from dateutil import parser as date_parser
import fire
import logging
import os
import subprocess
import time
import yaml
import googleapiclient.errors
from google.oauth2 import credentials
from google.api_core import exceptions as google_exceptions
from googleapiclient.discovery import build
from google_auth_oauthlib import flow
from google.cloud import secretmanager
from google.auth.transport import requests
from oauth2client import service_account
from pathlib import Path
import pickle
import json
# The public address of the kubeflow.org calendar
CALENDAR_ID = 'kubeflow.org_7l5vnbn8suj2se10sen81d9428@group.calendar.google.com'
SCOPES = ['https://www.googleapis.com/auth/calendar.events']
# The default GCP project and name of a secret in secret manager
# containing the oauth credentials for autobot@kubeflow.org.
DEFAULT_PROJECT = "kf-infra-gitops"
DEFAULT_SECRET = "autobot-at-kubeflow-oauth"
def update_meeting(service, meeting):
date = meeting['date']
time_start, time_end = meeting['time'].split('-')
day_of_week = datetime.strptime("{} {}".format(date, time_start), '%m/%d/%Y %I:%M%p').strftime('%A').upper()[0:2]
start_datetime = datetime.strptime("{} {}".format(date, time_start), '%m/%d/%Y %I:%M%p').strftime('%Y-%m-%dT%H:%M:%S%z')
end_datetime = datetime.strptime("{} {}".format(date, time_end), '%m/%d/%Y %I:%M%p').strftime('%Y-%m-%dT%H:%M:%S%z')
timezone = meeting.get("timezone", "America/Los_Angeles")
event = {
'summary': meeting['name'],
'id': meeting['id'],
'location': meeting.get("video"),
'description': meeting['description'],
'start': {
'dateTime': start_datetime,
'timeZone': timezone,
},
'end': {
'dateTime': end_datetime,
'timeZone': timezone,
},
'guestsCanSeeOtherGuests': 'false',
'reminders': {
'useDefault': False,
'overrides': [
{'method': 'popup', 'minutes': 10},
],
},
"creator": {
"displayName": "Kubeflow",
"email": "kubeflow-discuss@googlegroups.com",
},
"organizer": {
"displayName": meeting['organizer'],
},
}
if meeting.get("frequency"):
rec = 'RRULE:FREQ={};BYDAY={}'.format(meeting['frequency'].upper(), day_of_week)
if meeting['frequency'] == "bi-weekly":
rec += ';INTERVAL=2'
rec = rec.replace('BI-WEEKLY', 'WEEKLY')
elif meeting['frequency'] == "every-4-weeks":
rec += ';INTERVAL=4'
rec = rec.replace('EVERY-4-WEEKS', 'WEEKLY')
elif meeting['frequency'] == "monthly":
# In monthly meetings start date defines week number
# e.g. if start date is 09/09/2020 meetings are on every 2nd Wednesday
start_datetime = datetime.strptime("{} {}".format(date, time_start), '%m/%d/%Y %I:%M%p')
week_number = start_datetime.isocalendar()[1] - start_datetime.replace(day=1).isocalendar()[1] + 1
rec = rec.replace(day_of_week, "{}{}".format(week_number, day_of_week))
if meeting.get("until"):
until_day = date_parser.parse(meeting.get("until"))
until_time = date_parser.parse(end_datetime)
until = datetime.combine(until_day.date(), until_time.time())
# TODO(jlewi): Do we need to include a time zone correction?
rec += ";UNTIL=" + until.strftime("%Y%m%dT%H%M%SZ")
event["recurrence"] = [rec]
if meeting.get("attendees"):
event["attendees"] = meeting["attendees"]
try:
event = service.events().insert(calendarId=CALENDAR_ID, body=event).execute()
logging.info("Created Event: {}".format(meeting['name'][:100]))
logging.info(event.get('htmlLink'))
except googleapiclient.errors.HttpError as e:
content = json.loads(e.content)
if (content.get("error", {}).get("code") ==
requests.requests.codes.CONFLICT):
# It already exists so issue an update instead
event = service.events().update(calendarId=CALENDAR_ID, eventId=meeting['id'],
body=event).execute()
logging.info("Updated Event: {}".format(meeting['name'][:100]))
logging.info(event.get('htmlLink'))
else:
logging.error("Exception occurred trying to insert event:\n%s",
content)
except Exception as e:
logging.error("Error occurred creating the event: ", e)
def get_user_credentials(oauth_client_secret=None,
credentials_file=None):
"""Obtain user credentials through the web flow.
oauth_client_secret: Path to the json file containing an OAuth client id
for an OAuth application to use the Calendar API. Only required if
not using a service account
credentials_file: Where to save the credentials
credentials_secret:
"""
home = str(Path.home())
config_dir = os.path.join(home, ".config", "kubeflow", "calendar_import")
if not os.path.exists(config_dir):
os.makedirs(config_dir)
# File to store credentials
# Only used with the personal login flow.
if not credentials_file:
credentials_file = os.path.join(config_dir, "credentials.json")
# TODO(jlewi): Don't hardcode this
if not oauth_client_secret:
raise ValueError("An oauth_client_secret is required when using end user "
"credentials")
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists(credentials_file):
creds = credentials.Credentials.from_authorized_user_file(credentials_file)
# If there are no (valid) credentials available, let the user log in.
if not creds:
web_flow = flow.InstalledAppFlow.from_client_secrets_file(oauth_client_secret,
SCOPES)
creds = web_flow.run_local_server(port=0)
# Save the credentials for the next run
with open(credentials_file, 'w') as token:
token.write(creds.to_json())
if creds.expired and creds.refresh_token:
creds.refresh(requests.Request())
return creds
def _get_default_config():
this_file = __file__
repo_root = os.path.abspath(os.path.join(os.path.dirname(this_file), ".."))
config = os.path.join(repo_root, "calendar/calendar.yaml")
return config
def create_secret(client, project, secret):
"""
Create a new secret with the given name. A secret is a logical wrapper
around a collection of secret versions. Secret versions hold the actual
secret material.
"""
# Build the resource name of the parent project.
parent = f"projects/{project}"
# Create the secret.
try:
response = client.create_secret(
request={
"parent": parent,
"secret_id": secret,
"secret": {"replication": {"automatic": {}}},
}
)
except google_exceptions.AlreadyExists:
logging.info("Secret already exists")
return
# Print the new secret name.
logging.info("Created secret: {response.name}")
def load_secret(client, project, secret):
path = client.secret_path(project, secret)
name = f"{path}/versions/latest"
logging.info(f"Fetching secret {name}")
# Access the secret version.
response = client.access_secret_version(request={"name": name})
# Print the secret payload.
#
# WARNING: Do not print the secret in a production environment - this
# snippet is showing how to access the secret material.
payload = response.payload.data.decode("UTF-8")
return payload
def sync_calendar(config, creds):
service = build('calendar', 'v3', credentials=creds)
logging.info("Loading calendar data from %s", config)
with open(config) as cal:
meetings = yaml.safe_load(cal)
for meeting in meetings:
try:
update_meeting(service, meeting)
except Exception as e:
logging.error("Could not update meeting %s; Error:\n%s",
meeting.get("id"), e)
continue
class CalendarUpdater:
"""Class to update the calendar"""
@staticmethod
def sync(config=None, oauth_client_secret=None):
"""Sync the events in the YAML file to the calendar
Args:
config: Path to the YAML file containing the calendar data.
oauth_client_secret: Path to the json file containing an OAuth client id
for an OAuth application to use the Calendar API. Only required if
not using a service account
"""
# Since we are using the Google calendar API which isn't a Google Cloud API
# we can't use Google Cloud Platform Default Application credentials.
# There are two modes we want to support
# 1. Running locally using a personal account
# 2. Running in a cluster using a service account
# TODO(jlewi): Can we use Workload Identity with the calendar API or do
# we have to use a service account
creds = None
if config is None:
config = _get_default_config()
logging.info("config file not set resorting to default default")
if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
logging.info("GOOGLE_APPLICATION_CREDENTIALS is set using service "
"account")
# TODO(jlewi): How do we obtain credentials with a service account?
SERVICE_ACCOUNT_FILE = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
creds = service_account.ServiceAccountCredentials.from_json_keyfile_name(
SERVICE_ACCOUNT_FILE, SCOPES)
creds = creds.create_delegated("autobot@kubeflow.org")
else:
creds = get_user_credentials(oauth_client_secret=oauth_client_secret)
sync_calendar(config, creds)
@staticmethod
def periodic_sync(config=None, project=DEFAULT_PROJECT,
secret=DEFAULT_SECRET, period_seconds=15):
"""Run the sync periodically at the desired interval.
The sync only runs when the commit changes
Args:
config: Path to the YAML file containing the calendar config
project: The GCP project containing the secret containing OAuth
credentials for the bot account to run as
secret: The name of the secret in GCP secret manager
period_seconds: How frequently to check for changes to the file.
This should be pretty frequent since the sync only runs when changes
are detected.
"""
last_tag = None
if not config:
logging.info("Using default config file")
config = _get_default_config()
logging.info("Config: %s", config)
git_dir = os.path.dirname(config)
client = secretmanager.SecretManagerServiceClient()
secret_contents = load_secret(client, project, secret)
creds_json = json.loads(secret_contents)
creds = credentials.Credentials.from_authorized_user_info(creds_json)
while True:
# This is a bit of a hack. When relying on a side car (e.g. git-sync)
# to synchronize the config the file may not be available when the
# script first runs because the git sync hasn't completed yet.
if not os.path.exists(config):
logging.error("Config %s doesn't exist")
time.sleep(period_seconds)
continue
latest = subprocess.check_output(["git", "describe", "--dirty",
"--always"], cwd=git_dir)
latest = latest.strip()
logging.info("Current tag=%s; last run=%s", latest, last_tag)
if latest != last_tag:
logging.info("Running sync")
sync_calendar(config, creds)
else:
logging.info("No changes; not syncing")
last_tag = latest
time.sleep(period_seconds)
@staticmethod
def mint_credentials(project, secret, oauth_client_secret):
"""Mint credentials for a particular user:
The purpose of this command is to persist OAuth credentials for a
desktop app to a file:
https://developers.google.com/identity/protocols/oauth2/native-app
The purpose of this is to allow impersonating a bot account for
the purpose of automation.
Args:
project: GCP project to store the secret in
secret: The id of a secret in GCP secret manager to save the secret
to.
oauth_client_secret: Path to the json file containing an OAuth client id
for an OAuth application to use the Calendar API. Only required if
not using a service account
"""
# Create the Secret Manager client.
client = secretmanager.SecretManagerServiceClient()
create_secret(client, project, secret)
# Build the resource name of the parent secret.
parent = client.secret_path(project, secret)
# Go through the webflow
web_flow = flow.InstalledAppFlow.from_client_secrets_file(oauth_client_secret,
SCOPES)
creds = web_flow.run_local_server(port=0)
# Convert the string payload into a bytes. This step can be omitted if you
# pass in bytes instead of a str for the payload argument.
payload = creds.to_json().encode("UTF-8")
# Add the secret version.
response = client.add_secret_version(
request={"parent": parent, "payload": {"data": payload}}
)
# Print the new secret version name.
logging.info("Added secret version: {}".format(response.name))
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO,
format=('%(levelname)s|%(asctime)s'
'|%(pathname)s|%(lineno)d| %(message)s'),
datefmt='%Y-%m-%dT%H:%M:%S',
)
logging.getLogger().setLevel(logging.INFO)
fire.Fire(CalendarUpdater)
================================================
FILE: calendar/latest_image.yaml
================================================
image: hello
kf-infra-gitops:
calendar:
image: gcr.io/kf-infra-gitops/calendar-sync:2b76aca-dirty@sha256:49b7b067a354d6cdb898707eeb2a258405ce0e64e17b8b76f440a6d0b80defc2
================================================
FILE: calendar/manifests/deployment.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: calendar-sync
labels:
app: calendar-sync
spec:
replicas: 1
selector:
matchLabels:
app: calendar-sync
template:
metadata:
labels:
app: calendar-sync
spec:
containers:
- name: calendar-sync
image: calendar
command:
- python3.7
- /opt/kubeflow/calendar_import.py
- periodic-sync
- --config=/data/community.git/calendar/calendar.yaml
volumeMounts:
- name: data
mountPath: /data
- name: sync
image: k8s.gcr.io/git-sync:v3.1.6
args:
- --repo=https://github.com/kubeflow/community.git
- --branch=master
- --root=/data
# In seconds
- --wait=30
volumeMounts:
- name: data
mountPath: /data
serviceAccount: kf-autobot
volumes:
- name: data
emptyDir: {}
================================================
FILE: calendar/manifests/kustomization.yaml
================================================
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kf-autobot
resources:
- deployment.yaml
- service-account.yaml
images:
- digest: sha256:49b7b067a354d6cdb898707eeb2a258405ce0e64e17b8b76f440a6d0b80defc2
name: calendar
newName: gcr.io/kf-infra-gitops/calendar-sync:2b76aca-dirty
================================================
FILE: calendar/manifests/service-account.yaml
================================================
apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
iam.gke.io/gcp-service-account: kf-autobot@kf-infra-gitops.iam.gserviceaccount.com
name: kf-autobot
================================================
FILE: dco-signoff-hook/README.md
================================================
# Signing off commits
To automatically sign off on every commit, copy the [prepare-commit-msg](prepare-commit-msg) file to the `.git/hooks` directory in your repo or if you already have such a hook, merge the contents into your existing hook.
You can also configure it globally (for every repo on your machine) by copying the [prepare-commit-msg](prepare-commit-msg) file to the `${HOME}/.git-template/hooks` directory.
You can also sign off your contributions manually by doing ONE of the following:
* Use `git commit -s ...` with each commit to add the sign-off or
* Manually add a `Signed-off-by: Your Name <your.email@example.com>` to each commit message; please note that Name and Email of sign-off must match the commit's Author, in other words the `user.name` and `user.email` of the git configuration used when creating the commit. Using `git commit -s ...` does this automatically.
The email address must match your primary GitHub email. You do NOT need cryptographic (e.g. gpg) signing.
* Use `git commit -s --amend ...` to add a sign-off to the latest commit or
* Use `git rebase HEAD~N --signoff` to sign-off on the last `N` commits, if you forgot.
*Note*: Some projects will provide specific configuration to ensure all commits are signed-off. Please check the project's documentation for more details.
================================================
FILE: dco-signoff-hook/prepare-commit-msg
================================================
#!/bin/sh
NAME=$(git config user.name)
EMAIL=$(git config user.email)
if [ -z "$NAME" ]; then
echo "empty git config user.name"
exit 1
fi
if [ -z "$EMAIL" ]; then
echo "empty git config user.email"
exit 1
fi
git interpret-trailers --if-exists doNothing --trailer \
"Signed-off-by: $NAME <$EMAIL>" \
--in-place "$1"
================================================
FILE: devstats/Dockerfile.devstats
================================================
# Dockerfile for devstats
# Based on https://github.com/cncf/devstats/blob/master/INSTALL_UBUNTU17.md
FROM golang:1.11.5
RUN apt-get update -y && \
apt-get install -y apt-transport-https \
git psmisc jsonlint yamllint gcc
RUN mkdir -p /go/src
ENV GOPATH /go
RUN cd ${GOPATH}/src && \
git clone https://github.com/cncf/devstats.git devstats
RUN go get -u github.com/golang/lint/golint && \
go get golang.org/x/tools/cmd/goimports
RUN go get github.com/jgautheron/goconst/cmd/goconst && \
go get github.com/jgautheron/usedexports
RUN go get github.com/kisielk/errcheck && \
go get github.com/lib/pq
RUN go get golang.org/x/text/transform && \
go get golang.org/x/text/unicode/norm
RUN go get github.com/google/go-github/github && \
go get golang.org/x/oauth2
RUN go get gopkg.in/yaml.v2 && \
go get github.com/mattn/go-sqlite3
# Add /go/src/devstats to the path so that all the binaries will be on the path.
# This is needed by the devstats binary.
ENV PATH $PATH:/etc/gha2db:/${GOPATH}/bin
# Commit c905db8106d057f70a694ecd1276c9e32290152f is master on 02/14.
# Recommendation from Devstats folks was to us master.
RUN cd ${GOPATH}/src/devstats && \
git checkout c905db8106d057f70a694ecd1276c9e32290152f && \
make
RUN cd ${GOPATH}/src/devstats && \
make install
# TODO(jlewi): Do we need to fix the userid of postgres so we can run as that user?
# Create postgres user and group with fixed userid and groupid so we can run container as that user.
#
RUN groupadd -g 1000 postgres && \
useradd -r -u 1000 -g postgres --create-home --shell=/bin/bash postgres
# Install postgress
#
# This is based on
# https://github.com/docker-library/postgres/blob/master
RUN apt-get install -y postgresql-client postgresql sudo gosu
# Install emacs
RUN apt-get install -y emacs
# Install Ruby this is used by gitdm
# We set the environment because of this issue
# https://stackoverflow.com/questions/17031651/invalid-byte-sequence-in-us-ascii-argument-error-when-i-run-rake-dbseed-in-ra
ENV RUBYOPT "-KU -E utf-8:utf-8"
RUN apt-get install -y ruby gem
RUN gem install bundle pry octokit
# Keep this in sync with whatever package apt installs; maybe we should pin apt-install?
ENV PG_MAJOR 9.6
# make the sample config easier to munge (and "correct by default")
RUN mv -v "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample" /usr/share/postgresql/ \
&& ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/" \
&& sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/share/postgresql/postgresql.conf.sample
RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql
ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin
RUN mkdir -p /home/
COPY postgre-docker-entrypoint.sh /usr/local/bin/
RUN chmod a+x /usr/local/bin/postgre-docker-entrypoint.sh
# Add postgres to the sudoers group because some of the devstats scripts require it
RUN adduser postgres sudo
RUN addgroup --gid 472 grafana && \
adduser -u 472 --gid=472 grafana
# Workaround for https://github.com/cncf/devstats/issues/166
# devstats code assumes projects.yaml will be et /etc/gha2db
# but we will mount it from NFS
#
RUN rm -rf /etc/gha2db && \
ln -sf /mount/data/src/git_kubeflow-community/devstats/config /etc/gha2db
# TODO(jlewi): Per the instructions for devstats we should increase the number of default connections for postgres
================================================
FILE: devstats/Makefile
================================================
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requirements:
# https://github.com/mattrobenolt/jinja2-cli
# pip install jinja2-clie
# Update the Airflow deployment
IMG = gcr.io/devstats/devstats
TAG := $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)-$(shell git diff | sha256sum | cut -c -6)
DIR := ${CURDIR}
push: build
gcloud docker -- push $(IMG):latest
gcloud docker -- push $(IMG):$(TAG)
# To build without the cache set the environment variable
# export DOCKER_BUILD_OPTS=--no-cache
build:
docker build ${DOCKER_BUILD_OPTS} -f Dockerfile.devstats -t $(IMG):$(TAG) .
docker tag $(IMG):$(TAG) $(IMG):latest
@echo Built $(IMG):$(TAG) and tagged with latest
================================================
FILE: devstats/README.md
================================================
# Devstats
Instructions for deploying [cncf/devstats](https://github.com/cncf/devstats).
## Deploying on Kubernetes
We currently use
```
PROJECT=devstats
ZONE=us-east1-d
CLUSTER=devstats
NAMESPACE=devstats
```
### Setup the project with deployment manager.
```
gcloud deployment-manager --project=${PROJECT} deployments create devstats --config=devstats.yaml
gcloud deployment-manager --project=${PROJECT} deployments create devstats-gcfs --config=gcfs.yaml
```
This will create the resources
* GKE Cluster
* Static IP address for ingress
* A Cloud NFS file store
```
gcloud --project=${PROJECT} compute addresses list
```
* We use Cloud NFS to store grafana and postgres data. This way its easily accessible on multiple pods
### Create PVC for NVC
* Modify `k8s_manifests/nfs_pvc.yaml`
* Set IP address to ip address of cloud NFS
* Set namespace to namespace of your components
### Setup DNS
TODO(https://github.com/kubeflow/community/issues/228): Change host to devstats.kubeflow.org.
We used the name devstats2.kubeflow.org while we setup a new instance running in the devstats project.
Once its up and working we should turn down the existing instance in kubeflow-ci and change the hostname.
```
IPADDRESS=<..address from above...>
HOST=devstats
gcloud --project=kubeflow-dns dns record-sets transaction start -z=kubefloworg
gcloud --project=kubeflow-dns dns record-sets transaction add -z=kubefloworg \
--name="${HOST}.kubeflow.org." \
--type=A \
--ttl=300 "${IPADDRESS}"
gcloud --project=kubeflow-dns dns record-sets transaction execute -z=kubefloworg
```
* This uses domain kubeflow.org which is managed by Cloud DNS zone kubefloworg
### Create a secret with a GITHUB OAuth token
This token is only for rate quota so it doesn't need access to any services.
kubectl create secret generic github-oauth --from-literal=github-oauth=${GITHUB_TOKEN}
### Create a secret for the Grafana admin password
```
GRAFANA_PASSWORD=`< /dev/urandom tr -dc A-Za-z0-9 | head -c14; echo`
kubectl create secret generic grafana --from-literal=admin_password=${GRAFANA_PASSWORD}
```
If you need the password to login
```
kubectl get secrets grafana -o json | jq -r .data.admin_password | base64 -d && echo
```
** Important** Once created the password is stored in the database so changing
the secret won't change the password.
### Setup Devstats config Files
See config/README.md
### Create the K8s resources
```
kubectl apply -f k8s_manifest/nfs_pvc.yaml
kubectl apply -f k8s_manifest/cli_home_pvc.yaml
ks apply devstats2 -c cert-manager
ks apply devstats2 -c devstats
```
* The postgres and grafana containers will likely be crashing because we need to
setup the disk.
### How it all works
* Postgres has 3 databases
* **postgres** This database is created by `postgre-docker-entrypoint`
* I don't think this is used by devstats.
* psql needs to connect to a database even when psql is being used to create/delete other
databases. So we connect to the postgres database when executing commands to create/delete
the **kubeflow** and **devstats** dbs.
* **kubeflow** This is the DB where devstats data is actually stored.
* **devstats** This DB is used for logs
* List of environment variables used by [devstats code](https://github.com/cncf/devstats/blob/23ee872591a77f25c8832e0e46e0289cf22697a0/context.go)
* Grafana dashboards and datasources are defined in YAML files
* These are stored in source control and checked out into a NFS volume.
### Setup Postgres
1. We use the `devstats-cli-0` container to modify the NFS share; start a shell inside the container
using the `kubectl` command below and then proceed to run the other steps in the container.
```
kubectl exec -ti devstatsdb-0 -c postgres /bin/bash
```
1. Checkout the various source repositories onto NFS to get the scripts
```
mkdir /mount/data/src
cd /mount/data/src
git clone https://github.com/kubeflow/community.git git_kubeflow-community
```
1. Run the following script to set permissions on the NFS share
```
/mount/data/src/git_kubeflow-community/devstats/scripts/setup_nfs.sh
```
1. Set up the home directory for postgres in the CLI container
```
/mount/data/src/git_kubeflow-community/devstats/scripts/setup_postgres_home.sh
```
* We use a home diretory backed by PD in CLI so we need to do a onetime setup
1. Copy some binaries referenced by the scripts
```
cd /mount/data/src/git_kubeflow-community/devstats/config
./copy_devstats_binaries.sh
```
* TODO(jlewi): We should file a bug to get the devstats scripts updated to just assume the binaries are on the
path. I hink if we don't set GHA2DB_LOCAL it will use path.
1. Verify you can connect to the default database
```
psql -U postgres -d postgres -c "SELECT * FROM pg_catalog.pg_tables order by tablename;"
```
* The database postgres is created by the startup script `postgre-docker-entrypoint.sh`
* The kubeflow database won't exist at this point which is why we test using hte default database
1. Verify you can connect to the database and run queries.
```
sudo -E -u postgres psql -d postgres -U postgres -c "SELECT * FROM pg_catalog.pg_tables order by tablename;"
```
* If this doesn't work there might be a problem with the environment variables telling psql how to connect to the DB
* If this doesn't succeed most of the commands run by the scripts won't work.
1. Initialize the devstats DB
```
cd /mount/data/src/git_kubeflow-community/devstats/config
PGUSER=postgres PGDATABASE=postgres PG_PASS=${PG_PASS} PG_PASS_RO=${PG_PASS} PG_PASS_TEAM=kubeflow ./devel/init_database.sh
```
* We need to override `PGUSER` and `PGDATABASE` when running this command because the `kubeflow` database
doesn't exist it.
* So we connect to postgres using the database `postgres` and user `postgres` created by the `postgres` container
on startup
* Verify the devstats database exists
```
psql -U postgres -d postgres -c 'select * from pg_database;'
```
* devstats should be one of the listed databases.
* The devststats db is used for logs
1. Create the Kubeflow database
```
cd /mount/data/src/git_kubeflow-community/devstats/config
PGUSER=postgres PGDATABASE=postgres PROJ=kubeflow PROJDB=kubeflow PDB=1 TSDB=1 SKIPTEMP=1 ./devel/create_databases.sh
```
* This step creates the database table and backfills it based on the start date in `projects.yaml`
* Since the kubeflow database doesn't exist yet we override `PGUSER` and `PGDATABASE` so that we
connect to the postgres database in order to create the kubeflow database
* Verify the **kubeflow** database exists
```
psql -c 'select * from pg_database;'
```
* **kubeflow** should be one of the databases
* Check the `gha_*` tables were created
```
psql -d kubeflow -c "SELECT * FROM pg_catalog.pg_tables order by tablename;"
schemaname | tablename | tableowner | tablespace | hasindexes | hasrules | hastriggers | rowsecurity
--------------------+---------------------------------------+------------+------------+------------+----------+-------------+-------------
public | gha_actors | gha_admin | | t | f | f | f
public | gha_actors_affiliations | gha_admin | | t | f | f | f
public | gha_actors_emails | gha_admin | | t | f | f | f
public | gha_actors_names | gha_admin | | t | f | f | f
...
```
* Note that the tables containing metrics won't be created until later when we run `devstats`.
* `import_affs.sh` crashes see https://github.com/cncf/devstats/issues/166
* This step is related to importing user affiliations to generate company statistics
* For now I just skipped it and ran the next step manually
```
GHA2DB_PROJECT=kubeflow PG_DB=kubeflow GHA2DB_LOCAL=1 ./vars
```
1. Create tags
```
cd /mount/data/src/git_kubeflow-community/devstats/config
./shared/tags.sh
```
1. Run the following to create the annotations
```
cd /mount/data/src/git_kubeflow-community/devstats/config
./annotations
```
* This creates the time range selectors based on tags. So we need to run it along with previous step periodically
to get new tags.
* Verify that the table `tquick_ranges` now exists.
```
psql -d kubeflow -c "SELECT * FROM pg_catalog.pg_tables where tablename='tquick_ranges';"
```
* TODO(https://github.com/kubeflow/community/issues/230): Do we need to run this regularly?
1. Run devstats
```
cd /mount/data/src/git_kubeflow-community/devstats/config
devstats
```
* This should synchronize data and I think create metric and timeseries tables
* It checks out source for repos so it can get tags
* It should also be run as a cron job.
1. Known issues
* Looks like the relation `tcountries` wasn't created and some tables use this see https://github.com/kubeflow/community/issues/231
## Grafana
To access the admin ui port-forward to port `3000` and use the default admin account
which has username admin and password admin.
```
kubectl port-forward service/grafana 3000:3000
```
Dashboards are defined in `devstats/grafana/dashboards/kubeflow/`
These are checked out from git onto the NFS volume mounted on all pods.
### Troublehoosting Postgres
Check the databases using the following query
```
psql -c 'select * from pg_database;'
```
* Do the DB's **devstats** and **kubeflow** exist?
Check Kubeflow tables exist
```
psql -c -d kubeflow -c 'SELECT * FROM pg_catalog.pg_tables order by tablename;'
```
* There should be a bunch of tables named `gha_*`
### Loading data into the db
Use the gha2db program that is part of cncf/devstats
If we run devstats regularly that will syncronize the latest changes.
To backfill some range you can run gha2b directly
```
./gha2db 2018-04-16 00 2018-04-17 00 kubeflow
```
* Change the date range to the range you want
You can run this on K8s as a job by doing
```
ks param set backfill end_day 2018-01-01
ks param set backfill end_day 2018-04-17
ks apply default -c backfill
```
### Sync data
We use a cron job to run `devstats` regularly to pull in the latest data.
```
ks apply ${ENV} -c syncronjob
```
### Example running a query
After the backfill job completes you can run the following to verify data is in the SQL DB.
Run in the devstats-cli-0 container
```
./runq util_sql/top_unknowns.sql {{ago}} '1 month' {{lim}} 10
```
## Miscellaneous
Using psql from the CLI container (although you can also run from the postgre container and then you don't have to do a remote connect)
```
psql -h ${PG_HOST} -U gha_admin -d gha
```
List tables
```
SELECT * FROM pg_catalog.pg_tables order by tablename;
```
A simple query to look at events
```
select created_at, type from gha_events;
```
### Company Affilitations
We need to create a json file containing company affiliations for each user in order to get company stats.
These instructions are based on [sync.md](https://github.com/cncf/gitdm/blob/master/SYNC.md)
All commands should be run in the `devstats-cli-0` pod.
1. Make sure the repos are checked out on NFS
* Directory is set in ${GHA2DB_REPOS_DIR}
* To manually update
```
cd /mount/data/src/git_kubeflow-community/devstats/config
GHA2DB_PROCESS_REPOS=1 ./get_repos
```
1. Generate a list of all repos and the command to generate the repo log
```
cd /mount/data/src/git_kubeflow-community/devstats/config
GHA2DB_PROCESS_REPOS=1 GHA2DB_EXTERNAL_INFO=1 ./get_repos
```
1. Update `/mount/data/src/git_cncf-gitdm/src/repos.txt` with the list of repos outputted by the previous command
1. Setup gitdm
```
cd /mount/data/src/git_cncf-gitdm/src
gem install pry
gem install octokit
```
* TODO(jlewi): Should we do gem install in the image?
1. Generate a git log.
```
./all_repos_log.sh /mount/data/devstats_repos/kubeflow/*
```
* This should create a git log `/mount/data/src/git_cncf-gitdm/src/git.log`
1. To run cncf/gitdm on a generated git.log file do:
```
cd /mount/data/src/git_cncf-gitdm/src
./cncfdm.py -i git.log -r "^vendor/|/vendor/|^Godeps/" -R -n -b ./ -t -z -d -D -A -U -u -o all.txt -x all.csv -a all_affs.csv > all.out
```
1. Generate actors
```
cd /mount/data/src/git_kubeflow-community/devstats
bash -x ./scripts/generate_actors.sh /mount/data/src/git_cncf-gitdm/src/actors.txt
```
* TODO(jlewi): Should we store the actors file someplace other than git_cncf-gitdm? We currently put it there
because all the gitdm scripts make assumptions about the locations of the files
1. Create a secret containing
* A GitHub OAuth API token
* Can use the same token as before
* A GitHub OAuth client secret
* A GitHub client id
```
kubectl create secret generic gitdm-github-oauth --from-literal=oauth=${GITDM_GITHUB_OAUTH_TOKEN} --from-literal=client_id=${GITDM_GITHUB_CLIENT_ID} --from-literal=client_secret=${GITDM_GITHUB_CLIENT_SECRET}
```
1. Pull GitHub users
```
cd /mount/data/src/git_cncf-gitdm/src/
echo [] > github_users.json
ruby ghusers.rb
./encode_emails.rb github_users.json temp
mv temp github_users.json
```
* Ensure repos.txt doesn't include any repos that shouldn't count as contributors
* In particular ensure [kubeflow/homebrew-cask](https://github.com/kubeflow/homebrew-cask) and
[homebrew-core](https://github.com/kubeflow/homebrew-core) are excluded
* TODO(jlewi): I'm not sure we want to zero out github_users.json on each successive run
* I think we only wanted to do that once because github_users.json was originally for the CNCF projects
* ghusers.rb has to make API requests for each user so if we don't cache results we hit API limits.
* ghusers.rb appears to crash if github_users.json doesn't exist and doesn't have at least a json list
* See also these [instructions](https://github.com/cncf/gitdm/blob/master/README.md#github-users-can-be-pulled-using-octokit-gihub-api)
* The processing of repos.txt is very brittle
* I had to modify the code ala Ran into https://github.com/cncf/gitdm/issues/104
* I also had to remove the quotes around the repo names
* TODO(jlewi): Could we just use ghusesrs.sh? The reason I didn't was because it didn't seem to handle things
like the file github_users.json not existing
1. Update github_users.json
```
cd /mount/data/src/git_cncf-gitdm/src/
./enhance_json.sh
```
* Output is
```
Found 1, not found 420 from 425 additional actors
Processed 425 users, enchanced: 306, not found in CSV: 4, unknowns not found in JSON: 13614.
```
* I think this script sets affiliation field in `github_users.json`
* Check in `github_users.json` to `kubeflow/community/devststats/data`
* This makes it easy for people to check their affiliation.
1. See https://github.com/cncf/gitdm/blob/master/SYNC.md; there are a whole bunhch of steps that seem like they might be semi optional
* TODO(jlewi): We should create a script or something to run all the steps.
1. Import affiliations
```
cd /mount/data/src/git_kubeflow-community/devstats/config
./import_affs /mount/data/src/git_cncf-gitdm/src/github_users.json
2019-02-20 21:40:30 kubeflow/import_affs: Processing non-empty: 566 names, 707 emails lists and 116 affiliations lists
2019-02-20 21:40:30 kubeflow/import_affs: Empty/Not found: names: 142, emails: 0, affiliations: 612
2019-02-20 21:40:32 kubeflow/import_affs: 566 non-empty names, added actors: 0, updated actors: 314
2019-02-20 21:40:34 kubeflow/import_affs: 707 emails lists, added actors: 0, all emails: 735
2019-02-20 21:40:35 kubeflow/import_affs: 566 names lists, all names: 566
2019-02-20 21:40:35 kubeflow/import_affs: 116 affiliations, unique: 112, non-unique: 4, all user-company connections: 153
2019-02-20 21:40:35 kubeflow/import_affs: Processed 64 companies
2019-02-20 21:40:36 kubeflow/import_affs: Processed 153 affiliations, added 0 actors, cache hit: 153, miss: 0
2019-02-20 21:40:36 kubeflow/import_affs: Non-acquired companies: checked all regexp: 64, cache hit: 153
2019-02-20 21:40:36 kubeflow/import_affs: Time: 5.621040111s
```
* Verify there are companies
```
psql -c "select * from gha_companies;"
```
1. If affiliations are changed on a deployed setup, run
```
cd /mount/data/src/git_kubeflow-community/devstats/config
./shared/reinit.sh
```
This will regenerate the precomputed data for grafana without altering GH tables data.
1. TODO: Do we need to run devstats to compute various metrics?
### Deleting the Kubeflow database
If you want to delete the kubeflow database in the devstats cli pod run
```
psql -d postgres -c "drop database kubeflow"
```
* We need to set -d and change to a database other than kubeflow because we can't delete the current database
================================================
FILE: devstats/config/INSTALL_UBUNTU18.md
================================================
# devstats installation on Ubuntu
Prerequisites:
- Ubuntu 18.04.
- [golang](https://golang.org).
- `apt-get update`
- `sudo locale-gen "en_US.UTF-8"`
- Add to `/etc/environment`:
```
LC_ALL=en_US.UTF-8
LANG=en_US.UTF-8
```
- Check if language/locale settings are ok: `perl -e exit`.
- `apt install golang` - this installs Go 1.10.1.
- `apt install git psmisc jsonlint yamllint gcc`
- `mkdir /data; mkdir /data/dev`
1. Configure Go:
- For example add to `~/.bash_profile`:
```
GOPATH=/data/dev; export GOPATH
PATH=$PATH:$GOPATH/bin; export PATH
```
- Logout and login again.
- [golint](https://github.com/golang/lint): `go get -u github.com/golang/lint/golint`
- [goimports](https://godoc.org/golang.org/x/tools/cmd/goimports): `go get -u golang.org/x/tools/cmd/goimports`
- [goconst](https://github.com/jgautheron/goconst): `go get -u github.com/jgautheron/goconst/cmd/goconst`
- [usedexports](https://github.com/jgautheron/usedexports): `go get -u github.com/jgautheron/usedexports`
- [errcheck](https://github.com/kisielk/errcheck): `go get -u github.com/kisielk/errcheck`
- If you want to use ElasticSearch output: [elastic](https://github.com/olivere/elastic): `go get -u github.com/olivere/elastic`.
2. Go to `$GOPATH/src/` and clone devstats there:
- `git clone https://github.com/cncf/devstats.git`, cd `devstats`
- Set reuse TCP connections: `./cron/net_tcp_config.sh`
4. Go to devstats directory, so you are in `~/dev/go/src/devstats` directory and compile binaries:
- `make`
5. If compiled sucessfully then execute test coverage that doesn't need databases:
- `make test`
- Tests should pass.
6. Install binaries & metrics:
- `sudo make install` or `make install` as root.
7. Install Postgres database ([link](https://gist.github.com/sgnl/609557ebacd3378f3b72)):
- `apt install postgresql`.
- `devstats` repo directory must be available for postgres user. `chmod -R ugo+r /data/`.
- `sudo -i -u postgres`, `psql` and as root `sudo -u postgres psql` to test installation.
- Postgres only allows local connections by default so it is secure, we don't need to disable external connections:
- Config file is: `/etc/postgresql/10/main/pg_hba.conf`, instructions to enable external connections (not recommended): `http://www.thegeekstuff.com/2014/02/enable-remote-postgresql-connection/?utm_source=tuicool`
- Set bigger maximum number of connections, at least 4x number of your CPU cores or more: `/etc/postgresql/10/main/postgresql.conf`. Default is 100. `max_connections = 300`.
- You can also set `shared_buffers = ...` to something like 25% of your RAM. This is optional.
- `service postgresql restart`
- `swapoff -a` and remove any swap from `/etc/fstab`.
8. Clone `devstats-example`. It demonstrates DevStats setup for a `github.com/homebrew` org.
- `git clone https://github.com/cncf/devstats-example.git`, cd `devstats`
- See `SETUP_OTHER_PROJECT.md` to see how to enable DevStats on your own project.
9. Install Grafana.
- Go to: `https://grafana.com/grafana/download`. Prefer newest nightly build for your arch.
- `wget https://s3-us-west-2.amazonaws.com/grafana-releases/master/grafana_5.x.x_amd64.deb`.
- `sudo dpkg -i grafana_5.x.x_amd64.deb`
- `service grafana-server stop` - stop default Grafana, we only need it as a source of configuration, binaries etc.
10. Run automatic deploy
- `PG_PASS=... PG_PASS_RO=... PG_PASS_TEAM=... ./deploy.sh`.
- You can also take a look at DevStats' `ADDING_NEW_PROJECT.md` file for more info about setting up new projects.
- You should end up with Grafana running on port 3001 on your server's IP: `http://X.Y.Z.V:3001`.
- This deployment uses `147.75.105.130:3001`.
11. Configure Grafana
- Login as "admin/admin" to `http://X.Y.Z.V:3001`, change password to something more secure.
- Choose "Add data source" or Configuration -> data sources, then add PostgreSQL DB with those settings:
- Name "psql", Type "PostgreSQL", host "127.0.0.1:5432", database "your_project" (`homebrew` in this case), user "ro_user" (this is the select-only user for psql), password you used for `PG_PASS_RO`, ssl-mode "disabled".
- Run `devel/put_all_charts.sh`, then go to Home -> Manage: select "Dashboards" dashboard, and click star icon to make it favorite.
- Go to Configuration -> Preferences, change Organization name to "Your project" - this will allow anonymous access, change "Home dashboard" to "Dashboards".
- Go to User -> Preferences and and set Home dashboard to "Dashboards" (you can only choose from favorites).
- Sign out and ten remove `/login` part from the redirected URL. You should be able to access dashboards as an anonymous user.
- If all is fine, cleanup local Grafana DB copies: `Run `devel/put_all_charts_cleanup.sh`.
- If you have modified dashboards on the Grafana UI, download them as JSONs: `./devel/get_all_sqlite_jsons.sh`.
- cp devstats.sh $GOPATH/bin/
- You can use `crontab` file as an example of enabling hourly cron sync: `crontab -e`
================================================
FILE: devstats/config/README.md
================================================
A fork of https://github.com/cncf/devstats-example
* This directory contains the config files for Kubeflow
* It is based on https://github.com/cncf/devstats-example and then modified
* There are probably still references that need to be udpated
* Key files
* **projects.yaml** - Defines the project
* **grafana/dashboards** - Dashboards and grafana config
## Here's how we created this
1. We follow [Setup other project](https://github.com/cncf/devstats-example/blob/master/SETUP_OTHER_PROJECT.md)
1. We copied it to this directory
1. Copy over dashaboards
* I used the `knative` dashboards in cncf/devstats at commit 3c419a61ad130c80c7f1b7a5058eb0fd792b4201 (current devstats master)
rather than the homebrew ones in devstats-example
* The latter seemed outdated see https://github.com/cncf/devstats-example/issues/7
* I picked knative because I think it was added fairly recently and probably isn't doing special things like kubernetes
* Run the following script to update the dashboards
```
${GIT_KUBEFLOW_COMMUNITY}/devstats/modify_dashboards.sh ${GIT_DEVSTATS_EXAMPLE_FORK}/grafana/dashboards/kubeflow
```
1. Run `the following script to add the -E command to sudo commands
```
./scripts/modify_devstats_scripts.sh ./config/devel/
```
1. Copy over metrics from cncf/devstats
* We don't use cncf/devstats-examples because it has fewer metrics
* Copy `cncf/devstats/metrics/shared` to `metrics/shared`
* Copy `cncf/devstats/metrics/knative` to `metrics/kubeflow`
* Modify metrics/kubeflow to replace references to `knative` with `kubeflow`
1. Copy scripts/homebrew to scripts/kubeflow
* Update repo_groups.sql
1. Copy the K8s dashboards https://github.com/cncf/devstats/tree/master/grafana/dashboards/kubernetes
to `ks-app/components/grafana/dashboards/`
1. Run ` modify_dashboards.sh` to update all references to knative to kubeflow
1. Modified projects.yaml to define a Kubeflow project
================================================
FILE: devstats/config/SETUP_OTHER_PROJECT.md
================================================
# Setup you own project
This example deployment uses Homebrew project. To run DevStats on the other project do:
- In all steps: note that you have a lower-case project name `homebrew` and full name `Homebrew` (for example used as Grafana Org name etc).
- Update `grafana.sh`, vim: `s/homebrew/your_project/g`.
- Update `projects.yaml` to contain your new project data.
- Update `devel/deploy_all.sh` search for `omebrew` and replace with your project.
- Update `devel/get_all_sqlite_jsons.sh`, `devel/put_all_charts.sh`, `devel/add_single_metric_all.sh`, `devel/create_psql_user.sh`.
- Update `crontab`, change DB backup name from `homebrew` to `your_project` and uncomment if you need automatic backups.
- Rename `homebrew` folder to `your_project` and update `your_project/psql.sh`.
- Rename `grafana/img/homebrew*` to `grafana/img/your_project*` (provide your own SVG and PNG).
- Rename `grafana/homebrew` to `grafana/your_project` and update files in this directory.
- Rename `grafana/dashboards/homebrew` to `grafana/dashboards/your_project` and update files in this directory.
- Rename `metrics/homebrew` to `metrics/your_project` and update `metrics/your_project/vars.yaml`.
- Rename `scripts/homebrew` to `scripts/your_project` and update `repo_groups.sql` in this directory.
================================================
FILE: devstats/config/copy_devstats_binaries.sh
================================================
#!/bin/bash
binaries="structure runq gha2db calc_metric gha2db_sync import_affs annotations tags webhook devstats get_repos merge_dbs replacer vars ghapi2db columns hide_data website_data sync_issues devstats_api_server gha2es sqlitedb"
for f in $binaries
do
cp $GOPATH/bin/$f ./ || exit 1
done
echo "DevStats binaries copied."
================================================
FILE: devstats/config/cron/net_tcp_config.sh
================================================
#!/bin/bash
/sbin/sysctl net.ipv4.tcp_tw_reuse=1
================================================
FILE: devstats/config/crontab
================================================
7 * * * * PATH=$PATH:/data/dev/bin net_tcp_config.sh
8 * * * * PATH=$PATH:/data/dev/bin DEVSTATS_DIR="/data/dev/src/devstats-example" PG_PASS=... devstats.sh 2>> /tmp/devstats.err 1>> /tmp/devstats.log
#45 3 * * * PATH=$PATH:/data/dev/bin cron_db_backup.sh homebrew 2>> /tmp/backup.err 1>> /tmp/backup.log
================================================
FILE: devstats/config/deploy.sh
================================================
#!/bin/bash
mkdir /var/www 2>/dev/null
mkdir /var/www/html 2>/dev/null
./copy_devstats_binaries.sh || exit 1
cp cron/net_tcp_config.sh devel/sync_lock.sh devel/sync_unlock.sh $GOPATH/bin/ || exit 2
INIT=1 EXTERNAL=1 GHA2DB_GHAPISKIP=1 SKIPTEMP=1 ./devel/deploy_all.sh || exit 3
echo 'Deploy succeeded'
================================================
FILE: devstats/config/devel/add_single_metric.sh
================================================
#!/bin/bash
if [ -z "${PG_DB}" ]
then
echo "You need to set PG_DB environment variable to run this script"
exit 1
fi
function finish {
sync_unlock.sh
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
fi
GHA2DB_CMDDEBUG=1 GHA2DB_RESETTSDB=1 GHA2DB_METRICS_YAML=devel/test_metrics.yaml GHA2DB_TAGS_YAML=devel/test_tags.yaml GHA2DB_LOCAL=1 ./gha2db_sync
================================================
FILE: devstats/config/devel/add_single_metric_all.sh
================================================
#!/bin/bash
if [ -z "${PG_PASS}" ]
then
echo "You need to set PG_PASS environment variable to run this script"
exit 1
fi
function finish {
sync_unlock.sh
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
fi
if [ -z "$ONLY" ]
then
all="kubeflow"
else
all=$ONLY
fi
for proj in $all
do
db=$proj
GHA2DB_PROJECT=$proj PG_DB=$db ./devel/add_single_metric.sh || exit 2
done
echo 'OK'
================================================
FILE: devstats/config/devel/create_databases.sh
================================================
#!/bin/bash
# PDB=1 (will generate Postgres DB)
# TSDB=1 (will generate TS DB)
# PDROP=1 (will drop & create Postgres DB)
# GET=1 (will use Postgres DB backup from $HOST_SRC)
lim=70
set -x -o pipefail
if [ -z "$PG_PASS" ]
then
echo "$0: You need to set PG_PASS environment variable to run this script"
exit 1
fi
if ( [ -z "$PROJ" ] || [ -z "$PROJDB" ] )
then
echo "$0: You need to set PROJ, PROJDB environment variables to run this script"
exit 2
fi
function finish {
rm -rf "$PROJDB.dump" >/dev/null 2>&1
sync_unlock.sh
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
fi
if [ ! -z "$PDB" ]
then
exists=`sudo -E -u postgres psql -tAc "select 1 from pg_database WHERE datname = '$PROJDB'"` || exit 3
if ( [ ! -z "$PDROP" ] && [ "$exists" = "1" ] )
then
echo "dropping postgres database $PROJDB"
sudo -E -u postgres psql -c "select pg_terminate_backend(pid) from pg_stat_activity where datname = '$PROJDB'" || exit 4
sudo -E -u postgres psql -c "drop database $PROJDB" || exit 5
fi
exists=`sudo -E -u postgres psql -tAc "select 1 from pg_database WHERE datname = '$PROJDB'"` || exit 6
if [ ! "$exists" = "1" ]
then
echo "creating postgres database $PROJDB"
sudo -E -u postgres psql -c "create database $PROJDB" || exit 7
sudo -E -u postgres psql -c "grant all privileges on database \"$PROJDB\" to gha_admin" || exit 8
sudo -E -u postgres psql "$PROJDB" -c "create extension if not exists pgcrypto" || exit 23
if [ ! -z "$GET" ]
then
echo "attempt to fetch postgres database $PROJDB from backup"
wget "https://$HOST_SRC/$PROJDB.dump" || exit 9
sudo -E -u postgres pg_restore -d "$PROJDB" "$PROJDB.dump" || exit 10
rm -f "$PROJDB.dump" || exit 11
echo 'dropping and recreating postgres variables'
sudo -E -u postgres psql "$PROJDB" -c "delete from gha_vars" || exit 12
GHA2DB_PROJECT="$PROJ" PG_DB="$PROJDB" GHA2DB_LOCAL=1 ./vars || exit 13
GOT=1
else
echo "generating postgres database $PROJDB"
GHA2DB_MGETC=y ./$PROJ/psql.sh || exit 14
./devel/ro_user_grants.sh "$PROJDB" || exit 15
./devel/psql_user_grants.sh "devstats_team" "$PROJDB" || exit 16
dbcreated=1
cron_db_backup.sh "$PROJDB" || exit 17
fi
else
echo "postgres database $PROJDB already exists"
fi
else
echo "postgres database $PROJDB generation skipped"
fi
if [ ! -z "$TSDB" ]
then
exists=`sudo -E -u postgres psql -tAc "select 1 from pg_database WHERE datname = '$PROJDB'"` || exit 3
if [ ! "$exists" = "1" ]
then
echo "$0: '$PROJDB' must exist to initialize TSDB"
exit 21
fi
exists=`sudo -E -u postgres psql "$PROJDB" -tAc "select to_regclass('sevents_h')"` || exit 22
if [ "$exists" = "sevents_h" ]
then
echo "time series data already exists in $PROJDB"
else
echo "generating TSDB database $PROJDB"
if [ -f "./$proj/reinit.sh" ]
then
./$PROJ/reinit.sh || exit 18
else
GHA2DB_PROJECT=$PROJ PG_DB=$PROJDB ./shared/reinit.sh || exit 19
fi
REINIT=1
fi
if [ ! -z "$GOT" ]
then
GHA2DB_PROJECT="$PROJ" PG_DB="$PROJDB" ./gha2db_sync || exit 20
fi
if [ ! -z "$REINIT" ]
then
cron_db_backup.sh "$PROJDB" || exit 24
fi
else
echo "TS database $PROJDB generation skipped"
fi
echo "$0: $PROJ finished"
================================================
FILE: devstats/config/devel/create_grafana.sh
================================================
#!/bin/bash
# GGET=1 (Get grafana.db from the $HOST_SRC server)
# STOP=1 (Stops running grafana-server instance)
# RM=1 (only with STOP, get rid of all grafana data before proceeding)
# IMPJSONS=1 (will import all jsons defined for given project using sqlitedb tool), if used with GGET - it will first fetch from server and then import
# EXTERNAL=1 (will expose Grafana to outside world: will bind to 0.0.0.0 instead of 127.0.0.1, useful when no Apache proxy + SSL is enabled)
set -o pipefail
if ( [ -z "$PG_PASS" ] || [ -z "$PORT" ] || [ -z "$GA" ] || [ -z "$ICON" ] || [ -z "$ORGNAME" ] || [ -z "$PROJ" ] || [ -z "$PROJDB" ] || [ -z "$GRAFSUFF" ] )
then
echo "$0: You need to set PG_PASS, PROJ, PROJDB, PORT, GA, ICON, ORGNAME, GRAFSUFF environment variable to run this script"
exit 1
fi
function finish {
sync_unlock.sh
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
fi
host=`hostname`
if [ "$GA" = "-" ]
then
ga=";"
else
ga="google_analytics_ua_id = $GA"
fi
if [ ! -z "$EXTERNAL" ]
then
bind="0.0.0.0"
else
bind="127.0.0.1"
fi
if [ -z "$ARTWORK" ]
then
ARTWORK="$HOME/dev/cncf/artwork"
fi
pid=`ps -axu | grep grafana-server | grep $GRAFSUFF | awk '{print $2}'`
if [ ! -z "$STOP" ]
then
echo "stopping $PROJ grafana server instance"
if [ ! -z "$pid" ]
then
echo "stopping pid $pid"
kill $pid
else
echo "grafana-server $PROJ not running"
fi
if [ ! -z "$RM" ]
then
echo "shreding $PROJ grafana"
rm -rf "/usr/share/grafana.$GRAFSUFF/" 2>/dev/null
rm -rf "/var/lib/grafana.$GRAFSUFF/" 2>/dev/null
rm -rf "/etc/grafana.$GRAFSUFF/" 2>/dev/null
fi
fi
pid=`ps -axu | grep grafana-server | grep $GRAFSUFF | awk '{print $2}'`
if [ ! -z "$pid" ]
then
echo "$PROJ grafana-server is running, exiting"
exit 0
fi
if [ ! -d "$GRAF_USRSHARE.$GRAFSUFF/" ]
then
echo "copying /usr/share/grafana.$GRAFSUFF/"
cp -R "$GRAF_USRSHARE" "/usr/share/grafana.$GRAFSUFF/" || exit 3
if [ ! "$ICON" = "-" ]
then
wd=`pwd`
cd "$ARTWORK" || exit 4
git pull || exit 5
cd $wd || exit 6
icontype=`./devel/get_icon_type.sh "$PROJ"` || exit 7
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "/usr/share/grafana.$GRAFSUFF/public/img/grafana_icon.svg" || exit 8
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "/usr/share/grafana.$GRAFSUFF/public/img/grafana_com_auth_icon.svg" || exit 9
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "/usr/share/grafana.$GRAFSUFF/public/img/grafana_net_logo.svg" || exit 10
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "/usr/share/grafana.$GRAFSUFF/public/img/grafana_mask_icon.svg" || exit 11
convert "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.png" -resize 80x80 "/var/www/html/img/$PROJ-icon-color.png" || exit 12
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "/var/www/html/img/$PROJ-icon-color.svg" || exit 13
if [ ! -f "grafana/img/$GRAFSUFF.svg" ]
then
cp "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.svg" "grafana/img/$GRAFSUFF.svg" || exit 14
fi
if [ ! -f "grafana/img/${GRAFSUFF}32.png" ]
then
convert "$ARTWORK/$ICON/icon/$icontype/$ICON-icon-$icontype.png" -resize 32x32 "grafana/img/${GRAFSUFF}32.png" || exit 15
fi
fi
GRAFANA_DATA="/usr/share/grafana.$GRAFSUFF/" ./grafana/$PROJ/change_title_and_icons.sh || exit 16
fi
if [ ! -d "/var/lib/grafana.$GRAFSUFF/" ]
then
echo "copying /var/lib/grafana.$GRAFSUFF/"
cp -R "$GRAF_VARLIB" "/var/lib/grafana.$GRAFSUFF/" || exit 17
fi
if ( [ ! -f "/var/lib/grafana.$GRAFSUFF/grafana.db" ] && [ ! -z "$GGET" ] )
then
echo "attempt to fetch grafana database $GRAFSUFF from the test server"
wget "https://$HOST_SRC/grafana.$GRAFSUFF.db" || exit 18
mv "grafana.$GRAFSUFF.db" "/var/lib/grafana.$GRAFSUFF/grafana.db" || exit 19
fi
if [ ! -d "/etc/grafana.$GRAFSUFF/" ]
then
echo "copying /etc/grafana.$GRAFSUFF/"
cp -R "$GRAF_ETC" "/etc/grafana.$GRAFSUFF"/ || exit 20
cfile="/etc/grafana.$GRAFSUFF/grafana.ini"
cp ./grafana/etc/grafana.ini.example "$cfile" || exit 21
MODE=ss FROM='{{project}}' TO="$PROJ" replacer "$cfile" || exit 22
MODE=ss FROM='{{url}}' TO="$host" replacer "$cfile" || exit 23
MODE=ss FROM='{{bind}}' TO="$bind" replacer "$cfile" || exit 24
MODE=ss FROM='{{port}}' TO="$PORT" replacer "$cfile" || exit 25
MODE=ss FROM=';google_analytics_ua_id =' TO="-" replacer "$cfile" || exit 27
MODE=ss FROM='{{ga}}' TO="$ga" replacer "$cfile" || exit 28
MODE=ss FROM='{{test}}' TO="-" replacer "$cfile" || exit 29
MODE=ss FROM='{{org}}' TO="$ORGNAME" replacer "$cfile" || exit 32
fi
pid=`ps -axu | grep grafana-server | grep $GRAFSUFF | awk '{print $2}'`
if [ -z "$pid" ]
then
echo "starting $PROJ grafana-server"
./grafana/$PROJ/grafana_start.sh &
echo "started"
fi
if [ ! -z "$IMPJSONS" ]
then
while [ ! -f "/var/lib/grafana.$PROJ/grafana.db" ]
do
echo "Waiting for /var/lib/grafana.$PROJ/grafana.db to be created"
sleep 1
done
sleep 1
GRAFANA=$GRAFSUFF NOCOPY=1 ./devel/import_jsons_to_sqlite.sh ./grafana/dashboards/$PROJ/* || exit 37
fi
echo "$0: $PROJ finished"
================================================
FILE: devstats/config/devel/create_psql_user.sh
================================================
#!/bin/bash
if [ -z "${PG_PASS}" ]
then
echo "$0: You need to set PG_PASS environment variable to run this script"
exit 1
fi
if [ -z "$1" ]
then
echo "$0: user name required"
exit 1
fi
if [ -z "$ONLY" ]
then
all="kubeflow devstats"
else
all=$ONLY
fi
cp ./util_sql/drop_psql_user.sql /tmp/drop_user.sql || exit 1
FROM="{{user}}" TO="$1" MODE=ss ./replacer /tmp/drop_user.sql || exit 1
if [ ! -z "$DROP" ]
then
echo "Drop from public"
sudo -E -u postgres psql < /tmp/drop_user.sql || exit 1
for proj in $all
do
echo "Drop from $proj"
sudo -E -u postgres psql "$proj" < /tmp/drop_user.sql || exit 1
done
fi
if [ ! -z "$NOCREATE" ]
then
echo "Skipping create"
exit 0
fi
echo "Create role"
sudo -E -u postgres psql -c "create user \"$1\" with password '$PG_PASS'" || exit 1
for proj in $all
do
echo "Grants $proj"
./devel/psql_user_grants.sh "$1" "$proj" || exit 2
done
echo 'OK'
================================================
FILE: devstats/config/devel/deploy_all.sh
================================================
#!/bin/bash
# ARTWORK
# GET=1 (attempt to fetch Postgres database and Grafana database from the test server)
# INIT=1 (needs PG_PASS_RO, PG_PASS_TEAM, initialize from no postgres database state, creates postgres logs database and users)
# SKIPVARS=1 (if set it will skip final Postgres vars regeneration)
set -o pipefail
exec > >(tee run.log)
exec 2> >(tee errors.txt)
if [ -z "$PG_PASS" ]
then
echo "$0: You need to set PG_PASS environment variable to run this script"
exit 1
fi
if ( [ ! -z "$INIT" ] && ( [ -z "$PG_PASS_RO" ] || [ -z "$PG_PASS_TEAM" ] ) )
then
echo "$0: You need to set PG_PASS_RO, PG_PASS_TEAM when using INIT"
exit 1
fi
GRAF_USRSHARE="/usr/share/grafana"
GRAF_VARLIB="/var/lib/grafana"
GRAF_ETC="/etc/grafana"
export GRAF_USRSHARE
export GRAF_VARLIB
export GRAF_ETC
host=`hostname`
function finish {
sync_unlock.sh
rm -f /tmp/deploy.wip 2>/dev/null
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
> /tmp/deploy.wip
fi
if [ ! -z "$INIT" ]
then
./devel/init_database.sh || exit 1
fi
PROJ=kubeflow PROJDB=kubeflow PROJREPO="kubeflow/kubeflow" ORGNAME="kubeflow" PORT=3001 ICON="-" GRAFSUFF=kubeflow GA="-" ./devel/deploy_proj.sh || exit 2
if [ -z "$SKIPVARS" ]
then
./devel/vars_all.sh || exit 3
fi
echo "$0: All deployments finished"
================================================
FILE: devstats/config/devel/deploy_proj.sh
================================================
#!/bin/bash
# GET=1 (attempt to fetch Postgres database and Grafana database from the test server)
# SKIPDBS=1 (entirely skip project's database operations)
# SKIPADDALL=1 (skip adding/merging to allprj)
# SKIPGRAFANA=1 (skip all grafana related stuff)
set -o pipefail
if [ -z "$PG_PASS" ]
then
echo "$0: You need to set PG_PASS environment variable to run this script"
exit 1
fi
if ( [ -z "$PROJ" ] || [ -z "$PROJDB" ] || [ -z "$PROJREPO" ] || [ -z "$PORT" ] || [ -z "$GA" ] || [ -z "$ICON" ] || [ -z "$ORGNAME" ] || [ -z "$GRAFSUFF" ] )
then
echo "$0: You need to set PROJ, PROJDB, PROJREPO, PORT, GA, ICON, ORGNAME, GRAFSUFF environment variables to run this script"
exit 2
fi
function finish {
sync_unlock.sh
rm -f /tmp/deploy.wip 2>/dev/null
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
> /tmp/deploy.wip
fi
echo "$0: $PROJ deploy started"
if [ -z "$SKIPDBS" ]
then
PDB=1 TSDB=1 ./devel/create_databases.sh || exit 3
fi
if [ -z "$SKIPGRAFANA" ]
then
./devel/create_grafana.sh || exit 5
fi
echo "$0: $PROJ deploy finished"
================================================
FILE: devstats/config/devel/drop_psql_db.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: you need to provide db name"
exit 1
fi
sudo -E -u postgres psql -c "select pg_terminate_backend(pid) from pg_stat_activity where datname = '$1'"
sudo -E -u postgres psql -c "drop database $1"
================================================
FILE: devstats/config/devel/drop_ts_tables.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: need database name argument"
exit 1
fi
proj=$1
tables=`sudo -E -u postgres psql $proj -qAntc '\dt' | cut -d\| -f2`
for table in $tables
do
base=${table:0:1}
if ( [ "$base" = "t" ] || [ "$base" = "s" ] )
then
sudo -E -u postgres psql $proj -c "drop table $table" || exit 1
echo "dropped $table"
fi
done
================================================
FILE: devstats/config/devel/drop_tsdb_affs_tables.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: need database name argument"
exit 1
fi
proj=$1
# snum_stats scompany_activity shcom* shpr_comps* ssex ssexcum scountries scountriescum
# sudo -E -u postgres psql $proj -c "drop table snum_stats" || exit 1
# sudo -E -u postgres psql $proj -c "drop table scompany_activity" || exit 2
# sudo -E -u postgres psql $proj -c "drop table ssex" || exit 3
# sudo -E -u postgres psql $proj -c "drop table ssexcum" || exit 4
# sudo -E -u postgres psql $proj -c "drop table scountries" || exit 5
# sudo -E -u postgres psql $proj -c "drop table scountriescum" || exit 6
tables=`sudo -E -u postgres psql $proj -qAntc '\dt' | cut -d\| -f2`
for table in $tables
do
base1=${table:0:5}
base2=${table:0:10}
if ( [ "$base1" = "shcom" ] || [ "$base2" = "shpr_comps" ] )
then
sudo -E -u postgres psql $proj -c "drop table $table" || exit 1
echo "dropped $table"
fi
done
================================================
FILE: devstats/config/devel/get_all_sqlite_jsons.sh
================================================
#!/bin/bash
if [ -z "$ONLY" ]
then
all="kubeflow"
else
all=$ONLY
fi
mkdir sqlite 1>/dev/null 2>/dev/null
touch sqlite/touch
for proj in $all
do
db=$proj
rm -f sqlite/* 2>/dev/null
touch sqlite/touch
./sqlitedb /var/lib/grafana.$db/grafana.db || exit 1
rm -f grafana/dashboards/$proj/*.json || exit 2
mv sqlite/*.json grafana/dashboards/$proj/ || exit 3
done
echo 'OK'
================================================
FILE: devstats/config/devel/get_from_sqlite.sh
================================================
#!/bin/bash
if [ -z "$GRAFANA" ]
then
echo "$0: you need to set GRAFANA env variable"
exit 1
fi
if [ -z "$1" ]
then
echo "$0: you need to provide at least one dashboard JSON"
exit 2
fi
cp "/var/lib/grafana.$GRAFANA/grafana.db" a.db || exit 3
function finish {
rm -f a.db* 2>/dev/null
}
trap finish EXIT
./sqlitedb a.db $* || exit 4
./devel/update_from_sqlite.sh
================================================
FILE: devstats/config/devel/get_icon_type.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: requires project name parameter"
exit 1
fi
declare -A icontypes
icontypes=(
["cncf"]="color"
)
icontype=${icontypes[$1]}
if [ -z "$icontype" ]
then
echo "$0: project $1 is not defined"
exit 1
fi
echo $icontype
================================================
FILE: devstats/config/devel/grafana_start.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: you need to provide grafana name"
exit 1
fi
./devel/grafana_stop.sh $1 || exit 1
cd /usr/share/grafana.$1
grafana-server -config /etc/grafana.$1/grafana.ini cfg:default.paths.data=/var/lib/grafana.$1 1>/var/log/grafana.$1.log 2>&1 &
================================================
FILE: devstats/config/devel/grafana_stop.sh
================================================
#!/bin/sh
if [ -z "$1" ]
then
echo "$0: you need to provide grafana name"
exit 1
fi
pid=`ps -axu | grep grafana-server | grep $1 | awk '{print $2}'`
echo "stopping $1 grafana server instance"
if [ ! -z "$pid" ]
then
echo "stopping pid $pid"
kill $pid
else
echo "grafana-server $1 not running"
fi
================================================
FILE: devstats/config/devel/import_jsons_to_sqlite.sh
================================================
#!/bin/bash
if [ -z "$GRAFANA" ]
then
echo "$0: you need to set GRAFANA env variable (Grafana suffix). For example k8s, all, prometheus etc."
exit 1
fi
if [ -z "$1" ]
then
echo "$0: you need to provide at least one json to import"
exit 2
fi
cp /var/lib/grafana.$GRAFANA/grafana.db ./grafana.$GRAFANA.db || exit 4
./sqlitedb ./grafana.$GRAFANA.db $* || exit 5
./devel/grafana_stop.sh $GRAFANA || exit 6
cp ./grafana.$GRAFANA.db /var/lib/grafana.$GRAFANA/grafana.db || exit 7
./devel/grafana_start.sh $GRAFANA || exit 8
echo "OK, if all is fine delete grafana.$GRAFANA.db.* db backup files and *.was json backup files".
echo "Otherwise use grafana.$GRAFANA.db.* backup file to restore previous Grafana DB."
================================================
FILE: devstats/config/devel/init_database.sh
================================================
#!/bin/bash
# UDROP=1 attempt to drop users
# LDROP=1 attempt to drop devstats database
set -x -o pipefail
if ( [ -z "$PG_PASS" ] || [ -z "$PG_PASS_RO" ] || [ -z "$PG_PASS_TEAM" ] )
then
echo "$0: You need to set PG_PASS, PG_PASS_RO, PG_PASS_TEAM when using INIT"
exit 1
fi
function finish {
sync_unlock.sh
}
if [ -z "$TRAP" ]
then
sync_lock.sh || exit -1
trap finish EXIT
export TRAP=1
fi
echo "$0 start"
if [ ! -z "$UDROP" ]
then
echo "dropping users"
ONLY=devstats DROP=1 NOCREATE=1 devel/create_psql_user.sh gha_admin || exit 1
ONLY=devstats DROP=1 NOCREATE=1 devel/create_psql_user.sh ro_user || exit 2
ONLY=devstats DROP=1 NOCREATE=1 devel/create_psql_user.sh devstats_team || exit 3
echo "dropping done"
fi
exists=`sudo -E -u postgres psql -tAc "select 1 from pg_database WHERE datname = 'devstats'"` || exit 4
if ( [ ! -z "$LDROP" ] && [ "$exists" = "1" ] )
then
echo "dropping postgres database devstats (logs)"
sudo -E -u postgres psql -c "select pg_terminate_backend(pid) from pg_stat_activity where datname = 'devstats'" || exit 5
sudo -E -u postgres psql -c "drop database devstats" || exit 6
fi
if [ ! -z "$NOCREATE" ]
then
echo "skipping create"
exit 0
fi
exists=`sudo -E -u postgres psql -tAc "select 1 from pg_database WHERE datname = 'devstats'"` || exit 7
if [ ! "$exists" = "1" ]
then
echo "creating postgres database devstats (logs)"
sudo -E -u postgres psql -c "create database devstats" || exit 8
sudo -E -u postgres psql -c "create user gha_admin with password '$PG_PASS'" || exit 9
sudo -E -u postgres psql -c "create user ro_user with password '$PG_PASS_RO'" || exit 10
sudo -E -u postgres psql -c "create user devstats_team with password '$PG_PASS_TEAM'" || exit 11
sudo -E -u postgres psql -c "grant all privileges on database \"devstats\" to gha_admin" || exit 12
sudo -E -u postgres psql -c "alter user gha_admin createdb" || exit 13
sudo -E -u postgres psql devstats < ./util_sql/devstats_log_table.sql || exit 14
./devel/ro_user_grants.sh devstats || exit 15
./devel/psql_user_grants.sh "devstats_team" "devstats" || exit 16
else
echo "postgres database devstats (logs) already exists"
fi
echo "$0 OK"
================================================
FILE: devstats/config/devel/mass_replace.sh
================================================
#!/bin/bash
# Examples:
# MODE=rr FROM=`cat input` TO=`cat output` FILES=`find abc/ -type f -not -iname 'something.txt'` ./devel/mass_replace.sh
# MODE=ss FROM=`cat input` TO=`cat output` FILES=`ls grafana/dashboards/{all,cncf,cni,containerd,coredns,envoy,fluentd,grpc,jaeger,linkerd,kubernetes,notary,opencontainers,opentracing,prometheus,rkt,rook,tuf,vitess}/*` ./devel/mass_replace.sh
# MODE=ss0 FROM=CNCF TO='[[full_name]]' FILES=`find grafana/dashboards/cncf/ -type f -not -iname dashboards.json` ./devel/mass_replace.sh
# MODE=ss FROM=' "title": "' TO=' "title": "[[full_name]] ' FILES=`find grafana/dashboards/cncf/ -name "top_commenters.json" -or -name "project_statistics.json" -or -name "companies_summary.json" -or -name "prs_authors_companies_histogram.json" -or -name "developers_summary.json" -or -name "prs_authors_histogram.json"` ./devel/mass_replace.sh
# MODE=rs0 FROM='(?m)^.*"uid": "\w+",\n' TO='-' ./replacer input.json
# MODE=rr0 FROM='(?m)(^.*)"uid": "(\w+)",' TO='$1"uid": "placeholder",' ./replacer input.json
# MODE=rr FROM='(?m);;;(.*)$' TO=';;;$1 # {{repo_groups}}' FILES=`find metrics/ -iname "gaps.yaml"` ./devel/mass_replace.sh
if [ -z "${FROM}" ]
then
echo "You need to set FROM, example FROM=abc TO=xyz FILES='f1 f2' $0"
exit 1
fi
if [ -z "${TO}" ]
then
echo "You need to set TO, example FROM=abc TO=xyz FILES='f1 f2' $0"
exit 2
fi
if [ -z "${FILES}" ]
then
echo "You need to set FILES, example FROM=abc TO=xyz FILES='f1 f2' $0"
exit 3
fi
for f in ${FILES}
do
./replacer $f || exit 4
done
echo 'OK'
================================================
FILE: devstats/config/devel/net_tcp_config.sh
================================================
#!/bin/bash
/sbin/sysctl net.ipv4.tcp_tw_reuse=1
================================================
FILE: devstats/config/devel/psql_user_grants.sh
================================================
#!/bin/bash
if [ -z "$1" ]
then
echo "$0: need user name argument"
exit 1
fi
if [ -z "$2" ]
then
echo "$0: need database name argument"
exit 1
fi
proj=$2
sudo -E -u postgres psql -c "grant connect on database \"$proj\" to \"$1\"" || exit 1
sudo -E -u postgres psql -c "grant usage on schema \"public\" to \"$1\"" || exit 1
tables=`sudo -E -u postgres psql $proj -qAntc '\dt' | cut -d\| -f2`
for table in $tables
do
echo -n "$proj: $table "
sudo -E -u postgres psql $proj -c "grant select on $table to \"$1\"" || exit 1
done
================================================
FILE: devstats/config/devel/put_all_charts.sh
================================================
#!/bin/bash
if [ -z "$ONLY" ]
then
all="kubeflow"
else
all=$ONLY
fi
for proj in $all
do
suff=$proj
NOCOPY=1 GRAFANA=$suff devel/import_jsons_to_sqlite.sh grafana/dashboards/$proj/*.json || exit 2
done
echo 'OK'
================================================
FILE: devstats/config/devel/put
gitextract_nd5su600/ ├── .github/ │ ├── issue_label_bot.yaml │ └── workflows/ │ └── stale.yml ├── .gitignore ├── ADOPTERS.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── INCLUSIVITY.md ├── KUBEFLOW-GENERAL-TECHNICAL-REVIEW.md ├── KUBEFLOW-OUTREACH-COMMITTEE.md ├── KUBEFLOW-STEERING-COMMITTEE.md ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── calendar/ │ ├── Dockerfile │ ├── Makefile │ ├── OWNERS │ ├── README.md │ ├── calendar.yaml │ ├── calendar_import.py │ ├── latest_image.yaml │ └── manifests/ │ ├── deployment.yaml │ ├── kustomization.yaml │ └── service-account.yaml ├── dco-signoff-hook/ │ ├── README.md │ └── prepare-commit-msg ├── devstats/ │ ├── Dockerfile.devstats │ ├── Makefile │ ├── README.md │ ├── config/ │ │ ├── INSTALL_UBUNTU18.md │ │ ├── README.md │ │ ├── SETUP_OTHER_PROJECT.md │ │ ├── copy_devstats_binaries.sh │ │ ├── cron/ │ │ │ └── net_tcp_config.sh │ │ ├── crontab │ │ ├── deploy.sh │ │ ├── devel/ │ │ │ ├── add_single_metric.sh │ │ │ ├── add_single_metric_all.sh │ │ │ ├── create_databases.sh │ │ │ ├── create_grafana.sh │ │ │ ├── create_psql_user.sh │ │ │ ├── deploy_all.sh │ │ │ ├── deploy_proj.sh │ │ │ ├── drop_psql_db.sh │ │ │ ├── drop_ts_tables.sh │ │ │ ├── drop_tsdb_affs_tables.sh │ │ │ ├── get_all_sqlite_jsons.sh │ │ │ ├── get_from_sqlite.sh │ │ │ ├── get_icon_type.sh │ │ │ ├── grafana_start.sh │ │ │ ├── grafana_stop.sh │ │ │ ├── import_jsons_to_sqlite.sh │ │ │ ├── init_database.sh │ │ │ ├── mass_replace.sh │ │ │ ├── net_tcp_config.sh │ │ │ ├── psql_user_grants.sh │ │ │ ├── put_all_charts.sh │ │ │ ├── put_all_charts_cleanup.sh │ │ │ ├── restore_db.sh │ │ │ ├── ro_user_grants.sh │ │ │ ├── sync_lock.sh │ │ │ ├── sync_unlock.sh │ │ │ ├── test_metrics.yaml │ │ │ ├── test_tags.yaml │ │ │ ├── update_from_sqlite.sh │ │ │ └── vars_all.sh │ │ ├── devstats.sh │ │ ├── docs/ │ │ │ ├── annotations.md │ │ │ ├── dashboards/ │ │ │ │ ├── dashboards.md │ │ │ │ ├── dashboards_devel.md │ │ │ │ ├── kubernetes/ │ │ │ │ │ ├── blocked_prs.md │ │ │ │ │ ├── bot_commands.md │ │ │ │ │ ├── companies_table.md │ │ │ │ │ ├── company_stats.md │ │ │ │ │ ├── contributing_companies.md │ │ │ │ │ ├── developer_stats.md │ │ │ │ │ ├── episodic_issues.md │ │ │ │ │ ├── episodic_prs.md │ │ │ │ │ ├── gh_stats_commits.md │ │ │ │ │ ├── gh_stats_iclosed.md │ │ │ │ │ ├── gh_stats_icommenters.md │ │ │ │ │ ├── gh_stats_icomments.md │ │ │ │ │ ├── gh_stats_iopened.md │ │ │ │ │ ├── gh_stats_prclosed.md │ │ │ │ │ ├── gh_stats_prcommenters.md │ │ │ │ │ ├── gh_stats_prcomments.md │ │ │ │ │ ├── gh_stats_prmerged.md │ │ │ │ │ ├── gh_stats_propened.md │ │ │ │ │ ├── gh_stats_reviewers.md │ │ │ │ │ ├── ghstats_devel.md │ │ │ │ │ ├── issues_age.md │ │ │ │ │ ├── issues_opened_closed.md │ │ │ │ │ ├── pr_approval.md │ │ │ │ │ ├── pr_approve_to_merge.md │ │ │ │ │ ├── pr_authors.md │ │ │ │ │ ├── pr_comments.md │ │ │ │ │ ├── pr_labels.md │ │ │ │ │ ├── pr_reviews_by_contributor.md │ │ │ │ │ ├── pr_time_to_engagement.md │ │ │ │ │ ├── pr_workload.md │ │ │ │ │ ├── project_stats.md │ │ │ │ │ ├── prs_age.md │ │ │ │ │ ├── sig_mentions.md │ │ │ │ │ ├── sig_mentions_devel.md │ │ │ │ │ ├── sig_milestones.md │ │ │ │ │ └── stars_and_forks.md │ │ │ │ └── shared/ │ │ │ │ ├── activity.md │ │ │ │ ├── commits.md │ │ │ │ ├── community_stats.md │ │ │ │ ├── companies_stats.md │ │ │ │ ├── companies_summary.md │ │ │ │ ├── company_prs.md │ │ │ │ ├── contributing_companies.md │ │ │ │ ├── countries_stats.md │ │ │ │ ├── dashboards.md │ │ │ │ ├── developers_summary.md │ │ │ │ ├── gender_stats.md │ │ │ │ ├── github_events_docs_html.md │ │ │ │ ├── issues.md │ │ │ │ ├── issues_age.md │ │ │ │ ├── new_and_episodic_issues.md │ │ │ │ ├── new_and_episodic_prs.md │ │ │ │ ├── new_contributors.md │ │ │ │ ├── new_prs.md │ │ │ │ ├── non_author_activity.md │ │ │ │ ├── opened_to_merged.md │ │ │ │ ├── pr_authors.md │ │ │ │ ├── pr_comments.md │ │ │ │ ├── pr_companies.md │ │ │ │ ├── project_health.md │ │ │ │ ├── projects_health.md │ │ │ │ ├── projects_stats.md │ │ │ │ ├── prs_age.md │ │ │ │ ├── prs_approval.md │ │ │ │ ├── prs_authors_chart.md │ │ │ │ ├── prs_merged.md │ │ │ │ ├── repo_commenters.md │ │ │ │ ├── repo_comments.md │ │ │ │ ├── stub.md │ │ │ │ ├── time_metrics.md │ │ │ │ ├── top_commenters.md │ │ │ │ ├── tzs_stats.md │ │ │ │ ├── user_reviews.md │ │ │ │ └── users_stats.md │ │ │ ├── excluding_bots.md │ │ │ ├── github.md │ │ │ ├── periods.md │ │ │ ├── presentation/ │ │ │ │ ├── DevStatsPresentation.odp │ │ │ │ └── DevStatsPresentation.pptx │ │ │ ├── repository_aliases.md │ │ │ ├── repository_groups.md │ │ │ ├── tables/ │ │ │ │ ├── const_table.md │ │ │ │ ├── gha_actors.md │ │ │ │ ├── gha_comments.md │ │ │ │ ├── gha_commits.md │ │ │ │ ├── gha_commits_files.md │ │ │ │ ├── gha_events.md │ │ │ │ ├── gha_events_commits_files.md │ │ │ │ ├── gha_issues.md │ │ │ │ ├── gha_issues_events_labels.md │ │ │ │ ├── gha_issues_labels.md │ │ │ │ ├── gha_issues_pull_requests.md │ │ │ │ ├── gha_labels.md │ │ │ │ ├── gha_milestones.md │ │ │ │ ├── gha_orgs.md │ │ │ │ ├── gha_payloads.md │ │ │ │ ├── gha_postprocess_scripts.md │ │ │ │ ├── gha_pull_requests.md │ │ │ │ ├── gha_repos.md │ │ │ │ ├── gha_skip_commits.md │ │ │ │ ├── gha_texts.md │ │ │ │ ├── gha_vars.md │ │ │ │ └── variable_table.md │ │ │ ├── tags.md │ │ │ └── vars.md │ │ ├── find.sh │ │ ├── git/ │ │ │ ├── git_files.sh │ │ │ ├── git_reset_pull.sh │ │ │ └── git_tags.sh │ │ ├── github_users.json │ │ ├── grafana/ │ │ │ ├── dashboards/ │ │ │ │ └── kubeflow/ │ │ │ │ ├── activity-repository-groups.json │ │ │ │ ├── commits-repository-groups.json │ │ │ │ ├── community-stats.json │ │ │ │ ├── companies-stats.json │ │ │ │ ├── companies-summary.json │ │ │ │ ├── company-prs-in-repository-groups.json │ │ │ │ ├── contributing-companies.json │ │ │ │ ├── countries-stats.json │ │ │ │ ├── dashboards.json │ │ │ │ ├── developers-summary.json │ │ │ │ ├── first-non-author-activity.json │ │ │ │ ├── gender-stats.json │ │ │ │ ├── github-events.json │ │ │ │ ├── issues-age.json │ │ │ │ ├── issues-repository-group.json │ │ │ │ ├── new-and-episodic-issue-creators.json │ │ │ │ ├── new-and-episodic-pr-contributors.json │ │ │ │ ├── new-contributors-table.json │ │ │ │ ├── new-prs.json │ │ │ │ ├── opened-to-merged.json │ │ │ │ ├── pr-comments.json │ │ │ │ ├── project-statistics.json │ │ │ │ ├── prs-age.json │ │ │ │ ├── prs-approval.json │ │ │ │ ├── prs-authors-companies-histogram.json │ │ │ │ ├── prs-authors-histogram.json │ │ │ │ ├── prs-authors.json │ │ │ │ ├── prs-merged-repository-groups.json │ │ │ │ ├── repository-commenters.json │ │ │ │ ├── repository-comments.json │ │ │ │ ├── time-metrics.json │ │ │ │ ├── timezones-stats.json │ │ │ │ ├── top-commenters.json │ │ │ │ ├── user-reviews.json │ │ │ │ └── users-stats.json │ │ │ ├── etc/ │ │ │ │ └── grafana.ini.example │ │ │ ├── kubeflow/ │ │ │ │ ├── change_title_and_icons.sh │ │ │ │ └── grafana_start.sh │ │ │ └── provisioning/ │ │ │ ├── dashboards/ │ │ │ │ └── providers.yaml │ │ │ └── datasources/ │ │ │ └── datasource.yaml │ │ ├── grafana.sh │ │ ├── hide/ │ │ │ └── hide.csv │ │ ├── kubeflow/ │ │ │ └── psql.sh │ │ ├── metrics/ │ │ │ ├── kubeflow/ │ │ │ │ ├── sync_vars.yaml │ │ │ │ └── vars.yaml │ │ │ └── shared/ │ │ │ ├── activity_repo_groups.sql │ │ │ ├── all_prs_merged.sql │ │ │ ├── columns.yaml │ │ │ ├── columns_affs.yaml │ │ │ ├── commits_repo_groups.sql │ │ │ ├── committers_countries.sql │ │ │ ├── committers_countries_cum.sql │ │ │ ├── committers_gender.sql │ │ │ ├── committers_gender_cum.sql │ │ │ ├── committers_tz.sql │ │ │ ├── companies_tags.sql │ │ │ ├── company_activity.sql │ │ │ ├── company_activity_commits.sql │ │ │ ├── company_prs.sql │ │ │ ├── countries.sql │ │ │ ├── countries_cum.sql │ │ │ ├── countries_tags.sql │ │ │ ├── cumulative_periods.sql │ │ │ ├── episodic_contributors.sql │ │ │ ├── episodic_issues.sql │ │ │ ├── es_periods_tags.sql │ │ │ ├── event_types.sql │ │ │ ├── event_types_tags.sql │ │ │ ├── events.sql │ │ │ ├── first_non_author_activity.sql │ │ │ ├── gender.sql │ │ │ ├── gender_cum.sql │ │ │ ├── hist_commenters.sql │ │ │ ├── hist_pr_authors.sql │ │ │ ├── hist_pr_companies.sql │ │ │ ├── issues_age.sql │ │ │ ├── issues_closed.sql │ │ │ ├── issues_opened.sql │ │ │ ├── labels_priorities_tags_with_all.sql │ │ │ ├── metrics.yaml │ │ │ ├── metrics_affs.yaml │ │ │ ├── new_contributors.sql │ │ │ ├── new_contributors_data.sql │ │ │ ├── new_issues.sql │ │ │ ├── new_prs.sql │ │ │ ├── num_stats.sql │ │ │ ├── opened_to_merged.sql │ │ │ ├── pr_comments.sql │ │ │ ├── project_company_stats.sql │ │ │ ├── project_developer_stats.sql │ │ │ ├── project_stats.sql │ │ │ ├── projects_health.sql │ │ │ ├── prs_age.sql │ │ │ ├── prs_authors.sql │ │ │ ├── prs_merged_groups.sql │ │ │ ├── prs_state.sql │ │ │ ├── repo_commenters.sql │ │ │ ├── repo_comments.sql │ │ │ ├── repo_groups_tags.sql │ │ │ ├── repo_groups_tags_with_all.sql │ │ │ ├── reviewers_tags.sql │ │ │ ├── reviews_per_user.sql │ │ │ ├── tags.yaml │ │ │ ├── tags_affs.yaml │ │ │ ├── time_metrics.sql │ │ │ ├── top_repo_aliases_tags.sql │ │ │ ├── top_repos_tags_with_all.sql │ │ │ ├── tz.sql │ │ │ ├── tz_offset_tags.sql │ │ │ ├── user_activity.sql │ │ │ ├── user_activity_commits.sql │ │ │ ├── users_tags.sql │ │ │ └── watchers.sql │ │ ├── partials/ │ │ │ ├── projects.html │ │ │ └── projects_health.html │ │ ├── projects.yaml │ │ ├── run.sh │ │ ├── scripts/ │ │ │ ├── clean_affiliations.sql │ │ │ └── kubeflow/ │ │ │ └── repo_groups.sql │ │ ├── shared/ │ │ │ ├── get_repos.sh │ │ │ ├── import_affs.sh │ │ │ ├── reinit.sh │ │ │ ├── setup_repo_groups.sh │ │ │ ├── setup_scripts.sh │ │ │ ├── sync.sh │ │ │ └── tags.sh │ │ ├── sqlite/ │ │ │ └── touch │ │ └── util_sql/ │ │ ├── country_codes.sql │ │ ├── create_events_commits.sql │ │ ├── default_postprocess_scripts.sql │ │ ├── devstats_log_table.sql │ │ ├── drop_psql_user.sql │ │ ├── drop_ro_user.sql │ │ ├── exclude_bots.sql │ │ ├── list_unprocessed_commits.sql │ │ ├── postprocess_issues_prs.sql │ │ ├── postprocess_labels.sql │ │ ├── postprocess_repo_groups_from_repos.sql │ │ ├── postprocess_texts.sql │ │ ├── repo_groups_postprocess_script_from_repos.sql │ │ └── update_country_names.sql │ ├── data/ │ │ └── github_users.json │ ├── deploy-config/ │ │ ├── devstats.jinja │ │ ├── devstats.yaml │ │ └── gcfs.yaml │ ├── k8s_manifests/ │ │ ├── cli_home_pvc.yaml │ │ └── nfs_pvc.yaml │ ├── ks-app/ │ │ ├── .ksonnet/ │ │ │ └── registries/ │ │ │ ├── incubator/ │ │ │ │ └── 40285d8a14f1ac5787e405e1023cf0c07f6aa28c.yaml │ │ │ └── kubeflow/ │ │ │ └── e2c2c7c49e578472ba57f91f3b2a5f4d1a2d0289.yaml │ │ ├── app.yaml │ │ ├── components/ │ │ │ ├── backfill.jsonnet │ │ │ ├── cert-manager.jsonnet │ │ │ ├── devstats-parts.libsonnet │ │ │ ├── devstats.jsonnet │ │ │ ├── params.libsonnet │ │ │ └── syncronjob.jsonnet │ │ ├── environments/ │ │ │ ├── base.libsonnet │ │ │ └── devstats2/ │ │ │ ├── globals.libsonnet │ │ │ ├── main.jsonnet │ │ │ └── params.libsonnet │ │ ├── lib/ │ │ │ └── ksonnet-lib/ │ │ │ ├── v1.11.6/ │ │ │ │ ├── k.libsonnet │ │ │ │ ├── k8s.libsonnet │ │ │ │ └── swagger.json │ │ │ └── v1.7.0/ │ │ │ ├── k.libsonnet │ │ │ ├── k8s.libsonnet │ │ │ └── swagger.json │ │ └── vendor/ │ │ └── kubeflow/ │ │ └── core@e2c2c7c49e578472ba57f91f3b2a5f4d1a2d0289/ │ │ ├── README.md │ │ ├── all.libsonnet │ │ ├── ambassador.libsonnet │ │ ├── centraldashboard.libsonnet │ │ ├── cert-manager.libsonnet │ │ ├── cloud-endpoints.libsonnet │ │ ├── iap.libsonnet │ │ ├── jupyterhub.libsonnet │ │ ├── kubeform_spawner.py │ │ ├── nfs.libsonnet │ │ ├── parts.yaml │ │ ├── prototypes/ │ │ │ ├── all.jsonnet │ │ │ ├── cert-manager.jsonnet │ │ │ ├── cloud-endpoints.jsonnet │ │ │ ├── iap-ingress.jsonnet │ │ │ └── tensorboard.jsonnet │ │ ├── spartakus.libsonnet │ │ ├── tensorboard.libsonnet │ │ ├── tests/ │ │ │ ├── ambassador_test.jsonnet │ │ │ ├── iap_test.jsonnet │ │ │ ├── jupyterhub_test.jsonnet │ │ │ ├── nfs_test.jsonnet │ │ │ ├── spartakus_test.jsonnet │ │ │ ├── tf-job_test.jsonnet │ │ │ └── util_test.jsonnet │ │ ├── tf-job.libsonnet │ │ ├── util.libsonnet │ │ ├── version-info.json │ │ └── version.libsonnet │ ├── modify_dashboards.sh │ ├── postgre-docker-entrypoint.sh │ ├── print_imports.sh │ └── scripts/ │ ├── generate_actors.sh │ ├── modify_devstats_scripts.sh │ ├── setup_nfs.sh │ └── setup_postgres_home.sh ├── elections/ │ ├── eligible-candidates-and-voters-2023-ksc.md │ ├── eligible-candidates-and-voters-2024-ksc.md │ ├── eligible-candidates-and-voters-2025-ksc.md │ ├── kubeflow-steering-committee-elections-2023.md │ ├── kubeflow-steering-committee-elections-2024.md │ └── kubeflow-steering-committee-elections-2025.md ├── generator/ │ ├── README.md │ ├── aliases.tmpl │ ├── app.go │ ├── header.tmpl │ ├── list.tmpl │ ├── sig_readme.tmpl │ ├── ug_readme.tmpl │ └── wg_readme.tmpl ├── go.mod ├── go.sum ├── guidelines/ │ ├── README.md │ ├── application_requirements.md │ ├── auth.md │ └── creating_dockerfiles.md ├── how-to/ │ ├── README.md │ ├── community_meetings.md │ ├── github_admin.md │ ├── join_kubeflow_ecosystem.md │ ├── kubeflow_assets.md │ └── sharing_docs.md ├── issue-labels.md ├── ksc/ │ ├── README.md │ └── decision-log.md ├── labels-owners.yaml ├── member_organizations.yaml ├── proposal-workflow.md ├── proposals/ │ ├── 139-mpi-operator/ │ │ └── README.md │ ├── 141-chainer-operator/ │ │ └── README.md │ ├── 263-pvc-template/ │ │ └── README.md │ ├── 2655-kubeflow-data-cache/ │ │ └── README.md │ ├── 280-issue-triage/ │ │ └── README.md │ ├── 30-tf-operator-v1alpha2/ │ │ └── tf-operator-design-v1alpha2.md │ ├── 33-pytorch-operator/ │ │ └── README.md │ ├── 335-fate-operator/ │ │ └── README.md │ ├── 41-caffe2-operator/ │ │ └── README.md │ ├── 434-kubeflow-distribution/ │ │ └── README.md │ ├── 502-paddle-operator/ │ │ └── README.md │ ├── 524-kubeflow-conformance-program/ │ │ └── README.md │ ├── 525-kfserving-transition/ │ │ └── README.md │ ├── 585-kubeflow-governance/ │ │ ├── CONFORMANCE-COMMITTEE.md │ │ ├── GOVERNANCE.md │ │ └── TECH-OVERSIGHT-COMMITTEE.md │ ├── 645-kubeflow-steering-committee-election/ │ │ └── README.md │ ├── 648-spark-operator/ │ │ └── README.md │ ├── 748-expand-kubeflow-ecosystem/ │ │ └── README.md │ ├── 819-kubeflow-sdk/ │ │ └── README.md │ ├── 839-outreach-committee/ │ │ └── README.md │ ├── 841-release-process/ │ │ └── README.md │ ├── 867-kubeflow-documentation-ai/ │ │ └── README.md │ ├── 872-spark-history-server-mcp/ │ │ └── README.md │ ├── 897-experiment-tracking/ │ │ └── README.md │ ├── 907-model-registry-renaming/ │ │ └── README.md │ ├── 913-components-repo/ │ │ └── README.md │ ├── 936-kubeflow-mcp-server/ │ │ ├── DESIGN.md │ │ └── README.md │ ├── NNNN-template/ │ │ └── README.md │ ├── README.md │ ├── gsoc/ │ │ ├── README.md │ │ └── gsoc-proposal-template.md │ ├── kale-donation.md │ └── model-registry-proposal.md ├── psg/ │ └── ROADMAP.md ├── repository-setup.md ├── scripts/ │ ├── OWNERS │ ├── company_stats.ipynb │ ├── github_stats.ipynb │ ├── kf1.0_stats.ipynb │ ├── open_pr_stats.ipynb │ ├── print_companies.py │ ├── project_stats.ipynb │ ├── project_stats.py │ ├── prs_by_company.py │ └── requirements.txt ├── security/ │ ├── README.md │ └── self-assessment.md ├── sig-feature-store/ │ ├── README.md │ └── charter.md ├── sig-onprem/ │ ├── README.md │ └── charter.md ├── slack/ │ ├── OWNERS │ ├── README.md │ └── slack_channels.md ├── wg-automl/ │ ├── README.md │ └── charter.md ├── wg-data/ │ ├── README.md │ └── charter.md ├── wg-deployment/ │ ├── README.md │ └── charter.md ├── wg-list.md ├── wg-manifests/ │ ├── README.md │ └── charter.md ├── wg-ml-experience/ │ ├── README.md │ └── charter.md ├── wg-notebooks/ │ ├── README.md │ └── charter.md ├── wg-pipelines/ │ ├── README.md │ └── charter.md ├── wg-serving/ │ ├── README.md │ └── charter.md ├── wg-training/ │ ├── README.md │ └── charter.md ├── wgs/ │ ├── community-membership.md │ ├── overview.md │ ├── templates/ │ │ └── wg-charter-template.md │ ├── wg-charter.md │ ├── wg-foo/ │ │ └── OWNERS │ ├── wg-governance-requirements.md │ ├── wg-governance.md │ └── wg-lifecycle.md └── wgs.yaml
SYMBOL INDEX (69 symbols across 6 files)
FILE: calendar/calendar_import.py
function update_meeting (line 49) | def update_meeting(service, meeting):
function get_user_credentials (line 134) | def get_user_credentials(oauth_client_secret=None,
function _get_default_config (line 183) | def _get_default_config():
function create_secret (line 189) | def create_secret(client, project, secret):
function load_secret (line 216) | def load_secret(client, project, secret):
function sync_calendar (line 231) | def sync_calendar(config, creds):
class CalendarUpdater (line 247) | class CalendarUpdater:
method sync (line 251) | def sync(config=None, oauth_client_secret=None):
method periodic_sync (line 288) | def periodic_sync(config=None, project=DEFAULT_PROJECT,
method mint_credentials (line 343) | def mint_credentials(project, secret, oauth_client_secret):
FILE: devstats/config/util_sql/devstats_log_table.sql
type gha_logs (line 1) | CREATE TABLE gha_logs (
type logs_dt_idx (line 19) | CREATE INDEX logs_dt_idx ON gha_logs USING btree (dt)
type logs_id_idx (line 20) | CREATE INDEX logs_id_idx ON gha_logs USING btree (id)
FILE: devstats/ks-app/vendor/kubeflow/core@e2c2c7c49e578472ba57f91f3b2a5f4d1a2d0289/kubeform_spawner.py
class KubeFormSpawner (line 8) | class KubeFormSpawner(KubeSpawner):
method _options_form_default (line 11) | def _options_form_default(self):
method options_from_form (line 40) | def options_from_form(self, formdata):
method singleuser_image_spec (line 52) | def singleuser_image_spec(self):
method cpu_guarantee (line 59) | def cpu_guarantee(self):
method mem_guarantee (line 66) | def mem_guarantee(self):
method extra_resource_limits (line 73) | def extra_resource_limits(self):
FILE: generator/app.go
constant readmeTemplate (line 34) | readmeTemplate = "readme.tmpl"
constant listTemplate (line 35) | listTemplate = "list.tmpl"
constant aliasesTemplate (line 36) | aliasesTemplate = "aliases.tmpl"
constant headerTemplate (line 37) | headerTemplate = "header.tmpl"
constant wgsYamlFile (line 39) | wgsYamlFile = "wgs.yaml"
constant sigListOutput (line 40) | sigListOutput = "wg-list.md"
constant aliasesOutput (line 41) | aliasesOutput = "OWNERS_ALIASES"
constant indexFilename (line 42) | indexFilename = "README.md"
constant beginCustomMarkdown (line 44) | beginCustomMarkdown = "<!-- BEGIN CUSTOM CONTENT -->"
constant endCustomMarkdown (line 45) | endCustomMarkdown = "<!-- END CUSTOM CONTENT -->"
constant beginCustomYaml (line 46) | beginCustomYaml = "## BEGIN CUSTOM CONTENT"
constant endCustomYaml (line 47) | endCustomYaml = "## END CUSTOM CONTENT"
type FoldedString (line 56) | type FoldedString
method MarshalYAML (line 59) | func (x FoldedString) MarshalYAML() (interface{}, error) {
type Person (line 68) | type Person struct
type Meeting (line 75) | type Meeting struct
type Contact (line 87) | type Contact struct
type GithubTeam (line 95) | type GithubTeam struct
type Subproject (line 101) | type Subproject struct
type LeadershipGroup (line 110) | type LeadershipGroup struct
method PrefixToPersonMap (line 117) | func (g *LeadershipGroup) PrefixToPersonMap() map[string][]Person {
method Owners (line 126) | func (g *LeadershipGroup) Owners() []Person {
type Group (line 149) | type Group struct
method DirName (line 165) | func (g *Group) DirName(prefix string) string {
method LabelName (line 170) | func (g *Group) LabelName(prefix string) string {
type Context (line 175) | type Context struct
method PrefixToGroupMap (line 192) | func (c *Context) PrefixToGroupMap() map[string][]Group {
method Sort (line 202) | func (c *Context) Sort() {
method Validate (line 243) | func (c *Context) Validate() []error {
function index (line 182) | func index(groups []Group, predicate func(Group) bool) int {
function pathExists (line 299) | func pathExists(path string) bool {
function createDirIfNotExists (line 304) | func createDirIfNotExists(path string) error {
function getExistingContent (line 311) | func getExistingContent(path string, fileFormat string) (string, error) {
function tzURLEncode (line 358) | func tzURLEncode(tz string) string {
function writeTemplate (line 362) | func writeTemplate(templatePath, outputPath string, fileFormat string, d...
function writeCustomContentBlock (line 406) | func writeCustomContentBlock(f *os.File, content string, fileFormat stri...
function createGroupReadme (line 426) | func createGroupReadme(groups []Group, prefix string) error {
function readYaml (line 459) | func readYaml(path string, data interface{}) error {
function writeYaml (line 472) | func writeYaml(data interface{}, path string) error {
function main (line 483) | func main() {
FILE: scripts/project_stats.py
function run_query (line 20) | def run_query(query, headers): # A simple function to use requests.post ...
class ProjectStats (line 27) | class ProjectStats(object):
method __init__ (line 28) | def __init__(self, project):
method init_df (line 32) | def init_df(self, offset=0, size=300):
method grow_df (line 40) | def grow_df(self, df, offset=0, size=300):
method main (line 43) | def main(self):
method compute_stats (line 47) | def compute_stats(self):
method fetch_data (line 61) | def fetch_data(self):
FILE: scripts/prs_by_company.py
function get_company_from_email (line 22) | def get_company_from_email(email):
Copy disabled (too large)
Download .json
Condensed preview — 500 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (19,935K chars).
[
{
"path": ".github/issue_label_bot.yaml",
"chars": 119,
"preview": "label-alias:\n bug: 'kind/bug'\n feature_request: 'kind/feature'\n feature: 'kind/feature'\n question: 'kind/question'\n"
},
{
"path": ".github/workflows/stale.yml",
"chars": 2565,
"preview": "# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.\n#\n# You c"
},
{
"path": ".gitignore",
"chars": 92,
"preview": "flymd.*\n**.pyc\n**.swp\n**/.ipynb_checkpoints\n**/.cache\n.DS_Store\n\n# IntelliJ\n.idea/\n.vscode/\n"
},
{
"path": "ADOPTERS.md",
"chars": 3747,
"preview": "# Adopters of Kubeflow\r\n\r\nOur adopter files list organizations that have adopted Kubeflow in some way.\r\nSharing your ado"
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 316,
"preview": "# Kubeflow Community Code of Conduct\n\nKubeflow follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blo"
},
{
"path": "CONTRIBUTING.md",
"chars": 541,
"preview": "# Kubeflow Contributor Guide\n\nWelcome to the Kubeflow project! We'd love to accept your patches and \ncontributions to th"
},
{
"path": "INCLUSIVITY.md",
"chars": 5245,
"preview": "# Kubeflow Inclusivity\n\nThis document is a guide to inclusivity for leaders involved in the Kubeflow\nproject. It describ"
},
{
"path": "KUBEFLOW-GENERAL-TECHNICAL-REVIEW.md",
"chars": 50114,
"preview": "# General Technical Review - Kubeflow / Graduation\n\n- **Project:** Kubeflow\n\n- **Project Version:** Every Kubeflow sub-p"
},
{
"path": "KUBEFLOW-OUTREACH-COMMITTEE.md",
"chars": 7645,
"preview": "# Kubeflow Outreach Committee (KOC)\n\nThe Kubeflow Outreach Committee (KOC) is a committee dedicated to fostering growth,"
},
{
"path": "KUBEFLOW-STEERING-COMMITTEE.md",
"chars": 10904,
"preview": "# Kubeflow Steering Committee\n\nThe Kubeflow Steering Committee (KSC) is the governing body of the Kubeflow project, prov"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "MAINTAINERS.md",
"chars": 1271,
"preview": "# Maintainers\n\nThis document lists the maintainers under [the Kubeflow governance model](https://www.kubeflow.org/docs/a"
},
{
"path": "Makefile",
"chars": 299,
"preview": "IMAGE_NAME=golang:1.14\nexport GO111MODULE=on\nexport GOPROXY?=https://proxy.golang.org\n\ndefault: \\\n\tgenerate \\\n\nreset-doc"
},
{
"path": "OWNERS",
"chars": 188,
"preview": "approvers:\n - andreyvelich\n - chasecadet\n - franciscojavierarceo\n - juliusvonkohout\n - thesuperzapper\n\nemeritus_app"
},
{
"path": "OWNERS_ALIASES",
"chars": 1080,
"preview": "aliases:\n sig-feature-store-leads:\n - kevinstumpf\n - pyalex\n - woop\n sig-onprem-leads:\n - jtfogarty\n - "
},
{
"path": "README.md",
"chars": 1997,
"preview": "# Kubeflow Community\n\nWelcome to Kubeflow! We're glad to have you here.\n\n## Join us in discussions, forums, and meetings"
},
{
"path": "calendar/Dockerfile",
"chars": 389,
"preview": "FROM google/cloud-sdk\n\nRUN apt-get update -y && \\\n apt-get install -y python3-distutils\n\nRUN python3.7 -m pip install"
},
{
"path": "calendar/Makefile",
"chars": 891,
"preview": "GIT_VERSION:=$(shell git describe --dirty --always)\n\nPROJECT:=kf-infra-gitops\n\nIMAGE:=gcr.io/$(PROJECT)/calendar-sync:$("
},
{
"path": "calendar/OWNERS",
"chars": 94,
"preview": "approvers:\n - james-jwu\n - jtfogarty\n - nicholas-abad\n - yuzliu\n - zijianjoy\n - chensun\n"
},
{
"path": "calendar/README.md",
"chars": 4491,
"preview": "# Kubeflow community calendar\n\nThis is location for [Kubeflow community calendar](https://calendar.google.com/calendar/e"
},
{
"path": "calendar/calendar.yaml",
"chars": 18250,
"preview": "# This file controls the meetings on the \"Kubeflow Community Calendar\":\n# https://calendar.google.com/calendar/embed?src"
},
{
"path": "calendar/calendar_import.py",
"chars": 14458,
"preview": "\"\"\"\nImports meetings in 'caldendar/calendar.yaml' to the Kubeflow Community Calendar\nFor modifications please refer to t"
},
{
"path": "calendar/latest_image.yaml",
"chars": 176,
"preview": "image: hello\nkf-infra-gitops:\n calendar:\n image: gcr.io/kf-infra-gitops/calendar-sync:2b76aca-dirty@sha256:49b7b067a"
},
{
"path": "calendar/manifests/deployment.yaml",
"chars": 952,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calendar-sync\n labels:\n app: calendar-sync\nspec:\n replicas: "
},
{
"path": "calendar/manifests/kustomization.yaml",
"chars": 307,
"preview": "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: kf-autobot\nresources:\n- deployment.yaml\n- ser"
},
{
"path": "calendar/manifests/service-account.yaml",
"chars": 173,
"preview": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n annotations: \n iam.gke.io/gcp-service-account: kf-autobot@kf-infra"
},
{
"path": "dco-signoff-hook/README.md",
"chars": 1319,
"preview": "# Signing off commits\n\nTo automatically sign off on every commit, copy the [prepare-commit-msg](prepare-commit-msg) file"
},
{
"path": "dco-signoff-hook/prepare-commit-msg",
"chars": 342,
"preview": "#!/bin/sh\n\nNAME=$(git config user.name)\nEMAIL=$(git config user.email)\n\nif [ -z \"$NAME\" ]; then\n echo \"empty git conf"
},
{
"path": "devstats/Dockerfile.devstats",
"chars": 3411,
"preview": "# Dockerfile for devstats\n# Based on https://github.com/cncf/devstats/blob/master/INSTALL_UBUNTU17.md\nFROM golang:1.11.5"
},
{
"path": "devstats/Makefile",
"chars": 1238,
"preview": "# Copyright 2017 The Kubernetes Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may n"
},
{
"path": "devstats/README.md",
"chars": 17438,
"preview": "# Devstats\n\nInstructions for deploying [cncf/devstats](https://github.com/cncf/devstats).\n\n\n## Deploying on Kubernetes\n\n"
},
{
"path": "devstats/config/INSTALL_UBUNTU18.md",
"chars": 5147,
"preview": "# devstats installation on Ubuntu\n\nPrerequisites:\n- Ubuntu 18.04.\n- [golang](https://golang.org).\n - `apt-get update`"
},
{
"path": "devstats/config/README.md",
"chars": 1984,
"preview": "A fork of https://github.com/cncf/devstats-example\n\n* This directory contains the config files for Kubeflow\n* It is base"
},
{
"path": "devstats/config/SETUP_OTHER_PROJECT.md",
"chars": 1287,
"preview": "# Setup you own project\n\nThis example deployment uses Homebrew project. To run DevStats on the other project do:\n\n- In a"
},
{
"path": "devstats/config/copy_devstats_binaries.sh",
"chars": 330,
"preview": "#!/bin/bash\nbinaries=\"structure runq gha2db calc_metric gha2db_sync import_affs annotations tags webhook devstats get_re"
},
{
"path": "devstats/config/cron/net_tcp_config.sh",
"chars": 49,
"preview": "#!/bin/bash\n/sbin/sysctl net.ipv4.tcp_tw_reuse=1\n"
},
{
"path": "devstats/config/crontab",
"chars": 306,
"preview": "7 * * * * PATH=$PATH:/data/dev/bin net_tcp_config.sh\n8 * * * * PATH=$PATH:/data/dev/bin DEVSTATS_DIR=\"/data/dev/src/devs"
},
{
"path": "devstats/config/deploy.sh",
"chars": 302,
"preview": "#!/bin/bash\nmkdir /var/www 2>/dev/null\nmkdir /var/www/html 2>/dev/null\n./copy_devstats_binaries.sh || exit 1\ncp cron/net"
},
{
"path": "devstats/config/devel/add_single_metric.sh",
"chars": 395,
"preview": "#!/bin/bash\nif [ -z \"${PG_DB}\" ]\nthen\n echo \"You need to set PG_DB environment variable to run this script\"\n exit 1\nfi"
},
{
"path": "devstats/config/devel/add_single_metric_all.sh",
"chars": 430,
"preview": "#!/bin/bash\nif [ -z \"${PG_PASS}\" ]\nthen\n echo \"You need to set PG_PASS environment variable to run this script\"\n exit "
},
{
"path": "devstats/config/devel/create_databases.sh",
"chars": 3345,
"preview": "#!/bin/bash\n# PDB=1 (will generate Postgres DB)\n# TSDB=1 (will generate TS DB)\n# PDROP=1 (will drop & create Postgres DB"
},
{
"path": "devstats/config/devel/create_grafana.sh",
"chars": 5170,
"preview": "#!/bin/bash\n# GGET=1 (Get grafana.db from the $HOST_SRC server)\n# STOP=1 (Stops running grafana-server instance)\n# RM=1 "
},
{
"path": "devstats/config/devel/create_psql_user.sh",
"chars": 920,
"preview": "#!/bin/bash\nif [ -z \"${PG_PASS}\" ]\nthen\n echo \"$0: You need to set PG_PASS environment variable to run this script\"\n e"
},
{
"path": "devstats/config/devel/deploy_all.sh",
"chars": 1318,
"preview": "#!/bin/bash\n# ARTWORK\n# GET=1 (attempt to fetch Postgres database and Grafana database from the test server)\n# INIT=1 (n"
},
{
"path": "devstats/config/devel/deploy_proj.sh",
"chars": 1090,
"preview": "#!/bin/bash\n# GET=1 (attempt to fetch Postgres database and Grafana database from the test server)\n# SKIPDBS=1 (entirely"
},
{
"path": "devstats/config/devel/drop_psql_db.sh",
"chars": 238,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: you need to provide db name\"\n exit 1\nfi\nsudo -E -u postgres psql -c \"select"
},
{
"path": "devstats/config/devel/drop_ts_tables.sh",
"chars": 364,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: need database name argument\"\n exit 1\nfi\nproj=$1\ntables=`sudo -E -u postgres"
},
{
"path": "devstats/config/devel/drop_tsdb_affs_tables.sh",
"chars": 911,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: need database name argument\"\n exit 1\nfi\nproj=$1\n# snum_stats scompany_activ"
},
{
"path": "devstats/config/devel/get_all_sqlite_jsons.sh",
"chars": 396,
"preview": "#!/bin/bash\nif [ -z \"$ONLY\" ]\nthen\n all=\"kubeflow\"\nelse\n all=$ONLY\nfi\nmkdir sqlite 1>/dev/null 2>/dev/null\ntouch sqlit"
},
{
"path": "devstats/config/devel/get_from_sqlite.sh",
"chars": 374,
"preview": "#!/bin/bash\nif [ -z \"$GRAFANA\" ]\nthen\n echo \"$0: you need to set GRAFANA env variable\"\n exit 1\nfi\nif [ -z \"$1\" ]\nthen\n"
},
{
"path": "devstats/config/devel/get_icon_type.sh",
"chars": 263,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: requires project name parameter\"\n exit 1\nfi\ndeclare -A icontypes\nicontypes="
},
{
"path": "devstats/config/devel/grafana_start.sh",
"chars": 279,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: you need to provide grafana name\"\n exit 1\nfi\n./devel/grafana_stop.sh $1 || "
},
{
"path": "devstats/config/devel/grafana_stop.sh",
"chars": 306,
"preview": "#!/bin/sh\nif [ -z \"$1\" ]\nthen\n echo \"$0: you need to provide grafana name\"\n exit 1\nfi\npid=`ps -axu | grep grafana-serv"
},
{
"path": "devstats/config/devel/import_jsons_to_sqlite.sh",
"chars": 713,
"preview": "#!/bin/bash\nif [ -z \"$GRAFANA\" ]\nthen\n echo \"$0: you need to set GRAFANA env variable (Grafana suffix). For example k8s"
},
{
"path": "devstats/config/devel/init_database.sh",
"chars": 2196,
"preview": "#!/bin/bash\n# UDROP=1 attempt to drop users\n# LDROP=1 attempt to drop devstats database\nset -x -o pipefail\nif ( [ -z \"$P"
},
{
"path": "devstats/config/devel/mass_replace.sh",
"chars": 1556,
"preview": "#!/bin/bash\n# Examples:\n# MODE=rr FROM=`cat input` TO=`cat output` FILES=`find abc/ -type f -not -iname 'something.txt'`"
},
{
"path": "devstats/config/devel/net_tcp_config.sh",
"chars": 49,
"preview": "#!/bin/bash\n/sbin/sysctl net.ipv4.tcp_tw_reuse=1\n"
},
{
"path": "devstats/config/devel/psql_user_grants.sh",
"chars": 536,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: need user name argument\"\n exit 1\nfi\nif [ -z \"$2\" ]\nthen\n echo \"$0: need da"
},
{
"path": "devstats/config/devel/put_all_charts.sh",
"chars": 223,
"preview": "#!/bin/bash\nif [ -z \"$ONLY\" ]\nthen\n all=\"kubeflow\"\nelse\n all=$ONLY\nfi\nfor proj in $all\ndo\n suff=$proj\n NOCOPY=1 "
},
{
"path": "devstats/config/devel/put_all_charts_cleanup.sh",
"chars": 71,
"preview": "#!/bin/bash\nrm grafana.*.db*\nfind . -iname \"*.was\" -exec rm -f \"{}\" \\;\n"
},
{
"path": "devstats/config/devel/restore_db.sh",
"chars": 1248,
"preview": "#!/bin/bash\n# FROMBACKUP: if set, it will use /var/www/html/$1.dump as a restore source\n# TEMPRENAME: restore to $1_temp"
},
{
"path": "devstats/config/devel/ro_user_grants.sh",
"chars": 299,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"$0: need database name argument\"\n exit 1\nfi\nproj=$1\ntables=`sudo -E -u postgres"
},
{
"path": "devstats/config/devel/sync_lock.sh",
"chars": 407,
"preview": "#!/bin/bash\nif [ -f \"/tmp/deploy.wip\" ]\nthen\n echo \"another deploy process is running, exiting\"\n exit 1\nfi\nwait_for_co"
},
{
"path": "devstats/config/devel/sync_unlock.sh",
"chars": 189,
"preview": "#!/bin/bash\ncronctl.sh devstats on || exit 1\ncronctl.sh devstats.sh on || exit 2\nif [ -z \"$FROM_WEBHOOK\" ]\nthen\n cronct"
},
{
"path": "devstats/config/devel/test_metrics.yaml",
"chars": 221,
"preview": "---\nmetrics:\n - name: GitHub events\n series_name_or_func: multi_row_single_column\n sql: event_types\n periods: "
},
{
"path": "devstats/config/devel/test_tags.yaml",
"chars": 156,
"preview": "---\ntags:\n - name: GitHub event types\n sql: event_types_tags\n series_name: event_types\n name_tag: event_type_n"
},
{
"path": "devstats/config/devel/update_from_sqlite.sh",
"chars": 105,
"preview": "#!/bin/bash\nfor f in `find . -name \"*.was\"`\ndo\n f2=${f%.*}\n mv \"$f\" \"$f2\" || exit 1\ndone\necho 'OK'\n"
},
{
"path": "devstats/config/devel/vars_all.sh",
"chars": 528,
"preview": "#!/bin/bash\nif [ -z \"${PG_PASS}\" ]\nthen\n echo \"You need to set PG_PASS environment variable to run this script\"\n exit "
},
{
"path": "devstats/config/devstats.sh",
"chars": 136,
"preview": "#!/bin/bash\nif [ -z \"$DEVSTATS_DIR\" ]\nthen\n echo \"$0: you need to set DEVSTATS_DIR\"\n exit 1\n fi\ncd \"$DEVSTATS_DIR\"\n./r"
},
{
"path": "devstats/config/docs/annotations.md",
"chars": 7663,
"preview": "# Annotations\n\n- Most dashboards use Grafana's annotations query.\n- For example search for `\"annotations\": {` [here](htt"
},
{
"path": "devstats/config/docs/dashboards/dashboards.md",
"chars": 1366,
"preview": "<h1 id=\"-full_name-home-dashboard\">[[full_name]] Home dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Postgres <a href=\"https://gi"
},
{
"path": "devstats/config/docs/dashboards/dashboards_devel.md",
"chars": 4775,
"preview": "# Home dashboard\n\nLinks:\n- Postgres SQL file: [events.sql](https://github.com/cncf/devstats/blob/master/metrics/shared/e"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/blocked_prs.md",
"chars": 1595,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Blocked PRs by repository group dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/bot_commands.md",
"chars": 1954,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Bot commands repository group dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/companies_table.md",
"chars": 1425,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Companies table dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https:"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/company_stats.md",
"chars": 1982,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Company statistics by repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<l"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/contributing_companies.md",
"chars": 1786,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Companies contributing in repository groups dashboard</h1>\n<p>Links:</p>\n<ul"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/developer_stats.md",
"chars": 1948,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Developer Activity Counts by Repository Group dashboard</h1>\n<p>Links:</p>\n<"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/episodic_issues.md",
"chars": 1785,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] New And Episodic Issue Creators dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>New is"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/episodic_prs.md",
"chars": 1794,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] New And Episodic PR Contributors dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>New P"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_commits.md",
"chars": 2218,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (commits)</h1>\n<p>Links:</p>\n<ul>\n<li>Reposit"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_iclosed.md",
"chars": 2204,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (issues closed)</h1>\n<p>Links:</p>\n<ul>\n<li>R"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_icommenters.md",
"chars": 2232,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (Issue commenters)</h1>\n<p>Links:</p>\n<ul>\n<l"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_icomments.md",
"chars": 2210,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (Issue comments)</h1>\n<p>Links:</p>\n<ul>\n<li>"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_iopened.md",
"chars": 2204,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (issues opened)</h1>\n<p>Links:</p>\n<ul>\n<li>R"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_prclosed.md",
"chars": 2189,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (PRs closed)</h1>\n<p>Links:</p>\n<ul>\n<li>Repo"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_prcommenters.md",
"chars": 2220,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (PR commenters)</h1>\n<p>Links:</p>\n<ul>\n<li>R"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_prcomments.md",
"chars": 2198,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (PR comments)</h1>\n<p>Links:</p>\n<ul>\n<li>Rep"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_prmerged.md",
"chars": 2189,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (PRs merged)</h1>\n<p>Links:</p>\n<ul>\n<li>Repo"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_propened.md",
"chars": 2189,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (PRs opened)</h1>\n<p>Links:</p>\n<ul>\n<li>Repo"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/gh_stats_reviewers.md",
"chars": 2412,
"preview": "<h1 id=\"kubernetes-ghstats-dashboard\">[[full_name]] GitHub stats dashboard (reviewers)</h1>\n<p>Links:</p>\n<ul>\n<li>Repos"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/ghstats_devel.md",
"chars": 9443,
"preview": "# Kubernetes GitHub stats dashboard\n\nLinks:\n- Postgres SQL file: [github_stats_by_repos.sql](https://github.com/cncf/dev"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/issues_age.md",
"chars": 1769,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Issues age by SIG and repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<l"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/issues_opened_closed.md",
"chars": 1509,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Issues Opened/Closed by SIG dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Opened iss"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_approval.md",
"chars": 1442,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PRs approval repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric "
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_approve_to_merge.md",
"chars": 2466,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR Time to Approve and Merge dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_authors.md",
"chars": 1415,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PRs authors repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_comments.md",
"chars": 1164,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR comments dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://gi"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_labels.md",
"chars": 1309,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR labels repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a "
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_reviews_by_contributor.md",
"chars": 1681,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR Reviews by Contributor dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a hr"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_time_to_engagement.md",
"chars": 1590,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR Time to Engagment dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"h"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/pr_workload.md",
"chars": 2904,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] PR Workload per SIG dashboards (chart and table)</h1>\n<p>Links:</p>\n<ul>\n<li"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/project_stats.md",
"chars": 1528,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Overall Project Statistics dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Main metric"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/prs_age.md",
"chars": 1406,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Open PR Age By Repository Group dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/sig_mentions.md",
"chars": 2439,
"preview": "<h1 id=\"kubernetes-sig-mentions-dashboard\">[[full_name]] SIG mentions dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Postgres <a "
},
{
"path": "devstats/config/docs/dashboards/kubernetes/sig_mentions_devel.md",
"chars": 4829,
"preview": "# SIG mentions dashboard\n\nLinks:\n- Postgres SQL file: [sig_mentions.sql](https://github.com/cncf/devstats/blob/master/me"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/sig_milestones.md",
"chars": 1582,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] open issues/PRs by milestone and repository dashboard</h1>\n<p>Links:</p>\n<ul"
},
{
"path": "devstats/config/docs/dashboards/kubernetes/stars_and_forks.md",
"chars": 994,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Stars and Forks by Repository dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <"
},
{
"path": "devstats/config/docs/dashboards/shared/activity.md",
"chars": 1350,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] activity repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href="
},
{
"path": "devstats/config/docs/dashboards/shared/commits.md",
"chars": 1374,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] commits repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\""
},
{
"path": "devstats/config/docs/dashboards/shared/community_stats.md",
"chars": 949,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] community stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://gi"
},
{
"path": "devstats/config/docs/dashboards/shared/companies_stats.md",
"chars": 2014,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] companies stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://gi"
},
{
"path": "devstats/config/docs/dashboards/shared/companies_summary.md",
"chars": 1551,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] companies summary dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://"
},
{
"path": "devstats/config/docs/dashboards/shared/company_prs.md",
"chars": 1576,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] company PRs in repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a"
},
{
"path": "devstats/config/docs/dashboards/shared/contributing_companies.md",
"chars": 1736,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] contributing companies dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"htt"
},
{
"path": "devstats/config/docs/dashboards/shared/countries_stats.md",
"chars": 2370,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] countries stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://gi"
},
{
"path": "devstats/config/docs/dashboards/shared/dashboards.md",
"chars": 1166,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] stub dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github.com/cn"
},
{
"path": "devstats/config/docs/dashboards/shared/developers_summary.md",
"chars": 1690,
"preview": "<h1 id=\"kubernetes-dashboard\">[[full_name]] Developer Activity Counts by Repository Group dashboard</h1>\n<p>Links:</p>\n<"
},
{
"path": "devstats/config/docs/dashboards/shared/gender_stats.md",
"chars": 2372,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] gender stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://githu"
},
{
"path": "devstats/config/docs/dashboards/shared/github_events_docs_html.md",
"chars": 938,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] GitHub events dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://gith"
},
{
"path": "devstats/config/docs/dashboards/shared/issues.md",
"chars": 1384,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] issues repository group dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"ht"
},
{
"path": "devstats/config/docs/dashboards/shared/issues_age.md",
"chars": 1505,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] issues age dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github."
},
{
"path": "devstats/config/docs/dashboards/shared/new_and_episodic_issues.md",
"chars": 1790,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] new and episodic issue creators dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>New issues"
},
{
"path": "devstats/config/docs/dashboards/shared/new_and_episodic_prs.md",
"chars": 1821,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] new and episodic PR contributors dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>New contr"
},
{
"path": "devstats/config/docs/dashboards/shared/new_contributors.md",
"chars": 1140,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] new contributors table dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>New contributors me"
},
{
"path": "devstats/config/docs/dashboards/shared/new_prs.md",
"chars": 1122,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] new PRs dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github.com"
},
{
"path": "devstats/config/docs/dashboards/shared/non_author_activity.md",
"chars": 1473,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] first non-author activity dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\""
},
{
"path": "devstats/config/docs/dashboards/shared/opened_to_merged.md",
"chars": 1175,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] opened to merged dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://g"
},
{
"path": "devstats/config/docs/dashboards/shared/pr_authors.md",
"chars": 1410,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs authors histogram dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"http"
},
{
"path": "devstats/config/docs/dashboards/shared/pr_comments.md",
"chars": 1129,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PR comments dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github"
},
{
"path": "devstats/config/docs/dashboards/shared/pr_companies.md",
"chars": 1666,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs authors companies histogram dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a "
},
{
"path": "devstats/config/docs/dashboards/shared/project_health.md",
"chars": 737,
"preview": "<h1 id=\"dashboard-header\">Project health dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Projects health metric <a href=\"https://g"
},
{
"path": "devstats/config/docs/dashboards/shared/projects_health.md",
"chars": 1008,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] projects health dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Projects health metric <a "
},
{
"path": "devstats/config/docs/dashboards/shared/projects_stats.md",
"chars": 1525,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] project statistics dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Main metric <a href=\"ht"
},
{
"path": "devstats/config/docs/dashboards/shared/prs_age.md",
"chars": 1267,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs age dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github.com"
},
{
"path": "devstats/config/docs/dashboards/shared/prs_approval.md",
"chars": 1353,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs approval dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://githu"
},
{
"path": "devstats/config/docs/dashboards/shared/prs_authors_chart.md",
"chars": 1357,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs authors dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github"
},
{
"path": "devstats/config/docs/dashboards/shared/prs_merged.md",
"chars": 1652,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] PRs merged repository groups dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a hre"
},
{
"path": "devstats/config/docs/dashboards/shared/repo_commenters.md",
"chars": 1341,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] repository commenters dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"http"
},
{
"path": "devstats/config/docs/dashboards/shared/repo_comments.md",
"chars": 1319,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] repository comments dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https:"
},
{
"path": "devstats/config/docs/dashboards/shared/stub.md",
"chars": 1166,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] stub dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github.com/cn"
},
{
"path": "devstats/config/docs/dashboards/shared/time_metrics.md",
"chars": 1231,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] time metrics dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://githu"
},
{
"path": "devstats/config/docs/dashboards/shared/top_commenters.md",
"chars": 1380,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] top commenters dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://git"
},
{
"path": "devstats/config/docs/dashboards/shared/tzs_stats.md",
"chars": 2061,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] time zones stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>1st metric <a href=\"https"
},
{
"path": "devstats/config/docs/dashboards/shared/user_reviews.md",
"chars": 1331,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] user reviews dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://githu"
},
{
"path": "devstats/config/docs/dashboards/shared/users_stats.md",
"chars": 1775,
"preview": "<h1 id=\"dashboard-header\">[[full_name]] users stats dashboard</h1>\n<p>Links:</p>\n<ul>\n<li>Metric <a href=\"https://github"
},
{
"path": "devstats/config/docs/excluding_bots.md",
"chars": 646,
"preview": "# Excluding bots\n\n- You can put excluding bots partial `{{exclude_bots}}` anywhere in the metric SQL.\n- You should put e"
},
{
"path": "devstats/config/docs/github.md",
"chars": 582,
"preview": "# GitHub (GHAPI) datasource details\n\n- You need to have `/etc/github/oauth` file created on your server, this file shoul"
},
{
"path": "devstats/config/docs/periods.md",
"chars": 1962,
"preview": "# DevStats metrics periods definitions\n\n- Periods can be: h,d,w,m,q,y which means we should calculate this SQL for every"
},
{
"path": "devstats/config/docs/repository_aliases.md",
"chars": 1132,
"preview": "# DevStats repository aliases\n\n- Repository alias is usually defined as the most recent name of a given repository.\n- Gi"
},
{
"path": "devstats/config/docs/repository_groups.md",
"chars": 4986,
"preview": "# Repository groups\n\n- Most project use 'repository groups' to group data under them.\n- It is usually defined on the rep"
},
{
"path": "devstats/config/docs/tables/const_table.md",
"chars": 188,
"preview": "# Const table\n\n- Tables marked as `const` are not changing in time.\n- Their values are inserted once and are not updated"
},
{
"path": "devstats/config/docs/tables/gha_actors.md",
"chars": 1649,
"preview": "# `gha_actors` table\n\n- This table holds all GitHub actors (actor can be contributor, forker, commenter etc.)\n- This is "
},
{
"path": "devstats/config/docs/tables/gha_comments.md",
"chars": 1324,
"preview": "# `gha_comments` table\n\n- This is a table that holds GitHub comments state at a given point in time (`event_id` refers t"
},
{
"path": "devstats/config/docs/tables/gha_commits.md",
"chars": 2952,
"preview": "# `gha_commits` table\n\n- This table contains commits data.\n- This is a variable table, for details check [variable table"
},
{
"path": "devstats/config/docs/tables/gha_commits_files.md",
"chars": 1759,
"preview": "# `gha_commits_files` table\n\n- This table holds commit's files (added, removed, modified etc.)\n- We're listing all yet u"
},
{
"path": "devstats/config/docs/tables/gha_events.md",
"chars": 3464,
"preview": "# `gha_events` table\n\n- This is the main GHA (GitHub archives) table. It represents single event.\n- Each GHA JSON contai"
},
{
"path": "devstats/config/docs/tables/gha_events_commits_files.md",
"chars": 4026,
"preview": "# `gha_events_commits_files` table\n\n- This table holds commit's files connected with GitHub event additional data.\n- It "
},
{
"path": "devstats/config/docs/tables/gha_issues.md",
"chars": 2256,
"preview": "# `gha_issues` table\n\n- This is a table that holds GitHub issue state at a given point in time (`event_id` refers to [gh"
},
{
"path": "devstats/config/docs/tables/gha_issues_events_labels.md",
"chars": 3403,
"preview": "# `gha_issues_events_labels` table\n\n- This is a compute table, that contains shortcuts to issues labels connected with e"
},
{
"path": "devstats/config/docs/tables/gha_issues_labels.md",
"chars": 3002,
"preview": "# `gha_issues_labels` table\n\n- This is a table that holds labels set on an issue/PR in a given moment in time.\n- GitHub "
},
{
"path": "devstats/config/docs/tables/gha_issues_pull_requests.md",
"chars": 3075,
"preview": "# `gha_issues_pull_requests` table\n\n- This is a compute table, that contains data to connect issues with PRs.\n- It conta"
},
{
"path": "devstats/config/docs/tables/gha_labels.md",
"chars": 1507,
"preview": "# `gha_labels` table\n\n- This table holds GitHub labels.\n- This is a const table, values are inserted once and doesn't ch"
},
{
"path": "devstats/config/docs/tables/gha_milestones.md",
"chars": 1541,
"preview": "# `gha_milestones` table\n\n- This is a table that holds GitHub milestone state at a given point in time (`event_id` refer"
},
{
"path": "devstats/config/docs/tables/gha_orgs.md",
"chars": 923,
"preview": "# `gha_orgs` table\n\n- This table holds GitHub organizations.\n- This is a const table, values are inserted once and doesn"
},
{
"path": "devstats/config/docs/tables/gha_payloads.md",
"chars": 4359,
"preview": "# `gha_payloads` table\n\n- This is the main GHA (GitHub archives), every GitHub event contain payload. This event ID is t"
},
{
"path": "devstats/config/docs/tables/gha_postprocess_scripts.md",
"chars": 1989,
"preview": "# `gha_postprocess_scripts` table\n\n- This is a special table, not created by any GitHub archive (GHA) event.\n- It contai"
},
{
"path": "devstats/config/docs/tables/gha_pull_requests.md",
"chars": 2519,
"preview": "# `gha_pull_requests` table\n\n- This is a table that holds GitHub PR state at a given point in time (`event_id` refers to"
},
{
"path": "devstats/config/docs/tables/gha_repos.md",
"chars": 2697,
"preview": "# `gha_repos` table\n\n- This table holds GitHub repositories.\n- This is a const table, values are inserted once and doesn"
},
{
"path": "devstats/config/docs/tables/gha_skip_commits.md",
"chars": 2093,
"preview": "# `gha_skip_commits` table\n\n- Table is used to store invalid SHAs, to skip processing them again.\n- This is a const tabl"
},
{
"path": "devstats/config/docs/tables/gha_texts.md",
"chars": 3827,
"preview": "# `gha_texts` table\n\n- This is a special table, not created by any GitHub archive (GHA) event. Its purpose is to hold al"
},
{
"path": "devstats/config/docs/tables/gha_vars.md",
"chars": 814,
"preview": "# `gha_vars` table\n\n- This is a special table that holds PostgreS variables defined by [pdb_vars](https://github.com/cnc"
},
{
"path": "devstats/config/docs/tables/variable_table.md",
"chars": 369,
"preview": "# Variable table\n\n- Data in those tables can change between GitHub events, and `event_id` is a part of this tables prima"
},
{
"path": "devstats/config/docs/tags.md",
"chars": 2641,
"preview": "# Tags\n\n- You can use tags to define drop-down values (variables) in Grafana dashboards.\n- Some drop-downs can be hardco"
},
{
"path": "devstats/config/docs/vars.md",
"chars": 4826,
"preview": "# Postgres variables\n\n- Results are saved to [gha_vars](https://github.com/cncf/devstats/blob/master/docs/tables/gha_var"
},
{
"path": "devstats/config/find.sh",
"chars": 416,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"You need to provide path as first arument\"\n exit 1\nfi\nif [ -z \"$2\" ]\nthen\n ech"
},
{
"path": "devstats/config/git/git_files.sh",
"chars": 666,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"Arguments required: path sha, none given\"\n exit 1\nfi\nif [ -z \"$2\" ]\nthen\n echo"
},
{
"path": "devstats/config/git/git_reset_pull.sh",
"chars": 218,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"Argument required: path to call git-reset and the git-pull\"\n exit 1\nfi\n\ncd \"$1\""
},
{
"path": "devstats/config/git/git_tags.sh",
"chars": 173,
"preview": "#!/bin/bash\nif [ -z \"$1\" ]\nthen\n echo \"Argument required: repo path\"\n exit 1\nfi\n\ncd \"$1\" || exit 3\ngit tag -l --format"
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/activity-repository-groups.json",
"chars": 12874,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/commits-repository-groups.json",
"chars": 12810,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/community-stats.json",
"chars": 12151,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/companies-stats.json",
"chars": 13541,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/companies-summary.json",
"chars": 9561,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/company-prs-in-repository-groups.json",
"chars": 8197,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/contributing-companies.json",
"chars": 8879,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/countries-stats.json",
"chars": 22634,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/dashboards.json",
"chars": 7417,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/developers-summary.json",
"chars": 11435,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/first-non-author-activity.json",
"chars": 14201,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/gender-stats.json",
"chars": 22017,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/github-events.json",
"chars": 12781,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/issues-age.json",
"chars": 23758,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"datasource\": \"psql\",\n \"enable\": true,\n \"hide\": false,\n"
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/issues-repository-group.json",
"chars": 8728,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/new-and-episodic-issue-creators.json",
"chars": 11001,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/new-and-episodic-pr-contributors.json",
"chars": 11024,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/new-contributors-table.json",
"chars": 7089,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/new-prs.json",
"chars": 7947,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/opened-to-merged.json",
"chars": 14366,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
},
{
"path": "devstats/config/grafana/dashboards/kubeflow/pr-comments.json",
"chars": 10675,
"preview": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\""
}
]
// ... and 300 more files (download for full content)
About this extraction
This page contains the full source code of the kubeflow/community GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 500 files (31.4 MB), approximately 4.7M tokens, and a symbol index with 69 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.