Repository: HariSekhon/DevOps-Bash-tools Branch: master Commit: 318dd0e61bd4 Files: 1804 Total size: 4.8 MB Directory structure: gitextract_vlrxn056/ ├── .appveyor.yml ├── .bash.d/ │ ├── Makefile │ ├── README.md │ ├── aliases.sh │ ├── android.sh │ ├── ansible.sh │ ├── argocd.sh │ ├── aws-cloudshell.sh │ ├── aws.sh │ ├── azure.sh │ ├── bash_it.sh │ ├── circleci.sh │ ├── colors.sh │ ├── custom.sh │ ├── direnv.sh │ ├── docker.sh │ ├── env.sh │ ├── functions.sh │ ├── gcp.sh │ ├── git.sh │ ├── golang.sh │ ├── gpg-agent.sh │ ├── grype.sh │ ├── hadoop.sh │ ├── intellij.sh │ ├── java.sh │ ├── jenkins.sh │ ├── k3d.sh │ ├── kafka.sh │ ├── kubernetes.sh │ ├── linux.sh │ ├── lolcat.sh │ ├── mac.sh │ ├── mercurial.sh │ ├── mp3.sh │ ├── mysql.sh │ ├── network.sh │ ├── nodejs.sh │ ├── os_detection.sh │ ├── paths.sh │ ├── perl.sh │ ├── postgres.sh │ ├── prompt.sh │ ├── python.sh │ ├── rancid.sh │ ├── ruby.sh │ ├── screen.sh │ ├── skaffold.sh │ ├── spinner.sh │ ├── spotify.sh │ ├── ssh-agent.sh │ ├── ssh.sh │ ├── svn.sh │ ├── teamcity.sh │ ├── terraform.sh │ ├── title.sh │ ├── travis_ci.sh │ ├── trivy.sh │ ├── vagrant.sh │ ├── vim.sh │ ├── virtualbox.sh │ ├── vnc.sh │ ├── welcome.sh │ ├── when.sh │ └── z_final.sh ├── .bash_logout ├── .bash_profile ├── .bashrc ├── .buildkite/ │ └── pipeline.yml ├── .circleci/ │ └── config.yml ├── .cirrus.yml ├── .dockerignore ├── .drone.yml ├── .editorconfig ├── .envrc ├── .envrc-aws ├── .envrc-gcp ├── .envrc-java ├── .envrc-kubernetes ├── .envrc-python ├── .envrc-terraform ├── .git-templates/ │ └── git-secrets/ │ └── hooks/ │ ├── commit-msg │ ├── pre-commit │ └── prepare-commit-msg ├── .gitconfig ├── .gitconfig.local ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE.md │ └── workflows/ │ ├── actions-allowed.txt │ ├── alpine.yaml │ ├── alpine_3.yaml │ ├── centos.yaml.disabled │ ├── centos7.yaml.disabled │ ├── centos8.yaml.disabled │ ├── checkov.yaml │ ├── codeowners.yaml │ ├── commit_adjacent_repos.sh │ ├── debian.yaml │ ├── debian_10.yaml │ ├── debian_11.yaml │ ├── debian_12.yaml │ ├── debian_6.yaml.disabled │ ├── debian_7.yaml.disabled │ ├── debian_8.yaml.disabled │ ├── debian_9.yaml.disabled │ ├── docker_bash_alpine.yaml │ ├── docker_bash_centos.yaml │ ├── docker_bash_debian.yaml │ ├── docker_bash_fedora.yaml │ ├── docker_bash_ubuntu.yaml │ ├── dockerhub_status_alpine.yaml.disabled │ ├── dockerhub_status_centos.yaml.disabled │ ├── dockerhub_status_debian.yaml.disabled │ ├── dockerhub_status_fedora.yaml.disabled │ ├── dockerhub_status_ubuntu.yaml.disabled │ ├── fedora.yaml │ ├── fork-sync.yaml │ ├── fork-update-pr.yaml │ ├── ghcr_bash_ubuntu.yaml.disabled │ ├── grype.yaml │ ├── json.yaml │ ├── kics.yaml │ ├── mac.yaml │ ├── mac_11.yaml │ ├── mac_12.yaml │ ├── markdown.yaml │ ├── push_all_repos.sh │ ├── pypy2.yaml.disabled │ ├── pypy3.yaml.disabled │ ├── python2.7.yaml.disabled │ ├── python3.10.yaml │ ├── python3.11.yaml │ ├── python3.6.yaml.disabled │ ├── python3.7.yaml │ ├── python3.8.yaml │ ├── python3.9.yaml │ ├── self_hosted.yaml.disabled │ ├── semgrep-cloud.yaml │ ├── semgrep.yaml │ ├── shellcheck.yaml │ ├── sonarcloud.yaml │ ├── sync_to_adjacent_repos.sh │ ├── trivy.yaml │ ├── trivy_image.yaml │ ├── ubuntu.yaml │ ├── ubuntu_14.04.yaml.disabled │ ├── ubuntu_16.04.yaml.disabled │ ├── ubuntu_18.04.yaml.disabled │ ├── ubuntu_20.04.yaml │ ├── ubuntu_22.04.yaml │ ├── ubuntu_github.yaml │ ├── url_links.yaml.disabled │ ├── validate.yaml │ ├── xml.yaml │ └── yaml.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .gitmodules ├── .hound.yml ├── .mdl.rb ├── .mdlrc ├── .pre-commit-config.yaml ├── .pylintrc ├── .scrutinizer.yml ├── .semaphore/ │ └── semaphore.yml ├── .sonarcloud.properties ├── .sonarlint/ │ └── connectedMode.json ├── .terraformignore ├── .trivyignore ├── .zlogin ├── .zlogout ├── .zprofile ├── .zshenv ├── .zshrc ├── DOCKER_STATUS.md ├── Gemfile ├── Jenkinsfile ├── LICENSE ├── Makefile ├── Makefile.in ├── README.md ├── STARCHARTS.md ├── STATUS.md ├── ai/ │ └── openai_api.sh ├── applescript/ │ ├── app_names.sh │ ├── browser_close_tab.scpt │ ├── browser_get_default.scpt │ ├── com.harisekhon.wakeup_script.plist │ ├── get_application_names.scpt │ ├── get_frontmost_process.scpt │ ├── get_frontmost_process_title.scpt │ ├── get_mouse_coordinates.scpt │ ├── get_mouse_coordinates.sh │ ├── is_screen_locked.py │ ├── is_screensaver_running.scpt │ ├── keystrokes.sh │ ├── mouse_clicks.scpt │ ├── mouse_clicks.sh │ ├── mouse_clicks_remote_desktop.sh │ ├── mouse_random_movements.sh │ ├── reopen_app.sh │ ├── screensaver_activate.scpt │ ├── set_frontmost_process.scpt │ ├── set_mic_internal.sh │ ├── shazam_app_delete_track.sh │ ├── shazam_app_dump_tracks.sh │ ├── shazam_search_spotify.sh │ ├── shazam_search_spotify_then_delete_track.sh │ ├── shorten_text_selection.scpt │ ├── spotify_app_search.sh │ ├── start_app_at_login.sh │ ├── wakeup_script.sh │ └── world_clock_cities.txt ├── appveyor/ │ ├── appveyor_api.sh │ ├── appveyor_byoc.sh │ ├── appveyor_byoc_debian.sh │ └── appveyor_delete_offline_byoc.sh ├── aws/ │ ├── .aws_customize_environment │ ├── aws_account_summary.sh │ ├── aws_accounts_missing_from_config.sh │ ├── aws_batch_kill_stale_jobs.sh │ ├── aws_batch_stale_jobs.sh │ ├── aws_billing_alarm.sh │ ├── aws_budget.json │ ├── aws_budget_alarm.sh │ ├── aws_budget_notification.json │ ├── aws_budget_sns_access_policy.json │ ├── aws_cli_create_credential.sh │ ├── aws_cloudformation_stacks_pending.sh │ ├── aws_cloudfront_distribution_for_origin.sh │ ├── aws_cloudtrails_cloudwatch.sh │ ├── aws_cloudtrails_event_selectors.sh │ ├── aws_cloudtrails_s3_accesslogging.sh │ ├── aws_cloudtrails_s3_kms.sh │ ├── aws_cloudtrails_status.sh │ ├── aws_codecommit_csv_creds.sh │ ├── aws_config_all_types.sh │ ├── aws_config_recording.sh │ ├── aws_csv_creds.sh │ ├── aws_ec2_ami_boot.sh │ ├── aws_ec2_ami_boot_ssh.sh │ ├── aws_ec2_ami_create_from_instance.sh │ ├── aws_ec2_ami_ids.sh │ ├── aws_ec2_ami_name_to_id.sh │ ├── aws_ec2_ami_share_to_account.sh │ ├── aws_ec2_amis.sh │ ├── aws_ec2_ebs_create_snapshot_and_wait.sh │ ├── aws_ec2_ebs_resize_and_wait.sh │ ├── aws_ec2_ebs_volumes.sh │ ├── aws_ec2_ebs_volumes_unattached.sh │ ├── aws_ec2_instance_clone.sh │ ├── aws_ec2_instance_ip.sh │ ├── aws_ec2_instance_name_to_id.sh │ ├── aws_ec2_instance_terminate_by_name.sh │ ├── aws_ec2_instance_wait_for_ready.sh │ ├── aws_ec2_instances.sh │ ├── aws_ec2_launch_templates_ami_id.sh │ ├── aws_ecr_alternate_tags.sh │ ├── aws_ecr_delete_old_tags.sh │ ├── aws_ecr_delete_tag.sh │ ├── aws_ecr_docker_build_push.sh │ ├── aws_ecr_docker_login.sh │ ├── aws_ecr_list_repos.sh │ ├── aws_ecr_list_tags.sh │ ├── aws_ecr_newest_image_tags.sh │ ├── aws_ecr_tag_branch.sh │ ├── aws_ecr_tag_datetime.sh │ ├── aws_ecr_tag_image.sh │ ├── aws_ecr_tag_image_by_digest.sh │ ├── aws_ecr_tag_latest.sh │ ├── aws_ecr_tag_newest_as_latest.sh │ ├── aws_ecr_tags_old.sh │ ├── aws_ecr_tags_timestamps.sh │ ├── aws_eks_addon_versions.sh │ ├── aws_eks_ami_create.sh │ ├── aws_eks_available_ips.sh │ ├── aws_eks_cloudwatch_logs.sh │ ├── aws_eks_cluster_versions.sh │ ├── aws_eks_ssh_dump_logs.sh │ ├── aws_elasticache_serverless_list.sh │ ├── aws_emr_clusters_last_steps.sh │ ├── aws_foreach_profile.sh │ ├── aws_foreach_region.sh │ ├── aws_iam_generate_credentials_report_wait.sh │ ├── aws_iam_harden_password_policy.sh │ ├── aws_iam_password_policy.sh │ ├── aws_iam_policies_attached_to_users.sh │ ├── aws_iam_policies_granting_full_access.sh │ ├── aws_iam_policies_unattached.sh │ ├── aws_iam_policy_attachments.sh │ ├── aws_iam_policy_delete.sh │ ├── aws_iam_rename_user_accounts_domains.sh │ ├── aws_iam_replace_access_key.sh │ ├── aws_iam_users.sh │ ├── aws_iam_users_access_key_age.sh │ ├── aws_iam_users_access_key_age_report.sh │ ├── aws_iam_users_access_key_last_used.sh │ ├── aws_iam_users_access_key_last_used_report.sh │ ├── aws_iam_users_last_used_report.sh │ ├── aws_iam_users_mfa_active_report.sh │ ├── aws_iam_users_mfa_serials.sh │ ├── aws_iam_users_pw_last_used.sh │ ├── aws_iam_users_without_mfa.sh │ ├── aws_info.sh │ ├── aws_info_all_profiles.sh │ ├── aws_info_ec2.sh │ ├── aws_info_ec2_all_profiles_csv.sh │ ├── aws_info_ec2_csv.sh │ ├── aws_ip_ranges.sh │ ├── aws_kinesis_stream_names.sh │ ├── aws_kms_key_rotation_enabled.sh │ ├── aws_kube_creds.sh │ ├── aws_kubectl.sh │ ├── aws_logs.sh │ ├── aws_logs_batch_jobs.sh │ ├── aws_logs_ec2_spot.sh │ ├── aws_logs_ecs_tasks.sh │ ├── aws_meta.sh │ ├── aws_nat_gateways_public_ips.sh │ ├── aws_profile.sh │ ├── aws_profile_config_add_if_missing.sh │ ├── aws_profile_generate_direnvs.sh │ ├── aws_rds_get_version.sh │ ├── aws_rds_list.sh │ ├── aws_rds_open_port_to_my_ip.sh │ ├── aws_route53_check_ns_records.sh │ ├── aws_s3_access_logging.sh │ ├── aws_s3_account_block_public_access.sh │ ├── aws_s3_bucket.sh │ ├── aws_s3_buckets_block_public_access.sh │ ├── aws_s3_check_account_public_blocked.sh │ ├── aws_s3_check_buckets_public_blocked.sh │ ├── aws_s3_delete_bucket_with_versions.sh │ ├── aws_s3_sync.sh │ ├── aws_secret_add.sh │ ├── aws_secret_add_binary.sh │ ├── aws_secret_get.sh │ ├── aws_secret_list.sh │ ├── aws_secret_update.sh │ ├── aws_secret_update_binary.sh │ ├── aws_spot_when_terminated.sh │ ├── aws_sqs_check.sh │ ├── aws_sqs_delete_messages.sh │ ├── aws_ssm_put_param.sh │ ├── aws_ssm_wait_for_command.sh │ ├── aws_sso_account_id_names.sh │ ├── aws_sso_accounts.sh │ ├── aws_sso_accounts_missing_from_list.sh │ ├── aws_sso_cache_expires.sh │ ├── aws_sso_config_duplicate_profile_names.sh │ ├── aws_sso_config_duplicate_sections.sh │ ├── aws_sso_configs.sh │ ├── aws_sso_configs_save.sh │ ├── aws_sso_env_creds.sh │ ├── aws_sso_role_arn.sh │ ├── aws_sso_role_arns.sh │ ├── aws_sso_ssh.sh │ ├── aws_terraform_create_all.sh │ ├── aws_terraform_create_atlantis_role.sh │ ├── aws_terraform_create_credential.sh │ ├── aws_terraform_create_dynamodb_table.sh │ ├── aws_terraform_create_s3_bucket.sh │ ├── aws_terraform_iam_grant_s3_dynamodb.sh │ └── eksctl_cluster.sh ├── azure-pipelines.yml ├── azure_devops/ │ ├── azure_devops_api.sh │ ├── azure_devops_disable_repos.sh │ ├── azure_devops_foreach_repo.sh │ └── azure_devops_to_github_migration.sh ├── bigdata/ │ ├── beeline.sh │ ├── beeline_zk.sh │ ├── cloudera_manager_api.sh │ ├── cloudera_manager_impala_queries.sh │ ├── cloudera_manager_impala_queries_ddl.sh │ ├── cloudera_manager_impala_queries_exceptions.sh │ ├── cloudera_manager_impala_queries_failed.sh │ ├── cloudera_manager_impala_queries_metadata.sh │ ├── cloudera_manager_impala_queries_metadata_errors.sh │ ├── cloudera_manager_impala_queries_metadata_refresh.sh │ ├── cloudera_manager_yarn_apps.sh │ ├── cloudera_manager_yarn_apps_failed.sh │ ├── cloudera_navigator_api.sh │ ├── cloudera_navigator_audit_logs.sh │ ├── cloudera_navigator_audit_logs_download.sh │ ├── cloudera_navigator_audit_logs_download_retry.sh │ ├── cloudera_navigator_audit_logs_export_postgresql.sh │ ├── hadoop_random_node.sh │ ├── hdfs_checksum.sh │ ├── hdfs_checksum_crc.sh │ ├── hdfs_checksum_crc_parallel.sh │ ├── hdfs_checksum_parallel.sh │ ├── hdfs_file_size.sh │ ├── hdfs_file_size_including_replicas.sh │ ├── hdfs_find_replication_factor_1.sh │ ├── hdfs_set_replication_factor_3.sh │ ├── hive_foreach_table.sh │ ├── hive_list_databases.sh │ ├── hive_list_tables.sh │ ├── hive_tables_column_counts.sh │ ├── hive_tables_locations.sh │ ├── hive_tables_metadata.sh │ ├── hive_tables_row_counts.sh │ ├── impala_foreach_table.sh │ ├── impala_list_databases.sh │ ├── impala_list_tables.sh │ ├── impala_shell.sh │ ├── impala_tables_column_counts.sh │ ├── impala_tables_locations.sh │ ├── impala_tables_metadata.sh │ ├── impala_tables_row_counts.sh │ ├── zookeeper_client.sh │ └── zookeeper_shell.sh ├── bin/ │ ├── bash_most_used_commands.sh │ ├── bash_profile_bashrc.sh │ ├── center.sh │ ├── clean_caches.sh │ ├── cocomo_man_years_estimate.sh │ ├── copy_to_clipboard.sh │ ├── crt_hash.sh │ ├── crypto_dice_rolls.sh │ ├── curl_auth.sh │ ├── curl_with_cookies.sh │ ├── debian_netinstall_pxesetup.sh │ ├── decomment.sh │ ├── delete_duplicate_files.sh │ ├── delete_empty_dirs.sh │ ├── diff_line_threshold.sh │ ├── disable_swap.sh │ ├── disk-read-random.fio │ ├── disk-read-sequential.fio │ ├── disk-tests.fio │ ├── disk-write-random.fio │ ├── disk-write-sequential.fio │ ├── disk_speed_read_random_dd.sh │ ├── disk_speed_read_random_fio.sh │ ├── disk_speed_read_sequential_dd.sh │ ├── disk_speed_read_sequential_fio.sh │ ├── disk_speed_write_random_fio.sh │ ├── disk_speed_write_sequential_dd.sh │ ├── disk_speed_write_sequential_fio.sh │ ├── download_url_file.sh │ ├── elasticsearch_decommission_node.sh │ ├── exec_interactive.sh │ ├── file_extensions.sh │ ├── find_broken_links.sh │ ├── find_broken_symlinks.sh │ ├── find_duplicate_files_by_checksum.sh │ ├── find_duplicate_files_by_size.sh │ ├── find_duplicate_lines.sh │ ├── find_hanging_mount_point.sh │ ├── find_hardlinks.sh │ ├── find_lock.sh │ ├── find_symlinks_to_other_directories.sh │ ├── foreach_path_bin.sh │ ├── grep_or_append.sh │ ├── hackercase.sh │ ├── headtail.sh │ ├── hexencode.sh │ ├── htmldecode.sh │ ├── http_duplicate_urls.sh │ ├── jsondiff.sh │ ├── keycloak.sh │ ├── ldap_group_recurse.sh │ ├── ldap_user_recurse.sh │ ├── ldapsearch.sh │ ├── lint.sh │ ├── linux_distro_versions.sh │ ├── login.sh │ ├── lowercase_filename.sh │ ├── mac_backup_du_in_progress.sh │ ├── mac_backup_exclude_paths.sh │ ├── mac_backup_find_excluded_paths.sh │ ├── mac_delete_local_snapshots.sh │ ├── mac_diff_settings.sh │ ├── mac_gif_preview.sh │ ├── mac_iso_to_usb.sh │ ├── mac_ramdisk.sh │ ├── mac_restore_file.sh │ ├── mac_rmdir.sh │ ├── mv.sh │ ├── network_gateway.sh │ ├── open.sh │ ├── oreilly_cover_download.sh │ ├── organize_downloads.sh │ ├── paste_diff_settings.sh │ ├── paste_from_clipboard.sh │ ├── paste_from_clipboard_upon_changes.sh │ ├── path_revoke_world_writeable.sh │ ├── pldd.sh │ ├── processes_ram_sum.sh │ ├── progress_dots.sh │ ├── random_number.sh │ ├── random_select.sh │ ├── random_string.sh │ ├── retry.sh │ ├── run.sh │ ├── sbtw │ ├── scan_duplicate_macs.sh │ ├── screen_terminal_to_clipboard.sh │ ├── screen_terminal_to_stdout.sh │ ├── shorten_text_selection.sh │ ├── shred_file.sh │ ├── shred_free_space.sh │ ├── smart_quotes_replace.sh │ ├── spasticcase.sh │ ├── spasticcase2.sh │ ├── split.sh │ ├── sqlite.sh │ ├── ssl_get_cert.sh │ ├── ssl_verify_cert.sh │ ├── ssl_verify_cert_by_ip.sh │ ├── ssl_view_cert.sh │ ├── text_filter_ending_substrings.sh │ ├── tmux_horizontal.sh │ ├── tmux_square.sh │ ├── tmux_vertical.sh │ ├── ubuntu_release_version.sh │ ├── uniq_chars.sh │ ├── url_extract_redirects.sh │ ├── url_replace_redirects.sh │ ├── urldecode.sh │ ├── urlencode.sh │ ├── urlencode_utf.sh │ ├── urlextract.sh │ ├── urlopen.sh │ └── vault_pass.sh ├── bitbucket/ │ ├── bitbucket_api.sh │ ├── bitbucket_disable_pipelines.sh │ ├── bitbucket_enable_pipelines.sh │ ├── bitbucket_foreach_repo.sh │ ├── bitbucket_repo_disable_pipeline.sh │ ├── bitbucket_repo_enable_pipeline.sh │ ├── bitbucket_repo_set_description.sh │ ├── bitbucket_repo_set_env_vars.sh │ ├── bitbucket_ssh_add_public_keys.sh │ ├── bitbucket_ssh_delete_public_keys.sh │ ├── bitbucket_ssh_get_public_keys.sh │ └── bitbucket_workspace_set_env_vars.sh ├── bitbucket-pipelines.yml ├── buddy.yml ├── buildkite/ │ ├── buildkite_agent.sh │ ├── buildkite_agents.sh │ ├── buildkite_api.sh │ ├── buildkite_cancel_running_builds.sh │ ├── buildkite_cancel_scheduled_builds.sh │ ├── buildkite_create_pipeline.sh │ ├── buildkite_foreach_pipeline.sh │ ├── buildkite_get_pipeline.sh │ ├── buildkite_patch_pipeline.sh │ ├── buildkite_pipeline_disable_forked_pull_requests.sh │ ├── buildkite_pipeline_set_skip_settings.sh │ ├── buildkite_pipeline_skip_settings.sh │ ├── buildkite_pipelines.sh │ ├── buildkite_pipelines_vulnerable_forked_pull_requests.sh │ ├── buildkite_rebuild_all_pipelines_last_cancelled.sh │ ├── buildkite_rebuild_all_pipelines_last_failed.sh │ ├── buildkite_rebuild_cancelled_builds.sh │ ├── buildkite_rebuild_failed_builds.sh │ ├── buildkite_rebuild_last_cancelled.sh │ ├── buildkite_recreate_pipeline.sh │ ├── buildkite_retry_jobs_dead_agents.sh │ ├── buildkite_running_builds.sh │ ├── buildkite_save_pipelines.sh │ ├── buildkite_set_pipeline_description.sh │ ├── buildkite_set_pipeline_description_from_github.sh │ ├── buildkite_sync_pipeline_descriptions_from_github.sh │ ├── buildkite_trigger.sh │ ├── buildkite_trigger_all.sh │ └── buildkite_update_pipeline.sh ├── checks/ │ ├── check_all.sh │ ├── check_ansible_playbooks.sh │ ├── check_aws_no_git_credentials.sh │ ├── check_bash_arrays.sh │ ├── check_bash_duplicate_defs.sh │ ├── check_bash_references.sh │ ├── check_bash_syntax.sh │ ├── check_caches_clean.sh │ ├── check_circleci_config.sh │ ├── check_codefresh_config.sh │ ├── check_concourse_config.sh │ ├── check_cson.sh │ ├── check_docker_clean.sh │ ├── check_docker_compose.sh │ ├── check_dockerfiles.sh │ ├── check_drone_yml.sh │ ├── check_duplicate_dependencies.sh │ ├── check_duplicate_packages.sh │ ├── check_git_commit_authors.sh │ ├── check_git_no_merge_remnants.sh │ ├── check_github_actions_workflow_injection.sh │ ├── check_github_actions_workflows_without_checkout.sh │ ├── check_github_codeowners.sh │ ├── check_gitlab_ci_yml.sh │ ├── check_gradle_build.sh │ ├── check_groovyc.sh │ ├── check_internet.sh │ ├── check_javac.sh │ ├── check_javascript_eslint.sh │ ├── check_jenkinsfiles.sh │ ├── check_json.sh │ ├── check_kubernetes_yaml.sh │ ├── check_license_exists.sh │ ├── check_makefiles.sh │ ├── check_maven_pom.sh │ ├── check_no_suid_guid_shell_scripts.sh │ ├── check_no_tabs.sh │ ├── check_perl_syntax.sh │ ├── check_python3_compat.sh │ ├── check_python_asserts.sh │ ├── check_python_exception_pass.sh │ ├── check_python_misc.sh │ ├── check_python_pep8.sh │ ├── check_python_pylint.sh │ ├── check_pytools.sh │ ├── check_readme_badges.sh │ ├── check_readme_exists.sh │ ├── check_ruby_syntax.sh │ ├── check_sbt_build.sh │ ├── check_shebang_non_executable.sh │ ├── check_shell_commands_dash_protections.sh │ ├── check_shellcheck.sh │ ├── check_shippable_readme_ids.sh │ ├── check_sqlfluff.sh │ ├── check_srcdir_references.sh │ ├── check_ssh_keys_encrypted.sh │ ├── check_symlinks.sh │ ├── check_tests_run_qualified.sh │ ├── check_tld_chars.sh │ ├── check_travis_yml.sh │ ├── check_url_links.sh │ ├── check_vagrantfiles.sh │ ├── check_whitespace.sh │ ├── check_xml.sh │ └── check_yaml.sh ├── cicd/ │ ├── .concourse.yml │ ├── .gocd.yml │ ├── buildspec.yml │ ├── checkov_resource_count.sh │ ├── checkov_resource_count_all.sh │ ├── cloudbuild.yaml │ ├── codefresh_cancel_delayed_builds.sh │ ├── concourse.sh │ ├── coveralls_latest.sh │ ├── fly.sh │ ├── generate_status_page.sh │ ├── gerrit.sh │ ├── gerrit_projects.sh │ ├── gocd.sh │ ├── gocd_api.sh │ ├── octopus_api.sh │ ├── run_latest_tests.sh │ ├── run_tests.sh │ ├── selenium_hub_wait_ready.sh │ ├── sonarlint_generate_config.sh │ ├── sync_bootstraps_to_adjacent_repos.sh │ ├── sync_ci_to_adjacent_repos.sh │ ├── sync_configs_to_adjacent_repos.sh │ └── sync_github_actions_workflows_to_adjacent_repos.sh ├── circleci/ │ ├── circleci_api.sh │ ├── circleci_context_delete_env_vars.sh │ ├── circleci_context_set_env_vars.sh │ ├── circleci_local_execute.sh │ ├── circleci_project_delete_env_vars.sh │ ├── circleci_project_set_env_vars.sh │ └── circleci_public_ips.sh ├── cloudflare/ │ ├── cloudflare_api.sh │ ├── cloudflare_custom_certificates.sh │ ├── cloudflare_dns_record_create.sh │ ├── cloudflare_dns_record_delete.sh │ ├── cloudflare_dns_record_details.sh │ ├── cloudflare_dns_record_update.sh │ ├── cloudflare_dns_records.sh │ ├── cloudflare_dns_records_all_zones.sh │ ├── cloudflare_dnssec.sh │ ├── cloudflare_firewall_access_rules.sh │ ├── cloudflare_firewall_rules.sh │ ├── cloudflare_foreach_account.sh │ ├── cloudflare_foreach_zone.sh │ ├── cloudflare_ip_ranges.sh │ ├── cloudflare_purge_cache.sh │ ├── cloudflare_ssl_verified.sh │ ├── cloudflare_ssl_verified_all_zones.sh │ └── cloudflare_zones.sh ├── codefresh.yml ├── codeship/ │ ├── codeship.yml │ ├── codeship_api.sh │ └── codeship_api_token.sh ├── configs/ │ ├── .Codefresh/ │ │ └── cli-config/ │ │ └── config.yaml │ ├── .Xdefaults │ ├── .Xmodmap │ ├── .ansible.cfg │ ├── .athenacli/ │ │ └── athenaclirc │ ├── .aws/ │ │ ├── config │ │ └── shell/ │ │ └── awsshellrc │ ├── .checkov.yaml │ ├── .config/ │ │ ├── flake8 │ │ ├── htop/ │ │ │ └── htoprc │ │ ├── pycodestyle │ │ ├── terminalizer/ │ │ │ └── config.yml │ │ └── yamllint/ │ │ └── config │ ├── .gemrc │ ├── .grype.yaml │ ├── .inputrc │ ├── .luacheckrc │ ├── .my.cnf │ ├── .perlcritic_forbidden_modules │ ├── .perlcriticrc │ ├── .psqlrc │ ├── .sawsrc │ ├── .screenrc │ ├── .sdkman/ │ │ └── etc/ │ │ └── config │ ├── .sqliterc │ ├── .terraformrc │ ├── .tfdocs.d/ │ │ └── .terraform-docs.yml │ ├── .tmux.conf │ ├── .toprc │ ├── .vimrc │ ├── .wakatime.cfg │ ├── .yamllint.yaml │ ├── README.md │ ├── clamd.conf │ └── freshclam.conf ├── data/ │ ├── avro_tools.sh │ ├── csv_header_indices.sh │ ├── ini_config_add_if_missing.sh │ ├── ini_config_duplicate_section_names.sh │ ├── ini_config_duplicate_sections.sh │ ├── ini_grep_section.sh │ ├── json2yaml.sh │ ├── lines_to_end.sh │ ├── parquet_tools.sh │ ├── wordcloud.sh │ ├── wordcount.sh │ └── yaml2json.sh ├── diagrams/ │ ├── d2.sh │ ├── d2_generate_diagrams.sh │ ├── mermaidjs_generate_diagrams.sh │ └── python_mingrammer_generate_diagrams.sh ├── docker/ │ ├── docker_api.sh │ ├── docker_build_hashref.sh │ ├── docker_generate_status_page.sh │ ├── docker_mount_build_exec.sh │ ├── docker_package_check.sh │ ├── docker_registry_get_image_manifest.sh │ ├── docker_registry_list_images.sh │ ├── docker_registry_list_tags.sh │ ├── docker_registry_tag_image.sh │ ├── dockerhub_api.sh │ ├── dockerhub_build_status.sh │ ├── dockerhub_list_tags.sh │ ├── dockerhub_list_tags_by_last_updated.sh │ ├── dockerhub_repo_set_description.sh │ ├── dockerhub_repo_set_readme.sh │ ├── dockerhub_search.sh │ └── quay_api.sh ├── docker-compose/ │ ├── circleci.yml │ ├── concourse.yml │ ├── gerrit.yml │ ├── gocd.yml │ ├── jenkins.yml │ ├── keycloak.yml │ ├── octopus-deploy.env │ ├── octopus-deploy.yml │ ├── prometheus.yml │ ├── teamcity.yml │ └── wordpress.yml ├── drone/ │ ├── drone_api.sh │ ├── drone_docker_runner.sh │ └── drone_docker_server.sh ├── gcp/ │ ├── .customize_environment │ ├── .gcloudignore │ ├── bigquery_foreach_dataset.sh │ ├── bigquery_foreach_table.sh │ ├── bigquery_foreach_table_all_datasets.sh │ ├── bigquery_generate_query_biggest_tables_across_datasets_by_row_count.sh │ ├── bigquery_generate_query_biggest_tables_across_datasets_by_size.sh │ ├── bigquery_list_datasets.sh │ ├── bigquery_list_tables.sh │ ├── bigquery_list_tables_all_datasets.sh │ ├── bigquery_table_row_count.sh │ ├── bigquery_tables_row_counts.sh │ ├── bigquery_tables_row_counts_all_datasets.sh │ ├── firebase_foreach_project.sh │ ├── gce_foreach_vm.sh │ ├── gce_host_ips.sh │ ├── gce_instance_service_accounts.sh │ ├── gce_is_preempted.sh │ ├── gce_meta.sh │ ├── gce_ssh.sh │ ├── gce_ssh_keyscan.sh │ ├── gce_when_preempted.sh │ ├── gcp_ansible_create_credential.sh │ ├── gcp_ci_build.sh │ ├── gcp_ci_deploy_k8s.sh │ ├── gcp_cli_create_credential.sh │ ├── gcp_cloud_schedule_sql_exports.sh │ ├── gcp_find_orphaned_disks.sh │ ├── gcp_firewall_disable_default_rules.sh │ ├── gcp_firewall_risky_rules.sh │ ├── gcp_foreach_project.sh │ ├── gcp_iam_identities_in_use.sh │ ├── gcp_iam_roles_granted_to_identity.sh │ ├── gcp_iam_roles_granted_too_widely.sh │ ├── gcp_iam_roles_in_use.sh │ ├── gcp_iam_roles_with_direct_user_grants.sh │ ├── gcp_iam_serviceaccount_members.sh │ ├── gcp_iam_serviceaccounts_without_permissions.sh │ ├── gcp_iam_users_granted_directly.sh │ ├── gcp_iam_workload_identities.sh │ ├── gcp_info.sh │ ├── gcp_info_accounts_secrets.sh │ ├── gcp_info_all_projects.sh │ ├── gcp_info_auth_config.sh │ ├── gcp_info_bigdata.sh │ ├── gcp_info_cloud_sql.sh │ ├── gcp_info_cloud_sql_backups.sh │ ├── gcp_info_cloud_sql_databases.sh │ ├── gcp_info_cloud_sql_users.sh │ ├── gcp_info_compute.sh │ ├── gcp_info_gke.sh │ ├── gcp_info_networking.sh │ ├── gcp_info_projects.sh │ ├── gcp_info_services.sh │ ├── gcp_info_storage.sh │ ├── gcp_info_tools.sh │ ├── gcp_secret_add.sh │ ├── gcp_secret_add_binary.sh │ ├── gcp_secret_get.sh │ ├── gcp_secret_label_k8s.sh │ ├── gcp_secret_update.sh │ ├── gcp_secrets_labels.sh │ ├── gcp_secrets_to_kubernetes.sh │ ├── gcp_secrets_to_kubernetes_multipart.sh │ ├── gcp_secrets_update_label.sh │ ├── gcp_service_account_credential_to_secret.sh │ ├── gcp_service_account_members.sh │ ├── gcp_service_accounts_credential_keys.sh │ ├── gcp_service_accounts_credential_keys_age.sh │ ├── gcp_service_accounts_credential_keys_expired.sh │ ├── gcp_service_apis.sh │ ├── gcp_spinnaker_create_credential.sh │ ├── gcp_sql_backup.sh │ ├── gcp_sql_create_readonly_service_account.sh │ ├── gcp_sql_enable_automated_backups.sh │ ├── gcp_sql_enable_point_in_time_recovery.sh │ ├── gcp_sql_export.sh │ ├── gcp_sql_grant_instances_gcs_object_creator.sh │ ├── gcp_sql_list_databases.sh │ ├── gcp_sql_proxy.sh │ ├── gcp_sql_running_primaries.sh │ ├── gcp_sql_service_accounts.sh │ ├── gcp_terraform_create_credential.sh │ ├── gcr_alternate_tags.sh │ ├── gcr_delete_old_tags.sh │ ├── gcr_list_tags.sh │ ├── gcr_newest_image_tags.sh │ ├── gcr_tag_branch.sh │ ├── gcr_tag_datetime.sh │ ├── gcr_tag_latest.sh │ ├── gcr_tag_newest_image_as_latest.sh │ ├── gcr_tags_old.sh │ ├── gcr_tags_timestamps.sh │ ├── gcs_bucket_project.sh │ ├── gcs_curl_file.sh │ ├── gke_firewall_rule_cert_manager.sh │ ├── gke_firewall_rule_kubeseal.sh │ ├── gke_kube_creds.sh │ ├── gke_kubectl.sh │ ├── gke_nodepool_drain.sh │ ├── gke_nodepool_nodes.sh │ ├── gke_nodepool_nodes2.sh │ ├── gke_nodepool_taint.sh │ └── gke_persistent_volume_disk_mappings.sh ├── git/ │ ├── git_askpass.sh │ ├── git_branch_delete_squash_merged.sh │ ├── git_clean_repos.sh │ ├── git_diff_commit.sh │ ├── git_files_in_history.sh │ ├── git_files_last_modified.sh │ ├── git_files_no_uncommitted_changes.sh │ ├── git_filter_branch_fix_author.sh │ ├── git_filter_repo_replace_text.sh │ ├── git_foreach_branch.sh │ ├── git_foreach_modified.sh │ ├── git_foreach_repo.sh │ ├── git_foreach_repo_replace_readme_actions.sh │ ├── git_foreach_repo_update_readme.sh │ ├── git_graph_commit_history_gnuplot.sh │ ├── git_graph_commit_history_mermaidjs.sh │ ├── git_graph_commit_times_gnuplot.sh │ ├── git_graph_commit_times_gnuplot_all_repos.sh │ ├── git_graph_commit_times_mermaidjs.sh │ ├── git_graph_commit_times_mermaidjs_all_repos.sh │ ├── git_grep_env_vars.sh │ ├── git_log_empty_commits.sh │ ├── git_log_me.sh │ ├── git_log_me_added.sh │ ├── git_merge_all.sh │ ├── git_merge_branch.sh │ ├── git_merge_master.sh │ ├── git_merge_master_pull.sh │ ├── git_origin_commit_count_to_push.sh │ ├── git_origin_diff_to_push.sh │ ├── git_origin_files_to_push.sh │ ├── git_origin_line_count_to_push.sh │ ├── git_origin_log_to_push.sh │ ├── git_pull_make_repos.sh │ ├── git_push_stats.sh │ ├── git_remotes_add_origin_providers.sh │ ├── git_remotes_set_https_creds_helpers.sh │ ├── git_remotes_set_https_to_ssh.sh │ ├── git_remotes_set_multi_origin.sh │ ├── git_remotes_set_ssh_to_https.sh │ ├── git_repos.sh │ ├── git_repos_pull.sh │ ├── git_repos_update.sh │ ├── git_revert_line.sh │ ├── git_review_push.sh │ ├── git_set_dir_safe.sh │ ├── git_submodules_update.sh │ ├── git_submodules_update_repos.sh │ ├── git_summary_line.sh │ ├── git_sync_repos_upstream.sh │ ├── git_tag_release.sh │ ├── gitguardian_api.sh │ ├── gitignore.io_api.sh │ ├── precommit_run_changed_files.sh │ └── update_gitignore.io.sh ├── github/ │ ├── github_actions_aws_create_load_credential.sh │ ├── github_actions_delete_offline_runners.sh │ ├── github_actions_foreach_workflow.sh │ ├── github_actions_in_use.sh │ ├── github_actions_in_use_across_repos.sh │ ├── github_actions_in_use_repo.sh │ ├── github_actions_latest_log.sh │ ├── github_actions_log.sh │ ├── github_actions_repo_actions_allow.sh │ ├── github_actions_repo_env_set_secret.sh │ ├── github_actions_repo_restrict_actions.sh │ ├── github_actions_repo_secrets_overriding_org.sh │ ├── github_actions_repo_set_secret.sh │ ├── github_actions_repos_lockdown.sh │ ├── github_actions_runner.sh │ ├── github_actions_runner_local.sh │ ├── github_actions_runner_token.sh │ ├── github_actions_runners.sh │ ├── github_actions_workflow_enable.sh │ ├── github_actions_workflow_runs.sh │ ├── github_actions_workflows.sh │ ├── github_actions_workflows_cancel_all_runs.sh │ ├── github_actions_workflows_cancel_waiting_runs.sh │ ├── github_actions_workflows_disabled.sh │ ├── github_actions_workflows_enable_all.sh │ ├── github_actions_workflows_rerun_failed.sh │ ├── github_actions_workflows_state.sh │ ├── github_actions_workflows_status.sh │ ├── github_actions_workflows_status2.sh │ ├── github_actions_workflows_trigger_all.sh │ ├── github_api.sh │ ├── github_clone_or_pull_all_repos.sh │ ├── github_download_release_file.sh │ ├── github_download_release_jar.sh │ ├── github_foreach_repo.sh │ ├── github_forked_add_remote.sh │ ├── github_forked_checkout_branch.sh │ ├── github_generate_starcharts.md.sh │ ├── github_generate_status_page.sh │ ├── github_gpg_get_user_keys.sh │ ├── github_graph_commit_times_gnuplot.sh │ ├── github_graph_commit_times_mermaidjs.sh │ ├── github_install_binary.sh │ ├── github_invitations.sh │ ├── github_ip_ranges.sh │ ├── github_merge_branch.sh │ ├── github_mirror_repos_to_aws_codecommit.sh │ ├── github_mirror_repos_to_bitbucket.sh │ ├── github_mirror_repos_to_gcp_source_repos.sh │ ├── github_mirror_repos_to_gitlab.sh │ ├── github_public_lines_of_code.sh │ ├── github_pull_merge_trunk.sh │ ├── github_pull_request_create.sh │ ├── github_pull_request_preview.sh │ ├── github_purge_camo_cache.sh │ ├── github_push_pr.sh │ ├── github_push_pr_preview.sh │ ├── github_release.sh │ ├── github_remote_set_upstream.sh │ ├── github_repo_add_collaborator.sh │ ├── github_repo_check_pat_token.sh │ ├── github_repo_collaborators.sh │ ├── github_repo_description.sh │ ├── github_repo_find_files.sh │ ├── github_repo_fork_sync.sh │ ├── github_repo_fork_update.sh │ ├── github_repo_latest_release.sh │ ├── github_repo_latest_release_filter.sh │ ├── github_repo_protect_branches.sh │ ├── github_repo_stars.sh │ ├── github_repo_teams.sh │ ├── github_repos_disable_rebase.sh │ ├── github_repos_disable_wiki.sh │ ├── github_repos_find_files.sh │ ├── github_repos_not_in_terraform.sh │ ├── github_repos_public.sh │ ├── github_repos_sync_status.sh │ ├── github_repos_with_few_teams.sh │ ├── github_repos_with_few_users.sh │ ├── github_repos_without_branch_protections.sh │ ├── github_ssh_add_public_keys.sh │ ├── github_ssh_delete_public_keys.sh │ ├── github_ssh_get_public_keys.sh │ ├── github_ssh_get_user_public_keys.sh │ ├── github_ssh_get_user_public_keys2.sh │ ├── github_sync_repo_descriptions.sh │ ├── github_tag_hashref.sh │ ├── github_teams_not_idp_synced.sh │ ├── github_teams_not_in_terraform.sh │ ├── github_url_clipboard.sh │ ├── github_user_followers.sh │ ├── github_user_repos_count.sh │ ├── github_user_repos_forks.sh │ ├── github_user_repos_stars.sh │ └── gitio.sh ├── gitlab/ │ ├── gitlab_api.sh │ ├── gitlab_foreach_repo.sh │ ├── gitlab_get_user_ssh_public_keys.sh │ ├── gitlab_get_user_ssh_public_keys2.sh │ ├── gitlab_group_set_env_vars.sh │ ├── gitlab_install_binary.sh │ ├── gitlab_project_create_import.sh │ ├── gitlab_project_latest_release.sh │ ├── gitlab_project_mirrors.sh │ ├── gitlab_project_protect_branches.sh │ ├── gitlab_project_set_description.sh │ ├── gitlab_project_set_env_vars.sh │ ├── gitlab_pull_mirror.sh │ ├── gitlab_push_mr.sh │ ├── gitlab_push_mr_preview.sh │ ├── gitlab_ssh_add_public_keys.sh │ ├── gitlab_ssh_delete_public_keys.sh │ ├── gitlab_ssh_get_public_keys.sh │ └── gitlab_validate_ci_yaml.sh ├── hadolint.yaml ├── images/ │ └── README.md ├── install/ │ ├── download_avro_tools.sh │ ├── download_azul_openjdk.sh │ ├── download_bytecode_viewer_jar.sh │ ├── download_cfr_jar.sh │ ├── download_jd_gui_jar.sh │ ├── download_mssql_jdbc_jar.sh │ ├── download_mysql_jdbc_jar.sh │ ├── download_openjdk.sh │ ├── download_parquet_tools.sh │ ├── download_postgres_jdbc_jar.sh │ ├── download_procyon_jar.sh │ ├── download_vertica_jar.sh │ ├── getawless.sh │ ├── install_android_commandlinetools.sh │ ├── install_android_sdk.sh │ ├── install_ansible.sh │ ├── install_appveyor_byoc.sh │ ├── install_argocd.sh │ ├── install_awless.sh │ ├── install_aws_cli.sh │ ├── install_aws_ebcli.sh │ ├── install_aws_sam_cli.sh │ ├── install_azure_cli.sh │ ├── install_azure_devops_cli.sh │ ├── install_bazel.sh │ ├── install_bazelisk.sh │ ├── install_buildkite.sh │ ├── install_cert_manager_cli.sh │ ├── install_circleci.sh │ ├── install_circleci_runner.sh │ ├── install_clairctl.sh │ ├── install_cliclick.sh │ ├── install_cloud_sql_proxy.sh │ ├── install_cloudbees.sh │ ├── install_coder_cli.sh │ ├── install_container-diff.sh │ ├── install_crictl.sh │ ├── install_d2.sh │ ├── install_datree.sh │ ├── install_diff-so-fancy.sh │ ├── install_direnv.sh │ ├── install_docker_buildx.sh │ ├── install_docker_compose.sh │ ├── install_docker_scan.sh │ ├── install_dockerhub_cli.sh │ ├── install_dockle.sh │ ├── install_doctl.sh │ ├── install_drone.sh │ ├── install_eksctl.sh │ ├── install_eksup.sh │ ├── install_epel_repo.sh │ ├── install_etcd.sh │ ├── install_firebase_cli.sh │ ├── install_fly.sh │ ├── install_fossa_cli.sh │ ├── install_gcloud_sdk.sh │ ├── install_github_cli.sh │ ├── install_github_codeql.sh │ ├── install_github_ssh_keys.sh │ ├── install_gitlab_cli.sh │ ├── install_golang.sh │ ├── install_gonogo.sh │ ├── install_gradle.sh │ ├── install_groovy.sh │ ├── install_grype.sh │ ├── install_helm.sh │ ├── install_homebrew.sh │ ├── install_infoblox_ova.sh │ ├── install_intellij_plugins.sh │ ├── install_java.sh │ ├── install_jfrog_cli.sh │ ├── install_jx.sh │ ├── install_k3d.sh │ ├── install_k3s.sh │ ├── install_k6.sh │ ├── install_keeper_cli.sh │ ├── install_kics.sh │ ├── install_kind.sh │ ├── install_knative_cli.sh │ ├── install_kops.sh │ ├── install_kubectl.sh │ ├── install_kubectl_plugin_cert_manager.sh │ ├── install_kubectl_plugin_convert.sh │ ├── install_kubectl_plugin_krew.sh │ ├── install_kubent.sh │ ├── install_kubescape.sh │ ├── install_kubeseal.sh │ ├── install_kubevious.sh │ ├── install_kustomize.sh │ ├── install_maven.sh │ ├── install_mermaidjs.sh │ ├── install_minikube.sh │ ├── install_minishift.sh │ ├── install_mousetools.sh │ ├── install_ngrok.sh │ ├── install_nova.sh │ ├── install_octo.sh │ ├── install_oh-my-zsh.sh │ ├── install_openssh.sh │ ├── install_oracle_client.sh │ ├── install_oracle_sql_developer.sh │ ├── install_oracle_sqlcl.sh │ ├── install_packer.sh │ ├── install_parquet-tools.sh │ ├── install_pluto.sh │ ├── install_polaris.sh │ ├── install_powershell.sh │ ├── install_powershell_debian.sh │ ├── install_powershell_rhel.sh │ ├── install_powershell_ubuntu.sh │ ├── install_prometheus.sh │ ├── install_prometheus_alertmanager.sh │ ├── install_prometheus_blackbox_exporter.sh │ ├── install_prometheus_consul_exporter.sh │ ├── install_prometheus_graphite_exporter.sh │ ├── install_prometheus_memcached_exporter.sh │ ├── install_prometheus_mysqld_exporter.sh │ ├── install_prometheus_node_exporter.sh │ ├── install_prometheus_push_gateway.sh │ ├── install_prometheus_statsd_exporter.sh │ ├── install_promlens.sh │ ├── install_pulumi_cli.sh │ ├── install_rancher_cli.sh │ ├── install_rpmforge.sh │ ├── install_rvm.sh │ ├── install_sbt.sh │ ├── install_sdkman.sh │ ├── install_sdkman_all_sdks.sh │ ├── install_semaphore_ci.sh │ ├── install_serverless.sh │ ├── install_spotifycontrol.sh │ ├── install_squirrel_sql.sh │ ├── install_syft.sh │ ├── install_talosctl.sh │ ├── install_terraform.sh │ ├── install_terraformer.sh │ ├── install_terragrunt.sh │ ├── install_tfenv.sh │ ├── install_tfsec.sh │ ├── install_tgswitch.sh │ ├── install_tkn.sh │ ├── install_travis.sh │ ├── install_trivy.sh │ ├── install_vertica_vsql_client.sh │ ├── install_vertica_vsql_client_rpm.sh │ ├── install_vundle.sh │ ├── install_wercker_cli.sh │ └── install_yq.sh ├── internet/ │ ├── 0x0.sh │ ├── atlassian_ip_ranges.sh │ ├── catbox.sh │ ├── datadog_api.sh │ ├── digital_ocean_api.sh │ ├── dnsjson.sh │ ├── domains_subdomains_environments.sh │ ├── dpaste.sh │ ├── file.io.sh │ ├── google_maps_link.sh │ ├── imgur.sh │ ├── jira_api.sh │ ├── kong_api.sh │ ├── litterbox.sh │ ├── ngrok_api.sh │ ├── pastebin.sh │ ├── shields_embed_logo.sh │ ├── termbin.sh │ ├── traefik_api.sh │ ├── wordpress.htaccess │ ├── wordpress.sh │ ├── wordpress_api.sh │ ├── wordpress_plugins_markdown.sh │ └── wordpress_posts_without_category_tags.sh ├── ipaas/ │ └── make_api.sh ├── java/ │ ├── bytecode_viewer.sh │ ├── cfr.sh │ ├── java_decompile_jar.sh │ ├── java_show_classpath.sh │ ├── jd_gui.sh │ ├── jvm_heaps.sh │ ├── jvm_heaps_total_mb.sh │ └── procyon.sh ├── jenkins/ │ ├── README.md │ ├── jenkins.sh │ ├── jenkins_api.sh │ ├── jenkins_builds.sh │ ├── jenkins_clear_build_history.groovy │ ├── jenkins_clear_build_history_all_jobs.groovy │ ├── jenkins_cli.sh │ ├── jenkins_count_jobs.groovy │ ├── jenkins_create_job_check_gcp_serviceaccount.sh │ ├── jenkins_create_job_parallel_test_runs.sh │ ├── jenkins_create_run_job.sh │ ├── jenkins_cred_add_cert.sh │ ├── jenkins_cred_add_kubernetes_sa.sh │ ├── jenkins_cred_add_secret_file.sh │ ├── jenkins_cred_add_secret_text.sh │ ├── jenkins_cred_add_ssh_key.sh │ ├── jenkins_cred_add_user_pass.sh │ ├── jenkins_cred_cli_add_cert.sh │ ├── jenkins_cred_cli_add_kubernetes_sa.sh │ ├── jenkins_cred_cli_add_secret_file.sh │ ├── jenkins_cred_cli_add_secret_text.sh │ ├── jenkins_cred_cli_add_ssh_key.sh │ ├── jenkins_cred_cli_add_user_pass.sh │ ├── jenkins_cred_cli_delete.sh │ ├── jenkins_cred_cli_list.sh │ ├── jenkins_cred_cli_set_cert.sh │ ├── jenkins_cred_cli_set_kubernetes_sa.sh │ ├── jenkins_cred_cli_set_secret_file.sh │ ├── jenkins_cred_cli_set_secret_text.sh │ ├── jenkins_cred_cli_set_ssh_key.sh │ ├── jenkins_cred_cli_set_user_pass.sh │ ├── jenkins_cred_cli_update_cert.sh │ ├── jenkins_cred_cli_update_kubernetes_sa.sh │ ├── jenkins_cred_cli_update_secret_file.sh │ ├── jenkins_cred_cli_update_secret_text.sh │ ├── jenkins_cred_cli_update_ssh_key.sh │ ├── jenkins_cred_cli_update_user_pass.sh │ ├── jenkins_cred_delete.sh │ ├── jenkins_cred_get.sh │ ├── jenkins_cred_list.sh │ ├── jenkins_cred_set_cert.sh │ ├── jenkins_cred_set_kubernetes_sa.sh │ ├── jenkins_cred_set_secret_file.sh │ ├── jenkins_cred_set_secret_text.sh │ ├── jenkins_cred_set_ssh_key.sh │ ├── jenkins_cred_set_user_pass.sh │ ├── jenkins_cred_update_cert.sh │ ├── jenkins_cred_update_kubernetes_sa.sh │ ├── jenkins_cred_update_secret_file.sh │ ├── jenkins_cred_update_secret_text.sh │ ├── jenkins_cred_update_ssh_key.sh │ ├── jenkins_cred_update_user_pass.sh │ ├── jenkins_creds_cli_xml_dump.sh │ ├── jenkins_foreach_job.sh │ ├── jenkins_foreach_job_cli.sh │ ├── jenkins_job_config.sh │ ├── jenkins_job_description.sh │ ├── jenkins_job_disable.groovy │ ├── jenkins_job_disable.sh │ ├── jenkins_job_enable.sh │ ├── jenkins_job_trigger.sh │ ├── jenkins_job_trigger_with_params.sh │ ├── jenkins_jobs.groovy │ ├── jenkins_jobs.sh │ ├── jenkins_jobs_disable.sh │ ├── jenkins_jobs_disabled.groovy │ ├── jenkins_jobs_download_configs.sh │ ├── jenkins_jobs_download_configs_cli.sh │ ├── jenkins_jobs_enable.sh │ ├── jenkins_jobs_status.groovy │ ├── jenkins_password.sh │ └── jenkins_plugins_latest_versions.sh ├── kafka/ │ ├── kafka_acls.sh │ ├── kafka_cli_jaas.conf │ ├── kafka_configs.sh │ ├── kafka_console_consumer.sh │ ├── kafka_console_producer.sh │ ├── kafka_consumer_groups.sh │ ├── kafka_consumer_perf_test.sh │ ├── kafka_producer_perf_test.sh │ └── kafka_topics.sh ├── kics.config ├── kubernetes/ │ ├── argocd_apps_sync.sh │ ├── argocd_apps_wait_sync.sh │ ├── argocd_auto_sync.sh │ ├── argocd_generate_resource_whitelist.sh │ ├── argocd_namespace_resource_whitelist.sh │ ├── argocd_password.sh │ ├── curl_k8s_ingress.sh │ ├── datree_kustomize_all.sh │ ├── helm_template.sh │ ├── kubeadm_join_cmd.sh │ ├── kubeadm_join_cmd2.sh │ ├── kubectl.sh │ ├── kubectl_alpine.sh │ ├── kubectl_busybox.sh │ ├── kubectl_container_count.sh │ ├── kubectl_container_counts.sh │ ├── kubectl_create_namespaces.sh │ ├── kubectl_curl.sh │ ├── kubectl_delete_empty_namespaces.sh │ ├── kubectl_deployment_pods.sh │ ├── kubectl_diff_apply.sh │ ├── kubectl_dnsutils.sh │ ├── kubectl_empty_namespaces.sh │ ├── kubectl_exec.sh │ ├── kubectl_exec2.sh │ ├── kubectl_gcloud_sdk.sh │ ├── kubectl_get_all.sh │ ├── kubectl_get_annotation.sh │ ├── kubectl_image_counts.sh │ ├── kubectl_image_deployments.sh │ ├── kubectl_images.sh │ ├── kubectl_jobs_delete_stuck.sh │ ├── kubectl_jobs_stuck.sh │ ├── kubectl_kv_to_secret.sh │ ├── kubectl_logs.sh │ ├── kubectl_node_labels.sh │ ├── kubectl_node_taints.sh │ ├── kubectl_pod_count.sh │ ├── kubectl_pod_ips.sh │ ├── kubectl_pod_labels.sh │ ├── kubectl_pods_colocated.sh │ ├── kubectl_pods_dump_all.sh │ ├── kubectl_pods_dump_jstacks.sh │ ├── kubectl_pods_dump_logs.sh │ ├── kubectl_pods_dump_stats.sh │ ├── kubectl_pods_important.sh │ ├── kubectl_pods_per_node.sh │ ├── kubectl_pods_running_with_labels.sh │ ├── kubectl_port_forward.sh │ ├── kubectl_port_forward_spark.sh │ ├── kubectl_rerun_job.sh │ ├── kubectl_restart.sh │ ├── kubectl_rollout_history_all_deployments.sh │ ├── kubectl_run_sa.sh │ ├── kubectl_secret_values.sh │ ├── kubectl_secrets_annotate_to_be_sealed.sh │ ├── kubectl_secrets_download.sh │ ├── kubectl_secrets_not_sealed.sh │ ├── kubectl_secrets_to_be_sealed.sh │ ├── kubernetes_api.sh │ ├── kubernetes_autoscaler_release.sh │ ├── kubernetes_check_objects_namespaced.sh │ ├── kubernetes_delete_stuck_namespace.sh │ ├── kubernetes_etcd_backup.sh │ ├── kubernetes_foreach_context.sh │ ├── kubernetes_foreach_namespace.sh │ ├── kubernetes_info.sh │ ├── kubernetes_nodes_ssh_dump_logs.sh │ ├── kubernetes_resource_types.sh │ ├── kubernetes_secret_to_external_secret_gcp.sh │ ├── kubernetes_secret_to_sealed_secret.sh │ ├── kubernetes_secrets_compare_gcp_secret_manager.sh │ ├── kubernetes_secrets_to_external_secrets_gcp.sh │ ├── kubernetes_secrets_to_sealed_secrets.sh │ ├── kubernetes_yaml_strip_live_fields.sh │ ├── kustomize_check_objects_namespaced.sh │ ├── kustomize_diff_apply.sh │ ├── kustomize_diff_branch.sh │ ├── kustomize_install_helm_charts.sh │ ├── kustomize_materialize.sh │ ├── kustomize_parse_helm_charts.sh │ ├── kustomize_update_helm_chart_versions.sh │ ├── pluto_detect_helm_materialize.sh │ ├── pluto_detect_kubectl_dump_objects.sh │ ├── pluto_detect_kustomize_materialize.sh │ ├── rancher_api.sh │ └── rancher_kube_creds.sh ├── lib/ │ ├── README.md │ ├── args_extract.sh │ ├── aws.sh │ ├── bitbucket.sh │ ├── ci.sh │ ├── cloudera_manager.sh │ ├── cloudera_navigator.sh │ ├── dbshell.sh │ ├── docker.sh │ ├── dockerfile_keywords.txt │ ├── excluded.sh │ ├── gcp.sh │ ├── gcp_ci.sh │ ├── git.sh │ ├── github.sh │ ├── gitlab.sh │ ├── kubernetes.sh │ ├── mac.sh │ ├── mp3.sh │ ├── os.sh │ ├── packages.sh │ ├── perl.sh │ ├── python.sh │ ├── ruby.sh │ ├── spotify.sh │ ├── sql.sh │ ├── travis.sh │ ├── utils-bourne.sh │ └── utils.sh ├── markdown/ │ ├── markdown_columns_to_table.sh │ ├── markdown_generate_index.sh │ ├── markdown_list_indentations.sh │ ├── markdown_octocat_github_links.sh │ ├── markdown_replace_index.sh │ ├── markdown_replace_links_with_jsdelivr.sh │ ├── markdown_replace_repos.sh │ └── mdl_list_indentations.sh ├── media/ │ ├── asciinema.sh │ ├── avi_to_mp4.sh │ ├── avif_to_png.sh │ ├── image_join_vertical.sh │ ├── image_reduce_quality.sh │ ├── image_shrink.sh │ ├── image_to_png.sh │ ├── image_trim_pixels.sh │ ├── imageopen.sh │ ├── mkv_to_mp4.sh │ ├── mp3_set_album.sh │ ├── mp3_set_artist.sh │ ├── mp3_set_track_name.sh │ ├── mp3_set_track_order.sh │ ├── svg_to_png.sh │ ├── terminalizer.sh │ ├── ttygif.sh │ ├── video_to_720p_mp4.sh │ ├── vidopen.sh │ ├── webp_to_png.sh │ ├── youtube_download_channel.sh │ └── youtube_download_video.sh ├── monitoring/ │ ├── dump_stats.sh │ ├── log_timestamp_large_intervals.sh │ ├── prometheus.sh │ ├── prometheus_docker.sh │ ├── prometheus_node_exporter.sh │ ├── ssh_dump_logs.sh │ └── ssh_dump_stats.sh ├── mysql/ │ ├── mariadb.sh │ ├── mariadb_test_scripts.sh │ ├── mysql.sh │ ├── mysql_foreach_table.sh │ ├── mysql_list_databases.sh │ ├── mysql_list_tables.sh │ ├── mysql_tables_row_counts.sh │ ├── mysql_test_scripts.sh │ └── mysqld.sh ├── packages/ │ ├── apk_filter_installed.sh │ ├── apk_filter_not_installed.sh │ ├── apk_install_packages.sh │ ├── apk_install_packages_if_absent.sh │ ├── apk_remove_packages.sh │ ├── apk_upgrade_packages_if_outdated.sh │ ├── apt_install_packages.sh │ ├── apt_install_packages_if_absent.sh │ ├── apt_remove_packages.sh │ ├── apt_set_lock_timeout.sh │ ├── apt_upgrade_packages_if_outdated.sh │ ├── apt_wait.sh │ ├── brew_filter_in_setup.sh │ ├── brew_filter_installed.sh │ ├── brew_filter_not_in_setup.sh │ ├── brew_filter_not_installed.sh │ ├── brew_install_packages.sh │ ├── brew_install_packages_if_absent.sh │ ├── brew_package_owns.sh │ ├── brew_upgrade_packages.sh │ ├── brew_upgrade_packages_if_outdated.sh │ ├── debs_filter_installed.sh │ ├── debs_filter_not_installed.sh │ ├── golang_install.sh │ ├── golang_install_if_absent.sh │ ├── golang_rm_binaries.sh │ ├── install_binary.sh │ ├── install_packages.sh │ ├── install_packages_if_absent.sh │ ├── nodejs_npm_install.sh │ ├── nodejs_npm_install_if_absent.sh │ ├── rpms_filter_installed.sh │ ├── rpms_filter_not_installed.sh │ ├── ruby_gem_install.sh │ ├── ruby_gem_install_if_absent.sh │ ├── upgrade_packages_if_outdated.sh │ ├── yum_install_packages.sh │ ├── yum_install_packages_if_absent.sh │ ├── yum_remove_packages.sh │ └── yum_upgrade_packages_if_outdated.sh ├── perl/ │ ├── perl_cpanm_install.sh │ ├── perl_cpanm_install_if_absent.sh │ ├── perl_cpanm_reinstall_all.sh │ ├── perl_find_duplicate_cpan_requirements.sh │ ├── perl_find_library_executable.sh │ ├── perl_find_library_path.sh │ ├── perl_find_unused_cpan_modules.sh │ ├── perl_generate_fatpacks.sh │ ├── perl_generate_par_binaries.sh │ └── perlpath.sh ├── pingdom/ │ ├── pingdom_api.sh │ ├── pingdom_check_latency_by_hour.sh │ ├── pingdom_check_outages.sh │ ├── pingdom_checks.sh │ ├── pingdom_checks_average_response_times.sh │ ├── pingdom_checks_latency_by_hour.sh │ ├── pingdom_checks_outages.sh │ ├── pingdom_foreach_check.sh │ └── pingdom_sms_credits.sh ├── postgres/ │ ├── postgres.sh │ ├── postgres_foreach_table.sh │ ├── postgres_foreach_table_timeout.sh │ ├── postgres_list_databases.sh │ ├── postgres_list_schemas.sh │ ├── postgres_list_tables.sh │ ├── postgres_tables_row_counts.sh │ ├── postgres_test_scripts.sh │ ├── psql.sh │ └── psql_colorized.sh ├── python/ │ ├── pygmentize.sh │ ├── python_compile.sh │ ├── python_find_duplicate_pip_requirements.sh │ ├── python_find_library_executable.sh │ ├── python_find_library_path.sh │ ├── python_find_unused_pip_modules.sh │ ├── python_indices.sh │ ├── python_pip_install.sh │ ├── python_pip_install_for_script.sh │ ├── python_pip_install_if_absent.sh │ ├── python_pip_reinstall_all_modules.sh │ ├── python_pyinstaller.sh │ ├── python_pypi_versions.sh │ ├── python_translate_import_to_module.sh │ ├── python_translate_module_to_import.sh │ └── pythonpath.sh ├── requirements.txt ├── resources/ │ ├── oreilly-animals.json │ ├── pipreqs_mapping.txt │ ├── tabs_ignore.txt │ └── whitespace_ignore.txt ├── scalastyle_config.xml ├── scripts/ │ ├── README.md │ ├── git_capitalize_urls.sh │ ├── spotify_commit_playlists.sh │ ├── spotify_commit_rename_playlist.sh │ ├── spotify_rename_playlist_files.sh │ └── update_bash_tools_references.sh ├── search/ │ ├── solr_api.sh │ ├── solr_collection_check_exists.sh │ └── solr_collection_create_if_not_exists.sh ├── setup/ │ ├── Hari.terminal │ ├── R-packages.txt │ ├── README.md │ ├── alternatives_set_python.sh │ ├── apk-packages-desktop.txt │ ├── apk-packages-optional.txt │ ├── apk-packages.txt │ ├── atom-packages.txt │ ├── bootstrap.sh │ ├── brew-packages-desktop-casks.txt │ ├── brew-packages-desktop-taps.txt │ ├── brew-packages-desktop.txt │ ├── brew-packages-ignore.txt │ ├── brew-packages.txt │ ├── brew_fix_openssl_dependencies.sh │ ├── brew_packages_not_saved.sh │ ├── ccmenu_cp_plist.sh │ ├── ccmenu_setup.sh │ ├── ci.txt │ ├── ci_bootstrap.sh │ ├── ci_git_set_dir_safe.sh │ ├── cpan-packages-desktop.txt │ ├── cpan-requirements-optional.txt │ ├── cpan-requirements.txt │ ├── deb-packages-desktop.txt │ ├── deb-packages-optional.txt │ ├── deb-packages.txt │ ├── debian.experimental.pref │ ├── debian.stable.pref │ ├── debian.testing.pref │ ├── debian.unstable.pref │ ├── docker-images.txt │ ├── docker_bootstrap.sh │ ├── download_cassandra.sh │ ├── download_openjdk11.sh │ ├── files.txt │ ├── gem-packages-desktop.txt │ ├── gem-packages.txt │ ├── go-packages-desktop.txt │ ├── gocd_config_repo.json │ ├── intellij-plugins.txt │ ├── jenkins-job-check-gcp-serviceaccount.xml │ ├── jenkins-job-sleep-parallel-parameterized.xml │ ├── jenkins-job.xml │ ├── jenkins-plugins.txt │ ├── linux_desktop.sh │ ├── mac_delete_routes_on_interface.sh │ ├── mac_desktop.sh │ ├── mac_diff_settings.sh │ ├── mac_settings.sh │ ├── mac_spotlight_config_optimize.sh │ ├── mas-packages.txt │ ├── npm-packages-desktop.txt │ ├── npm-packages.txt │ ├── pip-packages-desktop.txt │ ├── pip-packages-mac.txt │ ├── pip_fix_version.sh │ ├── portage-packages-desktop.txt │ ├── portage-packages-dockapps.txt │ ├── portage-packages-extras.txt │ ├── portage-packages-server.txt │ ├── postgresql.conf │ ├── prometheus.yml │ ├── python_install_snakebite.sh │ ├── python_mac_upgrade_ssl_fix.sh │ ├── repos.txt │ ├── rpm-packages-desktop.txt │ ├── rpm-packages-optional.txt │ ├── rpm-packages.txt │ ├── setup_codefresh.sh │ ├── shell_link.sh │ ├── shell_unlink.sh │ ├── squirrelsql-install-options.xml │ ├── teamcity/ │ │ └── teamcity-database.properties │ ├── teamcity-mysql-setup.sql │ ├── upgrade_gradle_wrapper.sh │ └── which_python_installed.sh ├── shippable/ │ ├── shippable.yml │ ├── shippable_account_id.sh │ ├── shippable_api.sh │ ├── shippable_builds.sh │ ├── shippable_project_builds.sh │ └── shippable_projects.sh ├── sonar-project.properties ├── spotify/ │ ├── spotify_add_artist_to_backlog_playlist.sh │ ├── spotify_add_to_playlist.sh │ ├── spotify_api.sh │ ├── spotify_api_token.sh │ ├── spotify_artist_tracks.sh │ ├── spotify_artists_followed.sh │ ├── spotify_artists_followed_uri.sh │ ├── spotify_artists_followed_uri_name.sh │ ├── spotify_backup.sh │ ├── spotify_backup_artists_followed.sh │ ├── spotify_backup_playlist.sh │ ├── spotify_backup_playlists.sh │ ├── spotify_backup_playlists_list.sh │ ├── spotify_callback_openssl.cnf │ ├── spotify_create_playlist.sh │ ├── spotify_delete_any_duplicates_in_playlist.sh │ ├── spotify_delete_duplicate_track_uris_in_playlist.sh │ ├── spotify_delete_duplicate_tracks_in_playlists.sh │ ├── spotify_delete_from_playlist.sh │ ├── spotify_delete_from_playlist_if_in_other_playlists.sh │ ├── spotify_delete_from_playlist_if_track_in_other_playlists.sh │ ├── spotify_duplicate_tracks_in_playlist.sh │ ├── spotify_duplicate_uri_in_playlist.sh │ ├── spotify_filename_to_playlist.sh │ ├── spotify_follow_artists.sh │ ├── spotify_follow_liked_artists.sh │ ├── spotify_follow_top_artists.sh │ ├── spotify_foreach_playlist.sh │ ├── spotify_liked_artists.sh │ ├── spotify_liked_artists_uri.sh │ ├── spotify_liked_tracks.sh │ ├── spotify_liked_tracks_uri.sh │ ├── spotify_liked_uri_artist_track.sh │ ├── spotify_playlist_artists.sh │ ├── spotify_playlist_id_to_name.sh │ ├── spotify_playlist_json.sh │ ├── spotify_playlist_name_to_id.sh │ ├── spotify_playlist_snapshot_id.sh │ ├── spotify_playlist_to_filename.sh │ ├── spotify_playlist_top_artists.sh │ ├── spotify_playlist_tracks.sh │ ├── spotify_playlist_tracks_uri.sh │ ├── spotify_playlist_tracks_uri_artist_track.sh │ ├── spotify_playlist_tracks_uri_batch_by_year.sh │ ├── spotify_playlist_tracks_uri_by_year.sh │ ├── spotify_playlist_tracks_uri_in_year.sh │ ├── spotify_playlist_uri_offset.sh │ ├── spotify_playlists.sh │ ├── spotify_playlists_json.sh │ ├── spotify_release_year.sh │ ├── spotify_rename_playlist.sh │ ├── spotify_search.sh │ ├── spotify_search_alternate_track_uris.sh │ ├── spotify_search_json.sh │ ├── spotify_search_uri.sh │ ├── spotify_set_playlists_private.sh │ ├── spotify_set_playlists_public.sh │ ├── spotify_set_tracks_uri_to_liked.sh │ ├── spotify_top_artists.sh │ ├── spotify_top_artists_uri.sh │ ├── spotify_top_tracks.sh │ ├── spotify_top_tracks_uri.sh │ ├── spotify_uri_json.sh │ └── spotify_uri_to_name.sh ├── teamcity/ │ ├── .teamcity.vcs.json │ ├── .teamcity.vcs.oauth.json │ ├── .teamcity.vcs.ssh.json │ ├── teamcity.sh │ ├── teamcity_agents.sh │ ├── teamcity_api.sh │ ├── teamcity_builds.sh │ ├── teamcity_buildtype_create.sh │ ├── teamcity_buildtype_set_description_from_github.sh │ ├── teamcity_buildtypes.sh │ ├── teamcity_buildtypes_set_description_from_github.sh │ ├── teamcity_create_github_oauth_connection.sh │ ├── teamcity_create_project.sh │ ├── teamcity_create_vcs_root.sh │ ├── teamcity_export.sh │ ├── teamcity_export_buildtypes.sh │ ├── teamcity_export_project_config.sh │ ├── teamcity_export_vcs_roots.sh │ ├── teamcity_project_set_versioned_settings.sh │ ├── teamcity_project_vcs_versioning.sh │ ├── teamcity_projects.sh │ ├── teamcity_upload_ssh_key.sh │ └── teamcity_vcs_roots.sh ├── terraform/ │ ├── terraform_cloud_api.sh │ ├── terraform_cloud_ip_ranges.sh │ ├── terraform_cloud_organizations.sh │ ├── terraform_cloud_varset_delete_vars.sh │ ├── terraform_cloud_varset_set_vars.sh │ ├── terraform_cloud_varset_vars.sh │ ├── terraform_cloud_varsets.sh │ ├── terraform_cloud_workspace_delete_vars.sh │ ├── terraform_cloud_workspace_set_vars.sh │ ├── terraform_cloud_workspace_vars.sh │ ├── terraform_cloud_workspaces.sh │ ├── terraform_gcs_backend_version.sh │ ├── terraform_gitlab_download_backend_variable.sh │ ├── terraform_import.sh │ ├── terraform_import_aws_iam_groups.sh │ ├── terraform_import_aws_iam_policies.sh │ ├── terraform_import_aws_iam_users.sh │ ├── terraform_import_aws_sso_account_assignments.sh │ ├── terraform_import_aws_sso_managed_policy_attachments.sh │ ├── terraform_import_aws_sso_permission_set_inline_policies.sh │ ├── terraform_import_aws_sso_permission_sets.sh │ ├── terraform_import_foreach.sh │ ├── terraform_import_github_repos.sh │ ├── terraform_import_github_team.sh │ ├── terraform_import_github_team_repos.sh │ ├── terraform_import_github_teams.sh │ ├── terraform_managed_resource_types.sh │ ├── terraform_provider_count_sizes.sh │ ├── terraform_registry_url_extract.sh │ ├── terraform_registry_url_open.sh │ ├── terraform_registry_url_to_https.sh │ └── terraform_resources.sh ├── tests/ │ ├── README.md │ ├── azure_devops_url_conversion.sh │ └── test_spotify_uri_to_name.sh ├── travis/ │ ├── .travis.yml │ ├── travis_api.sh │ ├── travis_delete_cron.sh │ ├── travis_foreach_repo.sh │ ├── travis_lint.sh │ ├── travis_repo_build.sh │ ├── travis_repo_caches.sh │ ├── travis_repo_create_cron.sh │ ├── travis_repo_crons.sh │ ├── travis_repo_delete_caches.sh │ ├── travis_repo_delete_crons.sh │ ├── travis_repo_env_vars.sh │ ├── travis_repo_settings.sh │ ├── travis_repos.sh │ ├── travis_repos_caches.sh │ ├── travis_repos_create_cron.sh │ ├── travis_repos_crons.sh │ ├── travis_repos_delete_caches.sh │ ├── travis_repos_delete_crons.sh │ └── travis_repos_settings.sh ├── vagrant/ │ ├── vagrant_hosts.sh │ └── vagrant_total_mb.sh ├── wercker/ │ ├── .werckerignore │ ├── wercker.yml │ ├── wercker_api_app.sh │ ├── wercker_api_runs.sh │ ├── wercker_api_workflows.sh │ └── wercker_app_id.sh └── yamllint/ └── config ================================================ FILE CONTENTS ================================================ ================================================ FILE: .appveyor.yml ================================================ # # Author: Hari Sekhon # Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A p p V e y o r C I # ============================================================================ # # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu # workaround for default JDK9 have old CA certs: # # https://github.com/appveyor/ci/issues/3833 # # https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/#configuring-language-stack # stack: jdk 15 skip_commits: files: - docs/* - '**/*.md' # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ environment: APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension # enable SSH session accessible via my public key #init: # - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - # more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session on_finish: # set this in Settings -> Environment dynamically instead of here #- sh: export APPVEYOR_SSH_BLOCK=true # # workaround for https://github.com/appveyor/ci/issues/3373 # and https://github.com/appveyor/ci/issues/3384 # # has since been added to AppVeyor's own scripts: # # https://github.com/appveyor/ci/pull/3385 # #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Bash-tools/master/install/install_openssh.sh' | bash -e - # # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: # workaround for: # Some packages could not be installed. This may mean that you have # requested an impossible situation or if you are using the unstable # distribution that some required packages have not yet been created # or been moved out of Incoming. # The following information may help to resolve the situation: # # The following packages have unmet dependencies: # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. # bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed # make[2]: *** [apt-packages] Error 123 # make[2]: Leaving directory '/home/appveyor/projects/pylib' # bash-tools/Makefile.in:212: recipe for target 'system-packages' failed # # adding "|| :" to the end of these commands causes them to be silently ignored! - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list - sudo apt purge -yq --allow-change-held-packages mssql-server # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages - setup/ci_bootstrap.sh - make test_script: - make test build: off ================================================ FILE: .bash.d/Makefile ================================================ # # Author: Hari Sekhon # Date: 2016-01-17 12:56:53 +0000 (Sun, 17 Jan 2016) # # vim:ts=4:sts=4:sw=4:noet # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # include ../Makefile.in REPO := HariSekhon/DevOps-Bash-tools BASH_TOOLS := .. .PHONY: readme readme: @source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)/blob/master/.bash.d/README.md" ================================================ FILE: .bash.d/README.md ================================================ Advanced Bashrc Code - Interactive Functions, Aliases and Shell Customizations ============================================================== Advanced bashrc code I've been using for ~15 years, I've ported nearly 5000 lines to this public repo so far. All `*.sh` files in this directory are automatically sourced by `.bashrc` at the top level which is itself designed to be sourced in your `$HOME/.bashrc`. To disable any these source files, simply rename them to not match the `*.sh` glob, eg. => `*.sh.disabled`. - `aliases.sh` - general aliases - `functions.sh` - general functions eg. `pass` which prompts for a password which is saved to an environment variable and auto-populated in various top-level API querying scripts built on `curl_auth.sh` - `env.sh` - general environment variables and var/unvar functions for setting environment variables for the current and all new shell sessions - `paths.sh` - deduplicated adding to `$PATH` for lots of common places (eg. /usr/sbin, /usr/local/bin, ~/bin) and commands to clearly print one path per line for Bash `$PATH`, Perl `@INC` and Python `sys.path`. Also contains technology specific paths when there is no `.sh` file. All other includes use `add_PATH()` function defined here. - `.sh` - aliases, functions and environment variables to make interactive day-to-day use of a specific technologies easier - Cloud / Containerization / Virtualization: - `aws.sh` - [AWS](https://aws.amazon.com) functions: - `aws_env` - populates credentials from `~/.aws/credentials` / `~/.boto` section given as an argument to `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables and sets `$AWS_PROFILE` to the profile name (defaults to the 'default' profile and creds if no argument is specified) - `aws_envs` - prints the available envs configured in the aws credentials file and stars the one currently in use - `awk_token` - generates a 24-hour MFA session token, exports it as `$AWS_SESSION_TOKEN` for use with [AWS CLI](https://aws.amazon.com/cli/), and saves it to `~/.aws/token` for loading to other shells that call `aws_env` - `docker.sh` - [Docker](https://www.docker.com/) convenient aliases and functions like clearing old containers and dangling image layers to clean up space - `kubernetes.sh` - [Kubernetes](https://kubernetes.io/) aliases and functions, managing contexts and namespaces even for periodically regenerated `.kube/config` with refreshed embedded certificates, switching between open source [Kubernetes](https://kubernetes.io/) and Redhat [OpenShift](https://www.openshift.com/) `kubectl` and `oc` commands, automating getting authentication token and Kubernetes API endpoints - `vagrant.sh` - [Vagrant](https://www.vagrantup.com/) aliases and functions - Automation / Distributed Systems: - `ansible.sh` - [Ansible](https://www.ansible.com) aliases and environment variables - `kafka.sh` - [Kafka](http://kafka.apache.org/) environment variables for Kerberos security and CLI appropriate heap size (avoids heap allocation failures on VMs that otherwise default to using larger server configured heap size), avoiding need for common broker and zookeeper arguments when using `kafka_wrappers/` scripts by setting your Kafka broker and zookeeper addresses once instead of in every command - Coding: - `git.sh` - [Git](https://git-scm.com/) aliases and functions - `mercurial.sh` - [Mercurial](https://www.mercurial-scm.org/) aliases and functions - `svn.sh` - [Svn](https://subversion.apache.org) aliases and functions - `java.sh` - [Java](https://www.java.com/en/) detection and setting of `$JAVA_HOME` for Linux and Mac environments - OS: - `apple.sh` - [Apple Mac OS X / macOS](https://en.wikipedia.org/wiki/MacOS) specific tricks - `linux.sh` - [Linux](https://en.wikipedia.org/wiki/Linux) specific miscellaneous bits like X.org - `network.sh` - network aliases and functions - `ssh.sh` - SSH convenience functions and key management - `ssh-agent.sh` / `gpg-agent.sh` - auto-starts SSH and GPG agents if not already running, stores and auto-sources their details for new shells to automatically use them - `title.sh` - auto-title tricks for Screen and Terminals More script related functions can be found in the [lib/](https://github.com/HariSekhon/DevOps-Bash-tools/tree/master/lib) directory at the top level. ================================================ FILE: .bash.d/aliases.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230,SC2139 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G e n e r a l A l i a s e s # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/paths.sh" # manual local aliases # shellcheck disable=SC1090,SC1091 [ -f ~/.aliases ] && . ~/.aliases # bash_tools defined in .bashrc # shellcheck disable=SC2154 export bashrc=~/.bashrc export bashrc2="$bash_tools/.bashrc" alias reload='. $bashrc' alias r='reload' alias rq='set +x; . $bashrc; set -x' alias bashrc='$EDITOR $bashrc && reload' alias bashrc2='$EDITOR $bashrc2 && reload' alias bashrclocal='$EDITOR $bashrc.local; reload' alias bashrc3=bashrclocal alias v='vim' alias vimrc='$EDITOR ~/.vimrc' alias screenrc='$EDITOR ~/.screenrc' alias aliases='$EDITOR $bashd/aliases.sh' alias ae=aliases alias be=bashrc alias be2=bashrc2 alias be3=bashrc3 alias se=screenrc alias ve=vimrc alias creds='$EDITOR ~/.env/creds' alias pbc=pbcopy alias pbp=pbpaste # keep emacs with no window, use terminal, not X, otherwise I'd run xemacs... #alias emacs="emacs -nw" #em(){ emacs "$@" ; } #alias em=emacs #alias e=em #xe(){ xemacs $@ & } #alias x=xe # from DevOps-Perl-tools repo which must be in $PATH # done via functions.sh now #alias new=new.pl # not present on Mac #type tailf &>/dev/null || alias tailf="tail -f" alias tailf="tail -f" # tail -f is better than tailf anyway alias mv='mv -i' alias cp='cp -i' #alias rm='rm -i' # allows to re-use custommized less behaviour throughout profile without duplicating options #less='less -RFXig' #alias less='$less' export LESS="-RFXig --tabs=4" # will require LESS="-R" if type -P pygmentize &>/dev/null; then # shellcheck disable=SC2016 export LESSOPEN='| "$bash_tools/python/pygmentize.sh" "%s"' fi alias l='less' alias m='more' alias vi='vim' # used by vagrant now #alias v='vim' alias grep='grep --color=auto' # in functions.sh for multiple args now #alias envg="env | grep -i" alias dec="decomment.sh" alias dns='cat /etc/resolv.conf' alias hosts='sudo $EDITOR /etc/hosts' alias repos='$EDITOR $bash_tools/setup/repos.txt' alias path="echo \$PATH | tr ':' '\\n' | less" alias paths=path alias tmp="cd /tmp" alias mksupportdir="mkdir -v support-bundle-$(date '+%F_%H%M')" # not as compatible, better to call pypy explicitly or in #! line #if type -P pypy &>/dev/null; then # alias python=pypy #fi # shellcheck disable=SC2139 bt="$(dirname "${BASH_SOURCE[0]}")/.." export bt alias bt='sti bt; cd $bt' # shellcheck disable=SC2154 export bashd="$bash_tools/.bash.d" alias bashd='sti bashd; cd $bashd' #alias cleanshell='env - bash --rcfile /dev/null' alias cleanshell='env - bash --norc --noprofile' alias newshell='exec bash' alias rr='newshell' alias record=script alias dl="BACKGROUND_VIDEO=1 youtube_download_video.sh" #alias vidopen="vidopen.sh" #alias vidopenplay="PLAY_VIDEO=1 vidopen.sh" alias l33tmode='welcome; retmode=on; echo l33tm0de on' alias leetmode=l33tmode alias hist=history alias clhist='HISTSIZE=0; HISTSIZE=5000' alias nohist='unset HISTFILE' alias histgrep='history | grep' export LS_OPTIONS='-F' if is_mac; then export CLICOLOR=1 # equiv to using -G switch when calling else export LS_OPTIONS="$LS_OPTIONS --color=auto" export PS_OPTIONS="$LS_OPTIONS -L" fi alias ls='ls $LS_OPTIONS' # omit . and .. in listall with -A instead of -a alias lA='ls -lA $LS_OPTIONS' alias la='ls -la $LS_OPTIONS' alias ll='ls -l $LS_OPTIONS' alias lh='ls -lh $LS_OPTIONS' alias lr='ls -ltrh $LS_OPTIONS' alias ltr='lr' alias lR='ls -lRh $LS_OPTIONS' # shellcheck disable=SC2086 lw(){ ls -lh $LS_OPTIONS "$(type -P "$@")"; } # shellcheck disable=SC2086,SC2012 lll(){ ls -l "$(readlink -f "${@:-.}")" | less -R; } alias cd..='cd ..' alias ..='cd ..' alias ...='cd ../..' alias ....='cd ../../..' #up(){ # local times="${1:-1}" # if ! [[ "$times" =~ ^[[:digit:]]$ ]]; then # echo "How many directories to go up" # echo # echo "usage: up " # return 1 # fi # while [ "$times" -gt 0 ]; do # cd .. # times=$((times - 1)) # done #} # use bare 'cd' instead, it's more standard #alias ~='cd ~' alias screen='screen -T $TERM' #alias mt=multitail #alias halt='shutdown -h now -P' # my pytools github repo alias ht='headtail.py' alias run='run.sh' # ============================================================================ # # GitHub / GitLab / BitBucket / Azure DevOps repo checkouts # ============================================================================ # export github=~/github export gitlab=~/gitlab export azure_devops=~/azure-devops alias github="sti github; cd '$github'"; export work="$github/work" alias work="sti work; cd '$work'" alias btup="bt; u; cd -" export bitbucket=~/bitbucket alias bitb='cd $bitbucket' # clashes with bitbucket-cli #alias bitbucket='cd $bitbucket' # used to gitbrowse to bitbucket now in git.sh #alias bb=bitbucket alias diag=diagrams aliasdir(){ local directory="$1" local suffix="${2:-}" [ -d "$directory" ] || return 0 name="${directory##*/}" name="${name//-/_}" name="${name//./_}" name="${name// /}" # alias terraform /tf -> terra if [[ "$name" =~ ^(terraform|tf)$ ]]; then name="terra" fi if [ -z "${!name:-}" ]; then export "$name"="$directory" fi # don't clash with any binaries #if ! type -P "${name}${suffix}" &>/dev/null; then # don't clash with binaries or any previous defined aliases or functions if ! type "${name}${suffix}" &>/dev/null; then # shellcheck disable=SC2139,SC2140 alias "${name}${suffix}"="sti $name; cd $directory" elif ! type "g${name}${suffix}" &>/dev/null; then # shellcheck disable=SC2139,SC2140 alias "g${name}${suffix}"="sti $name; cd $directory" fi } for basedir in "$github" "$gitlab" "$bitbucket" "$azure_devops"; do if [ -d "$basedir" ]; then for directory in "$basedir/"*; do aliasdir "$directory" if [[ "$directory" =~ /work$ ]]; then for workdir in "$directory/"*; do aliasdir "$workdir" "w" # work dirs should have a w suffix done fi done fi done doc_alias(){ local docpath="$1" local prefix="${2:-d}" [ -f "$docpath" ] || return 1 docfile="${docpath##*/}" if ! [[ "$docfile" =~ \.(txt|md)$ ]]; then if [[ "$docfile" =~ \. ]]; then return 1 fi fi # slows down shell creation, will drain battery # if [ -L "$docpath" ]; then # # brew install coreutils to get greadlink since Mac doesn't have readlink -f # if type -P greadlink &>/dev/null; then # docfile="$(greadlink -f "$docpath")" # else # docfile="$(readlink -f "$docpath")" # fi # fi #local count=0 #[ -f ~/docs/$docfile ] && ((count+=1)) #[ -f "$github/docs/$docfile" ] && ((count+=1)) #[ -f "$bitbucket/docs/$docfile" ] && ((count+=1)) #if [ $count -gt 1 ]; then # echo "WARNING: $docfile conflicts with existing alias, duplicate doc '$docfile' among ~/docs, ~/github/docs, ~/bitbucket/docs?" # return #fi local shortname="${docfile%.md}" local shortname="${shortname%.txt}" # shellcheck disable=SC2139,SC2140 alias "${prefix}${shortname}"="ti ${docpath##*/}; \$EDITOR $docpath" } for x in ~/docs/* "$github"/docs/* "$bitbucket"/docs/*; do doc_alias "$x" || : done alias know="knowledge" for x in ~/knowledge/* "$github"/knowledge/* "$bitbucket"/knowledge/*; do doc_alias "$x" k || : done # ============================================================================ # # set in ansible.sh #alias a='ansible -Db' alias anonymize='anonymize.py' alias an='anonymize -a' alias bc='bc -l' alias chromekill='pkill -f "Google Chrome Helper"' alias eclipse='~/eclipse/Eclipse.app/Contents/MacOS/eclipse'; alias visualvm='~/visualvm/bin/visualvm' alias tmpl=templates # using brew version on Mac pmd_opts="-R rulesets/java/quickstart.xml -f text" if is_mac; then # yes evaluate $pmd_opts here # shellcheck disable=SC2139 pmd="pmd $pmd_opts" else for x in ~/pmd-bin-*; do if [ -f "$x/bin/run.sh" ]; then # yes evaluate $x here, and don't export it's lazy evaluated in alias below # shellcheck disable=SC2139,SC2034 pmd="$x/bin/run.sh pmd $pmd_opts" fi done fi alias pmd='$pmd' # from DevOps Perl tools repo - like uniq but doesn't require pre-sorting keeps the original ordering # Devops Golang tools uniq2 should be on path instead now #alias uniq2=uniq_order_preserved.pl # for piping from grep alias uniqfiles="sed 's/:.*//;/^[[:space:]]*$/d' | sort -u" export etc=~/etc alias etc='cd $etc' alias distro='cat /etc/*release /etc/*version 2>/dev/null' alias trace=traceroute alias t='$EDITOR ~/temp-notes.txt' # causes more problems than it solves on a slow machine missing the prompt #alias y=yes alias t2='$EDITOR ~/tmp2' alias t3='$EDITOR ~/tmp3' #alias tg='traceroute www.google.com' #alias sec='ps -ef| grep -e arpwatc[h] -e swatc[h] -e scanlog[d]' export lab=~/lab alias lab='cd $lab' # Auto-alias uppercase directories in ~ like Desktop and Downloads #for dir in $(find ~ -maxdepth 1 -name '[A-Z]*' -type d); do [ -d "$dir" ] && alias ${dir##*/}="cd '$dir'"; done export Downloads=~/Downloads export Documents=~/Documents alias Downloads='cd "$Downloads"' alias Documents='cd "$Documents"' export down="$Downloads" export docu="$Documents" alias down='cd "$Downloads"' alias docu='cd "$Documents"' alias doc='cd ~/docs' export desktop=~/Desktop export desk="$desktop" alias desktop='cd "$desktop"' alias desk=desktop export screenshots=~/Desktops/Screenshots alias screenshots='cd "$screenshots"' export bin=~/bin alias bin="cd $bin" alias todo='ti T; $EDITOR ~/TODO' alias TODO="todo" alias don='ti D; $EDITOR ~/DONE' alias DON=don # drive => Google Drive export google_drive=~/drive export drive="$google_drive" alias drive='cd "$drive"' for v in ~/github/pytools/validate_*.py; do z="${v##*/}" z="${z#validate_}" z="${z%.py}" # needs to expand now for dynamic alias creation # shellcheck disable=SC2139,SC2140 alias "v$z"="$v" done # in some environments I do ldap with Kerberos auth - see ldapsearch.sh script at top level which is more flexible with pre-tuned environment variables #alias ldapsearch="ldapsearch -xW" #alias ldapadd="ldapadd -xW" #alias ldapmodify="ldapmodify -xW" #alias ldapdelete="ldapdelete -xW" #alias ldappasswd="ldappasswd -xW" #alias ldapwhoami="ldapwhoami -xW" #alias ldapvi="ldapvi -b dc=domain,dc=local -D cn=admin,dc=domain,dc=local" alias fluxkeys='$EDITOR ~/.fluxbox/keys' alias fke=fluxkeys alias fluxmenu='$EDITOR ~/.fluxbox/mymenu' alias fme=fluxmenu alias mymenu=fluxmenu alias menu=mymenu # trigger script in ~/.config/mpv/scripts/delete-on-eof.lua to delete a video once it has been completely watched alias mpvd="MPV_DELETE_ON_EOF=1 mpv --speed=2" ================================================ FILE: .bash.d/android.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-02-27 02:08:34 +0700 (Thu, 27 Feb 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # export ANDROID_HOME="$HOME/Android/Sdk" # Doesn't work #export ANDROID_SDK_ROOT="$ANDROID_HOME" add_PATH "$ANDROID_HOME/platform-tools" add_PATH "$ANDROID_HOME/cmdline-tools/latest/bin" add_PATH "$ANDROID_HOME/cmdline-tools/bin" for x in "$ANDROID_HOME/build-tools/"*; do if [ -d "$x" ]; then add_PATH "$x" fi done ================================================ FILE: .bash.d/ansible.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # shellcheck disable=SC2139 # # Author: Hari Sekhon # Date: 2014-07-13 16:56:14 +0100 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A n s i b l e # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # order of precedence: # # $ANSIBLE_CONFIG # $PWD/ansible.cfg # $HOME/.ansible.cfg # /etc/ansible/ansible.cfg # # so don't set ANSIBLE_CONFIG because it'll cause issues in work repos # which would otherwise correctly default to $PWD/ansible.cfg # #export ANSIBLE_CONFIG=~/.ansible.cfg # symlinked to $bash_tools/configs/.ansible.cfg if [ -n "${ANSIBLE_HOME:-}" ]; then add_PATH PYTHONPATH "$ANSIBLE_HOME/lib" add_PATH ANSIBLE_LIBRARY "$ANSIBLE_HOME/library" # resets man search path, breaking man lookups #add_PATH MANPATH "$ANSIBLE_HOME/docs/man" fi # don't set this in case it causes issues in work repos #if [ -f ~/etc/ansible/hosts ]; then # export ANSIBLE_HOSTS=~/etc/ansible/hosts #fi # set in ~/.ansible.cfg now #export ANSIBLE_HOST_KEY_CHECKING=False # -D diff switch requires newish ansible, doesn't work on 1.7 # -b - matter of preference between using lots of sudo in manifests or not, better to remove it for tighter authz & logging purposes in governed environments ansible_opts="-D -b" alias a=ansible # expand now, no dynamic surprises alias ansible="ansible $ansible_opts" alias ansible_playbook="ansible-playbook $ansible_opts" #alias ansible_playbook_vault="ansible-playbook $ansible_opts --ask-vault-pass" alias ansible_playbook_vault="ansible-playbook $ansible_opts --vault-id '$bash_tools/bin/vault_pass.sh'" ================================================ FILE: .bash.d/argocd.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-01-07 16:03:39 +0000 (Fri, 07 Jan 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A r g o C D # ============================================================================ # # gets default admin pw and copies it to clipboard for quick pasting to UI alias argopass="argocd_password.sh | copy_to_clipboard.sh" # XXX: set the following in your local environment: # # ARGOCD_SERVER=localhost:8080 # without the http:// or https:// prefix # ARGOCD_AUTH_TOKEN='' if ! [[ "${ARGOCD_OPTS:-}" =~ --grpc-web ]]; then export ARGOCD_OPTS="$ARGOCD_OPTS --grpc-web" fi #export ARGOCD_OPTS="--grpc-web --insecure" # only in local dev argosync(){ local seconds="${1:-60}" shift || : if [ -z "${ARGOCD_APP:-}" ]; then namespace="${K8S_NAMESPACE:-$(kubectl_namespace)}" if argocd app list -o name | grep -Fxq "$namespace"; then ARGOCD_APP="$namespace" fi fi if [ -n "${ARGOCD_APP:-}" ]; then timestamp "ArgoCD Syncing App '$ARGOCD_APP'" echo argocd app sync "$ARGOCD_APP" --force "$@" argocd app wait "$ARGOCD_APP" --timeout "$seconds" "$@" echo if [ "$ARGOCD_APP" = argocd ]; then for x in projects apps; do if argocd app list -o name | grep -Fxq "$x"; then ARGOCD_APP="$x" argosync "$@" fi done fi else echo "\$ARGOCD_APP is not set" >&2 return 1 fi } ================================================ FILE: .bash.d/aws-cloudshell.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-12-17 18:54:23 +0000 (Thu, 17 Dec 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # This is designed to mimick the standard GCP CloudShell behaviour # of re-customizing a new CloudShell if ! [ "${AWS_EXECUTION_ENV:-}" = "CloudShell" ]; then return fi customize_script=~/.aws_customize_environment lockdir=/tmp/aws_customize_environment.lock completion_semaphore="/.aws_customize_environment_completed" if ! [ -f "$customize_script" ]; then return fi if [ -f "$completion_semaphore" ]; then return fi # XXX: if the lockdir is more than 2 hours old and the completion semaphore wasn't found, remove the lockdir to try again if [ -d "$lockdir" ] && ! [ -f "$completion_semaphore" ]; then lockdir_epoch="$(stat -c %Y "$lockdir")" current_epoch="$(date +%s)" if [ $((current_epoch - lockdir_epoch)) -gt 7200 ]; then rmdir "$lockdir" fi fi # used as a mutex lock mkdir "$lockdir" 2>/dev/null || return sudo bash < /var/log/customize_environment 2>&1 EOF ================================================ FILE: .bash.d/aws.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2012-09-01 13:01:11 +0100 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A W S - A m a z o n W e b S e r v i c e s # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # shellcheck disable=SC1090,SC1091 #type autocomplete &>/dev/null || . "$bash_tools/.bash.d/functions.sh" # ================== # AWS CLI completion aws_completer="$(type -P aws_completer 2>/dev/null)" if [ -n "$aws_completer" ]; then complete -C "$aws_completer" aws fi #autocomplete eksctl # ===================== # Elastic Beanstalk CLI (easier to use than AWS CLI) if [ -d ~/.ebcli-virtual-env/executables/ ]; then add_PATH ~/.ebcli-virtual-env/executables/ fi # ============================================================================ # # A l i a s e s a n d F u n c t i o n s # ============================================================================ # alias awsl='aws sso login' #alias s3='s3cmd' alias s3='aws s3' alias dockerecr='aws ecr get-login-password | docker login -u AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com' alias awscon='aws_consoler -o' alias awsc='awscon' alias aws_whoami="aws sts get-caller-identity" alias awhoami=aws_whoami # loads creds from a CLI cache file (eg. for AWS SSO) into environment variables # better done via direnv awscreds(){ # should be something like ~/.aws/cli/cached/[hash].json local cred_cache_file="$1" AWS_ACCESS_KEY_ID="$(jq -r .Credentials.AccessKeyId < "$cred_cache_file")" AWS_SECRET_ACCESS_KEY="$(jq -r .Credentials.SecretAccessKey < "$cred_cache_file")" AWS_SESSION_TOKEN="$(jq -r .Credentials.SessionToken < "$cred_cache_file")" export AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY export AWS_SESSION_TOKEN } # ================== # AWLess completion alias awl=awless alias assh="awless ssh" #autocomplete awless # make completion work with awl alias above #if ! [ -f ~/.bash.autocomplete.d/awl.sh ]; then # sed 's/awless/awl/g' ~/.bash.autocomplete.d/awless.sh > ~/.bash.autocomplete.d/awl.sh #fi #autocomplete awl # ================== # JAVA_HOME needs to be set to use EC2 api tools #[ -x /usr/bin/java ] && export JAVA_HOME=/usr # errors but still works # Shouldn't be needed any more, all these sorts of tools were unified on awscli # # link_latest '/usr/local/ec2-api-tools-*' #if [ -d /usr/local/ec2-api-tools/bin ]; then # export EC2_HOME=/usr/local/ec2-api-tools # this should be a link to the unzipped ec2-api-tools-1.6.1.4/ # add_PATH "$EC2_HOME/bin" #fi # ============================================================================ # # Old: new direnv now # # ec2dre - ec2-describe-regions - list regions you have access to and put them here # TODO: pull a more recent list and have aliases/functions auto-generated from that to export #aws_eu(){ # export EC2_URL=ec2.eu-west-1.amazonaws.com #} #aws_useast(){ # export EC2_URL=ec2.us-east-1.amazonaws.com #} #aws_eu # ============================================================================ # # https://github.com/remind101/assume-role assume-role(){ #eval "$(command assume-role "$@")" local output output="$(command assume-role "$@")" # shellcheck disable=SC2181 if [ $? -eq 0 ]; then eval "$output" fi } # ============================================================================ # aws_get_cred_path(){ # unreliable that HOME is set, ensure shell evaluates to the right thing before we use it [ -n "${HOME:-}" ] || HOME=~ local aws_credentials="${AWS_SHARED_CREDENTIALS_FILE:-$HOME/.aws/credentials}" local aws_config="${AWS_CONFIG_FILE:-$HOME/.aws/config}" local boto="${BOTO_CONFIG:-$HOME/.boto}" local credentials_file if [ -f "$aws_credentials" ]; then credentials_file="$aws_credentials" # older boto creds elif [ -f "$boto" ]; then credentials_file="$boto" elif [ -f "$aws_config" ]; then credentials_file="$aws_config" else echo "no credentials found - didn't find $aws_credentials or $boto or $aws_config" 2>/dev/null return 1 fi echo "$credentials_file" } # this rarely changes so just set it once as initialization instead of passing lots of params # or re-executing aws_get_cred_path() multiple times in different functions aws_credentials_file="$(aws_get_cred_path)" aws_clean_env(){ echo "clearing AWS_* environment variables" while read -r envvar; do unset "$envvar" done < <(env | sed -n '/^AWS_/ s/=.*// p') } # easily set a profile env var #aws_profile(){ # # false positive # # shellcheck disable=SC2317 # export AWS_PROFILE="$*" #} alias awsprofile=aws_profile.sh alias awsp=aws_profile.sh aws_get_profile_data(){ local profile="$1" local filename="${2:-$aws_credentials_file}" sed -n "/[[:space:]]*\\[\\(profile[[:space:]]*\\)*$profile\\]/,/^[[:space:]]*\\[/p" "$filename" } # Storing creds in one place in Boto creds file, pull them straight from there # if only using new creds, might want to just export AWS_PROFILE instead using aws_profile which provides validation aws_env(){ local profile="${1:-default}" # export AWS_ACCESS_KEY # export AWS_SECRET_KEY # export AWS_SESSION_TOKEN - for multi-factor authentication local aws_token=~/.aws/token aws_profile "$profile" || return 1 # section is checked for existence as part of aws_profile(), will return before here if not valid local profile_data profile_data="$(aws_get_profile_data "$profile")" echo "loading [$profile] creds from $aws_credentials_file" eval "$( for key in aws_access_key_id aws_secret_access_key aws_session_token; do awk -F= "/^[[:space:]]*$key/"'{gsub(/[[:space:]]+/, "", $0); gsub(/_id/, "", $1); gsub(/_secret_access/, "_secret", $1); print "export "toupper($1)"="$2}' <<< "$profile_data" done )" if [ -f "$aws_token" ]; then echo "sourcing $aws_token" # shellcheck disable=SC1090,SC1091 source "$aws_token" fi } alias awsenv=aws_env aws_envs(){ awk '/^[[:space:]]*\[.+\]/{print $1}' < "$aws_credentials_file" | sed 's/\[//;s/\]//' | while read -r profile; do default=0 if [ "$profile" = "$AWS_PROFILE" ]; then local default=1 elif [ -z "$AWS_PROFILE" ] && [ "$profile" = "default" ]; then local default=1 fi if [ "$default" = 1 ]; then echo -n "* " else echo -n " " fi echo -n "$profile" if [ "$default" = 1 ] && ! env | grep -q '^AWS_SECRET_KEY='; then echo -n " (keys not loaded to env)" fi echo done } alias awsenvs=aws_envs aws_unenv(){ unset AWS_ACCESS_KEY unset AWS_SECRET_KEY unset AWS_SESSION_TOKEN } alias awsunenv=aws_unenv aws_token(){ local output local token if [ $# -eq 0 ]; then echo "usage: aws_token []" return 1 fi if [ -z "${AWS_MFA_ARN:-}" ]; then echo "environment variable \$AWS_MFA_ARN not set - you need to" echo echo "export AWS_MFA_ARN=arn:aws:iam::<123456789012>:mfa/" echo echo "(you might want to put that in your ~/.bashrc.local or similar)" return 1 fi #aws sts get-session-token --serial-number arn-of-the-mfa-device --token-code code-from-token set -x output="$(aws sts get-session-token --serial-number "$AWS_MFA_ARN" --duration-seconds "${AWS_STS_DURATION_SECS:-129600}" --token-code "$@")" result=$? set +x echo "$output" if [ $result -ne 0 ]; then return $result fi if type -P jq &>/dev/null; then token="$(jq -r '.Credentials.SessionToken' <<< "$output")" else token-"$(awk -F: '/SessionToken/{print $2}' | sed 's/"//')" fi export AWS_SESSION_TOKEN="$token" echo "exported AWS_SESSION_TOKEN" echo echo "export AWS_SESSION_TOKEN=$token" > ~/.aws/token echo "saved to ~/.aws/token for other shells to source via aws_env()" echo echo "you can now use AWS CLI normally" } alias awstoken=aws_token ================================================ FILE: .bash.d/azure.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # shellcheck disable=SC1090,SC1091 # # Author: Hari Sekhon # Date: 2020-03-06 16:36:42 +0000 (Fri, 06 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A z u r e # ============================================================================ # srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh" # Azure CLI from script install, installs to $HOME/lib and $HOME/bin if [ -f ~/lib/azure-cli/az.completion ]; then source ~/lib/azure-cli/az.completion fi # assh is an alias to awless ssh azssh(){ local ip ip="$(az vm show --name "$1" -d --query "[publicIps]" -o tsv)" ssh azureuser@"$ip" } ================================================ FILE: .bash.d/bash_it.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-02-26 15:21:46 +0000 (Wed, 26 Feb 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # B a s h I T # ============================================================================ # return if ! [ -d ~/.bash_it ]; then git clone --depth=1 https://github.com/Bash-it/bash-it.git ~/.bash_it fi export BASH_IT=~/.bash_it export BASH_IT_THEME='bobby' ================================================ FILE: .bash.d/circleci.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-08-02 12:21:34 +0100 (Tue, 02 Aug 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C i r c l e C I # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" if ! type github_owner_repo &>/dev/null; then # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/git.sh" fi circleci_debug(){ circleci_project_set_env_vars.sh github/$(github_owner_repo) DEBUG=1 } circleci_undebug(){ circleci_project_delete_env_vars.sh github/$(github_owner_repo) DEBUG } ================================================ FILE: .bash.d/colors.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2034 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2012-06-25 15:20:39 +0100 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Terminal ANSI Escape Color Codes # ============================================================================ # # Show color codes - adapted from http://codesnippets.joyent.com/posts/show/1517 example 1 # # see also colors.pl from DevOps-Perl-tools repo which is slightly better colors(){ local text="hari" echo -e '\n 40m 41m 42m 43m 44m 45m 46m 47m'; for FGs in ' m' ' 1m' ' 30m' '1;30m' ' 31m' '1;31m' ' 32m' \ '1;32m' ' 33m' '1;33m' ' 34m' '1;34m' ' 35m' '1;35m' \ ' 36m' '1;36m' ' 37m' '1;37m'; do FG=${FGs// /} # shellcheck disable=SC1117 echo -en " $FGs \033[$FG $text " for BG in 40m 41m 42m 43m 44m 45m 46m 47m; do # shellcheck disable=SC1117 echo -en "$EINS \033[$FG\033[$BG $text \033[0m"; done echo done echo } # ============================================================================ # # For Gentoo stylish prompts # # Find or write a full colour output table like seen here: # Daniel Robbins prompt magic tip on ibm developerworks # # from http://wiki.archlinux.org/index.php/Color_Bash_Prompt # # would set 'readonly' but causes reloads to output readonly variable errors # replaced \e with \033 as it is more portable on Mac script includes for lib/utils.sh tick_msg() txtblk='\033[0;30m' # Black - Regular txtred='\033[0;31m' # Red txtgrn='\033[0;32m' # Green txtylw='\033[0;33m' # Yellow txtblu='\033[0;34m' # Blue txtpur='\033[0;35m' # Purple txtcyn='\033[0;36m' # Cyan txtwht='\033[0;37m' # White bldblk='\033[1;30m' # Black - Bold bldred='\033[1;31m' # Red bldgrn='\033[1;32m' # Green bldylw='\033[1;33m' # Yellow bldblu='\033[1;34m' # Blue bldpur='\033[1;35m' # Purple bldcyn='\033[1;36m' # Cyan bldwht='\033[1;37m' # White unkblk='\033[4;30m' # Black - Underline undred='\033[4;31m' # Red undgrn='\033[4;32m' # Green undylw='\033[4;33m' # Yellow undblu='\033[4;34m' # Blue undpur='\033[4;35m' # Purple undcyn='\033[4;36m' # Cyan undwht='\033[4;37m' # White bakblk='\033[40m' # Black - Background bakred='\033[41m' # Red bakgrn='\033[42m' # Green bakylw='\033[43m' # Yellow bakblu='\033[44m' # Blue bakpur='\033[45m' # Purple bakcyn='\033[46m' # Cyan bakwht='\033[47m' # White txtrst='\033[0m' # Text Reset ================================================ FILE: .bash.d/custom.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C u s t o m # ============================================================================ # # Stuff that's overly custom and only sourced for my own user # # eg. $USER specific env vars and too short less generic aliases if ! [[ $USER =~ hari|sekhon ]]; then return 0 fi # put secret tokens in vars() or ~/.bashrc.local instead export GITHUB_USER=HariSekhon export TRAVIS_USER="HariSekhon" export BUILDKITE_ORGANIZATION=hari-sekhon export SEMAPHORE_CI_ORGANIZATION=harisekhon alias tll="travis_last_log.py" # can't set this to just the shorter 'go' or 'perl' because it'll clash with the actual commands alias goto=go_tools alias pyt=pytools alias to=perl_tools # shellcheck disable=SC2154 export plugins="$github/nagios-plugins" export pl="$plugins" alias plugins='sti pl; cd $pl' alias pl=plugins # travis_last_log.py should be in $PATH from DevOps-Python-tools repo alias pll="travis_last_log.py HariSekhon/nagios-plugins" export pl2="${plugins}2" alias pl2='sti pl2; cd $pl2' alias pytl="tll /pytools" alias pyt2="pytools2" alias pyl="pylib" alias pyll="tll /pylib" alias tol="tll /tools" alias to2="tool2" # clashes with the D2 diagramming language #alias d2="Dockerfiles2" alias Dockerfilesl="tll /Dockerfiles" ================================================ FILE: .bash.d/direnv.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-04-10 13:02:46 +0100 (Fri, 10 Apr 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # if type -P direnv &>/dev/null; then if ! type _direnv_hook &>/dev/null || ! [[ "${PROMPT_COMMAND:-}" =~ _direnv_hook ]]; then eval "$(direnv hook bash)" fi fi # direnv seems to inserts a double semi-colon which breaks PROMPT_COMMAND #export PROMPT_COMMAND="${PROMPT_COMMAND%%;;*}" export PROMPT_COMMAND="${PROMPT_COMMAND//;;/;}" #alias envrc='$EDITOR .envrc && direnv allow .' # same effect as above alias envrc='direnv edit' # allow all .envrc under your current root - use only inside trusted repos alias direnvallowall='find . -name .envrc -exec direnv allow {} \;' alias da='direnv allow' alias daa='direnvallowall' ================================================ FILE: .bash.d/docker.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-05 20:53:32 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # D o c k e r # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # shellcheck disable=SC1090,SC1091 [ -f ~/.docker_vars ] && . ~/.docker_vars #if is_linux && type -P podman &>/dev/null; then # alias docker="podman" #fi export DOCKER_BUILDKIT=1 # for new M1 Macs which otherwise fail to build with errors like this: # # AWS CLI version: qemu-x86_64: Could not open '/lib64/ld-linux-x86-64.so.2': No such file or directory # export DOCKER_DEFAULT_PLATFORM=linux/amd64 alias dh=hub-tool alias dc=docker-compose alias dps='docker ps' alias dpsa='docker ps -a' alias dst="dockerhub_show_tags.py" # -l shows latest container, -q shows only ID #alias dl='docker ps -lq' alias dockerimg='$EDITOR "$bash_tools/setup/docker-images.txt"' # wipe out exited containers alias dockerrm='docker rm -- $(docker ps -qf status=exited)' alias dockerr=dockerrunrm alias dock=dockerr alias dockere=dockerexec alias de=dockere alias dockerrma=dockerrmall # wipe out dangling image layers #alias dockerrmi='docker rmi $(docker images -q --filter dangling=true)' dockerrmi(){ # want word splitting here # shellcheck disable=SC2046 docker rmi $(docker images -q --filter dangling=true) } # docker-compose -f ... dcf(){ local docker_compose_yaml="$1" if ! [ -f "$docker_compose_yaml" ] && [ -f "docker-compose.yaml" ]; then docker_compose_yaml=docker-compose.yaml fi shift docker-compose -f "$docker_compose_yaml" up "$@" docker-compose -f "$docker_compose_yaml" logs -f } # starts the docker VM, shows ASCII whale, but slow #alias dockershell="/Applications/Docker/Docker\ Quickstart\ Terminal.app/Contents/Resources/Scripts/start.sh" # better #alias dockervm="VBoxManage controlvm startvm default" #alias dockervm="docker-machine start default" #alias dm="docker-machine" #alias dockerrr="docker-machine restart default" #alias dockerreload="docker-machine env default > '$bash_tools/.docker_vars'; . '$bash_tools/.docker_vars'" #dockerstart(){ # if ! docker-machine status default | grep -q Running; then # docker-machine start default # sleep 20 # fi # docker start $(cat "$bash_tools/docker-start.txt") #} # avoid external commands per shell, slows down new shells and wastes battery # switched to using ~/.docker_vars file which is cheaper due to less forks and picked up in each new shell #if type -P docker-machine &>/dev/null; then # if docker-machine status default | grep -q -e Started -e Running; then # eval $(docker-machine env default) # fi #fi #alias dockerr="docker run --rm -ti" function dockerrunrm(){ local args=() local passed_first_non_switch_arg=0 # when this latch gets to level 3 we stop doing prefix processing to not adulterate ls -l / type args for x in "$@"; do if [ $passed_first_non_switch_arg -lt 3 ]; then if [ "${x:0:1}" = "-" ]; then passed_first_non_switch_arg=1 elif [ $passed_first_non_switch_arg -eq 1 ]; then passed_first_non_switch_arg=2 elif [ $passed_first_non_switch_arg -lt 3 ]; then if [ "${x:0:1}" = "/" ]; then if [[ "$x" != */Users/* && "$x" != */home/* ]] && [ "$(strLastIndexOf "$x" / )" -eq 1 ]; then x="harisekhon$x" fi fi passed_first_non_switch_arg=3 else ((passed_first_non_switch_arg+=1)) fi fi args+=("$x") done # Alpine 2 is dead in the water since the package list repos don't even load any more: # # # apk update # fetch http://dl-4.alpinelinux.org/alpine/v2.7/main/x86_64/APKINDEX.tar.gz # wget: server returned error: HTTP/1.1 404 Not Found # ERROR: http://dl-4.alpinelinux.org/alpine/v2.7/main: Bad address # WARNING: Ignoring APKINDEX.0f59c441.tar.gz: No such file or directory # #if [[ "$args" =~ alpine:2 ]] && ! [[ "$args" =~ [[:space:]] ]]; then # echo "warning: using alpine:2.* with args but alpine:2.* doesn't have a default CMD so adding 'sh' arg" >&2 # args="$args sh" #fi local basedir="${PWD##*/}" docker run --rm -ti -v "$PWD":"/$basedir" -w "/$basedir" "${args[@]}" } alias drun='docker run --rm -ti -v "${PWD}":/app' docker_get_container_ids(){ local exclude_file=~/docker-perm.txt local args=() # if exclude file doesn't exist, grep fails entirely and we get no IDs returned, even pre-emptively replacing with /dev/null doesn't work, so omit the option entirely if [ -f "$exclude_file" ]; then args=(-f "$exclude_file") fi docker ps -a --format "{{.ID}} {{.Names}}" | if [ ${#args} -gt 0 ]; then grep -vi "${args[@]}" 2>/dev/null else cat fi | awk '{print $1}' } dockerrmall(){ # would use xargs -r / --no-run-if-empty but that is GNU only, doesn't work on Mac local ids=() read -r -a ids <<< "$(docker_get_container_ids)" if [ ${#ids} -gt 0 ]; then docker rm -f -- "${ids[@]}" fi } dockerrmigrep(){ for x in "$@"; do docker images | awk "/$x/{print \$1\":\"\$2}" | sed '//d' | xargs -r docker rmi -- done } dockerrmgrep(){ for x in "$@"; do docker ps -a | grep "$x" | awk '{print $NF}' | xargs -r docker rm -f -- done } dockerip(){ docker inspect --format '{{ .NetworkSettings.IPAddress }}' "$@" } # this goes to the last created and sometimes exited container #alias dockere='docker exec -ti $(docker ps -lq) /bin/bash' dockerexec(){ if [ $# -gt 0 ]; then container="$(docker ps | grep -i "$1" | awk '{print $1}' | head -n1)" else container="$(docker ps -q | head -n1)" fi docker exec -ti "$container" /bin/sh } docker_get_images(){ # uniq_order_preserved.pl is in the DevOps-Perl-tools repo on github and should be in the $PATH # too many images on dockerhub to pull, fills up filesystem #echo "$(dockerhub_search.py harisekhon -n 1000 | tail -n +2 | awk '{print $1}' | sort) $(sed 's/#.*//;/^[[:space:]]*$/d' "$bash_tools/setup/docker-images.txt" | uniq_order_preserved.pl)" sed 's/#.*//;/^[[:space:]]*$/d' "$bash_tools/setup/docker-images.txt" | uniq_order_preserved.pl } dockerpull1(){ # pull only latest tag, mine first, then official local images="${*:-}" [ -z "$images" ] && images="$(docker_get_images)" images="$(grep -v ":" <<< "$images")" whendone "docker pull" # must be first arg so quoted, [l] trick not needed as grep -v grep's for image in $images; do #whendone "docker pull" # must be first arg so quoted, [l] trick not needed as grep -v grep's timestamp "docker pull $image" #docker pull "$image" | cat & docker pull "$image" # wipe out dangling image layers dockerrmi echo done } dockerpullgithub(){ dockerpull1 harisekhon/{nagios-plugins,pytools,tools,centos-github,debian-github,ubuntu-github,alpine-github} } dockerpull(){ local images="${*:-}" [ -z "$images" ] && images="$(docker_get_images)" dockerpull1 "$images" images="$(grep -i -e harisekhon -e ":" <<< "$images")" #local images="$(grep -i -e ":" <<< "$images")" # now pull all tags, mine first, then official whendone "docker pull" # must be first arg so quoted, [l] trick not needed as grep -v grep's for image in $images; do #whendone "docker pull" # must be first arg so quoted, [l] trick not needed as grep -v grep's if [[ "$image" = harisekhon/* && ! "$image" =~ ":" ]]; then [[ "$image" =~ presto.*-dev ]] && continue for tag in $(dockerhub_show_tags.py -q "$image" | grep -v '^latest$'); do timestamp "docker pull $image:$tag" #docker pull "$image":"$tag" | cat & docker pull "$image":"$tag" echo done else timestamp docker pull "$image" #docker pull "$image" | cat & docker pull "$image" echo fi # wipe out dangling image layers dockerrmi done } dockerpull1r(){ while true; do dockerpull1 "$@" wait echo -e '\n\nsleeping for 1 hour\n\n' sleep 3600 done } dockerpullr(){ while true; do dockerpull "$@" wait echo -e '\n\nsleeping for 1 hour\n\n' sleep 3600 done } # quick, only pull things for which we don't already have local images dockerpullq(){ for x in $(docker_get_images); do docker images | grep -q "^${x}[[:space:]]" && continue whendone "docker pull" # must be first arg so quoted, [l] trick not needed as grep -v grep's timestamp docker pull "$x" docker pull "$x" done # wipe out dangling image layers dockerrmi } ================================================ FILE: .bash.d/env.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # E n v i r o n m e n t V a r i a b l e s # ============================================================================ # # more environment variables defined next to the their corresponding aliases in aliases.sh bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # similar to what zsh does by default if [ -f ~/.bashenv ]; then # shellcheck disable=SC1090,SC1091 . ~/.bashenv fi #export DISPLAY=:0.0 #export TERM=xterm export EDITOR=vim export INPUTRC=~/.inputrc # allow programs to use $LINES and $COLUMNS export LINES export COLUMNS # sets directories to cyan on default bg so they stand out more in dark terminal - see 'man ls' for more details # works on Mac - you may need to see 'man 5 dir_colors' on Linux export LSCOLORS="gx" # ENV refers to the file that sh attempts to read as a startup file (done on my Mac OSX Snow Leopard) # Needs the following line added to sudoers for ENV to be passed through on sudo su #Defaults env_keep += "ENV" export ENV=~/.bashrc # ============================================================================ # cpenv(){ local env_var="$1" if [[ -z "${!env_var}" ]]; then echo "Error: Environment variable '$env_var' is not set" return 1 fi copy_to_clipboard.sh <<< "${!env_var}" echo "Value of '$env_var' has been copied to the clipboard" } # Autocomplete function for environment variables _cpenv_autocomplete() { # 'compgen -v' lists all environment variables # COMPREPLY is set to the autocomplete options local cur_word="${COMP_WORDS[COMP_CWORD]}" COMPREPLY=($(compgen -v -- "$cur_word")) } # Register autocomplete function for `cpenv` complete -F _cpenv_autocomplete cpenv # ============================================================================ # # L o c a l e I n t e r n a t i o n a l i z a t i o n # ============================================================================ # # Run this to see available locales: # # locale -a # # See details of a specific locale variable eg. time formats: # # LC_ALL=C locale -ck LC_TIME # aterm doesn't support UTF-8 and you get horrible chars here and there # so don't use utf and aterm together. xterm works ok with utf8 though #export LANG=en_GB # # LANG becomes default value for any LC_xxx variables not set #export LANG=C # # overrides all other LC_xxx variables #export LC_ALL=C # export LANG=en_US.UTF-8 export LC_ALL=en_US.UTF-8 export LANGUAGE=en_US.UTF-8 #export LC_ALL=en_GB # didn't seem to work #export LANG="en_GB.UTF-8" #export LC_ALL="en_GB.UTF-8" # ============================================================================ # # Clever dynamic environment variables, set using var() function sourced between shells export varfile=~/.bash_vars # shellcheck disable=SC1090,SC1091 [ -f "$varfile" ] && . "$varfile" # Secret Credentials # # separate cred files so if you accidentally expose it on a screen # to colleagues or on a presentation or screen share # you don't have to change all of your passwords # which you would have to if using the above ~/.bash_vars file if [ -d ~/.env/creds ]; then for credfile in ~/.env/creds/*; do if [ -f "$credfile" ]; then # shellcheck disable=SC1090,SC1091 . "$credfile" fi done fi #export DISTCC_DIR="/var/tmp/portage/.distcc/" # ============================================================================ # if is_mac; then #BROWSER=open unset BROWSER elif type -P google-chrome &>/dev/null; then BROWSER=google-chrome elif type -P firefox &>/dev/null; then BROWSER=firefox elif type -P konqueror &>/dev/null; then BROWSER=konqueror elif [ -n "${GOOGLE_CLOUD_SHELL:-}" ]; then : else : #BROWSER=UNKNOWN #echo "COULD NOT FIND ANY BROWSER IN PATH" fi # don't export BROWSER on Mac, trigger python bug: # AttributeError: 'MacOSXOSAScript' object has no attribute 'basename' # from python's webbrowser library if ! is_mac; then export BROWSER fi var(){ local var="${*%%=*}" local val="${*#*=}" if grep -i "export $var" "$varfile" &>/dev/null; then perl -pi -e 's/^export '"$var"'=.*$/export '"$var"'='"$val"'/' "$varfile" else echo "export $var=$val" >> "$varfile" fi export "$var"="$val" } vars(){ "$EDITOR" "$varfile" chmod 0600 "$varfile" # shellcheck disable=SC1090,SC1091 . "$varfile" } unvar(){ local var="${*%%=*}" [ -f "$varfile" ] || { echo "$varfile not found" ; return 1; } perl -pi -e 's/^export '"$var"'=.*\n$//' "$varfile" unset "$var" } # ============================================================================ # unsetall(){ local match="${1:-.*}" while read -r env_var; do if [ "$env_var" = PATH ]; then continue fi unset "$env_var" done < <( env | grep -i "$match" | sed 's/=.*//' ) } ================================================ FILE: .bash.d/functions.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # B a s h G e n e r a l F u n c t i o n s # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # Enables colourized return codes in prompt_func # better leave it as the same as already set. This way a reload of bashrc doesn't change the mode # could do retmode=${retmode:-off} but this is unnecessary overhead #retmode=off # safer to set it to off because otherwise it's possible to get in to a loop with set -u retmode=${retmode:-off} retmode(){ if [ "$retmode" = "on" ]; then retmode=off echo "retmode off" else retmode=on echo "retmode on" fi } cddir(){ cd "$(dirname "$1")" || return 1 } jq(){ command jq -CS "$@" } envg(){ env | eval grep -i "$(for arg; do echo -n " -e '$arg'"; done)" } new(){ if [ $# -eq 2 ]; then title "${2#modules/}" else title "$1" fi command new.pl "$@" title "$LAST_TITLE" } # generates bash autocompletion if not available # sources bash autocompletion from local standardized path autocomplete(){ local name="$1" shift || : if [ -f ~/.bash.autocomplete.d/"$name.sh" ]; then # shellcheck disable=SC1090,SC1091 . ~/.bash.autocomplete.d/"$name.sh" elif type -P "$name" &>/dev/null; then # doesn't work # shellcheck disable=SC1090,SC1091 #source <(command "$name" completion bash) mkdir -pv ~/.bash.autocomplete.d command "$name" completion "$@" bash > ~/.bash.autocomplete.d/"$name.sh" # shellcheck disable=SC1090,SC1091 . ~/.bash.autocomplete.d/"$name.sh" fi } pg(){ # don't want pgrep, want color coding # shellcheck disable=SC2009 ps -ef | grep -i --color=yes "$@" | grep -v grep } pstg(){ # want splitting of options # shellcheck disable=SC2086 pstree | grep -5 -i --color=always "$@" | less $LESS } # externalized to copy_to_clipboard.sh script #copy_to_clipboard(){ # if is_mac; then # cat | pbcopy # elif is_linux; then # cat | xclip # else # echo "ERROR: OS is not Darwin/Linux" # return 1 # fi #} unalias clip &>/dev/null || : # args are optional # shellcheck disable=SC2120 clip(){ if [ $# -gt 0 ]; then copy_to_clipboard.sh < "$1" else copy_to_clipboard.sh fi } dle(){ if [[ "$PWD" =~ $HOME(/Downloads(/Transmission)?)?$ ]]; then echo "Switching to $HOME/Downloads/YouTube" mkdir -p -v ~/Downloads/YouTube cd ~/Downloads/YouTube || return 1 if [ -f .envrc ]; then eval "$(direnv export bash)" fi fi while true; do if BACKGROUND_VIDEO=1 youtube_download_video.sh "$@"; then # doesn't persist past a pause/unpause, # and this starts playing which we don't want which is why it's backgrounded #osascript -e 'tell application "QuickTime Player" to set rate of document 1 to 2' && break fi local sleep_secs="$((RANDOM % 300))" echo "Sleeping for $sleep_secs secs before retrying..." sleep "$sleep_secs" done exit } alias ytp="cd ~/Downloads/YouTube && ./play.sh" deccp(){ # shellcheck disable=SC2119 decomment.sh "$@" | clip } decdiff(){ diff <(decomment.sh "$1" | sed 's/[[:space:]]*$//') <(decomment.sh "$2" | sed 's/[[:space:]]*$//') "${@:3}" } rmdirempty(){ find "${1:-.}" -type d -empty -exec rmdir "{}" \; } checkprog(){ if type -P "$1" &>/dev/null; then return 0 else echo "$1 could not be found in path" return 1 fi } function count() { local total="$1" for ((i = total; i > 0; i--)); do sleep 1 printf "Time remaining %d secs\r" "$i" done bell } dum(){ du -max "${@:-.}" | sort -k1n | tail -n 10000 } typer(){ local alias_target local type_output for x in "$@"; do type_output="$(type "$x")" # shellcheck disable=SC2119 alias_target="$( awk '/aliased to/{print $5}' <<< "$type_output" | unquote )" if [ -n "$alias_target" ]; then echo "$type_output" typer "$alias_target" else type "$x" fi done } findup(){ local arg="$1" current_dir="${PWD:-$(pwd)}" while [ "$current_dir" != "" ]; do if [ -e "$current_dir/$arg" ]; then echo "$current_dir/$arg" return 0 fi current_dir="${current_dir%/*}" done echo "Not found in above path: $arg" >&2 return 1 } cdup(){ local arg="$1" cd "$(findup "$arg")" } lld(){ { local target="$1" ls -ld "$target" [ "$target" = "/" ] && return lld "$(dirname "$target")" } | column -t } # shellcheck disable=SC2120 unquote(){ sed ' s/^[[:space:]]*[`'"'"'"]//; s/[`'"'"'"][[:space:]]*$//; ' "$@" } bell(){ echo -e '\a' } resolve_symlinks(){ local readlink=readlink if is_mac; then if type -P greadlink &>/dev/null; then readlink=greadlink else readlink="" fi fi if [ -z "$readlink" ]; then echo "$*" return fi for x in "$@"; do "$readlink" -m "$x" done } # for all files listed, return the highest directory - useful for pushd to the right git root following symlinks before doing git diff and commmits, used by gitu() in git.sh which is called in inline vimrc 'nmap ;;' basedir(){ local dir_list="" for x in "$@"; do dir_list="$dir_list $(dirname "$x")" done # assumes they share the same base and that the shortest one will be right - could put more comparison here and return error if not local output output="$(tr ' ' '\n' <<< "$dir_list" | grep -v '^[[:space:]]*$' | sort | head -n 1)" if [ -z "$output" ]; then echo "ERROR: empty basedir" return 1 fi echo "$output" } toLower(){ tr '[:upper:]' '[:lower:]' } toUpper(){ tr '[:lower:]' '[:upper:]' } trim(){ sed 's/^[[:space:]]*//; s/[[:space:]]*$//' "$@" } normalize_spaces(){ # not variant of + \+ works on Mac #sed 's/[[:space:]]\+/ /' # flattens out newlines, which changes behaviour in scripts like check_git_commit_authors.sh #perl -pe 's/\s+/ /g' # horizontal newlines, don't match \n perl -pe 's/\h+/ /g' } remove_last_column(){ awk '{$NF=""; print $0}' } strip_basedirs(){ local basedir="$1" shift while read -r filename; do filename="${filename#"${basedir%%/}"/}" filename="${filename##/}" echo "$filename" done <<< "$@" } user(){ read -r -p 'user: ' USERNAME export USERNAME if [ -z "${PASSWORD:-}" ]; then pass PASSWORD fi } pass(){ # doesn't echo, we can do better by making it star for each char #read -r -s -p 'password: ' PASSWORD # don't local PASSWORD or default case will not export PASSWORD, changing case to work around local password="" prompt="Enter password: " while IFS= read -p "$prompt" -r -s -n 1 char; do if [[ "$char" == $'\0' ]]; then break fi prompt='*' password="${password}${char}" done #passvar="${1:-PASSWORD}" for passvar in "${@:-PASSWORD}"; do export "$passvar"="$password" done echo } unpass(){ unset PASSWORD } hr(){ echo "# ============================================================================ #" } repeat(){ local i n n="$1" shift if [ -z "$n" ]; then echo "usage: repeat N command args" return 1 fi for ((i=1; i <= n; i++)); do "$@" done } loop(){ while true; do eval "${*//\$/\\$}" sleep 1 done } ptop(){ if [ -z "$1" ]; then echo "usage: ptop program1 program2 ..." return 1 fi local pids #pids="$(pgrep -f "$(sed 's/ /|/g' <<< "$*")")" pids="$(pgrep -f "${*// /|}")" local pid_args=() if is_mac; then # shellcheck disable=SC2001 read -r -a pid_args <<< "$(sed 's/^/-pid /' <<< "$pids")" else # shellcheck disable=SC2001 read -r -a pid_args <<< "$(sed 's/^/-p /' <<< "$pids")" fi if [ -z "${pids[*]}" ]; then echo "No matching programs found" return 1 fi top "${pid_args[@]}" } topcommands(){ # first awk print $2 but my advanced history records `date '+%F %T'` in between number and command for $2 and $3, making command $4 history | awk '{print $4}' | awk 'BEGIN {FS="|"} {print $1}' | sort | uniq -c | sort -n | tail -n "${1:-10}" | sort -nr } alias topcmds=topcommands # easy quick find recursing down current directory tree # #f(){ # [ -n "$*" ] || { echo "usage: f "; return 1; } # pattern="" # for x in $*; do # pattern+="*$x" # done # pattern+="*" # find -L . -iname "$pattern" #} # # shellcheck disable=SC2032 f(){ local grep="" # shellcheck disable=SC2013 for x in "${@//[^A-Za-z0-9_-]/.}"; do if [[ "$x" =~ [a-zA-Z0-9._-] ]]; then grep="$grep | grep -i --color=auto $x" fi done # times about the same #eval find -L . -type f -iname "\*$1\*" $grep eval find -L . -type f "$grep" } dgrep(){ local pattern="$*" # auto-exported in aliases.sh when iterating git repos # shellcheck disable=SC2154 ls "$docs/"*"${pattern// /}"* 2>/dev/null # shellcheck disable=SC2046,SC2033 grep -iER "$pattern" $(find ~/docs "$docs" -type f -maxdepth 1 2>/dev/null | grep -v '/\.') } diffl(){ diff "$@" | less } foreachfile(){ # not passing function f() # shellcheck disable=SC2033 find . -type f -maxdepth 1 | while read -r file; do [ ! -f "$file" ] && continue [ -b "$file" ] && continue [ -c "$file" ] && continue [ -d "$file" ] && continue [ -p "$file" ] && continue [ -S "$file" ] && continue [ -L "$file" ] && continue "$@" done } # vim which # vw() moved to vim.sh # file which fw(){ local path for x in "$@"; do path="$(which "$x")" if [ -z "$path" ]; then return 1 fi file "$path" echo # shellcheck disable=SC2086 ls -l $LS_OPTIONS "$path" done } cdwhich(){ local path local directory if [ $# -ne 1 ]; then echo "usage: cdwhich programname" return 1 fi path="$(which "$1")" if [ -z "$path" ]; then echo echo "$1 could not be found in \$PATH" return 1 fi directory="$(dirname "$path")" if [ -z "$directory" ]; then echo "cannot find directory for $path" return 2 fi echo "$directory" cd "$directory" || return 1 } whichall(){ local bin="$1" shift || : which -a "$bin" | while read -r bin; do echo -n "$bin: " "$bin" "$@" done } add_etc_host(){ local host_line="$*" # $sudo is set in .bashrc if needed # shellcheck disable=SC2154 $sudo grep -q "^$host_line" /etc/hosts || $sudo echo "$host_line" >> /etc/hosts } # vihosts() moved to vim.sh proxy(){ export proxy_host="${1:-${proxy_host:-localhost}}" export proxy_port="${2:-${proxy_port:-8080}}" export proxy_port_ssl="${3:-${proxy_port_ssl:-8443}}" export proxy_user="${4:-${proxy_user:-$USER}}" if [ -z "$proxy_password" ]; then read -r -s -p 'proxy password: ' proxy_password fi export http_proxy="http://$proxy_user:$proxy_password@$proxy_host:$proxy_port" export https_proxy="https://$proxy_user:$proxy_password@$proxy_host:$proxy_port_ssl" # MiniShift respects these next three export HTTP_PROXY="$http_proxy" export HTTPS_PROXY="$https_proxy" export NO_PROXY=".local,.localdomain,.intra,169.254.169.254" # works only on suffixes or IP addresses - ignore the EC2 Metadata API address export ftp_proxy="$http_proxy" # might need to replace protocol prefix here, would check, but who even uses ftp any more JAVA_NO_PROXY="$(sed 's/^/*/;s/,/|*/g' <<< "$NO_PROXY")" # strip the additions we just added off the end so that we don't end up with dups if running proxy more than once JAVA_OPTS="${JAVA_OPTS%%-Dhttp.proxyHost*}" export JAVA_OPTS="$JAVA_OPTS -Dhttp.proxyHost=$proxy_host -Dhttp.proxyPort=$proxy_port -Dhttp.proxyUser=$proxy_user -Dhttp.proxyPassword=$proxy_password -Dhttps.proxyHost=$proxy_host -Dhttps.proxyPort=$proxy_port_ssl -DnonProxyHosts='$JAVA_NO_PROXY'" export SBT_OPTS="$JAVA_OPTS" } readlink(){ if is_mac; then greadlink "$@" else command readlink "$@" fi } abspath(){ readlink --canonicalize-missing "$1" } #abspath(){ # if [ -z "$1" ]; then # echo "NO PATH GIVEN!" # return 1 # fi # # shellcheck disable=SC2001 # sed 's@^\./@'"$PWD"'/@; # s@^\([^\./]\)@'"$PWD"'/\1@; # s@^\.\./@'"${PWD%/*}"'/@; # s@/../@/@g; # s@/\./@/@g; # s@\(.*\/?\)\.\./?$@\1/@; # s@//@/@g; # s@/$@@;' <<< "$1" #} wcbash(){ # $github defined in aliases.sh # shellcheck disable=SC2154 wc ~/.bashrc \ ~/.bash_profile \ ~/.bash_logout \ ~/.alias* \ ~/.aliases* \ ~/.bashrc_dynamichosts \ "$github/bash-tools/.bashrc" \ "$github/bash-tools/.bash_profile" \ "$github/bash-tools/.bash.d/"*.sh 2>/dev/null } epoch2date(){ if is_mac; then date -r "$1" else date -d "@$1" fi } pdf(){ if ! [[ "$1" =~ .*.pdf$ ]]; then echo "'$1' does not end in .pdf!" return 1 fi if ! [ -f "$1" ]; then echo "file not found: $1" return 1 fi if is_mac; then open "$1" return $? fi for x in acroread evince xpdf; do if type -P "$x" &>/dev/null; then echo "opening with $x..." "$x" "$1" & return $? fi done echo "Error cannot find acroread, evince or xpdf in PATH." return 1 } currentScreenResolution(){ #xrandr | awk '/\*/ {print $1}' xdpyinfo | awk '/dimensions/ {print $2}' } yy(){ cal "$(date '+%Y')" } # ============================================================================ # timestamp(){ printf "%s" "$(date '+%F %T') $*" [ $# -gt 0 ] && printf '\n' } alias tstamp=timestamp timestampcmd(){ local output output="$("$@" 2>&1)" timestamp "$output" } alias tstampcmd=timestampcmd # ============================================================================ # bak(){ # TODO: switch this to a .backupstore folder for keeping this stuff instead for filename in "$@"; do [ -n "$filename" ] || { echo "usage: bak filename"; return 1; } [ -f "$filename" ] || { echo "file '$filename' does not exist"; return 1; } [[ $filename =~ .*\.bak\..* ]] && continue local bakfile bakfile="$filename.bak.$(date '+%F_%T' | sed 's/:/-/g')" until ! [ -f "$bakfile" ]; do echo "WARNING: bakfile '$bakfile' already exists, retrying with a new timestamp" sleep 1 bakfile="$filename.bak.$(date '+%F_%T' | sed 's/:/-/g')" done cp -av -- "$filename" "$bakfile" done } unbak(){ # restores the most recent backup of a file for filename in "$@"; do #[ -n "$filename" -o "${filename: -4}" != ".bak" ] || { echo "usage: unbak filename.bak"; return 1; } [ -n "$filename" ] || { echo "usage: unbak filename"; return 1; } #[ -f "$filename" ] || { echo "file '$filename' does not exist"; return 1; } local bakfile local dirname dirname="$(dirname "$filename")" filename="${filename##*/}" # don't use -t switch, we want the newest by name, not one that got touched recently bakfile="$(find "$dirname" -path "*/$filename.bak.*" -o -path "*/$filename.*.bak" 2>/dev/null | sort | tail -n 1)" echo "restoring $bakfile" cp -av -- "$bakfile" "$dirname/$filename" done } orig(){ if [ $# -lt 1 ]; then echo "usage: orig file1 file2 file3 ..." return 1 fi for filename in "$@"; do [ -f "$filename" ] || { echo "file '$filename' does not exist"; return 1; } [ -f "$filename.org" ] && { echo "$filename.orig already exists, aborting..."; return 1; } done for filename in "$@"; do cp -av -- "$filename" "$filename.orig" done } unorig(){ if [ $# -lt 1 ]; then echo "usage: unorig file1.orig file2.orig file3.orig ..." return 1 fi for filename in "$@"; do if [ -z "$filename" ] || [ "${filename: -5}" != ".orig" ]; then echo "usage: unorig filename.orig" return 1 fi if ! [ -f "$filename" ]; then echo "file '$filename' does not exist" return 1 fi done for filename in "$@"; do cp -av -- "$filename" "${filename%.orig}" done } # ============================================================================ # strLastIndexOf(){ local str="$1" local substr="$2" local remainder="${str##*"$substr"}" local lastIndex=$((${#str} - ${#remainder})) echo $lastIndex } # ============================================================================ # progs(){ # not passing function f() # shellcheck disable=SC2033 find "${@:-.}" -type f | grep -Evf ~/code_regex_exclude.txt | grep -v -e '/lib/' -e '.*-env.sh' -e '/tests/' } progs2(){ # not passing function f() # shellcheck disable=SC2033 find "${@:-.}" -type f -o -type l | grep -Evf ~/code_regex_exclude.txt } findpy(){ # not passing function f() # shellcheck disable=SC2033 find "${@:-.}" -type f -iname '*.py' -o -type f -iname '*.jy' | grep -vf ~/code_regex_exclude.txt } # ============================================================================ # ldapmaxuid(){ ldapsearch -x -W "uidNumber=*" uidNumber | sed 's/#.*//' | grep -v "^[[:space:]]*$" | grep uidNumber | sort -k2n | tail -n1 } ldapmaxuidgid(){ ldapsearch -xW -x -W "(|(objectClass=posixAccount)(objectClass=posixGroup))" uidNumber gidNumber | sed 's/#.*//' | grep --color=auto -v "^[[:space:]]*$" | grep -R --color=auto "(uidNumber|gidNumber)" | sort -k2n | tail -n1 } ================================================ FILE: .bash.d/gcp.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # shellcheck disable=SC1090,SC1091 # # Author: Hari Sekhon # Date: 2019-11-14 22:22:35 +0000 (Thu, 14 Nov 2019) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G C P - G o o g l e C l o u d P l a t f o r m # ============================================================================ # srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh" # adds GCloud CLI tools to $PATH if [ -f ~/google-cloud-sdk/path.bash.inc ]; then #source ~/google-cloud-sdk/path.bash.inc add_PATH ~/google-cloud-sdk/bin # appends rather than above which prepends to \$PATH, messing up kubectl version requirement for fluxcd fi # Bash completion for GCloud CLI tools if [ -f ~/google-cloud-sdk/completion.bash.inc ]; then source ~/google-cloud-sdk/completion.bash.inc fi export USE_GKE_GCLOUD_AUTH_PLUGIN=True # having to retype this way too much alias gal="gcloud auth login" # often bugs me to do this alias gcu="gcloud components update" alias gce="gcloud compute" alias gke="gcloud container clusters" alias gc="gcloud container" alias gbs="gcloud builds submit --tag" alias bqq="bq query" alias gssh="gcloud compute ssh" # open GCP Console in the current project and preferably on a relevant page if we can detect one gcpcon(){ local project local path # open in the compute instances page if we don't know where else to go path="compute/instances" if [[ "$PWD" =~ kubernetes|k8 ]]; then path="kubernetes/list/overview" elif [[ "$PWD" =~ iam ]]; then path="iam-admin/iam" fi project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config get core.project 2>/dev/null)}" open "https://console.cloud.google.com/$path?project=$project" } # when switching an alias to a function during re-source without un-aliasing, declare function explicitly to avoid errors function gcloudconfig(){ # configurations are usually called the same as the project name so export GOOGLE_PROJECT_ID for convenience too gcloud config configurations activate "$1" || return 1 export GOOGLE_PROJECT_ID="$1" } gsopen(){ local gspath="$1" gspath="${gspath#gs:\/\/}" browser "https://console.cloud.google.com/storage/browser/$gspath" } gcropen(){ local image="$1" if ! [[ "$image" =~ gcr\.io/ ]]; then echo "'$image' is not a GCR image name (requires gcr.io to know where to open)" return 1 fi image="${image%:*}" browser "https://$image" } ================================================ FILE: .bash.d/git.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # R e v i s i o n C o n t r o l - G i t # ============================================================================ # # Primary revision control system # # if svn.sh and hg.sh functions are enabled, detects and calls svn and mercurial commands if inside those repos so some of the same commands work dynamically bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" if [ -f ~/.github_token ]; then GITHUB_TOKEN="$(cat ~/.github_token)" export GITHUB_TOKEN fi if [ -f ~/.gitlab_token ]; then GITLAB_API_PRIVATE_TOKEN="$(cat ~/.gitlab_token)" export GITLAB_API_PRIVATE_TOKEN fi #if [ -z "${GITLAB_API_ENDPOINT:-}" ]; then # export GITLAB_API_ENDPOINT="https://gitlab.com/api/v4" #fi if ! type basedir &>/dev/null; then # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/functions.sh" fi if type -P gh &>/dev/null; then autocomplete gh -s fi # shellcheck disable=SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" #add_PATH ~/bin/codeql # find out who your 'gh' CLI is authenticating as - useful if you have multiple Personal Access Tokens for different environments alias githubwhoami='github_api.sh /user | jq -r .login' alias ghwhoami='github_api.sh /user | jq -r .login' # set location where you check out all the github repos export github=~/github export GIT_PAGER="less ${LESS:-}" # shellcheck disable=SC2230 #if [ -z "${GIT_PAGER:-}" ] && \ if type -P diff-so-fancy &>/dev/null; then # pre-loading a pattern to 'n' / 'N' / '?' / '/' search through will force you in to pager and disregard -F / --quit-if-one-screen #export GIT_PAGER="diff-so-fancy --color=yes | less -RFX --tabs=4 --pattern '^(Date|added|deleted|modified): '" export GIT_PAGER="diff-so-fancy --color=yes | $GIT_PAGER" fi alias gitconfig="\$EDITOR ~/.gitconfig" alias gitignore="\$EDITOR ~/.gitignore_global" alias gitrc=gitconfig # false positive, not calling this from xargs # shellcheck disable=SC2032 #alias add=gitadd add(){ gitadd "$@"; } addf(){ git add -f "$@" git ci -m "added $*" } alias gadd='git add' # beware covers up ImageMagick 'import' screenshot command (see HariSekhon/Knowledge-Base mac.md page) alias import=gitimport alias co=checkout alias commit="git commit" alias clone="git clone" alias cherry-pick="git cherry-pick" alias gitci=commit alias ci=commit alias gitco=checkout alias up=pull alias u=up alias uu="GIT_PULL_IN_BACKGROUND=1 u" alias pu=push alias gitp="git push" alias gdiff="git diff" # bypasses diff-so-fancy, could also just pipe through | cat to disable pager and color effects alias gdiff2="git --no-pager diff" alias gdiffc="git diff --cached" alias gdiffm="gdiff origin/master.." alias gd=gdiff alias gdc=gdiffc alias gdo=gdiffm alias branch="githg branch" alias br=branch alias fetch='git fetch' alias stash="git stash" alias tag="githg tag" alias tags='git tag' alias tagr='git_tag_release.sh' alias gitlogwc='git log --oneline | wc -l' alias um=git_submodules_update.sh #type browse &>/dev/null || alias browse=gbrowse alias gbrowse=gitbrowse alias gb='gitbrowse' alias prbrowse='gh pr view --web' # uses GitHub CLI alias prb=prbrowse #alias gh='gitbrowse github' # clashes with GitHub CLI alias gl='gitbrowse gitlab' alias bb='gitbrowse bitbucket' alias azd='gitbrowse azure' alias gha='gitbrowse github actions' alias ghw='github_workflows' alias wf='cd $(git_root)/.github/workflows/' alias ggrep="git grep" alias gfr='git_foreach_repo.sh' alias gfrur='git_foreach_repo_update_readme.sh' alias gfrrra='git_foreach_repo_replace_readme_actions.sh' alias remotes='git remote -v' alias remote='remotes' # much quicker to just 'cd $github; f ' #githubls(){ # # GitHub is svn compatible, use this to list files remotely # svn ls "https://github.com/$1.git/branches/master/" #} #githubgrep(){ # for repo in $(sed 's/#.*//;s/:.*//;/^[[:space:]]*$/d' "$srcdir/setup/repos.txt"); do # githubls "HariSekhon/$repo" # done | # grep "$@" #} # git fetch -p or git remote prune origin #alias prune="co master; git pull --no-edit; git remote prune origin; git branch --merged | grep -v -e '^\\*' -e 'master' | xargs git branch -d" removed_branches(){ git remote prune origin >&2 git branch -vv | cut -c 3- | awk '$4 ~ /gone\]/ {print $1}' } alias prune="removed_branches | xargs -r git branch -d" # don't use this unless you are a git pro and understand unwinding history and merge conflicts alias GRH="git reset HEAD^" alias gitnored='git status --ignored' alias master="switchbranch master" alias main="switchbranch main" alias prod="switchbranch prod" alias production="switchbranch production" alias staging="switchbranch staging" alias stage=staging alias dev="switchbranch dev" alias develop="switchbranch develop" alias grp="git_review_push.sh" # edit all GitHub READMEs alias readmes='$EDITOR $(git_foreach_repo.sh '"'"'echo $PWD/README.md'"')" alias readmesi='idea $(git_foreach_repo.sh '"'"'echo $PWD/README.md'"')" alias ureadmes='git_foreach_repo.sh '"'"'gitu README.md || :'"'" # equivalent of hg root # shellcheck disable=SC2120 git_root(){ local path="${1:-.}" local dir if [ -f "$path" ]; then dir="$(dirname "$path")" elif [ -d "$path" ]; then dir="$path" else echo "ERROR: arg passed is not a regular file or directory: $path" >&2 return 1 fi pushd "$dir" &>/dev/null || return 1 git rev-parse --show-toplevel popd &>/dev/null || return 1 } alias gitroot=git_root alias cdgitroot='cd "$(git_root)"' gitgc(){ cd "$(git_root)" || : if ! [ -d .git ]; then echo "not in a git repo, no .git/ directory and git root dir not found" return 1 fi du -sh .git git gc --aggressive du -sh .git } git_default_branch(){ git remote show origin | #awk '/^[[:space:]]*HEAD branch:[[:space:]]/{print $3}' sed -n '/HEAD branch/s/.*: //p' } git_url_base(){ local filter="${1:-.*}" git remote -v | { grep "$filter" || : ; }| awk '/git@|https:\/\/|ssh:\/\//{print $2}' | head -n1 | sed 's|^ssh://||; s|^https://.*@||; s|^https://||; s|:[[:digit:]][[:digit:]]*||; s/^git@ssh.dev.azure.com:v3/dev.azure.com/; s|^git@||; s|^|https://|; s/\.git$//; # Azure DevOps only puts this in https urls, not ssh, so strip for standardizing output # Update: actually dont do this because we cannot differentiate internal Azure DevOps # by internal fqdn urls so it is better to leave this in #s|/_git/|/|; ' | perl -pe 's/:(?!\/\/)/\//' } gitbrowse(){ local filter="${1:-origin}" local path="${2:-}" local url_base url_base="$(git_url_base "$filter")" if [ -z "$url_base" ] && [ "$filter" != origin ]; then url_base="$(git_url_base "origin")" fi if [ -z "$url_base" ]; then echo -n "git remote url not found for filter '$filter'" if [ "$filter" != "origin" ]; then echo " or 'origin'" else echo fi return 1 fi if [ -n "$path" ]; then path="$(git ls-files --full-name "$path")" fi if [[ "$url_base" =~ github.com ]]; then if [ -z "$path" ]; then path+="#readme" fi else if [ -z "$path" ]; then local default_branch default_branch="$(git_default_branch)" fi if [[ "$url_base" =~ gitlab.com ]]; then if [ -z "$path" ]; then url_base+="/-/blob/$default_branch/README.md" fi elif [[ "$url_base" =~ dev.azure.com ]]; then # don't re-add this as it's no longer stripped out in git_url_base # because we cannot differentiate internal Azure Devops by fqdn so need to leave it in #url_base="${url_base%/*}/_git/${url_base##*/}" if [ -z "$path" ]; then url_base+="?path=/README.md&_a=preview" fi elif [[ "$url_base" =~ bitbucket.org ]]; then if [ -z "$path" ]; then url_base+="/src/$default_branch/README.md" fi fi fi url="$url_base" if [ -n "$path" ]; then if [[ "$url_base" =~ github.com ]]; then local default_branch default_branch="$(git_default_branch)" url+="/blob/$default_branch" fi url+="/$path" fi browser "$url" } install_git_completion(){ if ! [ -f ~/.git-completion.bash ]; then wget -O ~/.git-completion.bash https://raw.githubusercontent.com/git/git/master/contrib/completion/git-completion.bash fi } # shellcheck disable=SC1090,SC1091 [ -f ~/.git-completion.bash ] && . ~/.git-completion.bash # usage: gi python,perl,go # gi list gitignore_api(){ local url local langs local options=() local args=() # noop - set to use 'tr' to separate items to newlines when given the 'list' arg local commas_to_newlines="cat" for arg; do if [ "$arg" = -- ]; then options+=("$arg") else args+=("$arg") fi done # take args 'python perl', store as 'python,perl' for the API call langs="$(IFS=, ; echo "${args[*]}")" url="https://www.gitignore.io/api/$langs" if [ "$langs" = "list" ]; then commas_to_newlines="tr ',' '\\n'" fi { if hash curl 2>/dev/null; then curl -sSL "${options[*]}" "$url" elif hash wget 2>/dev/null; then wget -O - "${options[*]}" "$url" fi } | eval "$commas_to_newlines" echo } alias gi=gitignore_api git_user_repo(){ git remote -v | awk '{print $2}' | head -n1 | git_repo_strip } git_repo(){ git_user_repo | sed 's|.*/||' } github_owner_repo(){ git remote -v | awk '/github.com/{print $2}' | head -n1 | git_repo_strip } github_user_repo(){ github_owner_repo } github_repo(){ github_user_repo | sed 's|.*/||' } git_repo_strip(){ git_repo_strip_auth | sed 's|.*\.[^:/]*[:/]||; s/\.git[[:space:]]*$//' } git_repo_strip_auth(){ sed 's/[[:alnum:]]*@//' } isGit(){ local target=${1:-.} # There aren't local .hg dirs everywhere only at top level so this is difficult in bash if [ -d "$target/.git" ]; then return 0 elif [ -f "$target" ] && [ -d "${target%/*}/.git" ]; then #-o "$target/../.git" -o "${target%/*}/../.git" ]; then return 0 else # This is because git command doesn't return correctly when running from outside git root, complains there is not .git if [ -d "$target" ]; then pushd "$target" >/dev/null || return 1 #if [ -n "$(git log -1 . 2>/dev/null)" ]; then # better because it will succeed in subdirectories of git repos which are not checked in yet if git status &>/dev/null; then # shellcheck disable=SC2164 popd &>/dev/null return 0 fi else pushd "$(dirname "$target")" >/dev/null || return 1 #if git log -1 "$target" 2>/dev/null | grep -q '.*'; then #if [ -n "$(git log -1 "$(basename "$target")" 2>/dev/null)" ]; then # better because it will succeed in subdirectories of git repos which are not checked in yet if git status &>/dev/null; then # shellcheck disable=SC2164 popd &>/dev/null return 0 fi fi # shellcheck disable=SC2164 popd &>/dev/null return 2 fi } git_revision(){ echo "Revision: $(git rev-parse HEAD)" } st(){ # shellcheck disable=SC2086 { local target="${1:-.}" shift if ! [ -e "$target" ]; then echo "$target does not exist" return 1 fi local target_basename local target_dirname target_basename="$(basename "$target")" target_dirname="$(dirname "$target")" #if [ -f "Vagrantfile" ]; then # echo "> vagrant status" # vagrant status # shellcheck disable=SC2166 if [ "$target" = "." ] && [ "$PWD" = "$HOME/github" ]; then hr for x in "$target"/*; do [ -d "$x" ] || continue pushd "$x" >/dev/null || { echo "failed to pushd to '$x'"; return 1; } if git remote -v | grep -qi harisekhon; then echo "> GitHub: git status $x $*" git status . "$@" echo hr echo fi # shellcheck disable=SC2164 popd &>/dev/null done elif { [ "$target" = "." ] && [ "${PWD##*/}" = work ] ; } || grep -Fxq "$PWD" <<< "${GIT_BASEDIRS:-}" || [ -f .iterate ]; then #ls ./*/.git &>/dev/null; then # matches inside repos with submodules unfortunately hr for x in "$target"/*; do [ -d "$x" ] || continue pushd "$x" >/dev/null || { echo "failed to pushd to '$x'"; return 1; } echo "> Work: git status $x $*" git status . "$@" echo hr echo # shellcheck disable=SC2164 popd &>/dev/null done elif isGit "$target"; then if [ -d "$target" ]; then pushd "$target" >/dev/null || { echo "Error: failed to pushd to $target"; return 1; } echo "> git stash list" >&2 git stash list && echo #"$bash_tools/git/git_summary_line.sh" echo "> git status $target $*" >&2 #git -c color.status=always status -sb . "$@" git -c color.status=always status . "$@" echo git_revision echo else pushd "$target_dirname" >/dev/null || { echo "Error: failed to pushed to '$target_dirname'"; return 1; } echo "> git status $target $*" >&2 #"$bash_tools/git/git_summary_line.sh" git -c color.status=always status "$target_basename" "$@" fi #git status "$target" "${*:2}" # shellcheck disable=SC2164 popd &>/dev/null elif type isHg &>/dev/null && isHg "$target"; then echo "> hg status $target $*" >&2 hg status "$target" "$@" | grep -v "^?" # to see relative paths instead of the default absolute paths #hg status "$(hg root)" elif type isSvn &>/dev/null && isSvn "$target"; then echo "> svn st $*" >&2 svn st --ignore-externals "$target" "$@" | grep -v -e "^?" -e "^x"; else echo "not a revision controlled resource as far as bashrc can tell" fi } | # more calls less on Mac, and gets stuck in interactive mode ignoring the less alias switches #more -R -n "$((LINES - 3))" #less -RFX eval ${GIT_PAGER:-cat} } stq(){ st "$@" | grep --color=no -e "=======" -e branch -e GitHub | eval "${GIT_PAGER:-cat}" } # disabling this as I don't use Mercurial or Svn any more, # replacing with simpler function below that will pass through more things like --rebase #pull(){ # local target="${1:-.}" # if ! [ -e "$target" ]; then # echo "$target does not exist" # return 1 # fi # local target_basename # target_basename="$(basename "$target")" # # shellcheck disable=SC2166 # if [ "$target_basename" = "github" ] || [ "$target" = "." -a "$(pwd)" = "$github" ]; then # for x in "$target"/*; do # [ -d "$x" ] || continue # # get last character of string # [ "${x: -1}" = 2 ] && continue # pushd "$x" >/dev/null || { echo "failed to pushd to '$x'"; return 1; } # if git remote -v | grep -qi harisekhon; then # echo "> GitHub: git pull $x ${*:2}" # git pull "${@:2}" # echo # echo "> GitHub: git submodule update --init --recursive" # git submodule update --init --recursive # echo # fi # # shellcheck disable=SC2164 # popd &>/dev/null # done # return # elif isGit "$target"; then # pushd "$target" >/dev/null && # echo "> git pull -v ${*:2}" >&2 # git pull -v "${@:2}" # echo "> git submodule update --init --recursive" # git submodule update --init --recursive # #local orig_branch=$(git branch | awk '/^\*/ {print $2}') # #for branch in $(git branch | cut -c 3- ); do # # git checkout -q "$branch" && # # echo -n "$branch => " && # # git pull -v # # echo # # echo # #done # #git checkout -q "$orig_branch" # # shellcheck disable=SC2164 # popd &>/dev/null # elif type isHg &>/dev/null && isHg "$target"; then # pushd "$target" >/dev/null && # echo "> hg pull && hg up" >&2 && # hg pull && hg up # # shellcheck disable=SC2164 # popd &>/dev/null # elif type isSvn &>/dev/null && isSvn "$target"; then # echo "> svn up $target" >&2 # svn up "$target" # else # echo "not a revision controlled resource as far as bashrc can tell" # return 1 # fi #} # simpler replacement function to above # shellcheck disable=SC2120 pull(){ # shellcheck disable=SC2166 if [ "$PWD" = "$HOME/github" ]; then for x in *; do [ -d "$x/.git" ] || continue # get last character of string - don't pull blah2, as I use them as clean checkouts [ "${x: -1}" = 2 ] && continue pushd "$x" >/dev/null || { echo "failed to pushd to '$x'"; return 1; } if git remote -v | grep -qi "${GITHUB_USER:-${GIT_USER:-${USER:-}}}"; then hr echo "> GitHub $x: git pull --no-edit $*" #echo "> GitHub $x: git submodule update --init --recursive" if [ -n "${GIT_PULL_IN_BACKGROUND:-}" ]; then git_pull "$@" & else git_pull "$@" fi fi # shellcheck disable=SC2164 popd &>/dev/null done elif [ "${PWD##*/}" = work ] || grep -Fxq "$PWD" <<< "${GIT_BASEDIRS:-}" || [ -f .iterate ]; then #ls ./*/.git &>/dev/null; then # matches inside repos with submodules unfortunately for x in *; do [ -d "$x/.git" ] || continue hr pushd "$x" >/dev/null || { echo "failed to pushd to '$x'"; return 1; } echo "> Work $x: git pull --no-edit $*" #echo "> work $x: git submodule update --init --recursive" if [ -n "${GIT_PULL_IN_BACKGROUND:-}" ]; then git_pull "$@" & else git_pull "$@" fi # shellcheck disable=SC2164 popd &>/dev/null done else echo "> git pull --no-edit $*" echo "> git submodule update --init --recursive" git_pull "$@" fi } git_pull(){ echo git pull --no-edit "$@" echo git submodule update --init --recursive echo } alias coj="git_branch_jira_ticket" git_branch_jira_ticket(){ local ticket="$1" local branch="${ticket##*/}" if git branch | sed 's/^..//' | grep -Fxq "$branch"; then git checkout "$branch" else git checkout -b "$branch" fi } checkout(){ if isGit "."; then git checkout "$@"; else echo "not a Git checkout, cannot switch to branch $*" return 1 fi } _gitaddimport() { local action="$1" shift; [ -z "$*" ] && return 1 local basedir local trap_codes="INT ERR" for filename in "$@"; do basedir="$(basedir "$filename")" # shellcheck disable=SC2064,SC2086 trap "popd &>/dev/null; trap - $trap_codes; return 1 2>/dev/null" $trap_codes; pushd "$basedir" > /dev/null || return 1; # shellcheck disable=SC2086 filename="$(strip_basedirs "$basedir" "$filename")"; if ! [ -e "$filename" ]; then echo "ERROR: $filename does not exist" >&2 #return 1 elif git status -s "$filename" | grep -q '^[?A]'; then git add "$filename" && git commit -m "$action $filename" "$filename" elif git status -s "$filename" | grep -q '^.M'; then echo "ERROR: '$filename' already in git, but has changes, commit as an update instead" >&2 #return 1 elif git status --ignored -s "$filename" | grep -q '^!!'; then echo "ERROR: '$filename' is ignored!!! => $(git check-ignore -v "$filename")" >&2 #return 1 else echo "ERROR: '$filename' already in git" >&2 #return 1 fi popd > /dev/null || return 1 done } gitadd(){ _gitaddimport added "$@" } gitimport(){ _gitaddimport imported "$@" } gitu(){ git_diff_commit.sh "$@" } gituu(){ # avoiding xargs due to function reference: # gxargs: gitu: No such file or directory eval gitu "$( git status --porcelain -s . | grep -e '^M' -e '^.M' | sed 's/^...//' | while read -r filename; do echo "\"$filename\"" done )" } #githgu(){ # target="${1:-.}" # #count=0 # while [ -L "$target" ]; do # #target="$(readlink "$target")" # #let count+=1 # #if [ $count -gt 10 ]; then # # echo "looping over links more than 10 times in hggitu! " # # exit 2 # #fi # echo "$target is a symlink! " # return 1 # done # if ! [ -e "$target" ]; then # echo "$target does not exist" # return 1 # fi # if isGit "$target"; then # echo "> git" >&2 # #if [ -d "$target" ]; then # # pushd "$target" >/dev/null # #else # # pushd "$(dirname "$target")" >/dev/null # #fi # #"$srcdir2/gitu" "${target##*/}" && # gitu "$target" # #popd &>/dev/null # elif type isHg &>/dev/null && isHg "$target"; then # echo "> hg" >&2 # #if [ -d "$target" ]; then # # pushd "$target" >/dev/null # #else # # pushd "$(dirname "$target")" >/dev/null # #fi # #"$srcdir2/hgu" "${target##*/}" && # hgu "$target" # #popd &>/dev/null # # Not supporting SVN any more # #elif type isSvn &>/dev/null && isSvn "$target"; then # # echo "> svn" >&2 # # svnu "$target" # else # echo "not a revision controlled resource as far as bashrc can tell" # return 1 # fi #} push(){ # shellcheck disable=SC2119 pull || return 1 if isGit .; then echo "> git push $*" #for remote in $(git remote); do # git push -v $remote $@ #done # can't be sure where we're pushing without parsing the command args, so omit for now if [ $# -eq 0 ]; then echo "pushing to:" # uniq_ordered.pl from my DevOps-Perl-tools repo or # uniq2 from my DevOps-Golang-tools repo would be better here # but not sure I want to create a dependency on that # unix's standard uniq unfortnately will only deduplicate adjacent lines but should be good enough in most cases git remote -v | awk '/^origin/{print $1"\t"$2}' | sed 's,://.*@,://,' | uniq echo fi # exposes your Github / GitLab / Bitbucket tokens on the screen, not secure, use printing above instead #git push -v "$@" git push "$@" echo st elif type isHg &>/dev/null && isHg .; then echo "> hg push $*" hg push "$@" else echo "not in a Git or Mercurial controlled directory" return 1 fi } unalias pushu 2>/dev/null || : pushu(){ if git remote -v | grep -qi '^origin[[:space:]].*gitlab\.'; then "$bash_tools/gitlab/gitlab_push_mr_preview.sh" else "$bash_tools/github/github_push_pr_preview.sh" fi } unalias pushup 2>/dev/null || : pushup(){ if git remote -v | grep -qi '^origin[[:space:]].*gitlab\.'; then "$bash_tools/gitlab/gitlab_push_mr.sh" else "$bash_tools/github/github_push_pr.sh" fi } unalias pushupmerge 2>/dev/null || : pushupmerge(){ if git remote -v | grep -qi '^origin[[:space:]].*gitlab\.'; then GITLAB_MERGE_PULL_REQUEST=true \ "$bash_tools/gitlab/gitlab_push_mr.sh" else GITHUB_MERGE_PULL_REQUEST=true \ "$bash_tools/github/github_push_pr.sh" fi } alias pushupm=pushupmerge pushr(){ for remote in $(git remote); do echo "> git push \"$remote\"" git push "$remote" echo done } alias pr=github_pull_request_create.sh alias mup=masterupdateprune masterupdateprune(){ #local master_branch="master" #if git branch | sed 's/^..//' | grep -Fx main; then # master_branch="main" #fi git checkout "$(git_default_branch)" pull prune git_revision echo } current_branch(){ git rev-parse --abbrev-ref HEAD } alias currentbranch=current_branch switchbranch(){ if isGit "."; then git checkout "$1"; elif type isHg &>/dev/null && isHg "."; then hg update "$1" else echo "not a Git / Mercurial checkout, cannot switch to branch $1" return 1 fi } gitrm(){ git rm -- "$@" && git commit -m "removed $*" -- "$@" } gitrename(){ if [ $# -ne 2 ]; then echo "usage: gitrename " return 1 fi if [ -f "$2" ]; then local file_already_exists=1 mv -iv -- "$2" "$2.tmp" fi git mv -- "$1" "$2" && git commit -m "renamed $1 to $2" "$1" "$2" if [ "${file_already_exists:-}" = 1 ]; then mv -fv -- "$2.tmp" "$2" fi } gitmv(){ if [ $# -ne 2 ]; then echo "usage: gitmv " return 1 fi if [ -f "$2" ]; then local file_already_exists=1 mv -iv -- "$2" "$2.tmp" fi git mv -- "$1" "$2" && git commit -m "moved $1 to $2" "$1" "$2" if [ "${file_already_exists:-}" = 1 ]; then mv -fv -- "$2.tmp" "$2" fi } gitd(){ git diff "${@:-.}" } gitadded(){ git log --name-status "$@" | grep -e '^A[^u]' -e '^Date' | grep -B 1 '^A' | less } # doesn't need pipe | less, git drops you in to less anyway gitl(){ git log --all --graph --decorate --name-status "$@" } gitlp(){ git log -p "$@" } alias gitlp1="gitlp -1" gitl2(){ git log --all --graph --decorate --stat "$@" } gitl3(){ git log --pretty=format:"%n%an => %ar%n%s" --name-status "$@" } githg(){ if isGit .; then git "$@" elif type isHg &>/dev/null && isHg .; then hg "$@" else echo "not a Git/Mercurial checkout" return 1 fi } retag(){ local tag1="$1" local checksum="$2" local additional_tags="${*:2}" for tag in $tag1 $additional_tags; do git tag -d "$tag" || : echo "Creating git tag '$tag'" # quoting checksum causes failure with unrecognized checksum '' git tag "$tag" "$checksum" git tag | grep -qF "$tag" || echo "FAILED" done } gitfind(){ local refids refids="$(git log --all --oneline | grep "$@" | awk '{print $1}')" printf 'Branches:\n\n' for refid in $refids; do git branch --contains "$refid" done | sort -u printf '\nTags:\n\n' for refid in $refids; do git tag --contains "$refid" done | sort -u } #stagemerge(){ # if isGit "."; then # git checkout prod && git pull && # git checkout staging && git pull && # git merge prod # git checkout prod # else # echo "Not a Git working copy"; # fi #} gitdiff(){ local filename="${1:-}" [ -n "$filename" ] || { echo "usage: gitdiff filename"; return 1; } git diff "$filename" > "/tmp/gitdiff.tmp" diffnet.pl "/tmp/hgdiff.tmp" } git_author_names(){ git log --all --pretty=format:"%an" | sort | uniq -c | sort -k1nr | less } git_author_emails(){ git log --all --pretty=format:"%ae" | sort | uniq -c | sort -k1nr | less } git_author_names_emails(){ git log --all --pretty=format:"%an %ae" | sort | uniq -c | sort -k1nr | less } git_authors(){ git_author_emails } git_commit_count(){ # interestingly, even on 10,000 commit repos, there are no duplicate short hashes shown from: # git log --all --pretty=format:"%h" | sort | uniq -d git log --all --pretty=format:"%h" | wc -l } git_revert_typechange(){ # want splitting to separate filenames # shellcheck disable=SC2046 co $(git status --porcelain -s "${1:-.}" | awk '/^.T/{print $2}') } git_rm_untracked(){ if [ $# -lt 1 ]; then echo "usage: rm_untracked " return 1 fi # iterate on explicit targets only # intentionally not including current directory to avoid accidentally wiping out untracked files - you must specify "rm_untracked ." if you really intend this for x in "${@:-}"; do git status --porcelain -s --untracked-files=all "$x" | # this breaks the correct spacings for Spotify playlist filenames #awk '/^\?\?/{$1=""; print}' | grep '^??' | sed 's/^?? //' | while read -r filename; do # git status --porcelain double quotes file paths when containing unicode chars which are representated in \xxx format # you must set 'git config --global core.quotePath false' for this to work properly # # this doesn't help because you are still stuck with \xxx chars throughout filename="${filename#\"}" filename="${filename%\"}" rm -v -- "$filename" || break done done } # example of usage of this in the function below - make sure to put '$repo' or "\$repo" somewhere in the argument body to make use of the iteration variable foreachrepo(){ local repolist="${REPOLIST:-$bash_tools/setup/repos.txt}" while read -r repo; do "$@" done < <(sed 's/#.*$//; s/.*://; /^[[:space:]]*$/d' "$repolist") } github_authors(){ # deferring expansion into loop # shellcheck disable=SC2016 foreachrepo 'echo "repo: $repo"; pushd "$github/$repo" >/dev/null || return 1; git_authors; popd >/dev/null || return 1; echo' | ${less:-less} } merge_conflicting_files(){ # merge conflicts: # # UU = both updated # AA = both added # git status --porcelain | awk '/^UU|^AA/{$1=""; print}' } merge_deleted_files(){ git status --porcelain | awk '/^DU/{$1=""; print}' } # useful for Dockerfiles merging lots of branches # # while ! make mergemasterpull; do fixmerge "merged master"; done # fixmerge(){ local msg="${*:-merged}" local merge_conflicted_files local merge_deleted_files merge_deleted_files="$(merge_deleted_files)" if [ -n "$merge_deleted_files" ]; then # false positive, not passing add function/alias add to git # shellcheck disable=SC2033 xargs git add <<< "$merge_deleted_files" fi merge_conflicted_files="$(merge_conflicting_files)" if [ -n "$merge_conflicted_files" ]; then # shellcheck disable=SC2086 "$EDITOR" $merge_conflicted_files && git add $merge_conflicted_files fi git ci -m "$msg" } buildkite_browse(){ if [ -z "${BUILDKITE_ORGANIZATION:-}" ]; then echo "\$BUILDKITE_ORGANIZATION not set" return 1 fi local repo repo="$(git_repo | tr '[:upper:]' '[:lower:]')" browser "https://buildkite.com/$BUILDKITE_ORGANIZATION/$repo" } # bk is used by buildkite cli now alias bkb=buildkite_browse ================================================ FILE: .bash.d/golang.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2015 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G o l a n g # ============================================================================ # # Golang bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # unreliable that HOME is set, ensure shell evaluates to the right thing before we use it [ -n "${HOME:-}" ] || HOME=~ github="${github:-$HOME/github}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/os_detection.sh" #export GOPATH="$github/go-tools" export GOPATH="$HOME/go" alias gopath='cd "$GOPATH"' alias gogo='gopath' alias cdgo='gopath' alias gosrc='cd "$GOPATH/src"' alias gobin='cd "$GOPATH/bin"' alias go-tools='cd "$github/go-tools"; export GOPATH="$github/go-tools"' alias gtools=go-tools alias gt=gtools # already added in paths.sh GitHub section #add_PATH "$github/go-tools" add_PATH "$github/go-tools/bin" if [ -d ~/go/bin ]; then add_PATH ~/go/bin fi # manual installation of 1.5 mismatches with HomeBrew 1.6 installed to $PATH and #export GOROOT="/usr/local/go" # causes: # imports runtime/internal/sys: cannot find package "runtime/internal/sys" in any of: # /usr/local/go/src/runtime/internal/sys (from $GOROOT) # /Users/hari/github/go-tools/src/runtime/internal/sys (from $GOPATH) if type -P go &>/dev/null; then if is_mac; then GOROOT="$(dirname "$(dirname "$(greadlink -f "$(type -P go)")")")" else GOROOT="$(dirname "$(dirname "$(readlink -f "$(type -P go)")")")" fi export GOROOT add_PATH "$GOROOT/bin" add_PATH "$GOROOT/libexec/bin" add_PATH "$GOPATH/bin" fi if type -P colorgo &>/dev/null; then alias go=colorgo fi alias lsgobin='ls -d ~/go/bin/* "$GOROOT"/{bin,libexec/bin}/* "$GOPATH/bin/"* 2>/dev/null' alias llgobin='ls -ld ~/go/bin/* "$GOROOT"/{bin,libexec/bin}/* "$GOPATH/bin/"* 2>/dev/null' ================================================ FILE: .bash.d/gpg-agent.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G P G A g e n t # ============================================================================ # # Pinentry is important, gpg-agent won't work without it. # pinentry intercepts and stores passphrase. gpg_agent(){ if [ $UID != 0 ]; then if type -P gpg-agent &>/dev/null; then # looks like gpg-agent not longer outputs the pid to stdout to capture #GPG_ENV_FILE=~/.gpg-agent.env #if [ -f "$GPG_ENV_FILE" ]; then # shellcheck disable=SC1090,SC1091 #. "$GPG_ENV_FILE" > /dev/null #GPG_AGENT_PID="${GPG_AGENT_INFO#*:}" #GPG_AGENT_PID="${GPG_AGENT_PID%:*}" #if ! kill -0 "$GPG_AGENT_PID" > /dev/null 2>&1; then # echo "Stale gpg-agent found. Spawning new agent..." # killall -9 gpg-agent # eval "$(gpg-agent --daemon | tee "$GPG_ENV_FILE")" #elif [ "$(ps -p "$GPG_AGENT_PID" -o comm=)" != "gpg-agent" ]; then # echo "gpg-agent PID does not belong to gpg-agent, spawning new agent..." # eval "$(gpg-agent --daemon | tee "$GPG_ENV_FILE")" #fi if type -P pgrep &>/dev/null; then if ! pgrep -qf gpg-agent.*--daemon; then echo "Starting gpg-agent..." killall -9 gpg-agent #eval "$(gpg-agent --daemon | tee "$GPG_ENV_FILE")" gpg-agent --daemon fi fi #clear fi fi } # don't really use this any more anyway so don't bother starting it #gpg_agent ================================================ FILE: .bash.d/grype.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-01-10 17:58:03 +0000 (Mon, 10 Jan 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G r y p e # ============================================================================ # #set -euo pipefail [ -n "${DEBUG:-}" ] && set -x #eval "$(grype completion bash)" # generates auto-completion file to avoid repeating running auto-completion command, and sources from there #autocomplete grype ================================================ FILE: .bash.d/hadoop.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2009+ (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # H a d o o p E c o s y s t e m # ============================================================================ # #srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh" # ============================================================================ # # E n v s # ============================================================================ # ## ln -s -- /usr/local/hadoop-x.y.z /usr/local/hadoop ## ln -s -- /usr/local/hbase-x.y.z /usr/local/hadoop ## ln -s -- /usr/local/zookeeper-x.y.z /usr/local/zookeeper # # #find /usr/local -type d -name 'hadoop-*' -o -type d -name 'hbase-*' -o -type d -name 'zookeeper-*' -maxdepth 1 | while read path; do sudo ln -vfsh "$path" "${path%%-*}"; done # link_latest '/usr/local/hadoop-*' '/usr/local/hbase-*' '/usr/local/pig-*' '/usr/local/zookeeper-*' # chown -R hari -- /usr/local/{hadoop,hbase,zookeeper} # re-enabled HADOOP_HOME for Kite SDK #export HADOOP_HOME="/usr/local/hadoop" # Deprecated. Annoying error msgs #export HADOOP_PREFIX="/usr/local/hadoop" # Hate this ## For OSX #export HADOOP_OPTS="$HADOOP_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc=" #export HBASE_OPTS=" $HBASE_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc=" #export HBASE_HOME=/usr/local/hbase #export PIG_HOME=/usr/local/pig #export ZOOKEEPER_HOME=/usr/local/zookeeper #add_PATH "$HADOOP_PREFIX/bin" #add_PATH "$HBASE_HOME/bin" #add_PATH "$PIG_HOME/bin" #add_PATH "$ZOOKEEPER_HOME/bin" #export MAHOUT_HOME=/usr/local/mahout ## indicates to run locally instead of on Hadoop #export MAHOUT_LOCAL=true #add_PATH "$MAHOUT_HOME/bin" # ============================================================================ # # C L I # ============================================================================ # # Hadoop CLI usability is weak so some conveniences for day to day alias dfs='hdfs dfs' alias dfsls='hdfs dfs -ls' alias yarnapp='yarn application' alias impala='impala_shell.sh' # nobody should use hive 1 cli any more, remap it to HS2 beeline alias hive='beeline.sh' alias hivezk='beeline_zk.sh' ================================================ FILE: .bash.d/intellij.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # # Author: Hari Sekhon # Date: 2023-07-26 00:10:06 +0100 # # vim:ts=4:sts=4:sw=4:et # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # I n t e l l i J # ============================================================================ # # requires git.sh for git_root() function # so that you can open files in IntelliJ from the command line on Mac like so: # # idea # if [ -d "/Applications/IntelliJ IDEA CE.app/Contents/MacOS" ]; then add_PATH "/Applications/IntelliJ IDEA CE.app/Contents/MacOS" fi alias i='idea' # wrote find_lock.sh to try to find if IntelliJ uses a lock file in the git project # whether it's open or not but it found not file existence changes at all # # after various experimentation, cannot find a reliable indicator via either process or lockfile # or any .idea/ file change if IntelliJ has a project dir open or not, surprising # #is_intellij_project_open(){ # local path="$1" # if [ -f "$path" ]; then # path="$(dirname "$path")" # fi # local git_root # git_root="$(git_root "$path")" # if [ -z "$git_root" ]; then # echo "Given path is not in a git checkout!!" >&2 # return 1 # fi # local pid # pid="$(lsof -n -c java | awk "/${git_root//\//\\/}/ {print \$2}" | sort -u)" # if [ -z "$pid" ]; then # return 1 # fi # if ps -ef | awk "\$2 == $pid { print }" | grep -q IntelliJ; then # return 0 # fi # return 1 #} # #open_intellij_project_if_not_already(){ # local path="$1" # local dir # if [ -e "$path" ]; then # if ! is_intellij_project_open "$path"; then # if [ -f "$path" ]; then # dir="$(dirname "$path")" # else # dir="$path" # fi # idea_bg_disown "$dir" # sleep 1 # fi # fi #} idea_bg_disown(){ nohup command idea "$@" & # disowns the first backgrounded command instead of the latest command, # so use $! to specify the pid of the latest command in this shell disown $! } # so that you can quickly open files without them holding your terminal open and spewing Java logs all over your screen idea(){ local dir for arg in "$@"; do # because otherwise README Markdown Preview will not render images with relative paths to images inside project: # # https://github.com/HariSekhon/Knowledge-Base/blob/69bb8d4220596e90e6c0e61c48dd8e1b9ffdf720/intellij.md#markdown-images-with-relative-paths-not-displaying-in-preview # # can't find any reliable method for this function, see comment just above function itself for more details #open_intellij_project_if_not_already "$arg" # XXX: caveat here - in order to not eat CLI args, we open all args after this loop, # which means multiple markdown files in one command will open in the last project # This will still result in broken markdown preview for any markdown files that are outside # the last project directory which will be ithe foreground window if [[ "$arg" =~ \.md$ ]]; then dir="$(git_root "$arg" || :)" if [ -n "$dir" ]; then idea_bg_disown "$dir" # give time to settle otherwise race condition of immediate idea_bg_own() call will open the file in the other existing project sleep 1 fi fi done idea_bg_disown "$@" } idearoot(){ idea "$(git_root)" } alias idear=idearoot # if a file does not already exist then IntelliJ opens it in a new light IDE instead of in the current project touch_idea(){ touch "$@" idea "$@" } alias tidea="touch_idea" ================================================ FILE: .bash.d/java.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Sun Sep 9 21:20:49 2012 +0100 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # J a v a # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" add_PATH CLASSPATH ~/bin/java # turn off those annoying Java 11 warnings when using Groovy scripting export GROOVY_TURN_OFF_JAVA_WARNINGS=true #alias rmclass='rm -fv *.class' alias rmclass='find . -type f -name "*.class" -exec rm -fv {} \;' if is_mac; then mac_export_java_home(){ local version="$1" local args=() local java_home local java_library_base="/Library/Java/JavaVirtualMachines" local java_home_variable="JAVA_HOME" # for cross compiling to be found by gradle build if [ -n "$version" ]; then args+=(-v "1.$version") java_home_variable="JAVA${version}_HOME" fi if [ -x /usr/libexec/java_home ]; then # want arg splutting java_home="$(/usr/libexec/java_home "${args[@]}" 2>/dev/null)" # $? is fine here thanks shellcheck # shellcheck disable=SC2181 if [ $? -eq 0 ] && [ -d "$java_home" ]; then export "$java_home_variable"="$java_home" if [ -n "$DEBUG" ]; then echo "Determined $java_home_variable from /usr/libexec/java_home to be '$java_home', update ~/.bashrc to optimize by setting this explicitly" >&2 fi fi else ## java_home=/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home ## JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Home ## JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/Current/Home ## JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home java_home="$(find "$java_library_base/"*1."$version"* -type d -name 'Home*' 2>/dev/null | tail -n1)" if [ -d "$java_home" ]; then export "$java_home_variable"="$java_home" if [ -n "$DEBUG" ]; then echo "Determined $java_home_variable from searching $java_library_base to be '$java_home', update ~/.bashrc to optimize by setting this explicitly" >&2 fi fi fi } if [ -z "$JAVA_HOME" ]; then mac_export_java_home mac_export_java_home 7 fi elif is_linux; then if [ -z "$JAVA_HOME" ]; then # RHEL / CentOS if type -P alternatives &>/dev/null; then java_home="$(alternatives --list | awk '/^java[[:space:]]/{print $3; exit}' | sed 's,\(/jre\)\?/bin/java$,,')" if [ -n "$java_home" ]; then export JAVA_HOME="$java_home" fi # Debian / Ubuntu elif type -P update-alternatives &>/dev/null; then java_home="$(update-alternatives --list java 2>/dev/null | sed 's,\(/jre\)\?/bin/java$,,' | head -n1)" if [ -n "$java_home" ]; then export JAVA_HOME="$java_home" fi # Alpine / Other / or if all else fails else # prefers Sun's JDK to OpenJDK, put it higher in the testing list # readlink -f => /etc/alternatives/java_sdk => /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.222.b10-0.el7_6.x86_64 for java_home in \ /usr/java/latest \ /usr/java/latest/jre \ /usr/lib/jvm/java \ /usr/lib/jvm/java-openjdk \ /usr/lib/jvm/jre-openjdk \ /usr/lib/jvm/jre \ /usr/lib/jvm/default-jvm \ ; do # default-jvm is on Alpine if [ -x "$java_home/bin/java" ]; then export JAVA_HOME="$java_home" break fi done if [ -z "$JAVA_HOME" ]; then if [ -n "$DEBUG" ]; then echo "WARNING: failed to find JAVA_HOME" >&2 fi # last ditch effort, this will work with warnings if [ -x /usr/bin/java ]; then export JAVA_HOME=/usr fi fi fi fi fi # haven't used this in many years #j(){ # for x in "$@"; do # echo "javac $x" && # javac "$x" && # echo "java ${x%.java} $x" && # java "${x%.java}" "$x" # done #} if ! type sdk &>/dev/null && [ -s ~/.sdkman/bin/sdkman-init.sh ]; then # shellcheck disable=SC1090,SC1091 source ~/.sdkman/bin/sdkman-init.sh fi ================================================ FILE: .bash.d/jenkins.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-02-22 17:18:09 +0000 (Mon, 22 Feb 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # J e n k i n s # ============================================================================ # #alias jenkins_cli='java -jar ~/jenkins-cli.jar -s http://jenkins:8080' alias jenkins-cli='jenkins_cli.sh' #alias backup_jenkins="rsync -av root@jenkins:/jenkins_backup/*.zip '~/jenkins_backup/'" # sets Jenkins URL to the local docker and finds and loads the current container's superuser token to the environment for immediate use with jenkins_api.sh jenkins_local(){ JENKINS_SUPERUSER_PASSWORD="$( docker-compose -p bash-tools -f "$(dirname "${BASH_SOURCE[0]}")/../docker-compose/jenkins.yml" \ exec -T jenkins-server cat /var/jenkins_home/secrets/initialAdminPassword /dev/null || . "$bash_tools/.bash.d/paths.sh" autocomplete k3d ================================================ FILE: .bash.d/kafka.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-07-28 14:46:37 +0100 (Sun, 28 Jul 2019) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # K a f k a # ============================================================================ # srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh" for directory in \ /usr/hdp/current/kafka-broker \ /usr/local/kafka \ ; do if [ -d "$directory" ]; then #export KAFKA_HOME=/usr/local/kafka export KAFKA_HOME=/usr/hdp/current/kafka-broker add_PATH "$KAFKA_HOME/bin" break fi done # kafka wrapper scripts use underscores instead of dashes so do not conflict with above kafka scripts which appear first in $PATH # moved to top level now #kafka_wrappers="$(dirname "${BASH_SOURCE[0]}")/../kafka_wrappers" #add_PATH "$kafka_wrappers" # HDP defaults to 8GB, on VMs that often breaks cli commands which try to claim too much ram and fail export KAFKA_OPTS="${KAFKA_OPTS:-} -Xms1G -Xmx1G" # there was another setting like KAFKA_KERBEROS_CLIENT I've used before but can't remember, this should work too #kafka_cli_jaas_conf="$(dirname "${BASH_SOURCE[0]}")/../kafka_wrappers/kafka_cli_jaas.conf" kafka_cli_jaas_conf="$(dirname "${BASH_SOURCE[0]}")/../kafka_cli_jaas.conf" export KAFKA_OPTS="${KAFKA_OPTS:-} -Djava.security.auth.login.config=$kafka_cli_jaas_conf" # ============================================================================ # # XXX: Enable KAFKA_BROKERS and KAFKA_ZOOKEEPER for convenience # of not having to specify them each time when using kafka_wrapper/ commands # XXX: Must use FQDNs to match Kerberos service principals # Apache / Cloudera #export KAFKA_BROKERS="$(hostname -f):9092" # Hortonworks #export KAFKA_BROKERS="$(hostname -f):6667" #export KAFKA_ZOOKEEPERS="$(hostname -f):2181" # optional - use if chrooting in zookeeper #export KAFKA_ZOOKEEPER_ROOT=/kafka # ============================================================================ # bootstrap_server="" if [ -n "${KAFKA_BROKERS:-}" ] && ! [[ "$*" =~ --bootstrap-server ]]; then # shellcheck disable=SC2034 bootstrap_server="--bootstrap-server $KAFKA_BROKERS" fi broker_list="" if [ -n "${KAFKA_BROKERS:-}" ] && ! [[ "$*" =~ --broker-list ]]; then # shellcheck disable=SC2034 broker_list="--broker-list $KAFKA_BROKERS" fi kafka_zookeeper="" if [ -n "${KAFKA_ZOOKEEPERS:-}" ] && ! [[ "$*" =~ --zookeeper ]]; then # shellcheck disable=SC2034 kafka_zookeeper="--zookeeper $KAFKA_ZOOKEEPERS" fi ================================================ FILE: .bash.d/kubernetes.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-07-28 14:56:41 +0100 (Sun, 28 Jul 2019) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # K u b e r n e t e s / O p e n S h i f t # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" for x in kubectl oc helm flux; do autocomplete "$x" done # minishift oc-env > ~/.minishift.env if [ -f ~/.minishift.env ]; then # remove .minishift.env if it causes errors, which can happen if it was generated when there was no MiniShift VM running # shellcheck disable=SC1090,SC1091 . ~/.minishift.env || rm -f -- ~/.minishift.env fi #if [ -f "/usr/local/opt/kube-ps1/share/kube-ps1.sh" ]; then # . "/usr/local/opt/kube-ps1/share/kube-ps1.sh" # # overriden in prompt.sh which is evaluated later so this is sourced there # #PS1='$(kube_ps1)'" $PS1" #fi # unreliable that HOME is set, ensure shell evaluates to the right thing before we use it [ -n "${HOME:-}" ] || HOME=~ add_PATH "${KREW_ROOT:-$HOME/.krew/bin}" for x in "$bash_tools"/kubernetes*.sh; do x="${x##*/}" name="${x#kubernetes_}" eval "k8s_${name}(){ '$x' \"\$@\" }" done # ============================================================================ # # replaced by function further down #alias k=kubectl # still need this autocomplete complete -F __start_kubectl k # 'k8s-app' label is set by dashboard creation but who uses that # false positive, the comma doesn't separate args # shellcheck disable=SC2054 k8s_get_pod_opts=(-o wide -L app,env --show-labels) #alias po='k get po "${k8s_get_pod_opts[@]}"' alias po='k get po' alias pow='po -o wide' alias powc='poc -o wide' alias pocw='poc -o wide' alias kapply='k apply -f' alias kapp=kapply alias kget='k get' alias kedit='k edit' alias kdel='k delete' alias kdelf='kdel -f' alias wp=watchpods alias kd=kdesc alias ke=kubectl_exec.sh alias kexec=kubectl_exec.sh alias ke2=kubectl_exec2.sh alias keg=kubectl_exec_grep.sh alias kg='k get' alias ka='k apply' alias kaf='ka -f' alias kl='k logs' alias ktp='k top po' alias kshell='kube-shell' alias kubesh='kube-shell' alias kubeconfig='$EDITOR "${KUBECONFIG:-~/.kube/config}"' alias kubeconf=kubeconfig #alias use="k config use-context" alias contexts="k config get-contexts" #alias context="k config current-context" context(){ k config current-context; } #alias con=context alias cons=contexts # contexts has this info and is more useful #alias clusters="k config get-clusters" # scripts in kubernetes/ directory that should be added to \$PATH (done automatically by sourcing this repo's .bashrc) alias kbusybox="kubectl_busybox.sh" alias kalpine="kubectl_alpine.sh" alias kcurl="kubectl_curl.sh" alias kdns="kubectl_dnsutils.sh" kube_config_isolate(){ local tmpdir="/tmp/.kube" mkdir -pv "$tmpdir" local default_kubeconfig="${HOME:-$(cd ~ && pwd)}/.kube/config" local original_kubeconfig="${KUBECONFIG:-$default_kubeconfig}" # reload safety - do not source from new tmpdir - not necessary for direnv but useful for local sourcing tests #if [[ "$original_kubeconfig" =~ $tmpdir ]]; then # echo "ignoring \$KUBECONFIG=$original_kubeconfig, using default home location $default_kubeconfig" # original_kubeconfig="$default_kubeconfig" #fi # isolate the kubernetes context to avoid a race condition affecting any other shells or scripts # epoch is added because $$ and $PPID are direnv sub-processes and may be reused later, so using epoch to add uniqueness local epoch epoch="$(date +%s)" export KUBECONFIG="$tmpdir/config.${EUID:-${UID:-$(id -u)}}.$$.$epoch" # load your real kube config to isolated staging area to source the context info if [ -f "$original_kubeconfig" ]; then cp -v -- "$original_kubeconfig" "$KUBECONFIG" elif [ -f "$default_kubeconfig" ]; then cp -v -- "$default_kubeconfig" "$KUBECONFIG" elif [ -f "$PWD/.kube/config" ]; then cp -v -- "$PWD/.kube/config" "$KUBECONFIG" elif [ -f "/etc/rancher/k3s/k3s.yaml" ]; then cp -v -- "/etc/rancher/k3s/k3s.yaml" "$KUBECONFIG" else echo "WARNING: failed to find one of: $original_kubeconfig $default_kubeconfig $PWD/.kube/config /etc/rancher/k3s/k3s.yaml " >&2 fi } # false positive, not using positional parameters # shellcheck disable=SC2142 alias namespace='k config get-contexts | grep -F "$(kubectl config current-context)" | awk "{print \$5}"' alias kwhere="{ echo -n 'context: '; context; echo -n 'namespace: '; namespace; }" alias con='kwhere' #alias kcd='k config set-context "$(kubectl config current-context)" --namespace' alias menv='eval $(minikube docker-env)' # scripts at top level, automatically included in $PATH alias labels="kubectl_node_labels.sh" alias taints="kubectl_node_taints.sh" unalias kcd 2>/dev/null kcd(){ if [ $# -lt 1 ] || [ $# -gt 2 ]; then echo "usage: kcd " >&2 return 1 fi local namespace="$1" echo "Switching to namespace '$namespace'" k config set-context "$(kubectl config current-context)" --namespace "$namespace" } unalias use 2>/dev/null use(){ if [ $# -lt 1 ] || [ $# -gt 2 ]; then echo "usage: use []" >&2 return 1 fi local context="$1" local namespace="${2:-}" local contexts contexts="$(k config get-contexts -o name)" if ! grep -Fxq "$context" <<< "$contexts"; then #echo "No matching contexts, inferring first partial match" context="$(grep -Em1 "$context" <<< "$contexts" || :)" if [ -z "$context" ]; then echo "Couldn't find any matching context name" >&2 return 1 fi #echo "Inferred context to be '$context'" fi #local args=() #if [ -n "$namespace" ]; then # args+=(--namespace "$namespace") #fi #k config use-context "$context" "${args[@]}" # less efficient, but more verbose k config use-context "$context" if [ -n "$namespace" ]; then kcd "$namespace" fi } kubectl_namespace(){ kubectl config get-contexts | awk '/^\*/{print $5}' } #alias poc='po | grep -v Completed' unalias poc &>/dev/null poc(){ po "$@" | grep -v Completed } #alias dat='datree test --only-k8s-files --ignore-missing-schemas' dat(){ if [ $# -eq 0 ]; then find . -type f -iname '*.y*ml' | # datree doesn't handle patches well grep -v patch | tr '\n' '\0' | xargs -0 datree test --only-k8s-files --ignore-missing-schemas else datree test --only-k8s-files --ignore-missing-schemas "$@" fi } datkust(){ datree_kustomize_all.sh . -- --enable-helm } # kustomize alias kbuild='kustomize build --enable-helm' alias kustomizebuilddiff='kbuild | kubectl_create_namespaces.sh; kbuild | kubectl diff -f -' alias kbuilddiff=kustomizebuilddiff alias kbuildd=kbuilddiff alias kbd=kbuildd alias kda=kustomize_diff_apply.sh # workaround for the fact that kustomize doesn't accept other filenames kustomize_build_file(){ local kustomization="$1" if [ -z "$kustomization" ]; then echo "usage: kustomize_build_file -kustomization.yaml" >&2 return 1 fi # because shell completion will stop at the prefix, so allow us to just enter and have it figure out what we're doing if ! [ -f "$kustomization" ];then if [ -f "${kustomization}kustomization.yaml" ]; then kustomization+="kustomization.yaml" elif [ -f "${kustomization}-kustomization.yaml" ]; then kustomization+="-kustomization.yaml" elif [ -f "${kustomization}kustomization.yml" ]; then kustomization+="kustomization.yml" elif [ -f "${kustomization}-kustomization.yml" ]; then kustomization+="-kustomization.yml" else echo "File not found: $kustomization" >&2 return 1 fi fi local prefix="${kustomization%kustomization.y*ml}" prefix="${prefix%-}" prefix="${prefix%_}" command cp -v -- "$prefix"*.yaml /tmp/ >&2 cd /tmp >&2 || return 1 echo >&2 command mv -v -- "${kustomization##*/}" kustomization.yaml >&2 echo >&2 kbuild local result=$? echo >&2 cd - >&2 || return 1 return $result } alias kbuildf=kustomize_build_file alias kbf=kbuildf kbfa(){ kbuildf "$@" >/dev/null || return 1 cd /tmp >&2 || return 1 kustomize_diff_apply.sh cd - >&2 || return 1 } # copies kustomization and values files while stripping their comments and filename prefixes kustcp(){ local name="$1" local dir="$2" echo "Copying $name-kustomization.yaml to $dir/kustomization.yaml" >&2 decomment "$name-kustomization.yaml" > "$dir/kustomization.yaml" echo "Copying $name-values.yaml to $dir/values.yaml" >&2 decomment "$name-values.yaml" > "$dir/values.yaml" echo "Replacing values filename reference in kustomization.yaml" >&2 perl -pi -e "s/$name-values\\.yaml/values.yaml/" "$dir/kustomization.yaml" echo "Done" >&2 } # ============================================================================ # # results in a blank arg which breaks kubectl command #kubectl_opts=("${KUBECTL_OPTS:-}") # split KUBECTL_OPTS to array properly read -r -a kubectl_opts <<< "${KUBECTL_OPTS:-}" # set K8S_NAMESPACE in local .bashrc or similar files for environments where your ~/.kube/config # gets regenerated daily with certification authentication from a kerberos login script, which # resets the 'kcd bigdata' namespace change. This way you automatically send the right namespace every time if [ "${K8S_NAMESPACE:-}" ]; then kubectl_opts+=(-n "$K8S_NAMESPACE") fi # TODO: might split this later oc_opts=("${kubectl_opts[@]:-}") # ============================================================================ # # oc() and kubectl() fix future invocations of k() to the each command if you want to explicitly switch between them oc(){ export KUBERNETES_CLI=oc command oc "${oc_opts[@]}" "$@" } kubectl(){ export KUBERNETES_CLI=kubectl # if empty causes 'bash: kubectl_opts[@]: unbound variable', and can't use "${kubectl_opts[@]:-}" default because this results in a blank arg which ruins commands if [ -n "${kubectl_opts[*]:-}" ]; then command kubectl "${kubectl_opts[@]}" "$@" else command kubectl "$@" fi } k(){ local opts=() # more efficient than forking to check history every time if [ -n "$KUBERNETES_CLI" ]; then case "$KUBERNETES_CLI" in kubectl) opts+=("${kubectl_opts[@]}") ;; oc) opts+=("${oc_opts[@]:-}") ;; *) echo "invalid command '$KUBERNETES_CLI' listed in \$KUBERNETES_CLI (must be either 'kubectl' or 'oc' depending on whether you are using straight Kubernetes or OpenShift). Fix the variable or unset it to auto-detect when calling the k() function" return ;; esac command "$KUBERNETES_CLI" "${opts[@]}" "$@" else # shellcheck disable=SC2086 case "$(k8s_or_openshift)" in openshift) command oc "${oc_opts[@]}" "$@" export KUBERNETES_CLI=oc ;; k8s|*) command kubectl "${kubectl_opts[@]}" "$@" export KUBERNETES_CLI=kubectl ;; esac fi } krun(){ local image="$1" local name="${image//\//-}" shift # sleep infinity only works on some distros k run --generator=run-pod/v1 "$name" --image "$image" -ti -- /bin/sh } # use ../kubernetes/kubectl_exec.sh via alias instead #kexec(){ # local lines # local name="${1//\//-}" # if [ -z "$name" ]; then # echo "usage: kexec " # return 1 # fi # for ((i=0;i<100;i++)); do # lines="$(k get po | grep -F "$name")" # if [ -z "$lines" ]; then # echo "No pods matching name $name found!" # return 1 # fi # name="$(awk '$3 ~ /Running/{print $1; exit}' <<< "$lines")" # if [ -n "$name" ]; then # break # fi # echo "waiting for pod to start running..." # sleep 1 # done # local cmd=(kubectl exec -ti "$name" "$@" -- /bin/sh -c 'if type bash >/dev/null 2>&1; then exec bash; else exec sh; fi') # echo "${cmd[*]}" # "${cmd[@]}" #} klog(){ local name="$1" k logs -f -n "$name" "deploy/$name" } klogs(){ local lines local name="${1//\//-}" shift || : if [ -z "$name" ]; then echo "usage: klogs " return 1 fi for ((i=0;i<100;i++)); do lines="$(k get po | grep -F "$name")" if [ -z "$lines" ]; then echo "No pods matching name $name found!" return 1 fi # often want to see the logs of the last pod restart in 'Crashing' status #name="$(awk '$3 ~ /Running/{print $1; exit}' <<< "$lines")" name="$(awk '{print $1; exit}' <<< "$lines")" if [ -n "$name" ]; then break fi echo "waiting for pod to start running..." sleep 1 done echo kubectl logs "$@" "\"$name\"" k logs "$@" "$name" } kfwd(){ local filter="$1" local port="$2" local hostport="$3" shift shift shift # mind need splitting if it's a filter # shellcheck disable=SC2086 kubectl port-forward $filter "$port" "$hostport" & open "http://localhost:$hostport" } # looks like both of these work on OpenShift context # # 'kubectl get pods' # # 'oc get pods' # figure out if we're using k8s or openshift via most recent commands - return either 'k8s' or 'openshift' k8s_or_openshift(){ local last_k8s_cmd last_k8s_cmd="$( history | grep -v history | grep -Eo -e '\' \ -e '\' \ -e '\' \ -e '\' | tail -n 1 )" case "$last_k8s_cmd" in oc|minishift) echo openshift # these end up in a subshell so aren't really useful, set in k() instead #export KUBERNETES_CLI=oc ;; kubectl|minikube) echo k8s #export KUBERNETES_CLI=kubectl ;; *) echo unknown ;; esac } oc_get_pods(){ # shellcheck disable=SC2086 oc get pods "${k8s_get_pod_opts[@]}" } k8s_get_pods(){ # shellcheck disable=SC2086 k get pods "${k8s_get_pod_opts[@]}" } get_pods(){ #case "$(k8s_or_openshift)" in # openshift) oc_get_pods # ;; # k8s) k8s_get_pods # ;; # *) k8s_get_pods # ;; #esac # # k8s functions now include k8s vs oc detection, no need for above or would end up double calling k8s_or_openshift k8s_get_pods } export -f get_pods get_pod(){ local filter="${1:-.*}" get_pods | grep -v '^NAME[[:space:]]' | grep Running | awk "/$filter/{print \$1; exit}" } watchpods(){ # watch on Mac (brew installed) doesn't have -x switch and doesn't work on even 'export -f function' # leave using kubectl call for now as that works on openshift too watch " echo 'Context: ' echo kubectl config current-context echo echo echo 'Pods:' echo kubectl " "${kubectl_opts[@]}" " get pods " "${k8s_get_pod_opts[@]:-}" " 2>&1 echo " } kdesc(){ k describe "$@" | less } # kdesc pod with grep filter on name for fast describing a pod in the current or given namespace kdp(){ local filter="${1:-.*}" shift || : pod="$(k get po -o name "$@" | grep -Em 1 "$filter")" || return kdesc "$pod" "$@" } kdelp(){ k delete pod "$@" } # Getting token works on stock Kubernetes but not OpenShift due to stricter defaults # # Error from server (Forbidden): secrets is forbidden: User "developer" cannot list secrets in the namespace "kube-system": no RBAC policy matched # error: resource name may not be empty # ## even after 'oc login' as system/admin # # Error from server (Forbidden): secrets is forbidden: User "system" cannot list secrets in the namespace "kube-system": no RBAC policy matched # error: resource name may not be empty # k8s_get_token(){ kubectl describe secret -n kube-system "$(kubectl get secrets -n kube-system | awk '/^default-token/ {print $1}')" | awk '/^token/ {print $2}' } # better than: kubectl config view | grep server k8s_get_api(){ local context local cluster context="$(context)" cluster="$(k config view -o jsonpath="{.contexts[?(@.name == \"$context\")].context.cluster}")" k config view -o jsonpath="{.clusters[?(@.name == \"$cluster\")].cluster.server}" # or if you have jq installed: # k get --raw=/api | jq -r '.serverAddressByClientCIDRs[0].serverAddress' echo } # TODO: path like above to get the current context's cluster k8s_get_client_cert(){ awk '/^[[:space:]]*client-cert/{print $2}' ~/.kube/config | head -n 1 } k8s_get_client_key(){ awk '/^[[:space:]]*client-key-data/{print $2}' ~/.kube/config | head -n 1 } k8s_get_ca_cert(){ awk '/^[[:space:]]*certificate-authority-data/{print $2}' ~/.kube/config | head -n 1 } # generates files for authenticating to kube-apiserver via curl: # # curl --cert client_cert.pem --key client_key.pem --cacert ca_cert.pem https://k8smaster:6443/api/v1/pods # curl --cert client_cert.pem --key client_key.pem --cacert ca_cert.pem https://k8smaster:6443/api/v1/pods/namespaces/default/pods -XPOST -H 'Content-Type: application/json' -d @pod_defintion.json k8s_get_keys(){ # use --decode not -d / -D which varies between Linux and Mac k8s_get_client_cert | base64 --decode - > client_cert.pem echo "generated client_cert.pem" k8s_get_client_key | base64 --decode - > client_key.pem echo "generated client_key.pem" k8s_get_ca_cert | base64 --decode - > ca_cert.pem echo "generated ca_cert.pem" } # run kubectl commands against multiple clusters kclusters(){ for context in $(kubectl config get-contexts -o=name --kubeconfig clusters.yaml); do kubectl "$@" --kubeconfig clusters.yaml --context="$context" done } # to kubectl apply manifests to both clusters for multi-cluster deployments kclustersapply(){ kclusters apply -f "$@" # eg. manifests } # inspired by my class 'when' functions in when.sh whenpodup(){ local name="${1:-}" shift || : if [ -z "$name" ]; then echo "usage: whenpodup " return 1 fi local count=0 while ! kubectl get pods "$name" -o 'jsonpath={.status.phase}' | grep -q 'Running'; do ((count+=1)) timestamp "waiting for pod '$name' to come up..." if [ $count -gt 22 ]; then sleep 10 else sleep 5 fi done timestamp "pod '$name' is up" "$@" } ================================================ FILE: .bash.d/linux.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # L i n u x # ============================================================================ # # Linux specific bits to not include on Mac # most of the regular stuff is in the other bash.d/*.sh files bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" is_linux || return open(){ if type -P xdg-open &>/dev/null; then xdg-open "$@" elif sensible-browser &>/dev/null; then sensible-browser "$@" elif x-www-browser &>/dev/null; then x-www-browser "$@" elif gnome-open &>/dev/null; then gnome-open "$@" else echo "Neither 'xdg-open' nor 'sensible-browser' were found in \$PATH - install one of them to automatically open this URL:" echo echo "$*" echo fi } alias reloadXdefaults="xrdb ~/.Xdefaults" #setxkbmap us # Assign middle mouse to my Alt Gr key # TODO: change this to keysym as keycodes can change between keyboards, to find keymaps do # xmodmap -pkie if [ -n "$DISPLAY" ] && ! is_mac; then # This caused the left to be remapped, must test and handle better #xmodmap -e 'keycode 113 = Pointer_Button2' #xmodmap -e 'keycode 113 = Left NoSymbol Left' xkbset m # Ubuntu 12.04.1 LTS had a bug where it turned off repeat keys on my down arrow, this fixed it xkbset repeatkeys 116 fi rpmqf(){ rpm -qf "$(readlink -m "$1")" } fixtime(){ # $sudo defined in .bashrc # shellcheck disable=SC2154 $sudo /etc/init.d/ntp stop $sudo ntpdate pool.ntp.org $sudo /etc/init.d/ntp start } getmounts(){ #grep -e "ext" -e "reiser" -e "fat" -e "ntfs" < /proc/mounts | #awk '{ print $2 }' awk '/ext|reiser|fat|ntfs|btrfs|xfs/{print $2}' } findsuid(){ for x in $(getmounts); do echo "Searching $x for suid programs:" # $sudo defined in .bashrc if not root # shellcheck disable=SC2154 $sudo find "$x" -xdev -type f -perm -u+s -exec ls -l {} \; done } findguid(){ for x in $(getmounts); do echo "Searching $x for guid programs:" $sudo find "$x" -xdev -type f -perm -g+s -exec ls -l {} \; done } findsguid(){ for x in $(getmounts); do echo "Searching $x for suid and guid programs:" $sudo find "$x" -xdev -type f \( -perm -u+s -o -perm -g+s \) -exec ls -l {} \; done } findwritable(){ for x in $(getmounts); do echo "Searching $x for world writeable files:" $sudo find "$x" -xdev -type f -perm -o+w -exec ls -l {} \; done } # ========================== # When using Samba WinPopups on Linux in Windows workgroups - convenient back in the day but shouldn't be needed today with the plethora of better chat tools # #netsend(){ smbclient -M "$1" <<< "${*:2}"; } #alias ns=netsend # # clear pop-ups and alerts if sending instant security alerts #clearnetsend(){ # sudo pkill -f sambapopup #} #alias cns=clearnetsend # #clearxmessage(){ # while pkill xmessage; do # sleep 0.1 # done # while pkill gmessage; do # sleep 0.1 # done #} # ========================= ================================================ FILE: .bash.d/lolcat.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-03 11:31:02 +0000 (Sun, 03 Nov 2019) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # lolz(){ exec 1> >(lolcat >&2) } ================================================ FILE: .bash.d/mac.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2011 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A p p l e M a c O S X # ============================================================================ # # More Mac specific stuff in adjacent *.sh files, especially network.sh bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" is_mac || return export HOMEBREW_DISPLAY_INSTALL_TIMES=1 #export HOMEBREW_DEBUG=1 #export HOMEBREW_CLEANUP_MAX_AGE_DAYS=30 # default: 120 # Stops Mac calling update_terminal_cwd() which causes a tonne of noise during set -x tracing export INSIDE_EMACS=1 export icloud="$HOME/Library/Mobile Documents/com~apple~CloudDocs" alias icloud="cd '$icloud'" alias osash="osascript -i" alias osashell=osash if [ -x /opt/homebrew/bin/brew ]; then # shellcheck disable=SC2046 eval $(/opt/homebrew/bin/brew shellenv) fi date(){ gdate "$@" } xargs(){ # because --no-run-if-empty is useful command gxargs "$@" } if ! type tac &>/dev/null; then tac(){ gtac "$@" } fi # used for Shazaming while on headphones - see: # # https://github.com/HariSekhon/Knowledge-Base/blob/master/audio.md#shazam-songs-while-using-headphones-on-mac # # Switches to Multi-Output Device which should already be configured as above and contain your headphones and BlackHole 2ch # #alias mshazam='SwitchAudioSource -s "Multi-Output Device"; open -a Shazam' #alias msound='SwitchAudioSource -s "Multi-Output Device"' unalias msound 2>/dev/null || : msound(){ # because if you have 2 multi-output devices eg. # # - one using AirPods + Blackhole # - another using Speakers + Blackhole # # then Mac automatically renames "Multi-Output Device" to "Multi-Output Device 1", # even if the other multi-output device has already been differentiated, it refuses to let you set # it back to the default name, so just determine what the first one is called and use that # local multi_output_device multi_output_device="$(SwitchAudioSource -a | grep -m1 '^Multi-Output Device')" #echo "Found first multi-output device: $multi_output_device" echo "Using first found multi-output device" SwitchAudioSource -s "$multi_output_device" } alias restartsound='sudo killall coreaudiod' alias mshazam='msound; open -a Shazam' vol(){ if [ $# -ne 1 ]; then echo "usage: vol " return 1 fi osascript -e "set volume output volume $1" } # put in inputrc for readline #set completion-ignore-case on # Apple default in Terminal is xterm #export TERM=xterm # not sure why I set it to linux #export TERM=linux #ulimit -u 512 dhcprenew(){ local interface="${1:-en0}" watch -q 1 ifconfig "$interface" sudo scutil <<< "add State:/Network/Interface/$interface/RefreshConfiguration temporary" watch ifconfig "$interface" } dhcpdiscover(){ local interface="${1:-en0}" watch -q 1 ifconfig "$interface" sudo ipconfig set "$interface" BOOTP sudo ipconfig set "$interface" DHCP watch ifconfig "$interface" } macsleep(){ sudo pmset sleepnow } nosleep(){ echo "Running: caffeinate -s $*" echo "(works even if you close the Macbook lid but will still sleep on battery power)" caffeinate -s "$@" } silence_startup(){ sudo nvram SystemAudioVolume=%80 } top(){ local opts=(-F -R -o) if [ $# -eq 1 ]; then command top "${opts[@]}" "$1" elif [ $# -gt 1 ]; then command top "$@" else command top "${opts[@]}" cpu fi } fixvbox(){ sudo /Library/StartupItems/VirtualBox/VirtualBox restart } fixaudio(){ sudo kextunload /System/Library/Extensions/AppleHDA.kext sudo kextload /System/Library/Extensions/AppleHDA.kext } showhiddenfiles(){ defaults write com.apple.finder AppleShowAllFiles YES # must killall Finder after this } alias reloadprefs='killall -u $USER cfprefsd' alias strace="dtruss -f" alias usbinfo='system_profiler SPUSBDataType' alias vlc="/Applications/VLC.app/Contents/MacOS/VLC" # clear paste buffer clpb(){ copy_to_clipboard.sh < /dev/null } macmac(){ ifconfig | awk ' /^en[[:digit:]]+:/{gsub(":", "", $1); printf "%s:\t", $1} /^[[:space:]]ether[[:space:]]/{print $2} ' | # filters to only the lines with prefixed interfaces from first match grep '\t' } duall(){ # bash_tools defined in .bashrc # shellcheck disable=SC2154 du -ax "$bash_tools" | sort -k1n | tail -n 2000 sudo du -ax / | sort -k1n | tail -n 50 } alias dua=duall if type -P brew &>/dev/null; then brew_prefix="$(brew --prefix)" if [ -f "$brew_prefix/etc/bash_completion" ]; then # shellcheck disable=SC1090,SC1091 . "$brew_prefix/etc/bash_completion" fi fi brewupdate(){ if ! brew update; then echo "remove the following to brew update" brew update 2>&1 | tee /dev/stderr | grep '^[[:space:]]*Library/Formula/' | while read -r formula; do echo rm -fv -- "/usr/local/$formula" done return 1 fi } brewinstall(){ brewupdate && sed 's/#.*// ; /^[[:space:]]*$/d' < ~/mac-list.txt | while read -r pkg; do brew install "$pkg" #|| #{ echo "FAILED"; break; } done } brew_find_unlinked_bins(){ for x in /usr/local/Cellar/*/*/bin/*; do if ! [ -f "/usr/local/bin/${x##*/}" ]; then echo "$x" fi done } # don't export BROWSER on Mac, trigger python bug: # AttributeError: 'MacOSXOSAScript' object has no attribute 'basename' # from python's webbrowser library #export BROWSER="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" #export BROWSER="/Applications/Firefox.app/Contents/MacOS/firefox" # MacPorts - using HomeBrew instead in recent years #if [ -e "/sw/etc/bash_completion" ]; then # . /sw/etc/bash_completion #fi # seems Mac OS X has a native pkill now #pkill(){ # local args="" # local regex="" # local grep_args="" # while [ -n "$1" ]; do # case "$1" in # -i) grep_args="$grep_args -i" # shift # ;; # -*) args="$args $1" # shift # ;; # *) regex="$1" # shift # ;; # esac # done # # TODO: check this a few times and then remove the echo # local proclist=$(ps -e | awk '{printf $1 OFS;for(i=4;i<=NF;i++)printf $i OFS;print""}' | grep $grep_args "$regex") # if [ -n "$proclist" ]; then # echo "$proclist" # awk '{print $1}' <<< "$proclist" | xargs echo kill $args # read -r -p "Kill all these processes? [y/N] " answer # if [ "$answer" = "y" ]; then # awk '{print $1}' <<< "$proclist" | xargs kill $args # fi # fi #} ================================================ FILE: .bash.d/mercurial.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # R e v i s i o n C o n t r o l - M e r c u r i a l # ============================================================================ # type -P hg &>/dev/null || return 0 alias hgi=hgignore alias hgrc='$EDITOR ~/.hgrc' # HG doesn't record dirs and there is no .hg per subdir, rather than traverse upwards checking filesystem boundaries use hg tools themselves isHg(){ local target="${1:-.}" # There aren't local .hg dirs everywhere only at top level so this is difficult in bash #if [ -d "$target/.hg" -o -d "$(dirname "$target")/.hg" ]; then # shortcut for efficiency if [ -d "$target/.hg" ] || [ -d "${target%/*}/.hg" ]; then # || hg parents "$target" &>/dev/null; then # can only call this on files not dirs anyway since Hg doesn't track dirs return 0 #elif [ -f "$target" ] && hg parents "$target"; then # return 0 #elif hg status "$target" &>/dev/null; then # Doesn't work at all always returns 0 if a subdir of a repo and just blank even if in ignores # Unfortunately the -P switch only supports a single pattern so we can't use --file #grep -qP -f "$(hg root)/.hgignore" # algorithm horribly inefficient, was going to rewrite in perl isHg.pl but see futher down #while read regex; do # grep "^[[:space:]]*$" <<< "$regex" && continue # abs_path="$(abs_path "$target")" # abs_path="${abs_path/$(hg root)\/}" # grep -qP "$regex" <<< "$abs_path" && return 1 #done < "$(hg root)/.hgignore" # return 0 #elif [ -d "$target" ]; then # echo "WARNING: cannot call hg on a dir, $target is a dir so this returns false and will fall through" # return 1 # finally found a reasonably efficient way to handle all cases # trick to return False on subdirs which are not handled by Mercurial elif [ -n "$(hg log --limit 1 "$target" 2>/dev/null)" ]; then return 0 else return 1 fi } hgignore(){ #pushd "$srcdir" &>/dev/null local hgroot hgroot="$(hg root)" [ -n "$hgroot" ] || return 1 "$EDITOR" "$hgroot/.hgignore" #popd &>/dev/null } hgci(){ local hgcimsg="" for x in "$@"; do if hg st "$x" | grep -q "^[?A]"; then hgcimsg+="$x, " fi done [ -z "$hgcimsg" ] && return 1 hgcimsg="${hgcimsg%, }" hgcimsg="added $hgcimsg" hg add -- "$@" && echo "committing $*" hg ci -m "$hgcimsg" -- "$@" } hgrm(){ hg rm -- "$@" && hg ci -m "removed $*" -- "$@" } hgrevertrm(){ hg revert "$@" rm -v -- "$@" } hgrename(){ hg mv -- "$1" "$2" && hg ci -m "renamed $1 to $2" -- "$1" "$2" } hgmv(){ hg mv -- "$1" "$2" && hg ci -m "moved $1 to $2" -- "$1" "$2" } hgl(){ hg log "$@" | less } hgu(){ [ -n "$1" ] || { echo "ERROR: must supply arg"; return 1; } [ "$(hg diff "$@" | wc -l)" -gt 0 ] || return hg diff -- "$@" | more && read -r && echo "committing $*" && hg ci -m "updated $*" -- "$@" } #hhgu(){ # # all playlists end in \n from now on via paste_playlists.sh fix # [ -n "$1" ] || { echo "ERROR: must supply arg"; return 1; } # pushd "$music" >/dev/null # spotify/validate_playlists.sh "$1" || { echo "Playlist validation failed"; return 1; } # spotify/validate_playlist_lengths.sh "$1" || { echo "Playlist dump length validation failed"; return 1; } # [ `hg st "$1" "spotify/$1" | wc -l` -gt 0 ] || { echo "No changes in either uri or track lists"; return 0; } # local target="${1##*/}" # local target_tip="$(dirname "$target")/.$(basename "$target").tip" # hg cat "$target" | spotify/normalize_tracknames.pl > "$target_tip" # cat "$target" | spotify/normalize_tracknames.pl > ".$target" # if [ -z "$(diff -iwu "$target_tip" ".$target")" ]; then # echo "Noop changes only, committing..." # hg mydiff "$target" | # #egrep '^\+' | tee /dev/stderr | # grep -v '^[+-][+-][+-]' # | sl --no-locking # hg ci -m "updated $target" "$target" "spotify/$target" # return $? # elif diff -iwu "$target_tip" ".$target" | grep -q '^-[^-]'; then # local diffs="$( # { hg mydiff "$target" # hg mydiff "spotify/$target" # })" # local removals="$(grep -c "^-[^-]" <<< "$diffs")" # local additions="$(grep -c "^+[^+]" <<< "$diffs")" # diffs="$(echo "$diffs" | # egrep "^[+-]" | # spotify/normalize_tracknames.pl | # diffnet.pl -iw # )" # if [ -z "$diffs" ]; then # echo "Noop changes to tracks, committing..." # elif ! echo "$diffs" | grep -q '^-[^-]'; then # echo "Net diff shows only playlist additions, committing..." # echo "$diffs" | # more # else # { # echo "$additions additions $removals removals" # echo "$diffs" # } | # more && # read || return # fi # hg ci -m "updated $target" "$target" "spotify/$target" # else # echo "Only playlist additions detected, committing..." # hg mydiff "$target" | # #grep -v '^[+-][+-][+-]' # egrep "^[+-]" # hg ci -m "updated $target" "$target" "spotify/$target" # return $? # # echo "No additions or removals detected, playlist dump must currently be in progress" # # return 1 # fi # popd &>/dev/null #} # equiv to using the 3rd party shelve extension since HG doesn't have this Git Stash functionality hgshelve(){ local hgroot hgroot="$(hg root)" [ -f "$hgroot/shelve.diff" ] && { echo "$hgroot/shelve.diff already exists, aborting for safety to not lose changes"; return 1; } hg diff > "$hgroot/shelve.diff" hg revert -a } # Then merge, hg up etc, then unshelve hgunshelve(){ hg import --no-commit "$hgroot/shelve.diff" # && rm -v "$srcdir/shelve.diff" } hgdiff(){ local filename="${1:-}" [ -n "$filename" ] || { echo "usage: hgdiff filename"; return 1; } hg diff -- "$filename" > "/tmp/hgdiff.tmp" diffnet.pl "/tmp/hgdiff.tmp" } ================================================ FILE: .bash.d/mp3.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-07-21 11:36:49 +0100 (Tue, 21 Jul 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # set the Track number metadata on mp3 files in the order that they are given # see much better mp3_set_track_order.sh at top level of this repo now #mp3_renumber(){ # local i=0 # for x in "$@"; do # ((i+=1)) # id3v2 --track "$i" "$x" # done #} mp3info(){ find "${@:-.}" -type f -iname '*.mp3' | head -n 1 | while read -r filename; do mediainfo "$filename" done } mp3infotail(){ find "${@:-.}" -type f -iname '*.mp3' | tail -n 1 | while read -r filename; do mediainfo "$filename" done } mp3infoheadtail(){ find "${@:-.}" -type f -iname '*.mp3' | sed -n '1p;$p' | while read -r filename; do mediainfo "$filename" done } mp3set(){ if [ $# != 2 ]; then echo "usage: mp3set " return 1 fi local artist="$1" local album="$2" mp3_set_artist.sh "$artist" mp3_set_album.sh "$album" mp3_set_track_order.sh mp3_set_track_name.sh } ================================================ FILE: .bash.d/mysql.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-16 13:26:50 +0000 (Mon, 16 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # M y S Q L # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # shellcheck disable=SC1090,SC1091 type pass &>/dev/null || . "$bash_tools/.bash.d/functions.sh" # highest priority env var first, common one second - export as both alias mysqlpass='pass MYSQL_PWD MYSQL_PASSWORD' ================================================ FILE: .bash.d/network.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # N e t w o r k # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" alias 4="ping 4.2.2.1" alias 8="ping 8.8.8.8" alias ping="ping -n" alias p="ping" pingwait="-w" is_mac && pingwait="-W" alias ping_google="while true; do ping www.google.com && sleep 1 || break; done" alias g=ping_google # watch_url.pl is in DevOps-Perl-tools repo which should be in $PATH alias watchu="watch_url.pl" # watch google # https because http often gets intercepted by routers + proxies giving false 200 OKs where there is an internet issue alias wg="watch_url.pl https://google.com" alias speedtesturl="browser https://speedtest.net" # ============================================================================ # # Y o u r I P A d d r e s s # ============================================================================ # # local/internal IP address myip(){ ifconfig | grep 'inet[[:space:]]' | grep -v 127.0.0.1 | awk '{print $2}' } # public IP address ifconfigco(){ curl ifconfig.co # something else to consider with jq for lat/long coordinates, ASN, country etc #curl ifconfig.co/json } ipinfo(){ # returns json without /ip with region, reverse dns hostname, city, region, country, lat/long coordinates, org, postcode, timezone curl ipinfo.io/ip } ipify(){ curl http://api.ipify.org/ echo } # doesn't welcome automation / curl - requires captchas now so obsolete #whatismyip(){ # #lynx -dump $(lynx -dump www.whatismyip.com | tail -n 1) # lynx -useragent="Mozilla" -dump www.whatismyip.com 2>/dev/null | awk '/Your Public IPv[46] is:/ {print $6}' #} # ============================================================================ # # F u n c t i o n s # ============================================================================ # checkhost(){ if [ -z "$1" ]; then echo "usage: checkhost hostname/ip" return 1 fi if grep -qi "unknown host" <<< "$(ping -c 1 "$pingwait" 1 "$1" 2>&1)"; then echo "Unknown host" return 1 fi } n(){ if type -P host &>/dev/null; then host "$@" elif type -P nslookup &>/dev/null; then nslookup "$@" else echo "neither host nor nslookup were found in the path" return 1 fi } alias h=n getip(){ host "$@" | grep "has address" | awk '{print $4; exit}' } tping(){ while true; do #echo -n "`date '+%F %T'` " local output output="$(ping -c 1 "$pingwait" 2 "$@" | grep -v -e statistics \ -e "transmitted" \ -e "rtt min/avg/max/mdev" \ -e "bytes of data" \ -e "^[[:space:]]*$" \ -e "^PING " \ -e "round-trip" )" echo "$(date '+%F %T') ${output:-no response from $1}" sleep 1 done } tpinggw(){ tping "$(get_gw)" "$@" } # for trying to find those damn wifi capture portals that disappear but block your internet http proxying opengw(){ local gateway gateway="$(get_gw)" open "http://$gateway" open "https://$gateway" } port(){ if [ -z "$2" ]; then echo "You must supply a hostname/ip address to test followed by a port number" return 1 fi #sudo nmap $1 -p $2 ${@:3} -P0 | grep tcp nc -zv "$1" "$2" 2>&1 | grep -v "DNS fwd/rev misma" | sed 's/[^]]*\] //' } testport(){ if [ -z "$2" ]; then echo "You must supply a hostname/ip address to test followed by a port number" return 1 fi while true; do timestampcmd port "$1" "$2" sleep 1 done } hammerport(){ for i in {1..500}; do printf "%-3s: " "$i" nc -z -v "$1" "$2" done } halfopen(){ while true; do echo -n "half-open connections: " netstat -ant | grep c SYN_RECV sleep 1 done } get_gw(){ local gw #gw="$(netstat -rn | awk '/^default.*\./ {print $2;exit}')" gw="$( netstat -rn | awk ' /^default.*\./ { print $2; exit } $1 == "Internet:" { inet = 1; next } $1 == "Internet6:" { inet = 0 } inet && ($1 == "default" || $1 == "0.0.0.0") && $2 ~ /^[0-9.]+$/ { print $2 exit } ' )" if [ -z "$gw" ]; then echo "Could not find gateway, no default route! " >&2 return 1 fi echo "$gw" } gw(){ local gw gw="$(get_gw)" [ -n "$gw" ] || return 1 ping "$gw" } z(){ local gw gw="$(get_gw)" if [ -n "$gw" ]; then whenup "$gw" && whenup 4.2.2.1 && whenup www.google.com echo "INTERNET OK" else echo "Couldn't find gateway, cannot test upstream connectivity!" return 1 fi } browser(){ if [ -n "${BROWSER:-}" ]; then "$BROWSER" "$@" elif is_mac; then open "${*:-http://google.com}" else echo "\$BROWSER environment variable not set and not on Mac OSX, not sure which browser to use, aborting..." return 1 fi } browse(){ if isGit . &>/dev/null && git remote -v | grep -qi http; then gitbrowse "$@" else browser "$@" fi } downorjustme(){ browser "http://www.downforeveryoneorjustme.com/$1" } # directs to the same as downorjustme isupme(){ browser "http://www.isup.me/$1" } chrome(){ if is_mac; then # opens in most recent Chrome window # could use one of these: --new --args --incognito --new-window open -a 'Google Chrome' "${*:-http://www.google.com}" else checkprog google-chrome || return 1 google-chrome "${*:-http://www.google.com}" & fi } ff(){ if is_mac; then open -a 'Firefox' "http://${*:-www.google.com}" else checkprog firefox || return 1 firefox "${*:-http://www.google.com}" & fi } gg(){ if [ -z "$*" ]; then browser & else searchterm="${*// /%20}" browser "http://www.google.com/search?q=$searchterm" & fi } netcraft(){ checkprog firefox || return 1 browser "http://uptime.netcraft.com/up/graph?site=$*" & } wikipedia(){ checkprog "firefox" || return 1 local searchterm searchterm="${*// /%20}" browser "http://en.wikipedia.org?search=$searchterm&go=Go" & } alias wiki=wikipedia definition(){ checkprog "firefox" || return 1 local searchterm searchterm="${*// /%20}" # hl=en&q=test&btnI=I%27m+Feeling+Lucky&meta=&aq=f browser "http://www.google.co.uk/search?hl=en&q=definition+$searchterm&btnI=I%27m+Feeling+Lucky" & } # alias def=definition # gh(){ # url="http://www.google.com/search?q=" # browser "${url}site%3A$*" & # browser "${url}site%3A$* login" & # browser "${url}link%3A$*" & # browser "${url}related%3A$*" & # } retry(){ local cmd="$1" local host="${2##*@}" #local user="${2%%@*}" local args=("${@:3}") if [ -z "$host" ]; then echo "You must supply a hostname or ip address to connect to" return 2 fi if [ "$cmd" = "ssh" ] || [ "$cmd" = "rdp" ]; then whenup "$host" || return 1 fi #[ "$cmd" = "ssh" ] && host="root@$host" if [ "$cmd" = "ssh" ]; then until port "${host##*@}" 22 >/dev/null; do tstamp "trying $host port 22" sleep 1 done elif [ "$cmd" = "rdp" ]; then until port "$host" 3389 >/dev/null; do tstamp "trying $host port 3389" sleep 1 done fi [ "$cmd" = "ssh" ] && printargs="" || printargs="${args[*]}" until "$cmd" "$host" "${args[@]}"; do sleep 1 tstamp "trying $cmd $host $printargs" done echo >/dev/null } rdp(){ if is_mac; then "/Applications/Remote Desktop Connection.app/Contents/MacOS/Remote Desktop Connection" "$@" & else [ -n "$1" ] || return 1 local resolution="800x600" if [ "$(xdpyinfo | awk '/dimensions/ {print $2}' | sed 's/x.*//')" -gt 1024 ]; then resolution="1024x768" fi if type -P krdc &>/dev/null; then krdc "rdp:/$WINDOWSDOMAIN\\$WINDOWSUSER@$*" & exit 0 elif type -P rdesktop &>/dev/null; then rdesktop -u "$WINDOWSUSER" -d "$WINDOWSDOMAIN" "$@" -g "$resolution" & exit 0 else echo "Could not find krdc or rdesktop in path" return 1 fi fi } rerdp(){ retry "whenport $1 3389; rdp" "$1" } # ============================================================================ # # L i n u x # ============================================================================ # if is_linux; then ipl(){ iptables -L | nl } fi # ============================================================================ # # M a c O S X # ============================================================================ # if ! is_mac; then return fi dnsservers(){ scutil --dns | grep 'nameserver\[[0-9]*\]' | sort -u } flushdns(){ dscacheutil -flushcache sudo killall -HUP mDNSResponder } alias flushcache=flushdns #APPLE_INTERFACES="Ethernet Airport" #APPLE_INTERFACES="$(networksetup -listallnetworkservices | grep -E 'Ethernet|Wi-Fi')" unset APPLE_INTERFACES get_apple_interfaces(){ networksetup -listallnetworkservices | grep -E 'Ethernet|Wi-Fi' } # Cisco AnyConnect set these rules which mess up my ability to connect directly to VirtualBox VMs on HostOnly Networking cleardeny(){ sudo ipfw delete "$(sudo ipfw list | grep deny | awk '{print $1}')" } ipfwqflush(){ sudo ipfw -q flush } isMacNetworkService(){ local interface="$1" if [ "$interface" != "Thunderbolt Ethernet" ] && [ "$interface" != "Wi-Fi" ]; then echo "interface must be one of Thunderbolt Ethernet or Wi-Fi" return 1 fi } set_dns(){ get_apple_interfaces | while read -r interface; do sudo networksetup -setdnsservers "$interface" "$@" done } set_dns_search(){ get_apple_interfaces | while read -r interface; do sudo networksetup -setsearchdomains "$interface" "$@" done } set_dns_search_empty(){ set_dns_search "Empty" } # this wasn't found as an alias from another function clear_dns_search(){ set_dns_search_empty } function publicdns(){ set_dns 4.2.2.1 4.2.2.2 4.2.2.3 4.2.2.4 4.2.2.5 4.2.2.6 set_dns_search_empty } function dhcpdns(){ clear_dns_search # hangs without this as I think it tries to query DNS for all the suffixes in the list set_dns "Empty" #networksetup -setsearchdomains [domain2] } get_wifi_interface(){ networksetup -listnetworkserviceorder | grep "Hardware.*Wi-Fi" | sed 's/.*: //;s/)$//' } get_wifi_network(){ networksetup -getairportnetwork "$(get_wifi_interface)" | sed 's/^Current Wi-Fi Network: //' } set_wifi_network(){ networksetup -setairportnetwork "$(get_wifi_interface)" "$*" } wifi(){ if [ $# -eq 1 ]; then airport on set_wifi_network "$1" elif [ $# -eq 0 ]; then get_wifi_network else echo "usage: wifi " return 1 fi } wifi_networks_preferred(){ networksetup -listpreferredwirelessnetworks "$(get_wifi_interface)" } airport(){ networksetup -setairportpower "$(get_wifi_interface)" "$1" } alias air=airport airportr(){ airport off airport on } alias airr=airportr alias ag="airr; g" watchwifi(){ scnum 39 while true; do checkwifi sleep 30 || break done } checkwifi(){ # needs to be global otherwise will be forgotten between runs of this program [ -z "$wifi_failures" ] && wifi_failures=0 for((i=1;i<=3;i++)); do if ping -c1 -W1 4.2.2.1 >/dev/null; then if [ "$wifi_failures" -gt 0 ]; then tstamp "wifi recovered from $wifi_failures failures" fi wifi_failures=0 return else ((wifi_failures+=1)) tstamp "$wifi_failures wifi failures" fi done timestamp "RESTARTING WIFI" airportr } setdhcp(){ isMacNetworkService "$1" || return 1; sudo networksetup -setdhcp "$1" } renewdhcp(){ sudo ipconfig set "$1" DHCP } #sethomenet(){ # isMacNetworkService "$1" || return 1; # sudo networksetup -setmanual "$1" x.x.x.x 255.255.255.0 x.x.x.1 # sudo route delete 0.0.0.0 # sudo route add 0.0.0.0 x.x.x.1 # publicdns #} ================================================ FILE: .bash.d/nodejs.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2019 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # N o d e # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/os_detection.sh" #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # output from 'npm bin' if [ -d ~/node_modules/.bin ]; then add_PATH ~/node_modules/.bin fi if [ -d "$bash_tools/node_modules/.bin" ]; then add_PATH "$bash_tools/node_modules/.bin" fi alias lsnodebin='ls -d ~/node_modules/.bin/* 2>/dev/null' alias llnodebin='ls -ld ~/node_modules/.bin/* 2>/dev/null' ================================================ FILE: .bash.d/os_detection.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2010 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # type is_linux &>/dev/null && type isMac &>/dev/null && type isGoogleCloudShell &>/dev/null && type isAzureCloudShell &>/dev/null && return get_os(){ if [ -z "${operating_system:-}" ] || ! [[ "$operating_system" =~ ^(Linux|Darwin)$ ]]; then operating_system="$(uname -s)" export operating_system fi } isLinux(){ [ -n "${LINUX:-}" ] && return 0 get_os if [ "$operating_system" = Linux ]; then export LINUX=1 return 0 fi return 1 } isMac(){ [ -n "${OSX:-}" ] && return 0 get_os if [ "$operating_system" = Darwin ]; then export APPLE=1 export OSX=1 return 0 fi return 1 } isGoogleCloudShell(){ [ -n "${GOOGLE_CLOUD_SHELL:-}" ] && return 0 get_os [ "$operating_system" = Linux ] || return 1 # DEVSHELL_PROJECT_ID is more likely to be unique to GCP Cloud Shell environment #if [ -n "${GOOGLE_CLOUD_PROJECT:-}" ]; then if [ -n "${DEVSHELL_PROJECT_ID:-}" ]; then export GOOGLE_CLOUD_SHELL=1 return 0 fi return 1 } isAzureCloudShell(){ [ -n "${AZURE_CLOUD_SHELL:-}" ] && return 0 get_os [ "$operating_system" = Linux ] || return 1 if [ -n "${ACC_TERM_ID:-}" ]; then export AZURE_CLOUD_SHELL=1 return 0 fi return 1 } # for compatibility to use the same names as non-interactive lib/ is_linux(){ isLinux } is_mac(){ isMac } is_google_cloud_shell(){ isGoogleCloudShell "$@" } is_azure_cloud_shell(){ isAzureCloudShell "$@" } # make this safe to import in set -e scripts is_linux || : isMac || : isGoogleCloudShell || : isAzureCloudShell || : ================================================ FILE: .bash.d/paths.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # $ P A T H # ============================================================================ # # general path additions that aren't big enough to have their own .sh file # this is sourced in .bashrc before .bash.d/*.sh because add_PATH() is used extensively everywhere to deduplicate $PATHs across disparate code and also reloads before it gets to this point in the .bash.d/*.sh lexically ordered list if type add_PATH &>/dev/null && [ -n "${PATHS_SET:-}" ]; then return fi bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # unreliable that HOME is set, ensure shell evaluates to the right thing before we use it [ -n "${HOME:-}" ] || HOME=~ github="${github:-$HOME/github}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # ============================================================================ # repaths(){ unset PATHS_SET # shellcheck disable=SC1091 source "$bash_tools/.bash.d/paths.sh" } #export PATH="${PATH%%:~/github*}" add_PATH(){ export PATH="$PATH:$1" # this clever stuff kills performance and I want my shell to open faster # it's not worth saving a few duplicates in $PATH # was used by dedupe paths at the end of this file #local env_var #local path #if [ $# -gt 1 ]; then # env_var="$1" # path="$2" #else # env_var=PATH # path="${1:-}" #fi #path="${path%/}" #path="${path//[[:space:]]/}" #if [[ "$path" =~ \$ ]]; then # echo "WARNING: skipping add path '$path' for safety" # return #fi #if ! [[ "${!env_var}" =~ (^|:)$path(:|$) ]]; then # # shellcheck disable=SC2140 # eval "$env_var"="${!env_var}:$path" #fi ## to prevent Empty compile time value given to use lib at /Users/hari/perl5/lib/perl5/perl5lib.pm line 17. ##PERL5LIB="${PERL5LIB##:}" ## fix for Codeship having a space after one of the items in their $PATH, causing the second half of the $PATH to error out as a command #eval "$env_var"="${!env_var//[[:space:]]/}" #eval "$env_var"="${!env_var##:}" #export "${env_var?env_var not defined in add_PATH}" } # use 'which -a' # #binpaths(){ # if [ $# != 1 ]; then # echo "usage: binpaths " # return 1 # fi # local bin="$1" # tr ':' '\n' <<< "$PATH" | # while read -r path; do # if [ -x "$path/$bin" ]; then # echo "$path/$bin" # fi # done #} # need newer Homebrew curl for --cookies-from-browser functionality # Homebrew build wasn't compiled with it anyway - see curl_with_cookies.sh script for a more generic workaround # that works with any version of curl #if [ -d /opt/homebrew/opt/curl/bin ]; then # PATH="/opt/homebrew/opt/curl/bin:$PATH" #fi add_PATH "/bin" add_PATH "/usr/bin" add_PATH "/sbin" add_PATH "/usr/sbin" add_PATH "/usr/local/sbin" add_PATH "/usr/local/bin" add_PATH "/usr/local/opt/python/libexec/bin" # Mac brew installed Python, must be ahead of ~/anaconda/bin below add_PATH "/opt/homebrew/bin/" # on new M1 Macs add_PATH "$bash_tools" add_PATH ~/bin add_PATH ~/.local/bin add_PATH ~/venv/bin while read -r x; do # much less noisy to just just find the right dirs instead of testing lots of files #[ -d "$x" ] || continue #if [ -d "$x/bin" ]; then # add_PATH "$x/bin" #else add_PATH "$x" #fi done < <(for x in "$bash_tools" ~/bin; do find "$x" -maxdepth 2 -type d -name bin; done) # Serverless.com framework if [ -d ~/.serverless/bin ]; then add_PATH ~/.serverless/bin fi # HomeBrew on Linux if [ -d /opt/homebrew/bin ]; then add_PATH /opt/homebrew/bin fi # HomeBrew on Linux if [ -d ~/.linuxbrew/bin ]; then add_PATH ~/.linuxbrew/bin fi # AWS CLI Linux install location if [ -d ~/.local/bin ]; then add_PATH ~/.local/bin fi # AWS SAM CLI Linux install location if [ -d "/home/linuxbrew/.linuxbrew/bin" ]; then add_PATH "/home/linuxbrew/.linuxbrew/bin" fi # Rancher Desktop if [ -d ~/.rd/bin ]; then add_PATH ~/.rd/bin fi if [ -d ~/.pulumi/bin ]; then add_PATH ~/.pulumi/bin fi #add_PATH "${JX_HOME:-$HOME/.jx}/bin" add_PATH ~/.jx/bin # do the same with MANPATH if [ -d ~/man ]; then MANPATH=~/man:"${MANPATH:-}" export MANPATH fi # added to .bash_profile by SnowSQL installer #if [ -d /Applications/SnowSQL.app/Contents/MacOS ]; then # add_PATH /Applications/SnowSQL.app/Contents/MacOS #fi # so that you can open files in IntelliJ from the command line: idea if [ -d "/Applications/IntelliJ IDEA CE.app/Contents/MacOS" ]; then add_PATH "/Applications/IntelliJ IDEA CE.app/Contents/MacOS" fi if [ -d "/Applications/Visual Studio Code.app" ]; then # don't need this one as you can just 'code /path/to/filename' to open the file in VS Code #add_PATH "/Applications/Visual Studio Code.app/Contents/MacOS" # Electron IDE is here add_PATH "/Applications/Visual Studio Code.app/Contents/Resources/app/bin" # code CLI is here fi # ============================================================================ # # A n a c o n d a # ============================================================================ # # Make sure to customize Anaconda installation and de-select Modify Path otherwise it'll change the bash profile # XXX: WARNING - this will appear earlier in the $PATH than the python bin paths, so if you have it installed, you should use it # otherwise pylint for example may be called from anaconda/bin but not have the pip modules necessary to check files, leading to CI breakages # for the 'conda' command add_PATH ~/anaconda/bin # version installed by HomeBrew add_PATH /usr/local/anaconda3/bin # ============================================================================ # # P a r q u e t T o o l s # ============================================================================ # for x in ~/bin/parquet-tools-*; do if [ -d "$x" ]; then add_PATH "$x" fi done if [ -d /usr/local/parquet-tools ]; then add_PATH "/usr/local/parquet-tools" fi # ============================================================================ # # M y G i t H u b r e p o s # ============================================================================ # # $github defined in aliases.sh # shellcheck disable=SC2154 add_PATH "$bash_tools" while read -r x; do add_PATH "$x" done < <(find "$bash_tools" -maxdepth 1 -type d) add_PATH "$github/go-tools" add_PATH "$github/go" add_PATH "$github/go-tools/bin" add_PATH "$github/go/bin" add_PATH "$github/perl-tools" add_PATH "$github/perl" add_PATH "$github/pytools" add_PATH "$github/tools" #add_PATH "$github/tool" add_PATH "$github/nagios-plugins" add_PATH "$github/nagios-plugin-kafka" add_PATH "$github/spotify" add_PATH "$github/spotify-tools" if is_linux; then add_PATH ~/.buildkite-agent/bin fi # ============================================================================ # link_latest(){ # -p suffixes / on dirs, which we grep filter on to make sure we only link dirs # shellcheck disable=SC2010 ls -d -p "$@" | grep "/$" | tail -n 1 | while read -r path; do [ -d "$path" ] || continue #local path_noversion="$( echo "$path" | perl -pn -e 's/-\d+(\.v?\d+)*(-\d+|-[a-z]+)?\/?$//' )" local path_noversion path_noversion="$(perl -pn -e 's/-\d+[\.\w\d-]+\/?$//' <<< "$path")" if [ "$path_noversion" = "$path" ]; then echo "FAILED to strip version, linking back on itself will create a link in subdir" return 1 fi [ -e "$path_noversion" ] && [ ! -L "$path_noversion" ] && continue if is_mac; then local ln_opts="-h" else local ln_opts="-T" fi # if you're in 'admin' group on Mac you don't really need to sudo here # shellcheck disable=SC2154 $sudo ln -vfs $ln_opts -- "$path" "$path_noversion" done } # ============================================================================ # # ============================================================================ # # O l d S t u f f # ============================================================================ # # Most of the stuff below has been migrated to Docker rather than /usr/local installs # ============================================================================ # # A p a c h e D r i l l # ============================================================================ # #link_latest /usr/local/apache-drill-* #export DRILL_HOME=/usr/local/apache-drill #add_PATH "$DRILL_HOME/bin" # ============================================================================ # # M i s c # ============================================================================ # #add_PATH "/usr/local/etcd" #add_PATH "/usr/local/artifactory-oss/bin" #add_PATH "/usr/local/jmeter/bin" #add_PATH "/usr/local/jruby/bin" #add_PATH "/usr/local/jython/bin" #add_PATH "/usr/local/mysql/bin" #add_PATH ~/bin/expect #add_PATH "$RANCID_HOME/bin" #add_PATH /usr/lib/bin/distcc #add_PATH "/usr/lib/nagios/plugins" #add_PATH "/usr/nagios/libexec" #add_PATH "/usr/nagios/libexec/contrib" #if is_mac; then # # MacPort and Octave installation # add_PATH /opt/local/bin # # if [ -d "/Applications/VMware Fusion.app/Contents/Library" ]; then # add_PATH "/Applications/VMware Fusion.app/Contents/Library" # fi #fi # ============================================================================ # # C a s s a n d r a # ============================================================================ # #export CASSANDRA_HOME=/usr/local/cassandra #export CCM_HOME=/usr/local/ccm #add_PATH "$CASSANDRA_HOME/bin" #add_PATH "$CASSANDRA_HOME/tools/bin" #add_PATH "$CCM_HOME/bin" # ============================================================================ # # E l a s t i c s e a r c h # ============================================================================ # #export ELASTICSEARCH_HOME=/usr/local/elasticsearch #add_PATH "$ELASTICSEARCH_HOME/bin" # ============================================================================ # # C o u c h b a s e # ============================================================================ # #export COUCHBASE_HOME="/Applications/Couchbase Server.app/Contents/Resources/couchbase-core" #alias cbq="$COUCHBASE_HOME/bin/cbq" #add_PATH "$COUCHBASE_HOME/bin" # ============================================================================ # # G r o o v y # ============================================================================ # # brew install groovy #export GROOVY_HOME=/usr/local/opt/groovy/libexec # brew uninstall groovy # brew install groovysdk #export GROOVY_HOME=/usr/local/opt/groovysdk/libexec # using SDK Man now, sourced at end of private .bashrc # ============================================================================ # # 0 x d a t a H 2 O # ============================================================================ # #export H2O_HOME=/usr/local/h2o #alias h2o="cd $H2O_HOME && java -jar h2o.jar -Xmx1g" # ============================================================================ # # J e t t y # ============================================================================ # #export JETTY_HOME="/usr/local/jetty-hightide" #alias jetty="cd $JETTY_HOME/ && java -jar start.jar" # ============================================================================ # # M e s o s # ============================================================================ # # this breaks parsing if supplying without port and causes duplicate --master switch if supplying the switch manually to mesos-slave or mesos-slave.sh #export MESOS_MASTER=$HOST:5050 # link_latest /usr/local/mesos #export MESOS_HOME=/usr/local/mesos #add_PATH "$MESOS_HOME/bin" #if is_mac; then # export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/mesos/src/.libs/libmesos.dylib #else # # check this path # export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/mesos/lib/libmesos.so #fi # deprecated old var #export MESOS_NATIVE_LIBRARY="$MESOS_NATIVE_JAVA_LIBRARY" # ============================================================================ # # M o n g o D B # ============================================================================ # #export MONGO_HOME=/usr/local/mongo #add_PATH "$MONGO_HOME/bin" #add_PATH "$github/mtools" # ============================================================================ # # N e o 4 J # ============================================================================ # #export NEO4J_HOME="/usr/local/neo4j" #add_PATH "$NEO4J_HOME/bin" # ============================================================================ # # S o l r # ============================================================================ # # find /usr/local -type d -name 'apache-solr-*' -maxdepth 1 | while read path; do sudo ln -vfsh "$path" "${path%%-*}"; done # link_latest '/usr/local/apache-solr-*' # ln -vsf /usr/local/apache-solr /usr/local/solr # 3.x #export SOLR_HOME=/usr/local/apache-solr # 4.x #export SOLR_HOME=/usr/local/solr #export APACHE_SOLR_HOME="$SOLR_HOME" #add_PATH "$SOLR_HOME/bin" #add_PATH "$SOLR_HOME/example/scripts/cloud-scripts" # ============================================================================ # # S t o r m # ============================================================================ # #export STORM_HOME=/usr/local/storm #add_PATH "$STORM_HOME/bin" # ============================================================================ # # T a c h y o n # ============================================================================ # #export TACHYON_HOME=/usr/local/tachyon #add_PATH "$TACHYON_HOME/bin" # ============================================================================ # # B a s h o R i a k # ============================================================================ # #export RIAK_HOME=/usr/local/riak #add_PATH "$RIAK_HOME/bin" # ============================================================================ # # S c a l a # ============================================================================ # #add_PATH "/usr/local/scala/bin" # ============================================================================ # # S p a r k # ============================================================================ # #export SPARK_HOME=/usr/local/spark #add_PATH "$SPARK_HOME/bin" # ============================================================================ # # S o n a r Q u b e # ============================================================================ # #export SONAR_SCANNER_HOME=/usr/local/sonar-scanner #add_PATH "$SONAR_SCANNER_HOME/bin" # ============================================================================ # # TypeSafe Activator - Akka, Play # ============================================================================ # # link_latest /usr/local/activator-dist-* #export ACTIVATOR_HOME=/usr/local/activator-dist #add_PATH "$ACTIVATOR_HOME" # slows down new shells #dedupe_paths(){ # local var="${1:-PATH}" # local path_tmp="" # # <( ) only works in Bash, but breaks when sourced from sh # # <( ) also ignores errors which don't get passed through the /dev/fd # # while read -r path; do # #done < <(tr ':' '\n' <<< "$PATH") # local IFS=':' # for path in ${!var}; do # if [[ "$path" =~ ^[[:space:]]*$ ]]; then # continue # fi # if ! [[ "$path_tmp" =~ :$path(:|$) ]]; then # path_tmp="$path_tmp:$path" # fi # done # eval export "$var"="\"$path_tmp\"" #} # call in z_final.sh #dedupe_paths export PATHS_SET=1 ================================================ FILE: .bash.d/perl.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P e r l # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/os_detection.sh" #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" if [ -d ~/perl5/bin ]; then add_PATH ~/perl5/bin fi # see the effect of inserting a path like so # PERL5LIB=/path/to/blah perlpath perlpath(){ perl -e 'print join("\n", @INC) . "\n";' } # XXX: Perl Taint mode resets and restricts the Perl Path to not use PERL5LIB for security # # scripts like those in the Advanced Nagios Plugins Collection and DevOps-Perl-tools will need to add lib # #if [ -d /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Perl/ ]; then # add_PATH PERL5LIB /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Perl #fi if [ -d ~/perl5/lib/perl5 ]; then #add_PATH PERL5LIB ~/perl5/lib/perl5 #export PERL5LIB="$PERL5LIB:$HOME/perl5/lib/perl5" if ! [[ "${PERL5LIB:-}" == *"$HOME/perl5/lib/perl5"* ]]; then export PERL5LIB=~/perl5/lib/perl5"${PERL5LIB+:$PERL5LIB}" fi fi if ! [[ "${PERL_LOCAL_LIB_ROOT:-}" == *"$HOME/perl5"* ]]; then export PERL_LOCAL_LIB_ROOT="$HOME/perl5${PERL_LOCAL_LIB_ROOT+:$PERL_LOCAL_LIB_ROOT}" fi export PERL_MB_OPT="--install_base '$HOME/perl5'" export PERL_MM_OPT="INSTALL_BASE=$HOME/perl5" alias lsperlbin='ls -d ~/perl5/bin/* 2>/dev/null' alias llperlbin='ls -ld ~/perl5/bin/* 2>/dev/null' # cpanm --local-lib=~/perl5 local::lib # populates a bunch of Perl env vars pointing to ~/perl5/... # eval "$(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)" ================================================ FILE: .bash.d/postgres.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-16 13:20:17 +0000 (Mon, 16 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P o s t g r e S Q L # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/functions.sh" # highest priority env var alias pgpass='pass PGPASSWORD' # mac alias postgresd='postgres -D /usr/local/var/postgres' ================================================ FILE: .bash.d/prompt.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P r o m p t M a g i c # ============================================================================ # # XXX: Warning: This must be perfect - edit at your own peril as imperfect PS1 prompt codes cause terminal wrap around to the same line [ -n "${GOOGLE_CLOUD_SHELL:-}" ] && return #[ "${USER:-}" = vagrant ] && return # \[\033k\033\\\] is required for Screen auto title feature to detect prompt # replace \033 with \e as it's directly supported in PS1 # need 'Defaults env_keep=STY' for this to not trigger on sudo su # shellcheck disable=SC1117 SCREEN_ESCAPE="\[\ek\e\\\\\]" PS1="" # if inside Screen, set the screen escape inside PS1 [ -n "$STY" ] && PS1="$SCREEN_ESCAPE" # defined in adjacent colors.sh # shellcheck disable=SC2154 PS1_COLOUR="$txtgrn" # shellcheck disable=SC2154 PS1_USER_COLOUR="$txtcyn" if [ $EUID -eq 0 ]; then # shellcheck disable=SC2154 PS1_COLOUR="$bldred" PS1_USER_COLOUR="$bldred" fi # XXX: important that every single escape sequence is enclosed in \[ \] to make sure it isn't included in the line wrapping calculcation otherwise the lines wrap back on to themselves # \W basename of cwd # \w full path of cwd # \h host # shellcheck disable=SC2154 # export PS1+="\[$PS1_COLOUR\]\t \[$bldblu\]\w \[$PS1_COLOUR\]> \[$txtrst\]" # shellcheck disable=SC1117 export PS1+="\[$PS1_COLOUR\]\t \[$PS1_USER_COLOUR\]\u\[$txtwht\]@\[$bldcyn\]\h:\[$bldpur\]\$(git branch 2>/dev/null | grep '^*' | sed 's/^*//') \[$bldgrn\]\w \[$PS1_COLOUR\]> \[$txtrst\]" #if type kube_ps1 &>/dev/null; then # PS1='$(kube_ps1)'" $PS1" #fi # Screen relies on prompt having a dollar sign to detect the next word to set the screen title dynamically - .screenrc needs the following setting # # shelltitle ' > |' # TODO: make screen auto title detect # for root # For passing PS1 around: # # base64 <<< "$PS1" # # and then pass the result through base64 --decode # # base64 <<< "$PS1" | base64 --decode # # is a noop to demonstrate if [ -z "$BASH_TIMING" ]; then export PS4="--> " fi # Bash performance profiling, can be heavy on performance, &>/tmp/bash_perf.out then use a profiling script # combine with DEBUG=1 or set -x bash_timing(){ export BASH_TIMING=1 export PS4='$(date "+%s.%N ($LINENO) + ") --> ' } debug_bashrc(){ PS4='+ $BASH_SOURCE:$LINENO:' bash -xic '' 2>&1 | less } debug_bash_profile(){ PS4='+ $BASH_SOURCE:$LINENO:' bash -xlic '' 2>&1 | less } if is_mac; then # turn this off on mac thing - it heavily pollutes $DEBUG output and does nothing because we have a custom prompt if [[ "${PROMPT_COMMAND:-}" =~ update_terminal_cwd ]]; then # this unsets direnv's hook #unset PROMPT_COMMAND PROMPT_COMMAND="${PROMPT_COMMAND//update_terminal_cwd;/}" fi # stripping update_terminal_cwd can leave some weird broken PROMPT_COMMAND start due to interaction with direnv, so strip it without forking to sed/perl while [[ "${PROMPT_COMMAND:-}" =~ ^[[:space:]]|^\; ]]; do PROMPT_COMMAND="${PROMPT_COMMAND##[[:space:]]}" PROMPT_COMMAND="${PROMPT_COMMAND##;}" done export PROMPT_COMMAND export SHELL_SESSION_HISTORY=0 fi ================================================ FILE: .bash.d/python.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P y t h o n # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # silence those annoying Python 2 cryptography warnings that mess up our programs outputs export PYTHONWARNINGS=ignore::UserWarning # shellcheck disable=SC1090,SC1091 type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # see the effect of inserting a path like so # PYTHONPATH=/path/to/blah pythonpath pythonpath(){ python -c 'from __future__ import print_function; import sys; [print(_) for _ in sys.path if _]' } # enable this to avoid creating .pyc files (sometimes they trip you up executing outdated python code) # export PYTHONDONTWRITEBYTECODE=1 if is_mac; then # try to find pip in brew installed Python versions since it is # not in /System/Library/Frameworks/Python.framework/Versions/2.7/bin for dir in /usr/local/Cellar/python*; do if [ -d "$dir" ]; then add_PATH "$dir/bin" fi done fi if [ -d ~/Library/Python ]; then for x in ~/Library/Python/*/bin; do [ -d "$x" ] || continue add_PATH "$x" done fi alias lspythonbin='ls -d ~/Library/Python/*/bin/* 2>/dev/null' alias llpythonbin='ls -ld ~/Library/Python/*/bin/* 2>/dev/null' alias lspybin=lspythonbin alias llpybin=llpythonbin # RHEL8 has split python2 / python3 and removed default 'python' :-( if ! type -P python &>/dev/null; then if type -P python2 &>/dev/null; then python(){ python2 "$@"; } elif type -P python3 &>/dev/null; then python(){ python3 "$@"; } fi fi ================================================ FILE: .bash.d/rancid.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2010 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # R a n c i d # ============================================================================ # export RANCID_HOME=~/rancid #flogin(){ # title "$1" # command flogin "${@:1}" # title " " #} # l33ter way of generating functions for all of the rancid programs #for x in $(ls "$RANCID_HOME/bin/"*login | sed 's/.*\///;s/*$//'); do # eval "$x"'(){ title "$1" # command '"$x"' ${@:1} # title " " # }' #done # More prim and proper abstracted function with minimal function code rancidlogin_func(){ local prog="$1" local host="$2" shift title "$host" command "$prog" "$@" title "$LAST_TITLE" } #for x in "$RANCID_HOME/bin/"*login; do y=${x##*/}; alias "$y"="rancidlogin_func $y"; done for x in "$RANCID_HOME/bin/"*login; do y="${x##*/}" # needs to be evaluated here to build dynamic aliases # shellcheck disable=SC2139,SC2140 alias "$y"="rancidlogin_func $y" done # for x in "$RANCID_HOME/bin/"*login; do y="${x##*/}"; which "${y%ogin}" &>/dev/null || alias "${y%ogin}"="$y"; done # This is slow to do every time so just past the echo output from: # for x in "$RANCID_HOME/bin/"*login; do y="${x##*/}"; which "${y%ogin}" &>/dev/null || echo alias "${y%ogin}"="$y"; done #alias al=alogin #alias avol=avologin #alias bl=blogin #alias cl=clogin #alias el=elogin #alias fl=flogin #alias hl=hlogin #alias htl=htlogin #alias jl=jlogin #alias mrvl=mrvlogin # nl is a real program so skipped nlogin #alias nsl=nslogin #alias rivl=rivlogin #alias tl=tlogin #alias tntl=tntlogin ================================================ FILE: .bash.d/ruby.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # R u b y / G e m # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/os_detection.sh" type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # gems will be installed to ~/.gem/ruby/x.y.z/bin # ============================================================================ # # R V M # ============================================================================ # # RVM # usually /usr/local/rvm/bin if [ -n "${rvm_bin_path:-}" ];then add_PATH "$rvm_bin_path" fi if [ -d /usr/local/rvm/bin ]; then add_PATH /usr/local/rvm/bin fi if [ -d ~/.rvm/bin ]; then add_PATH ~/.rvm/bin fi # ============================================================================ # # R b E n v # ============================================================================ # # RbEnv if [ -f ~/.rbenv/bin ]; then add_PATH ~/.rbenv/bin fi #eval "$(rbenv init - --no-rehash bash)" # ============================================================================ # # R u b y U s e r D i r s # ============================================================================ # # Ruby User Dirs # add newest ruby to path first ruby_bins="$( find ~/.gem/ruby -maxdepth 2 -name bin -type d 2>/dev/null find ~/.local/share/gem/ruby -maxdepth 2 -name bin -type d 2>/dev/null )" #if is_mac; then # ruby_bins_newest="$(tail -r <<< "$ruby_bins")" #else # ruby_bins_newest="$(tac <<< "$ruby_bins")" #fi ruby_bins_newest="$(sort -Vr <<< "$ruby_bins")" for ruby_bin in $ruby_bins_newest; do add_PATH "$ruby_bin" done unset ruby_bins unset ruby_bins_newest alias lsrubybin='ls -d ~/.gem/ruby/*/bin/* 2>/dev/null' alias llrubybin='ls -ld ~/.gem/ruby/*/bin/* 2>/dev/null' # ============================================================================ # # HomeBrew install on Linux (for AWS SAM CLI) if [ -d ~/.linuxbrew/Homebrew/Library/Homebrew/vendor/portable-ruby/current/bin ]; then add_PATH ~/.linuxbrew/Homebrew/Library/Homebrew/vendor/portable-ruby/current/bin fi ================================================ FILE: .bash.d/screen.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S c r e e n # ============================================================================ # # quickly open new screen terminal in the same $PWD alias scbash="screen bash" sc(){ checkprog screen || return 1 isscreen && { echo "I am already in a screen, aborting"; return 1; } screen -wipe local session=main local detached_screens detached_screens="$(screen -ls | grep Detached)" if [ -n "$detached_screens" ] && [ "$(wc -l <<< "$detached_screens" | awk '{print $1}')" = 1 ]; then session="$(awk '{print $1;exit}' <<< "$detached_screens")" fi screen -aARRD -S "$session" "$@" } screencmd(){ screen -X "$@" } screensleep(){ screen "$@" sleep 0.1 } alias scnum="screen -X number" #screen_get_pid(){ # # Mac ps doesn't have --noheaders # ps -p "${PPID}" -o ppid | tail -n +2 | sed 's/[[:space:]]//g' #} # #screen_get_session_name(){ # local ppid # ppid="$(screen_get_pid)" # screen -ls | awk "/^[[:blank:]]$ppid/{print \$1}" | cut -d. -f2 #} # needs modern version of screen for -Q switch - on Mac you must brew install screen to get recent version, then start new screen # when installing GNU screen you will lose Mac's screen since /usr/bin/screen uses a different /var/folders/...../.screen directory for screen sessions screen_renumber_windows(){ local windowlist windowlist="$(screen -Q windows | grep -Eo '(^|[[:blank:]])[[:digit:]]+')" i=0 for windownum in $windowlist; do screen -p "$windownum" -X number "$i" ((i+=1)) done } alias screnum=screen_renumber_windows screenbuf(){ local tmp tmp="$(mktemp /tmp/screen-exchange.XXXXXX)" cat > "$tmp" screen -X readbuf "$tmp" rm -- "$tmp"; } alias sb=screenbuf sh_server_real(){ for x in "$@"; do echo "sh server real $x | i $x|Weight|Total" done | tee /dev/stderr | screenbuf echo } alias fsr=sh_server_real alias ssr=sh_server_real # this idea's is a bust so far... #function c(){ # screen -t "$@" bash -c ". ~/.bashrc && eval $@" #} ================================================ FILE: .bash.d/skaffold.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-07-05 16:31:14 +0100 (Mon, 05 Jul 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S k a f f o l d # ============================================================================ # #bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" #autocomplete skaffold ================================================ FILE: .bash.d/spinner.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2012-06-25 15:20:39 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S p i n n e r # ============================================================================ # spinner(){ local msg="$* " #local num=${2:-100} local num=1000 #local delay=${3:-0.00001} local delay=0.00001 spin='-\|/' #printf "${msg//?/ }" printf "%s" "$msg " for ((i=0; i < num; i++)); do sleep $delay # This way results in more flashing #printf "\r${msg}${spin:$((${i}%${#spin})):1}" # TODO: naughty allowing variables in printf format string but fiddly with msg var replaced backspace otherwise, clean up later... # shellcheck disable=SC2059 printf "\\b${msg//?/\\b}${msg}${spin:$((i % ${#spin})):1}" done printf '\b ' echo echo "done" } ================================================ FILE: .bash.d/spotify.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-07-18 19:43:00 +0100 (Sat, 18 Jul 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # exports SPOTIFY_ACCESS_TOKEN so all the user private data spotify tools (../spotify_*.sh) can use it easily without re-prompting ia browser spotifysession(){ local SPOTIFY_ACCESS_TOKEN # this would prevent it from being exported to the shell as we want to make it easier to use full spotify tools #local SPOTIFY_ACCESS_TOKEN # SECONDS cannot be reset in the background in spotify_token_expire_timer() function SECONDS=0 spotify_token_expire_timer & # defined in ../.bashrc # shellcheck disable=SC2154 SPOTIFY_ACCESS_TOKEN="$(SPOTIFY_PRIVATE=1 "$bash_tools/spotify/spotify_api_token.sh")" export SPOTIFY_ACCESS_TOKEN timestamp "starting spotify session shell" "$SHELL" timestamp "exiting spotify session shell" unset SPOTIFY_ACCESS_TOKEN } spotify_token_expire_timer(){ # we have the same $$ as our foreground shell local ppid=$$ while true; do if [ "$SECONDS" -ge 3600 ]; then # XXX: this would never work as it'd only affect this background thread #unset SPOTIFY_ACCESS_TOKEN # instead kill the shell session and handle the unset in the spotifysession() function timestamp "Spotify Token expired - killing spotify shell" pgrep -P "$ppid" | # protect own shell so we can finish this code pipe grep -v $$ | # bash needs a -HUP signal, ignores TERM xargs kill -HUP break fi sleep 1 done } ================================================ FILE: .bash.d/ssh-agent.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S S H A g e n t # ============================================================================ # # keychain id_rsa # . .keychain/$HOSTNAME-sh ssh_agent(){ #if [ $UID != 0 ]; then local SSH_ENV_FILE=~/.ssh-agent.env if [ -f "${SSH_ENV_FILE:-}" ]; then # shellcheck source=~/.agent.env # shellcheck disable=SC1090,SC1091 . "$SSH_ENV_FILE" > /dev/null if ! kill -0 "$SSH_AGENT_PID" >/dev/null 2>&1; then echo "Stale ssh-agent found. Spawning new agent..." killall -9 ssh-agent eval "$(ssh-agent | tee "$SSH_ENV_FILE")" #| grep -v "^Agent pid [[:digit:]]\+$" # lazy evaluated ssh func now so it's not prompted until used #ssh-add elif [ "$(ps -p "$SSH_AGENT_PID" -o comm=)" != "ssh-agent" ]; then echo "ssh-agent PID does not belong to ssh-agent, spawning new agent..." eval "$(ssh-agent | tee "$SSH_ENV_FILE")" #| grep -v "^Agent pid [[:digit:]]\+$" # lazy evaluated ssh func now so it's not prompted until used #ssh-add fi else echo "Starting ssh-agent..." killall -9 ssh-agent eval "$(ssh-agent | tee "$SSH_ENV_FILE")" # lazy evaluated ssh func now so it's not prompted until used #ssh-add fi #clear #fi } [ -n "${GOOGLE_CLOUD_SHELL:-}" ] && return # do not launch SSH Agent if we are inheriting an SSH_AUTH_SOCK from an 'ssh -A' agent forwarding connection [ -n "${SSH_AUTH_SOCK:-}" ] && return ssh_agent ================================================ FILE: .bash.d/ssh.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S S H # ============================================================================ # #ssh() { set -o xtrace ; command ssh "$@" <<< "$(cat .bashrc_remote)" ; } alias sshconfig='$EDITOR ~/.ssh/config' alias sshcfg=sshconfig # ssh-add ssha(){ ssh_agent #num_keys="$(ssh-add -l | grep -Ec "(rsa|dsa)")" #if [ "$num_keys" -lt 1 ]; then # ssh-add ~/.ssh/id_[rd]sa #else # return 0 #fi for key in ~/.ssh/id_[rd]sa; do key_fingerprint="$(ssh-keygen -lf "$key" | awk '{print $2}')" if ! ssh-add -l | grep -Fq "$key_fingerprint"; then ssh-add "$key" fi done } ssh_func(){ ssha if [ "$1" = "ssh" ] || [ "$1" = "vagrant" ]; then local n=2 [ "$1" = "vagrant" ] && ((n+=1)) until [ "$n" -gt "$#" ]; do case ${!n} in -*) : ;; *) grep -Eq "^[0-9]+$" <<< "${!n}" || break ;; esac ((n+=1)) done fi local t="${!n}" # indirect reference to variable number by evaluating t="${t##*@}" t="${t%%.*}" title "$t" command "$1" "${@:2}" [ "$1" != "scp" ] && title "$LAST_TITLE" } alias ssh="ssh_func ssh" alias sshl="ssh-add -l" alias sshni="ssh_func ssh -oPreferredAuthentications=publickey -oStrictHostKeyChecking=no" alias scp="ssh_func scp" alias sftp="ssh_func sftp" #alias sshc="ssh_custom" safe_ssh(){ if [ $# -lt 1 ]; then echo "usage: safe_ssh [user@]hostname" return 1 fi host="${1##*@}" if grep -q '@' <<< "$1"; then user="${1%%@*}" else user=root fi #keys=`ssh -oStrictHostKeyChecking=no $@ 'for x in rsa dsa; do ssh-keygen -l -f /etc/ssh/ssh_host_${x}_key ; done' 2>&1` #keys=`ssh -oStrictHostKeyChecking=no $@ 'for x in rsa dsa; do cat /etc/ssh/ssh_host_${x}_key ; done' 2>&1` keys="$(ssh "$user@$host" 'for x in rsa dsa; do cat /etc/ssh/ssh_host_${x}_key.pub ; done')" rsa_key="$(echo "$keys" | awk '/ssh-rsa/ {print $1" "$2}')" dsa_key="$(echo "$keys" | awk '/ssh-dsa/ {print $1" "$2}')" known_key="$(grep "${@##*@}" ~/.ssh/known_hosts | awk '{print $2" "$3}' | sort -u)" echo if [ "$rsa_key" = "$known_key" ]; then echo "OK: Host rsa key matches known key" elif [ "$dsa_key" = "$known_key" ]; then echo "OK: Host dsa key matches known key" else echo 'WARNING: known ssh key for '"$1"' does not match either rsa or dsa keys obtained from server!!!' echo "Known Key: $known_key" echo "RSA Key: $rsa_key" echo "DSA Key: $dsa_key" fi } alias sssh=safe_ssh check_sshkey(){ for x in "$@"; do grep "$x" ~/.ssh/known_hosts | sort | while read -r host id known_key; do scanned_key="$(ssh-keyscan "$host" | awk "/^$host $id / {print \$3}")" if [ "$scanned_key" != "$known_key" ]; then echo -e "\\nMISMATCH: $host\\nknown key: $known_key\\nscanned_key:$scanned_key\\n\\n" fi done done } #update_sshkey(){ # host="$1" # id="$2" # key="$3" # perl -pi -e "s/^$host .*/$host $id $key/" ~/.ssh/known_hosts #} ressh(){ # FIXME: a cheat, it should be set properly but ssh in retry doesn't look like it's triggers ssh_func properly title "$1" retry ssh "$@" title "$LAST_TITLE" } rissh(){ if [ $# -lt 1 ]; then echo "rissh " return 1 fi cleankey "$@" ressh "$@" -oStrictHostKeyChecking=no } issh(){ if [ $# -lt 1 ]; then echo "issh " return 1 fi cleankey "$@" ssh -oStrictHostKeyChecking=no "$@" } bouncessh(){ checkhost "$1" || return 1 title "$1" whendown "$1" ressh "$@" } alias bssh=bouncessh bouncerissh(){ checkhost "$1" || return 1 title "$1" whendown "$1" rissh "$@" } alias brissh=bouncerissh alias bissh=bouncerissh rekey(){ [ -n "$1" ] || { echo "usage: rekey host"; return 1; } cleankey "$1" ssh-keyscan -t rsa "$1" | grep "^$1 ssh-rsa" >> ~/.ssh/known_hosts ssh-keyscan -t dsa "$1" | grep "^$1 ssh-dss" >> ~/.ssh/known_hosts } sshkey(){ local key=~/.ssh/id_rsa.pub # now available on Mac, but my tried and tested function of years gone by dedupes the keys # if type -P ssh-copy-id; then # ssh-copy-id -i "$key" "$@" # else ssh "$@" ' umask 077; [ -d ~/.ssh ] || mkdir -p ~/.ssh; key=`cat`; # my version is better than what ssh-copy-id did it would add duplicate keys if ! grep "$key" ~/.ssh/authorized_keys >/dev/null 2>&1; then echo $key >> ~/.ssh/authorized_keys; fi; chmod 0600 ~/.ssh/authorized_keys # this line was the only advantage ssh-copy-id script had test -x /sbin/restorecon && /sbin/restorecon ~/.ssh ~/.ssh/authorized_keys >/dev/null 2>&1 || true ' < "$key" # fi } sshkeygo(){ sshkey "$@" ssh "$@" } sshkey2(){ sshkeygo "$@" } cleankey(){ if [ $# -lt 1 ]; then echo "usage: cleankey regex" return 1 fi for x in "$@"; do ssh-keygen -R "$x" local aliasname aliasname="$(host "$x" | awk '/is an alias for/ {print $6}')" if [ -n "$aliasname" ]; then ssh-keygen -R "$aliasname" ssh-keygen -R "${aliasname%%.*}" fi continue # local ip # ip="$(host -W 1 "$x" | grep address)" # if [ $? -eq 0 ]; then # ip="$(cut -d" " -f 4 <<< "$ip")" # perl -pi -e 's/^\[?[^,]+\]?(:\d+)?,\[?'"$ip"'\]?(:\d)? .*$//;s/^'"$ip"' .*$//' ~/.ssh/known_hosts # fi # # need to leave second deletion just in case as you may want to specify just the ip address # #perl -pi -e 's/^\[?'"$x"'\]?(:\d+)?\[,\s.*$//;s/^.*[^,]+,'"$x"' .*$//' ~/.ssh/known_hosts # perl -pi -e 's/^'"$x"'\s.*$//;s/^.*[^,]+,'"$x"' .*$//' ~/.ssh/known_hosts done } keyremove(){ for x in "$@"; do # shellcheck disable=SC1117 ssh -o "PasswordAuthentication no" "$x" ' for y in ~/.ssh/authorized_keys*; do if [ -f "$y" ]; then perl -pi -e '"'s/ssh-rsa .*= hari@.*\n//'"' "$y" fi done ' done } #keyremoveall(){ # for x in "$@"; do # for y in root hari oracle; do # keyremove "$y@$x" # done # done #} ================================================ FILE: .bash.d/svn.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # R e v i s i o n C o n t r o l - S u b v e r s i o n # ============================================================================ # # I don't use SVN any more so a lot of the convenient aliases for daily use are commented out bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # superceded by hg and then git pull #alias u="svn up" #alias sd="svn diff" #alias svnl=svnlog #alias svnr="svn revert" # remapped to screen #s(){ svn st "$@" | more; } #stx(){ svn st "$@" | grep -v "^?"; } isSvn(){ local target=${1:-.} if [ -d "$target/.svn" ]; then return 0 elif [ -f "$target" ] && [ -d "$(dirname "$target")/.svn" ]; then return 0 else return 1 fi } svn(){ if is_mac; then export stat_formatopt=f else export stat_formatopt=c fi local svn_owner for x in "$@"; do if [ -d "$x/.svn" ]; then local dir_tested=true svn_owner="$(stat -$stat_formatopt "%u" "$x/.svn")" if [ "$EUID" != "$svn_owner" ]; then echo "YOU ARE RUNNING SVN AS THE WRONG USER ON $x" return 1 fi fi done if [ "$dir_tested" != "true" ]; then if [ -d "./.svn" ]; then svn_owner="$(stat -$stat_formatopt "%u" ./.svn)" if [ "$EUID" != "$svn_owner" ]; then echo "YOU ARE RUNNING SVN AS THE WRONG USER HERE" return 1 fi fi fi command svn "$@" } svnst(){ svn st "$@" } svnkw(){ svn ps svn:keywords "LastChangedBy LastChangedDate Revision URL Id" "$@" } svnadd(){ svn add -- "$@" && svnkw "$@" } svni(){ svn pe svn:ignore -- "${@:-.}" } svnaddci(){ svnadd "$@" && svn ci -m "added $*" -- "$@" } svnci() { local svncimsg="" for x in "$@"; do if svn st "$x" | grep -q "^[?A]"; then svncimsg+="$x, " fi done [ -z "$svncimsg" ] && return 1 svncimsg="${svncimsg%, }" svncimsg="added $svncimsg" svn add -- "$@" && svn ci -m "$svncimsg" "$@" } svnrm(){ svn rm -- "$@" && svn ci -m "removed $*" -- "$@" } svnrmf(){ svn rm --force -- "$@" && svn ci -m "removed $*" -- "$@" } svnrename(){ svn up "$(dirname "$1")" "$(dirname "$2")" svn mv -- "$1" "$2" && svn ci -m "renamed $1 to $2" -- "$1" "$2" } svnrename2(){ local svn_url svn_url=$(svn info "$1" | grep "^URL: " | sed 's/^URL: //') [ -n "$svn_url" ] || return svn rename -m "renamed $1 to $2" -- "$svn_url" "$(dirname "$svn_url")/$2" svn up -- "$1" "$2" } svnmkdir(){ svn mkdir --parents -- "$@" && svn ci -m "created directory $*" -- "$@" } svnmv(){ svn up "$(dirname "$1")" "$(dirname "$2")" svn mv -- "$1" "$2" && svn ci -m "moved $1 to $2" -- "$1" "$2" } svnrevert(){ svn revert -- "$@" } svnlog(){ local args=() local args2=() until [ $# -lt 1 ]; do case "$1" in -*) args+=("$1") ;; *) args2+=("$1") ;; esac shift done svn up "${args2[@]}" && read -r -p "press enter to see log" && svn log "${args[@]}" "${args2[@]}" | less } svnu(){ [ -n "$1" ] || { echo "ERROR: must supply arg"; return 1; } [ "$(svn diff -- "$@" | wc -l)" -gt 0 ] || return svn diff -- "$@" | more && read -r && svn ci -m "updated $*" -- "$@" } svne(){ svn ps svn:executable on "$@" } svnec(){ svne "$@"; for x in "$@"; do svn ci -m "set executable on $x" done } svncommitauthors(){ svn log | awk '/^r[[:digit:]]+[[:space:]]/ {print $3}' | sort | uniq -c | sort -k1nr } rmnonsvn(){ svn st | grep "^?" | awk '{print $2}' | xargs rm -f -- } svndifflast(){ local rev=HEAD local rev_last=PREV if grep -q '^[[:digit:]]\+' <<< "$1"; then rev="$1" shift; fi if [ "$rev" != "HEAD" ]; then #let rev_last=$rev-1 (( rev_last = rev - 1 )) fi svn diff -r "$rev_last:$rev" -- "$@" | more } #alias sdl=svndifflast svndiff(){ local filename="${1:-}" [ -n "$filename" ] || { echo "usage: svndiff filename"; return 1; } svn diff "$filename" > "/tmp/svndiff.tmp" diffnet.pl "/tmp/svndiff.tmp" } svndiffcumulative(){ local url svn up url="$(svn info | awk '/^URL/ {print $2}')" HEAD="$(svn info | awk '/Revision/ {print $2}')" for x in $(eval echo "{25470..$HEAD}"); do ((y=x+1)) echo -n "svn $x => $y: " svn diff -r -- "$x:$y" "$url" done } alias svndiffcum="svndiffcumulative" svncommmitcount(){ svn up svn log -r 25470:HEAD | grep -E "^r[[:digit:]]+ |" | wc } ================================================ FILE: .bash.d/teamcity.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-11-24 17:09:11 +0000 (Tue, 24 Nov 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T e a m C i t y # ============================================================================ # # sets TeamCity URL to the local docker and finds and loads the current container's superuser token to the environment for immediate use with teamcity_api.sh teamcity_local(){ TEAMCITY_SUPERUSER_TOKEN="$( # project name must match COMPOSE_PROJECT_NAME from teamcit.sh otherwise will fail to find token docker-compose -p bash-tools -f "$(dirname "${BASH_SOURCE[0]}")/../docker-compose/teamcity.yml" \ logs teamcity-server | \ grep -E -o 'Super user authentication token: [[:alnum:]]+' | \ tail -n1 | \ awk '{print $5}' || : )" export TEAMCITY_SUPERUSER_TOKEN export TEAMCITY_URL="http://localhost:8111" } ================================================ FILE: .bash.d/terraform.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-09-24 18:02:04 +0100 (Thu, 24 Sep 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T e r r a f o r m # ============================================================================ # if ! [ -e ~/.tfenv/bin ] && is_mac; then mkdir -p -v ~/.tfenv tfenv_bin="$(find /usr/local/Cellar/tfenv -type d -name bin 2>/dev/null | head -n1)" if [ -d "$tfenv_bin" ]; then ln -sfv -- "$tfenv_bin" ~/.tfenv/bin fi fi add_PATH ~/.tfenv/bin # don't get this wrong or you'll get this error: # # ERRO[0000] fork/exec /Users/hari/.tfenv/bin: no such file or directory # ERRO[0000] Unable to determine underlying exit code, so Terragrunt will exit with error code 1 # if [ -x ~/.tfenv/bin/terraform ]; then #export TERRAGRUNT_TFPATH=~/.tfenv/bin/ # it's full path to binary executable not a search $PATH! # neweer versions of Mac seems to not create ~/.tfenv/bin/ terraform link and instead use /opt/homebrew/bin/terraform link to the tfenv Cellar path export TERRAGRUNT_TFPATH=~/.tfenv/bin/terraform elif ! [ -x "$TERRAGRUNT_TFPATH" ]; then unset TERRAGRUNT_TFPATH fi export TG_PROVIDER_CACHE=1 export TG_PROVIDER_CACHE_HOST=172.0.0.1 alias tf=terraform alias tfp='tf plan' alias tfa='tf apply' alias tfip='tf init && tfp' alias tfia='tf init && tfa' alias tfaa='tfa -auto-approve' alias tfiaa='tfia -auto-approve' #complete -C /Users/hari/bin/terraform terraform unalias tffu &>/dev/null || : tffu(){ local lock_id="${1:-}" # self-determine the lock if not provided if [ -z "$lock_id" ]; then lock_id="$(terraform plan -input=false -no-color 2>&1 | grep -A 1 'Lock Info:' | awk '/ID:/{print $NF; exit}')" fi if [ -z "$lock_id" ]; then echo "Failed to determine lock ID" >&2 return 1 fi terraform force-unlock -force "$lock_id" } alias tg=terragrunt alias tgp='tg plan' alias tga='tg apply' alias tgaa='tga -auto-approve' alias tgip='tg init && tgp' alias tgia='tg init && tga' # the fix for .terraform.lock.hcl: # # the cached package for registry.terraform.io/hashicorp/aws 5.80.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file # alias tfprov='terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=linux_amd64' alias tgprov='terragrunt providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=linux_amd64' unalias tgfu &>/dev/null || : tgfu(){ local lock_id="${1:-}" # self-determine the lock if not provided if [ -z "$lock_id" ]; then lock_id="$(terragrunt plan -input=false -no-color 2>&1 | grep -A 1 'Lock Info:' | awk '/ID:/{print $NF; exit}')" fi if [ -z "$lock_id" ]; then echo "Failed to determine lock ID" >&2 return 1 fi terragrunt force-unlock -force "$lock_id" } if [ -n "${github:-}" ]; then for x in terraform-templates terraform tf; do if [ -d "$github/$x" ]; then # shellcheck disable=SC2139 alias tft="cd '$github/$x'" break fi done fi #generate_terraform_autocomplete(){ # local terraform_bin # local terraform_version_number # # for terraform_bin in ~/bin/terraform[[:digit:]]*; do # [ -x "$terraform_bin" ] || continue # terraform_version_number="${terraform_bin##*/terraform}" # # expand now # # shellcheck disable=SC2139,SC2140 # alias "tf${terraform_version_number}"="$terraform_bin" # complete -C "$terraform_bin" terraform # complete -C "$terraform_bin" tf # done # # terraform_bin="$(type -P terraform)" # complete -C "$terraform_bin" terraform # complete -C "$terraform_bin" tf #} # #generate_terraform_autocomplete ================================================ FILE: .bash.d/title.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T i t l e M a g i c # ============================================================================ # # Sets the Screen and Terminal titles alias ti="title" # static title - turn dynamic prompt escape codes off, while optionally setting new title alias sti="dpoff >/dev/null; ti" termtitle(){ # in tmux sets the secondary title at bottom right, duplicating info if ! istmux; then printf '\033]0;%s\007' "${*:- }" fi } isscreen(){ # $STY is only set in screen it seems so this determines if we're in screen [ -n "${STY:-}" ] } screentitle(){ if isscreen; then # shellcheck disable=SC1003 printf '\033k%s\033\\' "${*:- }" # or # screen -X title "$*" fi } istmux(){ [ -n "${TMUX:-}" ] } tmuxtitle(){ if istmux; then # window name appears in bottom left as a secondary name #printf "\033]2;%s\033\\" "${*:-}" # this is actually what we want to act like screen tmux rename-window "${*:-}" fi } title(){ export LAST_TITLE="$TITLE" #if [ $# -eq 0 ]; then # return # various commands will reset the title after their commands so skip those calls if NO_SCREEN_UPDATES is set if [ -z "$*" ] && [ "$NO_SCREEN_UPDATES" = "1" ]; then return fi export TITLE="$*" termtitle "$TITLE" screentitle "$TITLE" tmuxtitle "$TITLE" } # .bashrc reload causes title loss if enabling this #title # so instead just reset the termtitle which we don't see anyway and get rid of that annoying Unnamed in the title bar termtitle " " # ============================================================================ # # toggle dynamic prompt on / off dpstatus(){ if grep -q '^\\\[\\ek\\e\\\\\\]' <<< "$PS1"; then echo "enabled" return 0 else echo "disabled" return 1 fi } # dynamic prompt escape codes off dpoff(){ if dpstatus >/dev/null; then title " " PS1="${PS1#\\[\\ek\\e\\\\\\\]}" export PS1 export NO_SCREEN_UPDATES=1 echo "disabled" fi } # dynamic prompt escape codes on dpon(){ if ! dpstatus >/dev/null; then PS1="${SCREEN_ESCAPE}${PS1}" export PS1 export NO_SCREEN_UPDATES=0 echo "enabled" fi } # toggle dynamic prompt on/off dp(){ if dpstatus; then printf '\b\renabled => ' dpoff else printf '\b\rdisabled => ' dpon fi title } # ============================================================================ # man(){ title "man $1" command man "$@" title "$LAST_TITLE" } sudo(){ title "sudo $1" command sudo "$@" title "$LAST_TITLE" } vim(){ local title="" #until [ -z "$1" ]; do for x in "$@"; do case "$x" in -*) : ;; +*) : ;; *) #title="$title$x " title="$x" break ;; esac #shift done local num=10 if [[ "${TITLE_SHORT:-}" =~ ^[0-9]+$ ]]; then num=$TITLE_SHORT fi if [ "$num" -lt 3 ]; then num=3 fi title="${title//.txt/}" if dpstatus >/dev/null; then if echo "$title" | grep -q docs/; then title="$(basename "$title")" title "d${title:0:$num}" else title="$(basename "$title")" title "${title:0:$num}" fi fi command vim "$@" #if dpstatus >/dev/null; then title; fi } ================================================ FILE: .bash.d/travis_ci.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Jan 20 15:28:12 2016 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T r a v i s C I # ============================================================================ # # Travis bash autocomplete # adapted from Travis ruby gem auto-added to end of ~/.bashrc # shellcheck disable=SC1090,SC1091 #[ -f ~/.travis/travis.sh ] && source ~/.travis/travis.sh bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" type git_repo &>/dev/null || . "$bash_tools/.bash.d/git.sh" type browse &>/dev/null || . "$bash_tools/.bash.d/network.sh" alias trav='travis_browse' travis_browse(){ local repo repo="$(github_repo)" url="https://travis-ci.org/${TRAVIS_USER:-${USER:-$(whoami)}}/$repo" browser "$url" } # for auto authentication using Travis CI tools like travis_last_log.py and travis_debug_session.py #export TRAVIS_TOKEN=... travis_debug(){ # code has better automatic handling, doesn't need this now #if grep '/' <<< "$1" &>/dev/null; then # travis_debug_session.py -r "$1" ${@:2} #else # travis_debug_session.py -J "$1" ${@:2} #fi opts=() if [[ "$PWD" =~ /github/ ]]; then local repo repo="$(git_repo)" if [ -n "$repo" ]; then opts+=(--repo "$repo") fi fi travis_debug_session.py "${opts[@]}" "$@" } travis_log(){ local repo repo="$(git_repo)" travis_last_log.py --failed "$repo" "$@" } ================================================ FILE: .bash.d/trivy.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2023-05-12 06:50:49 +0100 (Fri, 12 May 2023) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T r i v y # ============================================================================ # #bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" #autocomplete trivy ================================================ FILE: .bash.d/vagrant.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Thu Mar 14 12:42:17 2013 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # V a g r a n t # ============================================================================ # #export VAGRANT_HOME=~/vagrant #export VAGRANT_BOXES=~/boxes # $bash_tools defined in adjacent shell modules # shellcheck disable=SC2154 export vagrant="$bash_tools/vagrant" #alias cd_vagrant='[ "$PWD" = "$VAGRANT_HOME" ] || cd "$VAGRANT_HOME"' #alias cd_vagrant='cd "$VAGRANT_HOME"' alias cd_vagrant='cd "$vagrant"' alias cdv=cd_vagrant alias vhome=cd_vagrant #alias v='cd_vagrant; vagrant' # 'v' is aliased to vim now as it's used much more often #alias v='vagrant' alias vag='vagrant' #alias vf='cd_vagrant; vim Vagrantfile; vagrant_gen_etc_hosts; eval "$(vagrant_gen_aliases)"' alias vst='vagrant status' alias vrun='vst | grep running' # vr is aliased to vbox_running in virtualbox.sh alias vrr='vrun' #alias vssh='cd_vagrant; ssh_func vagrant ssh' #alias vssh='cd_vagrant; vagrant ssh' alias vssh='vagrant ssh' alias boxes='cd $VAGRANT_BOXES' while read -r directory; do # shellcheck disable=SC2139,SC2140 alias "v${directory##*/}"="cd $directory" done < <(find "$vagrant" -maxdepth 1 -type d) # see ../vagrant_hosts.sh for similar parsing # #vagrant_parse_hosts(){ # if ! [ -f "$VAGRANT_HOME/Vagrantfile" ]; then # return # fi # #grep '[^#]*config.vm.define' "$VAGRANT_HOME/Vagrantfile" | awk -F'"' '{print $2}' # sed 's/#.*//;/^[[:space:]]*$/d' "$VAGRANT_HOME/Vagrantfile" | # grep -e host_name -e network | # grep -B1 -e network | # grep -v -e "^--" -e "default_hostname" | # sed 's/^.*[[:space:]]"//;s/"//' | # tr '\n' ' ' | # perl -pn -e 's/(\.\d+)\s/$1\n/g' #} # #vagrant_gen_aliases(){ # vagrant_parse_hosts | # while read -r host ip rest; do # if ! type "$host" &>/dev/null; then # #echo "alias $host='ssh root@$host'" # #echo "alias $host='vups $host'" # echo "alias $host='vssh $host'" # fi # done #} # # don't really use these aliases to vagrant boxes any more #eval "$(vagrant_gen_aliases)" #vagrant_gen_etc_hosts(){ # vagrant_parse_hosts | # while read -r host ip rest; do # if [ -n "${rest:-}" ]; then # echo "error third token '$rest' detected from pipe line '$ip $host $rest'" # return 1 # fi # [ "${host}" = "localhost" ] && continue # host_record="$ip $host.local $host" # # sudo auto-defined in .bashrc # # shellcheck disable=SC2154 # $sudo perl -pi -e "s/^$ip\\s+.*/$host_record/" /etc/hosts # if [ -n "$ip" ]; then # grep -q "^$host_record" /etc/hosts || # $sudo sh -c "echo '$host_record' >> /etc/hosts" # else # echo "no ip passed in pipe for host '$host'! " # return 1 # fi # done #} vagrant_usage(){ if [ -z "$1" ]; then echo "usage: ${FUNCNAME[1]} " return 1 fi } vup(){ #vagrant_usage "$1" || return 1 vagrant up "$@" } vre(){ vagrant_usage "$1" || return 1 vagrant reload "$@" } vressh(){ vagrant_usage "$1" || return 1 vagrant reload "$1" vagrant ssh "$1" } vsus(){ #vagrant_usage "$1" || return 1 vagrant suspend "$@" } vres(){ #vagrant_usage "$1" || return 1 vagrant resume "$@" } #alias vsusall="vsus $(vst | grep running | awk '{print $1}')" vsusall(){ local running_vms=() read -r -a running_vms <<< "$(vst | grep running | awk '{print $1}')" [ ${#running_vms} -gt 0 ] || return 0 vsus "${running_vms[@]}" } alias vsusa=vsusall vupssh(){ vagrant_usage "$1" || return 1 local status status="$(vst "$1")" if grep -Eq "^$1[[:space:]]" <<< "$status"; then grep -Eq "^$1[[:space:]]+running" <<< "$status" || vup "$1" else echo "VM not found: $1" return 1 fi #vup $1 vssh "$1" } alias vups="vupssh" vhalt(){ #vagrant_usage "$1" || return 1 vagrant halt "$@" } vrhalt(){ # want splitting # shellcheck disable=SC2046 vhalt "$@" $(vagrant status | awk '/running/ {print $1}') } vrsus(){ # want splitting # shellcheck disable=SC2046 vsus $(vagrant status | awk '/running/ {print $1}') } vdestroy(){ #vagrant_usage "$1" || return 1 vagrant destroy --force "$@" } vdestroyup(){ vdestroy "$@" || : vup "$@" } vdestroyups(){ vdestroy "$@" || : vups "$@" } vprovision(){ #vagrant_usage "$1" || return 1 vagrant provision "$@" } vprov(){ vprovision "$@" } whenvdown(){ vst | while grep "$1.*running"; do sleep 0.1 done } ================================================ FILE: .bash.d/vim.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc and later functions.sh) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # V I M # ============================================================================ # bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # vim() function is in title.sh as it is tightly integrated with the functions in there and not necessary otherwise vimhome(){ # must expand in vim, not in shell # shellcheck disable=SC2016 vim -Nesc '!echo $VIMRUNTIME' -c qa | #tr -dc '[:alnum:]/\r\n' sed 's,[^/]*/,/,' } cdvimhome(){ # shellcheck disable=SC2164 cd "$(vimhome)" } vimfiletypes(){ cdvimhome || return 1 find syntax ftplugin -iname '*.vim' -exec basename -s .vim {} + | sort -u } gitgrepvim(){ if [ $# -lt 2 ]; then echo "usage: gitgrepvim " return 3 fi # want splitting # shellcheck disable=SC2046 vim $(git grep -i "$*" | sed 's/:.*//' | sort -u) } alias ggrepv=gitgrepvim grepvim(){ if [ $# -lt 2 ]; then echo "usage: grepvim " return 3 fi # want splitting # shellcheck disable=SC2046 vim $(grep -l "$1" "$@" | sort -u) } alias grepv=grepvim alias vimgrep=grepvim vimchanged(){ local git_root git_root="$(git_root)" # want splitting # shellcheck disable=SC2046 vim "$@" $(git status --porcelain | awk '/^.M/ {$1=""; print}' | sed "s|^[[:space:]]|$git_root/|") } filesvim(){ local files=() # mapfile not available on Mac and read -a only takes first result # shellcheck disable=SC2207 IFS=$'\n' files=($(find . -iname "$@" | sort -u)) if [ -n "${files[*]}" ]; then vim "${files[@]}" fi } alias fvim=filesvim alias vimf=fvim # vim which vw(){ local path if [ -z "$1" ]; then echo "usage: vw " return 1 fi path="$(which "$1")" if [ -z "$path" ]; then echo "File not found in \$PATH: $1" return 1 fi "$EDITOR" "$path" } vihosts(){ [ $EUID -eq 0 ] && sudo="" || sudo=sudo $sudo vim /etc/hosts $sudo pkill -1 dnsmasq } vimup(){ local arg="$1" up_target="$(findup "$arg")" [ -n "$up_target" ] || return 1 vim "$(findup "$1")" } ================================================ FILE: .bash.d/virtualbox.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-01-14 22:13:51 +0000 # Original: circa 2011 - 2012 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # V i r t u a l B o x # ============================================================================ # vboxmanage_list_vms="VBoxManage list runningvms" # shellcheck disable=SC2139 alias vbox_running="echo $vboxmanage_list_vms; $vboxmanage_list_vms" alias vr=vbox_running unset vboxmanage_list_vms alias startvm="VBoxManage startvm" alias stopvm="VBoxManage controlvm acpipowerbutton" alias poweroff="VBoxManage controlvm poweroff" alias savestate="VBoxManage savestate" alias controlvm="VBoxManage controlvm" #docker(){ # local vm='boot2docker-vm' # VBoxManage list runningvms | grep -q "$vm" || VBoxManage startvm "$vm" # command docker "$@" #} # fixvbox() in .bash.d/apple.sh restarts VirtualBox on Mac OSX only fixvboxnet(){ sudo ifconfig vboxnet0 down sudo ifconfig vboxnet0 up } ================================================ FILE: .bash.d/vnc.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # V N C # ============================================================================ # #bash_tools="${bash_tools:-$(dirname "${BASH_SOURCE[0]}")/..}" # shellcheck disable=SC1090,SC1091 #. "$bash_tools/.bash.d/os_detection.sh" if [ -d "/Applications/VNC Viewer.app/Contents/MacOS" ]; then export PATH+=":/Applications/VNC Viewer.app/Contents/MacOS" fi vncwho() { netstat -tW | grep ".*:5900 .*:.*" | awk '{a=$5; split(a,b,":"); print b[1]}' } vnc(){ local host_port="$1" # if just a port number is given, then it's shorthand for localhost: eg. for copying and pasting Qemu's randomly generated VNC port if [[ "$host_port" =~ ^[[:digit:]]+$ ]]; then host_port="localhost:$1" fi if test -x "/Applications/VNC Viewer.app/Contents/MacOS/vncviewer"; then "/Applications/VNC Viewer.app/Contents/MacOS/vncviewer" "$host_port" & elif type -P krdc &>/dev/null; then krdc "vnc:/$host_port" & elif type -P vncviewer &>/dev/null; then vncviewer "$host_port" & else echo "could not find krdc or vncviewer in \$PATH" return 1 fi } revnc(){ local host_port="$1" local host="$host_port" if [[ "$host" =~ : ]]; then host="${host%%:*}" fi if [ -z "$1" ]; then echo "You must supply a hostname or ip address to connect to" return 1 fi # $pingwait is defined in network.sh # shellcheck disable=SC2154 while ! ping -c 1 "$pingwait" 1 "$host" &>/dev/null; do sleep 1 done timestamp "machine is up" until vnc "$host_port"; do sleep 1 timestamp "retrying $host_port" done } ================================================ FILE: .bash.d/welcome.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2012-06-25 15:20:39 +0100 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # W e l c o m e # ============================================================================ # # Original version was in Perl many years ago, but defaulting to Python version now # Bash version further down is for interest of if you don't have the other repos # welcome should be found in $PATH from DevOps-Golang-Tools repo # welcome.py should be found in $PATH from DevOps-Python-Tools repo welcome(){ if type -P welcome &>/dev/null; then command welcome elif type -P welcome.py &>/dev/null; then welcome.py fi } # set this instead to use bash only version if you don't have the other repos #alias welcome=bash_welcome bash_welcome(){ local msg msg="Welcome Hari - your last access was $(last|head -n2|tail -n1|sed 's/[^ ]\+ \+[^ ]\+ \+[^ ]\+ \+//;s/ *$//')" #local msg="Welcome Hari" # generated by for x in {A..z}; do printf "%s" $x; done #charmap="ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_\`abcdefghijklmnopqrstuvwxyz" # generated by: for x in {1..128}; do printf \\$(printf '%03o' $x); done # shellcheck disable=SC1117 charmap="!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_\`abcdefghijklmnopqrstuvwxyz{|}~ \t\n" for ((i=0; i<"${#msg}"; i++)); do local x="${msg:i:1}" #echo "x == $x" printf " " local j=0 while true; do #for ((j=0; j<"${#charmap}"; j++)); do #while true; do #set -x if [ $j -gt 2 ]; then local y=$x else local y=${charmap:$((RANDOM%${#charmap})):1} fi #local y="${charmap:j:1}" printf "\\b%s" "$y" # This does not have enough precision, re-implement in Perl # This is because it's an external being called, otherwise pure bash # is so fast that you don't see any effect... sleep 0.000000000001 #perl -e 'sleep 0.0000000000000000000000000001' [ "$y" = "$x" ] && break ((j+=1)) #set +x done done #printf "\\n" printf "\\n" } ================================================ FILE: .bash.d/when.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 - 2012 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # W h e n F u n c t i o n s # ============================================================================ # # interactive time latch alternative to 'at' command when(){ # should be in the format HH:MM:SS local time="$1" shift || : if ! grep -Eq "^[012]?[0-9]:[0-5]?[0-9]:[0-5]?[0-9]$" <<< "$time"; then echo "invalid time format - must be in format HH:MM:SS" return 1 fi while true; do if [ "$(date '+%T')" = "$time" ]; then break fi sleep 1 done "$@" } whenup(){ local host="$1" shift || : checkhost "$host" || return 1 local count=0 # defined in network.sh # shellcheck disable=SC2154 while ! ping -c 1 "$pingwait" 1 "$host" &>/dev/null; do ((count+=1)) timestamp "waiting for $host to come up..." if [ $count -gt 22 ]; then sleep 10 else sleep 5 fi done timestamp "$host is up" "$@" } # HTTP(s) version of whenup because corporate firewalls block ping whenurl(){ local url="$1" shift || : local count=0 while ! curl -s --connect-timeout 2 "$url" &>/dev/null; do ((count+=1)) timestamp "waiting for $url to come up..." if [ $count -gt 22 ]; then sleep 10 else sleep 5 fi done timestamp "$url is up" "$@" } whendown(){ local host="$1" shift || : checkhost "$host" || return 1 local count=0 while ping -c 1 "$pingwait" 1 "$host" &>/dev/null; do ((count+=1)) timestamp "waiting for machine to go down..." if [ $count -gt 22 ]; then sleep 10 else sleep 5 fi done timestamp "machine is down" "$@" } whenport(){ local host="$1" local port="$2" shift || : shift || : checkhost "$host" || return 1 local count=0 timestamp "checking port $port open..." checkprog nc while ! nc -z "${host#*@}" "$port" &>/dev/null; do ((count+=1)) timestamp "waiting for port $port to open..." if [ $count -gt 22 ]; then sleep 10 else sleep 5 fi done timestamp "port $port is open" "$@" } whendone(){ local search="$1" shift || : if [ -z "$search" ]; then echo "usage: when " return 1 fi while true; do if ! pgrep -qf "$search"; then echo break else echo -n . sleep 1 fi done "$@" } ================================================ FILE: .bash.d/z_final.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2230 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: circa 2006 (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # F i n a l i z a t i o n # ============================================================================ # # place to add last minute clean ups like path deduping # slows down new shells #dedupe_paths ================================================ FILE: .bash_logout ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # B a s h L o g o u t # ============================================================================ # # read all history lines not already read from the history file and append them to the history list history -n # destroy kerberos tickets in all caches (-A), quietly don't beep (-q) kdestroy -A -q clear # From Ubuntu ## when leaving the console clear the screen to increase privacy # #if [ "$SHLVL" = 1 ]; then # [ -x /usr/bin/clear_console ] && /usr/bin/clear_console -q #fi ================================================ FILE: .bash_profile ================================================ # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # # ~/.bash_profile: executed by bash(1) for login shells. # see /usr/share/doc/bash/examples/startup-files for examples. # the files are located in the bash-doc package. trap clear EXIT # the default umask is set in /etc/login.defs #umask 022 if [ -f ~/.bashrc ]; then . ~/.bashrc fi # not supported in the tmux terminal in GCP Cloud Shell #if ! is_mac && # ! isGoogleCloudShell && # [ "${TERM:-}" != "xterm-256color" ]; then # setterm -blank 0 #fi # prints a cool spinning welcome message which shows the time of last login # this is available in the Python / Perl and Golang Devops Tools repos, # as well as a function in .bash.d/welcome.sh in the DevOps Bash Tools repo #if type welcome &>/dev/null; then # welcome #fi #eval "$(rbenv init -)" # from brew install bash-completion [[ -r "/usr/local/etc/profile.d/bash_completion.sh" ]] && . "/usr/local/etc/profile.d/bash_completion.sh" #sudo setmixer -V pcm 100 complete -C /usr/local/bin/terragrunt terragrunt # ============================================================================ # # This should be automatically added to ~/.bash_profile when you install SDKman (install/install_sdkman.sh): # #THIS MUST BE AT THE END OF THE FILE FOR SDKMAN TO WORK!!! export SDKMAN_DIR="$HOME/.sdkman" [[ -s "/Users//.sdkman/bin/sdkman-init.sh" ]] && source "/Users//.sdkman/bin/sdkman-init.sh" ================================================ FILE: .bashrc ================================================ #!/usr/bin/env bash # shellcheck disable=SC1091 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # BASH - Heavily Customized Environment # ============================================================================ # # Sources thousands of lines of Bash code written over the course of ~15+ years # some of which is now found in this GitHub repo's .bash.d/*.sh # ============================================================================ # # # put this at the top of your ~/.bashrc to inherit the goodness here (assuming you've checked out this repo to ~/github/bash-tools): # # if [ -f ~/github/bash-tools/.bashrc ]; then # . ~/github/bash-tools/.bashrc # fi # # ============================================================================ # # Use with PS4 further down + profile-bash.pl (still in private repos) for performance profiling this bashrc #set -x # If not running interactively, don't do anything: [ -z "${PS1:-}" ] && return [ -n "${PERLBREW_PERL:-}" ] && return # Another alternative #case $- in # *i*) ;; # *) return 0;; #esac # Another variation #if [[ $- != *i* ]] ; then # # Shell is non-interactive. Be done now! # return #fi # ============================================================================ # # after cleanshell, not even $HOME is set, this messes up things that base off $HOME, like SDKman if [ -z "${HOME:-}" ]; then export HOME=~ fi bash_tools="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # needed to inherit by things like vim script execution from within files using libraries rooted at this location export bash_tools # shellcheck disable=SC1090,SC1091 . "$bash_tools/.bash.d/os_detection.sh" # enable color support for ls if [ "$TERM" != "dumb" ] && \ ! is_mac; then eval "$(dircolors -b)" fi # shut up Mac, Bash still rocks export BASH_SILENCE_DEPRECATION_WARNING=1 # ============================================================================ # # technically should get called only for new login shells #[ -f /etc/profile ] && . /etc/profile [ -f /etc/bash/bashrc ] && . /etc/bash/bashrc [ -f /etc/bashrc ] && . /etc/bashrc [ -f /etc/bash_completion ] && . /etc/bash_completion [ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" # shellcheck disable=SC1090,SC1091 [ -f "$HOME/.aliases" ] && source "$HOME/.aliases" # ============================================================================ # # SECURITY TO STOP STUFF BEING WRITTEN TO DISK #unset HISTFILE #unset HISTFILESIZE export HISTSIZE=50000 export HISTFILESIZE=50000 rmhist(){ history -d "$1"; } histrm(){ rmhist "$1"; } histrmlast(){ history -d "$(history | tail -n 2 | head -n 1 | awk '{print $1}')"; } # This adds a time format of "YYYY-mm-dd hh:mm:ss command" to the bash history export HISTTIMEFORMAT="%F %T " # stop logging duplicate successive commands to history #HISTCONTROL=ignoredups:ignorespace HISTCONTROL=ignoredups # Neat trick "[ \t]*" to exclude any command by just prefixing it with a space. Fast way of going stealth for pw entering on cli # & here means any duplicate patterns, others are simple things like built-ins and ls and stuff you don't need history for #export HISTIGNORE="[ \t]*:&:ls:[bf]g:exit" # append rather than overwrite history shopt -s histappend # check window size and update $LINES and $COLUMNS after each command shopt -s checkwinsize shopt -s cdspell # prevent core dumps which can leak sensitive information ulimit -c 0 # tighten permissions except for root where library installations become inaccessible to my user account if [ $EUID = 0 ]; then umask 0022 else # caused no end of problems when doing sudo command which retained 0077 and broke library access for user accounts #umask 0077 umask 0022 fi # make less more friendly for non-text input files, see lesspipe(1) [ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" # ============================================================================ # sudo=sudo if [ $EUID -eq 0 ]; then # used throughout .bash.d/*.sh # shellcheck disable=SC2034 sudo="" fi # shellcheck disable=SC1090,SC1091 type add_PATH &>/dev/null || . "$bash_tools/.bash.d/paths.sh" # ============================================================================ # # want this to fail is there is no match because we should always have local .bash.d/*.sh in this repo # shopt -s nullglob for src in "$bash_tools/.bash.d/"*.sh; do # shellcheck disable=SC1090,SC1091 . "$src" done # shopt -u nullglob # added by travis gem - should be in ~/.bashrc so not needed to duplicate here #[ -f /Users/hari.sekhon/.travis/travis.sh ] && source /Users/hari.sekhon/.travis/travis.sh # shellcheck disable=SC1090,SC1091 [ -f "$HOME/.bashrc.local" ] && . "$HOME/.bashrc.local" if [ -d "$HOME/.bash.d" ]; then shopt -s nullglob for src in "$HOME/.bash.d/"*.sh; do # shellcheck disable=SC1090,SC1091 . "$src" done shopt -u nullglob fi if [ -d "$HOME/.bash.autocomplete.d" ]; then shopt -s nullglob for src in "$HOME/.bash.autocomplete.d/"*.sh; do # shellcheck disable=SC1090,SC1091 . "$src" done shopt -u nullglob fi ================================================ FILE: .buildkite/pipeline.yml ================================================ # # Author: Hari Sekhon # Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # B u i l d K i t e C I # ============================================================================ # # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: # # - command: buildkite-agent pipeline upload # Yaml Anchors workaround to BuildKite's lack of global retries configuration - credit to Jason @ BuildKite for this workaround: # # https://forum.buildkite.community/t/reschedule-builds-on-other-agents-rather-than-fail-builds-when-agents-time-out-or-are-killed-machine-shut-down-or-put-to-sleep/1388/5 # anchors: std_retries: &std_retries retry: automatic: - exit_status: -1 # Agent was lost limit: 2 - exit_status: 255 # Forced agent shutdown limit: 2 steps: - command: setup/ci_bootstrap.sh label: ci bootstrap timeout: 30 # brew can take 10 mins just to do a brew update branches: master <<: [*std_retries] - wait - command: make init label: init timeout: 2 branches: master <<: [*std_retries] - wait - command: make ci label: build timeout: 60 branches: master <<: [*std_retries] - wait - command: make test label: test timeout: 120 branches: master <<: [*std_retries] ================================================ FILE: .circleci/config.yml ================================================ # # Author: Hari Sekhon # Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020) # Original: H1 2016 (Circle CI 1.x) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C i r c l e C I # ============================================================================ # # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml # Reference: # # https://circleci.com/docs/2.0/configuration-reference version: 2.1 workflows: version: 2 workflow: jobs: - build jobs: build: docker: - image: cimg/base:2024.12 resource_class: small steps: # CLI is too old - config validate breaks in test - install new version to fix # doesn't work - existing version is too old to update #- run: circleci update - run: | curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout #- setup_remote_docker: # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make - run: make test ================================================ FILE: .cirrus.yml ================================================ # # Author: Hari Sekhon # Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C i r r u s C I # ============================================================================ # # https://cirrus-ci.org/guide/writing-tasks/ container: image: ubuntu:18.04 task: env: TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init - make ci test ================================================ FILE: .dockerignore ================================================ # # Author: Hari Sekhon # Date: Tue Sep 12 13:06:25 2017 +0200 # # vim: filetype=conf # # ============================================================================ # # . d o c k e r i g n o r e # ============================================================================ # # https://docs.docker.com/engine/reference/builder/#dockerignore-file # ================================================= # Don't send things you don't need to Docker server and avoid uploading your secrets / keys!! # # XXX: RULES: # - .dockerignore must be at top-level root context from which you invoke 'docker build' # - unfortunately .dockerignore doesn't match basenames like .gitignore so you need to prefix **/ for recursive matching to be safe # - Last Match Wins - you must put more specific whitelisting matches after the general exclusion pattern # ==================================================================== # Smaller more concise .dockerignore files are found in the builds at: # # https://github.com/HariSekhon/Dockerfiles # ========================================================= # This is a huge list of exclusions which covers most cases # # XXX: Best Practice - enable '*' ignore and create short concise whitelist of inclusions # # * # # Second option is to rely on this pretty extensive blacklist, which excludes most known credentials files, all hidden dot files and major RCS repos like Git / Mercurial / Subversion # # Sloppy Docker builds which do 'COPY/ADD .' are a ** Security Risk ** - see Dockerfiles repo's tests/ which checks for their existence in all Dockerfile's # # https://github.com/HariSekhon/Dockerfiles # * # ========================================================================================= # Whitelist - must come after the more general blacklist pattern * above as Last Match Wins # # always include package.json and requirements.txt for standard dependency installations on NodeJS and Python !package.json !requirements.txt !LICENSE # =============== # Docker specific # still sent to the daemon as needed to do the job but aren't included in the image by ADD/COPY . commands **/Dockerfile **/.dockerignore **/docker-compose.yml **/.gitignore **/.gcloudignore **/.ssh/ # don't accidentally publish your whole code base via Docker like Twitter Vine did!! **/.git/ **/.svn/ **/.hg/ **/git/ **/github/ **/gitolite*/ **/gitroot/ **/mercurial/ **/hg/ **/hgroot/ **/svn/ **/svnroot/ **/node_modules/ **/dist/ **/fatpacks/ **/logs/ **/vendor/ **/vagrant/ **/venv/ **/wordlists/ **/pytools_checks/ **/debs/ **/rpms/ **/drive/ **/Google Drive/ **/Dropbox/ # XXX: don't include any hidden files unless we explicitly override and include them with a !.filename **/.* # don't include CI configs not covered by ignoring dotfiles **/Jenkinsfile **/azure-pipelines.y*ml **/bitbucket-pipelines.y*ml **/buddy.y*ml **/codefresh.y*ml **/shippable.y*ml **/wercker.y*ml **/gocd_config_repo.json **/jenkins-job.xml **/hadolint.y*ml **/scalastyle_config.xml **/yamllint/ # contains webhook URL which should not be committed publicly **/buildkite-pipeline*.json # leave our README.md in case we want to include it in the image but filter out other .md files *.md !README.md # ======================================== # Based on the massive adjacent .gitignore, modified for Docker ignore's Go filepath.Match() function *#*# **/*.a **/*.avi **/*.bak **/*.bak.* **/*.bin **/*.bkp **/*.class **/*.dump **/*.flv **/*.kdb **/*.lock **/*.log **/*.macports-saved_* **/*.mp3 **/*.mp4 **/*.mpeg **/*.mpg **/*.o **/*.orig **/*.out **/*.part **/*.pyc **/*.pyo **/*.stderr **/*.stdout **/*.swo **/*.swp **/*.tmp **/*.wmv **/*~ **/tmp.* **/~* **/*.doc **/*.docx **/*.msg **/*.pages **/*.ppt **/*.pptx **/*.rtf **/*.wpd **/*.wps **/*.xls **/*.xlsx # ============================================================================ # # # regenerate all sections below in to a single arg for API call via: # # grep '[C]reated by https://' .dockerignore | sed 's,.*/,,' | tr ',' '\n' | sort -u | tr '\n' ',' | sed 's/,$//' | xargs echo gitignore.io_api.sh # # then pipe through perl to add recursive prefix '**/': # # gitignore.io_api.sh ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh | perl -p -e 's/^([^#\s\/])/**\/$1/; s/^\//**\//' >> .dockerignore # # Find new or missing tags you aren't using yet: # # grep '[C]reated by https://' .dockerignore | sed 's,.*/,,' | tr ',' '\n' | sort -u | tr '\n' ',' | sed 's/,$//' | gitignore.io_api.sh missing # # ============================================================================ # # Created by https://www.toptal.com/developers/gitignore/api/ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh # Edit at https://www.toptal.com/developers/gitignore?templates=ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh ### Ansible ### **/*.retry ### ApacheHadoop ### **/*.iml **/*.ipr **/*.iws **/*.orig **/*.rej **/.idea **/.svn **/.classpath **/.project **/.settings **/target **/hadoop-common-project/hadoop-kms/downloads/ **/hadoop-hdfs-project/hadoop-hdfs/downloads **/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads **/hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml **/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml ### AppCode ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff **/.idea/**/workspace.xml **/.idea/**/tasks.xml **/.idea/**/usage.statistics.xml **/.idea/**/dictionaries **/.idea/**/shelf # Generated files **/.idea/**/contentModel.xml # Sensitive or high-churn files **/.idea/**/dataSources/ **/.idea/**/dataSources.ids **/.idea/**/dataSources.local.xml **/.idea/**/sqlDataSources.xml **/.idea/**/dynamic.xml **/.idea/**/uiDesigner.xml **/.idea/**/dbnavigator.xml # Gradle **/.idea/**/gradle.xml **/.idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake **/cmake-build-*/ # Mongo Explorer plugin **/.idea/**/mongoSettings.xml # File-based project format # IntelliJ **/out/ # mpeltonen/sbt-idea plugin **/.idea_modules/ # JIRA plugin **/atlassian-ide-plugin.xml # Cursive Clojure plugin **/.idea/replstate.xml # Crashlytics plugin (for Android Studio and IntelliJ) **/com_crashlytics_export_strings.xml **/crashlytics.properties **/crashlytics-build.properties **/fabric.properties # Editor-based Rest Client **/.idea/httpRequests # Android studio 3.1+ serialized cache file **/.idea/caches/build_file_checksums.ser ### AppCode Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint **/.idea/**/sonarlint/ # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin **/.idea/**/sonarIssues.xml # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced **/.idea/**/markdown-navigator.xml **/.idea/**/markdown-navigator-enh.xml **/.idea/**/markdown-navigator/ # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 **/.idea/$CACHE_FILE$ # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream **/.idea/codestream.xml ### AppEngine ### # Google App Engine generated folder **/appengine-generated/ ### Archive ### ### Mostly from https://en.wikipedia.org/wiki/List_of_archive_formats ## Archiving only # The traditional archive format on Unix-like systems, now used mainly for the creation of static libraries. **/*.a **/*.ar # RPM files consist of metadata concatenated with (usually) a cpio archive. Newer RPM systems also support other archives, as cpio is becoming obsolete. cpio is also used with initramfs. **/*.cpio # A self-extracting archive that uses the Bourne shell (sh). **/*.shar # A system for storing multiple files. LBR archives typically contained files processed by SQ, or the archive itself was compressed with SQ. LBR archives that were compressed with SQ ended with the extension .LQR **/*.LBR # An archive format originally used mainly for archiving and distribution of the exact, nearly-exact, or custom-modified contents of an optical storage medium such as a CD-ROM or DVD-ROM. However, it can be used to archive the contents of other storage media, selected partitions, folders, and/or files. The resulting archive is typically optimized for convenient rendering to (re-)writable CD or DVD media. **/*.iso # A library format used primarily on the Commodore 64 and 128 lines of computers. This bears no resemblance to the DOS LBR format. While library files were quick to implement (a number of programs exist to work with them) they are crippled in that they cannot grow with use: once a file has been created it cannot be amended (files added, changed or deleted) without recreating the entire file. **/*.lbr # An archive format used by Mozilla for storing binary diffs. Used in conjunction with bzip2. **/*.mar # A common archive format used on Unix-like systems. Generally used in conjunction with compressors such as gzip, bzip2, compress or xz to create .tar.gz, .tar.bz2, .tar.Z or tar.xz files. **/*.tar # Package managers # Red Hat Package Manager **/*.rpm # Debian package **/*.deb # MicroSoft Installer **/*.msi **/*.msm **/*.msp # Mozilla package installer **/*.xpi # Ruby Package **/*.gem ### Archives ### # It's better to unpack these files and commit the raw source because # git has its own built in compression methods. **/*.7z **/*.jar **/*.rar **/*.zip **/*.gz **/*.gzip **/*.tgz **/*.bzip **/*.bzip2 **/*.bz2 **/*.xz **/*.lzma **/*.cab **/*.xar # Packing-only formats # Package management formats **/*.dmg **/*.egg **/*.txz ### ArchLinuxPackages ### **/*.tar.* **/*.exe **/*.log **/*.log.* **/*.sig **/pkg/ **/src/ ### Audio ### **/*.aif **/*.iff **/*.m3u **/*.m4a **/*.mid **/*.mp3 **/*.mpa **/*.ra **/*.wav **/*.wma **/*.ogg **/*.flac ### Autotools ### # http://www.gnu.org/software/automake **/Makefile.in **/ar-lib **/mdate-sh **/py-compile **/test-driver **/ylwrap **/.deps/ # http://www.gnu.org/software/autoconf **/autom4te.cache **/autoscan.log **/autoscan-*.log **/aclocal.m4 **/compile **/config.guess **/config.h.in **/config.log **/config.status **/config.sub **/configure **/configure.scan **/depcomp **/install-sh **/missing **/stamp-h1 # https://www.gnu.org/software/libtool/ **/ltmain.sh # http://www.gnu.org/software/texinfo **/texinfo.tex # http://www.gnu.org/software/m4/ **/m4/libtool.m4 **/m4/ltoptions.m4 **/m4/ltsugar.m4 **/m4/ltversion.m4 **/m4/lt~obsolete.m4 # Generated Makefile # (meta build system like autotools, # can automatically generate from config.status script # (which is called by configure script)) **/Makefile ### Autotools Patch ### ### Backup ### **/*.bak **/*.gho **/*.ori **/*.tmp ### Basic ### # Apples Build **/*.build **/*.apples # Initialized files **/*.ini **/*.basic ### BitTorrent ### **/*.torrent ### C ### # Prerequisites **/*.d # Object files **/*.o **/*.ko **/*.obj **/*.elf # Linker output **/*.ilk **/*.map **/*.exp # Precompiled Headers **/*.gch **/*.pch # Libraries **/*.lib **/*.la **/*.lo # Shared objects (inc. Windows DLLs) **/*.dll **/*.so **/*.so.* **/*.dylib # Executables **/*.out **/*.app **/*.i*86 **/*.x86_64 **/*.hex # Debug files **/*.dSYM/ **/*.su **/*.idb **/*.pdb # Kernel Module Compile Results **/*.mod* **/*.cmd **/.tmp_versions/ **/modules.order **/Module.symvers **/Mkfile.old **/dkms.conf ### C++ ### # Prerequisites # Compiled Object files **/*.slo # Precompiled Headers # Compiled Dynamic libraries # Fortran module files **/*.mod **/*.smod # Compiled Static libraries **/*.lai # Executables ### Zsh ### # Zsh compiled script + zrecompile backup **/*.zwc **/*.zwc.old # Zsh completion-optimization dumpfile **/*zcompdump* # Zsh zcalc history **/.zcalc_history # A popular plugin manager's files **/._zplugin **/.zplugin_lstupd # zdharma/zshelldoc tool's files **/zsdoc/data # robbyrussell/oh-my-zsh/plugins/per-directory-history plugin's files # (when set-up to store the history in the local directory) **/.directory_history # MichaelAquilina/zsh-autoswitch-virtualenv plugin's files # (for Zsh plugins using Python) **/.venv # Zunit tests' output **/tests/_output/* **/!/tests/_output/.gitkeep ### certificates ### **/*.pem **/*.key **/*.crt **/*.cer **/*.priv ### ChefCookbook ### **/.vagrant **/cookbooks # Bundler **/bin/* **/.bundle/* **/.kitchen/ **/.kitchen.local.yml **/.kitchen.*.local.yml **/kitchen.local.yml **/kitchen.*.local.yml ### CMake ### **/CMakeLists.txt.user **/CMakeCache.txt **/CMakeFiles **/CMakeScripts **/Testing **/cmake_install.cmake **/install_manifest.txt **/compile_commands.json **/CTestTestfile.cmake **/_deps ### CMake Patch ### # External projects **/*-prefix/ ### Clojure ### **/pom.xml **/pom.xml.asc **/*.class **/lib/ **/classes/ **/target/ **/checkouts/ **/.lein-deps-sum **/.lein-repl-history **/.lein-plugins/ **/.lein-failures **/.nrepl-port **/.cpcache/ ### Code-Java ### # Language Support for Java(TM) by Red Hat extension for Visual Studio Code - https://marketplace.visualstudio.com/items?itemName=redhat.java **/factoryConfiguration.json ### Cloud9 ### # Cloud9 IDE - http://c9.io **/.c9revisions **/.c9 ### Compressed ### **/*.pkg **/*.sit **/*.sitx **/*.zipx ### CompressedArchive ### ## Archiving and compression # Open source file format. Used by 7-Zip. # Mac OS X, restoration on different platforms is possible although not immediate Yes Based on 7z. Preserves Spotlight metadata, resource forks, owner/group information, dates and other data which would be otherwise lost with compression. **/*.s7z # Old archive versions only Proprietary format **/*.ace # A format that compresses and doubly encrypt the data (AES256 and CAS256) avoiding brute force attacks, also hide files in an AFA file. It has two ways to safeguard data integrity and subsequent repair of the file if has an error (repair with AstroA2P (online) or Astrotite (offline)). **/*.afa # A mainly Korean format designed for very large archives. **/*.alz # Android application package (variant of JAR file format). **/*.apk # ?? **/*.arc # Originally DOS, now multiple **/*.arj # Open archive format, used by B1 Free Archiver (http://dev.b1.org/standard/archive-format.html) **/*.b1 # Binary Archive with external header **/*.ba # Proprietary format from the ZipTV Compression Components **/*.bh # The Microsoft Windows native archive format, which is also used by many commercial installers such as InstallShield and WISE. # Originally DOS, now DOS and Windows Created by Yaakov Gringeler; released last in 2003 (Compressia 1.0.0.1 beta), now apparently defunct. Free trial of 30 days lets user create and extract archives; after that it is possible to extract, but not to create. **/*.car # Open source file format. **/*.cfs # Compact Pro archive, a common archiver used on Mac platforms until about Mac OS 7.5.x. Competed with StuffIt; now obsolete. **/*.cpt # Windows, Unix-like, Mac OS X Open source file format. Files are compressed individually with either gzip, bzip2 or lzo. **/*.dar # DiskDoubler Mac OS obsolete **/*.dd # ?? **/*.dgc # Apple Disk Image upports "Internet-enabled" disk images, which, once downloaded, are automatically decompressed, mounted, have the contents extracted, and thrown away. Currently, Safari is the only browser that supports this form of extraction; however, the images can be manually extracted as well. This format can also be password-protected or encrypted with 128-bit or 256-bit AES encryption. # Enterprise Java Archive archive **/*.ear # ETSoft compressed archive # The predecessor of DGCA. **/*.gca # Originally DOS Yes, but may be covered by patents DOS era format; uses arithmetic/Markov coding **/*.ha # MS Windows HKI **/*.hki # Produced by ICEOWS program. Excels at text file compression. **/*.ice # Java archive, compatible with ZIP files # Open sourced archiver with compression using the PAQ family of algorithms and optional encryption. **/*.kgb # Originally DOS, now multiple Multiple Yes The standard format on Amiga. **/*.lzh **/*.lha # Archiver originally used on The Amiga. Now copied by Microsoft to use in their .cab and .chm files. **/*.lzx # file format from NoGate Consultings, a rival from ARC-Compressor. **/*.pak # A disk image archive format that supports several compression methods as well as splitting the archive into smaller pieces. **/*.partimg # An experimental open source packager (http://mattmahoney.net/dc) **/*.paq* # Open source archiver supporting authenticated encryption, volume spanning, customizable object level and volume level integrity checks (form CRCs to SHA-512 and Whirlpool hashes), fast deflate based compression **/*.pea # The format from the PIM - a freeware compression tool by Ilia Muraviev. It uses an LZP-based compression algorithm with set of filters for executable, image and audio files. **/*.pim # PackIt Mac OS obsolete **/*.pit # Used for data in games written using the Quadruple D library for Delphi. Uses byte pair compression. **/*.qda # A proprietary archive format, second in popularity to .zip files. # The format from a commercial archiving package. Odd among commercial packages in that they focus on incorporating experimental algorithms with the highest possible compression (at the expense of speed and memory), such as PAQ, PPMD and PPMZ (PPMD with unlimited-length strings), as well as a proprietary algorithms. **/*.rk # Self Dissolving ARChive Commodore 64, Commodore 128 Commodore 64, Commodore 128 Yes SDAs refer to Self Dissolving ARC files, and are based on the Commodore 64 and Commodore 128 versions of ARC, originally written by Chris Smeets. While the files share the same extension, they are not compatible between platforms. That is, an SDA created on a Commodore 64 but run on a Commodore 128 in Commodore 128 mode will crash the machine, and vice versa. The intended successor to SDA is SFX. **/*.sda # A pre-Mac OS X Self-Extracting Archive format. StuffIt, Compact Pro, Disk Doubler and others could create .sea files, though the StuffIt versions were the most common. **/*.sea # Scifer Archive with internal header **/*.sen # Commodore 64, Commodore 128 SFX is a Self Extracting Archive which uses the LHArc compression algorithm. It was originally developed by Chris Smeets on the Commodore platform, and runs primarily using the CS-DOS extension for the Commodore 128. Unlike its predecessor SDA, SFX files will run on both the Commodore 64 and Commodore 128 regardless of which machine they were created on. **/*.sfx # An archive format designed for the Apple II series of computers. The canonical implementation is ShrinkIt, which can operate on disk images as well as files. Preferred compression algorithm is a combination of RLE and 12-bit LZW. Archives can be manipulated with the command-line NuLib tool, or the Windows-based CiderPress. **/*.shk # A compression format common on Apple Macintosh computers. The free StuffIt Expander is available for Windows and OS X. # The replacement for the .sit format that supports more compression methods, UNIX file permissions, long file names, very large files, more encryption options, data specific compressors (JPEG, Zip, PDF, 24-bit image, MP3). The free StuffIt Expander is available for Windows and OS X. # A royalty-free compressing format **/*.sqx # The "tarball" format combines tar archives with a file-based compression scheme (usually gzip). Commonly used for source and binary distribution on Unix-like platforms, widely available elsewhere. **/*.tar.gz **/*.tar.Z **/*.tar.bz2 **/*.tbz2 **/*.tar.lzma **/*.tlz # UltraCompressor 2.3 was developed to act as an alternative to the then popular PKZIP application. The main feature of the application is its ability to create large archives. This means that compressed archives with the UC2 file extension can hold almost 1 million files. **/*.uc **/*.uc0 **/*.uc2 **/*.ucn **/*.ur2 **/*.ue2 # Based on PAQ, RZM, CSC, CCM, and 7zip. The format consists of a PAQ, RZM, CSC, or CCM compressed file and a manifest with compression settings stored in a 7z archive. **/*.uca # A high compression rate archive format originally for DOS. **/*.uha # Web Application archive (Java-based web app) **/*.war # File-based disk image format developed to deploy Microsoft Windows. **/*.wim # XAR # Native format of the Open Source KiriKiri Visual Novel engine. Uses combination of block splitting and zlib compression. The filenames and pathes are stored in UTF-16 format. For integrity check, the Adler-32 hashsum is used. For many commercial games, the files are encrypted (and decoded on runtime) via so-called "cxdec" module, which implements xor-based encryption. **/*.xp3 # Yamazaki zipper archive. Compression format used in DeepFreezer archiver utility created by Yamazaki Satoshi. Read and write support exists in TUGZip, IZArc and ZipZag **/*.yz1 # The most widely used compression format on Microsoft Windows. Commonly used on Macintosh and Unix systems as well. # application/x-zoo zoo Multiple Multiple Yes **/*.zoo # Journaling (append-only) archive format with rollback capability. Supports deduplication and incremental update based on last-modified dates. Multi-threaded. Compresses in LZ77, BWT, and context mixing formats. Open source. **/*.zpaq # Archiver with a compression algorithm based on the Burrows-Wheeler transform method. **/*.zz ### Compression ### ### From https://en.wikipedia.org/wiki/List_of_archive_formats ## Compression only # An open source, patent- and royalty-free compression format. The compression algorithm is a Burrows-Wheeler transform followed by a move-to-front transform and finally Huffman coding # Old compressor for QNX4 OS. The compression algorithm is a modified LZSS, with an adaptive Huffman coding. **/*.F # GNU Zip, the primary compression format used by Unix-like systems. The compression algorithm is DEFLATE. # An alternate LZMA algorithm implementation, with support for checksums and ident bytes. **/*.lz # The LZMA compression algorithm as used by 7-Zip # An implementation of the LZO data compression algorithm **/*.lzo # A compression program designed to do particularly well on very large files containing long distance redundancy. **/*.rz # Windows compress/decompress- Linux and Mac OS X decompress only A compression program designed to do high compression on SF2 files (SoundFont) **/*.sfark # A compression format invented by Google and open-sourced in 2011. Snappy aims for very high speeds, reasonable compression, and maximum stability rather than maximum compression or compatibility with any other compression library. **/*.sz # Squeeze: A program which compressed files. A file which was "squeezed" had the middle initial of the name changed to "Q", so that a squeezed text file would end with .TQT, a squeezed executable would end with .CQM or .EQE. Typically used with .LBR archives, either by storing the squeezed files in the archive, or by storing the files decompressed and then compressing the archive, which would have a name ending in ".LQR". **/*.?Q? # A compression program written by Steven Greenberg implementing the LZW algorithm. For several years in the CP/M world when no implementation was available of ARC, CRUNCHed files stored in .LBR archives were very popular. CRUNCH's implementation of LZW had a somewhat unique feature of modifying and occasionally clearing the code table in memory when it became full, resulting in a few percent better compression on many files. **/*.?Z? # A compression format using LZMA2 to yield very high compression ratios. # The traditional Huffman coding compression format. **/*.z # The traditional LZW compression format. **/*.Z # Joke compression program, actually increasing file size **/*.infl # Compression format(s) used by some DOS and Windows install programs. MS-DOS includes expand.exe to decompress its install files. The compressed files are created with a matching compress.exe command. The compression algorithm is LZSS. **/*.??_ ### Data ### **/*.csv **/*.dat **/*.efx **/*.gbr **/*.pps **/*.ppt **/*.pptx **/*.sdf **/*.tax2010 **/*.vcf **/*.xml ### Code ### **/.vscode/* **/!.vscode/settings.json **/!.vscode/tasks.json **/!.vscode/launch.json **/!.vscode/extensions.json **/*.code-workspace ### DataRecovery ### ## Data recovery # File format used by dvdisaster to be used for data recovery when discs become damaged or partially unreadable. **/*.ecc # File format used in conjunction with any archive format to provide redundancy and data recovery, most often in newsgroup distribution of binary files. **/*.par **/*.par2 ### Diff ### **/*.patch **/*.diff ### direnv ### **/.direnv **/.envrc ### CodeBlocks ### # specific to CodeBlocks IDE **/*.layout **/*.depend # generated directories **/bin/ **/obj/ ### DocFx ### **/.cache **/**/_site/ ### Docpress ### # docpress documentation generator: https://docpress.github.io/index.html **/_docpress/ ### Docz ### **/.docz ### dotenv ### **/.env ### DotfilesSh ### **/local-patch **/patched-src ### DotSettings ### **/*.DotSettings ### Dropbox ### # Dropbox settings and caches **/.dropbox **/.dropbox.attr **/.dropbox.cache ### Eclipse ### **/.metadata **/tmp/ **/*.swp **/*~.nib **/local.properties **/.settings/ **/.loadpath **/.recommenders # External tool builders **/.externalToolBuilders/ # Locally stored "Eclipse launch configurations" **/*.launch # PyDev specific (Python IDE for Eclipse) **/*.pydevproject # CDT-specific (C/C++ Development Tooling) **/.cproject # CDT- autotools **/.autotools # Java annotation processor (APT) **/.factorypath # PDT-specific (PHP Development Tools) **/.buildpath # sbteclipse plugin **/.target # Tern plugin **/.tern-project # TeXlipse plugin **/.texlipse # STS (Spring Tool Suite) **/.springBeans # Code Recommenders **/.recommenders/ # Annotation Processing **/.apt_generated/ **/.apt_generated_test/ # Scala IDE specific (Scala & Java development for Eclipse) **/.cache-main **/.scala_dependencies **/.worksheet # Uncomment this line if you wish to ignore the project description file. # Typically, this file would be tracked if it contains build/dependency configurations: #.project ### Eclipse Patch ### # Spring Boot Tooling **/.sts4-cache/ ### Emacs ### # -*- mode: gitignore; -*- **/*~ **/\#*\# **/.emacs.desktop **/.emacs.desktop.lock **/*.elc **/auto-save-list **/tramp **/.\#* # Org-mode **/.org-id-locations **/*_archive # flymake-mode **/*_flymake.* # eshell files **/eshell/history **/eshell/lastdir # elpa packages **/elpa/ # reftex files **/*.rel # AUCTeX auto folder **/auto/ # cask packages **/.cask/ **/dist/ # Flycheck **/flycheck_*.el # server auth directory **/server/ # projectiles files **/.projectile # directory configuration **/.dir-locals.el # network security **/network-security.data ### Database ### **/*.accdb **/*.db **/*.dbf **/*.mdb **/*.sqlite3 ### Executable ### **/*.bat **/*.cgi **/*.com **/*.gadget **/*.pif **/*.vb **/*.wsf ### Firebase ### **/**/node_modules/* **/**/.firebaserc ### Firebase Patch ### **/.runtimeconfig.json **/.firebase/ ### Flask ### **/instance/* **/!instance/.gitignore **/.webassets-cache ### Flask.Python Stack ### # Byte-compiled / optimized / DLL files **/__pycache__/ **/*.py[cod] **/*$py.class # C extensions # Distribution / packaging **/.Python **/build/ **/develop-eggs/ **/downloads/ **/eggs/ **/.eggs/ **/lib/ **/lib64/ **/parts/ **/sdist/ **/var/ **/wheels/ **/pip-wheel-metadata/ **/share/python-wheels/ **/*.egg-info/ **/.installed.cfg **/MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. **/*.manifest **/*.spec # Installer logs **/pip-log.txt **/pip-delete-this-directory.txt # Unit test / coverage reports **/htmlcov/ **/.tox/ **/.nox/ **/.coverage **/.coverage.* **/nosetests.xml **/coverage.xml **/*.cover **/*.py,cover **/.hypothesis/ **/.pytest_cache/ **/pytestdebug.log # Translations **/*.mo **/*.pot # Django stuff: **/local_settings.py **/db.sqlite3 **/db.sqlite3-journal # Flask stuff: **/instance/ # Scrapy stuff: **/.scrapy # Sphinx documentation **/docs/_build/ **/doc/_build/ # PyBuilder **/target/ # Jupyter Notebook **/.ipynb_checkpoints # IPython **/profile_default/ **/ipython_config.py # pyenv **/.python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow **/__pypackages__/ # Celery stuff **/celerybeat-schedule **/celerybeat.pid # SageMath parsed files **/*.sage.py # Environments **/env/ **/venv/ **/ENV/ **/env.bak/ **/venv.bak/ **/pythonenv* # Spyder project settings **/.spyderproject **/.spyproject # Rope project settings **/.ropeproject # mkdocs documentation **/site # mypy **/.mypy_cache/ **/.dmypy.json **/dmypy.json # Pyre type checker **/.pyre/ # pytype static type analyzer **/.pytype/ # profiling data **/.prof ### Git ### # Created by git for backups. To disable backups in Git: # $ git config --global mergetool.keepBackup false # Created by git when using merge tools for conflicts **/*.BACKUP.* **/*.BASE.* **/*.LOCAL.* **/*.REMOTE.* **/*_BACKUP_*.txt **/*_BASE_*.txt **/*_LOCAL_*.txt **/*_REMOTE_*.txt ### GitBook ### # Node rules: ## Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) **/.grunt ## Dependency directory ## Commenting this out is preferred by some people, see ## https://docs.npmjs.com/misc/faq#should-i-check-my-node_modules-folder-into-git **/node_modules # Book build output **/_book # eBook build output **/*.epub **/*.mobi **/*.pdf ### Go ### # Binaries for programs and plugins **/*.exe~ # Test binary, built with `go test -c` **/*.test # Output of the go coverage tool, specifically when used with LiteIDE # Dependency directories (remove the comment below to include it) # vendor/ ### Go Patch ### **/vendor/ **/Godeps/ ### GPG ### **/secring.* ### DiskImage ### **/*.toast **/*.vcd ### Grails ### # .gitignore for Grails 1.2 and 1.3 # Although this should work for most versions of grails, it is # suggested that you use the "grails integrate-with --git" command # to generate your .gitignore file. # web application files **/web-app/WEB-INF/classes # default HSQL database files for production mode **/prodDb.* # general HSQL database files **/*Db.properties **/*Db.script # logs **/stacktrace.log **/test/reports **/logs # project release file **/*.war # plugin release files **/*.zip **/plugin.xml # older plugin install locations **/plugins **/web-app/plugins # "temporary" build files **/target ### Groovy ### # .gitignore created from Groovy contributors in https://github.com/apache/groovy/blob/master/.gitignore **/user.gradle **/.gradle/ **/*.DS_Store **/.shelf ### grunt ### # Grunt usually compiles files inside this directory # Grunt usually preprocesses files such as coffeescript, compass... inside the .tmp directory **/.tmp/ ### Haskell ### **/dist **/dist-* **/cabal-dev **/*.hi **/*.hie **/*.chi **/*.chs.h **/*.dyn_o **/*.dyn_hi **/.hpc **/.hsenv **/.cabal-sandbox/ **/cabal.sandbox.config **/*.prof **/*.aux **/*.hp **/*.eventlog **/.stack-work/ **/cabal.project.local **/cabal.project.local~ **/.HTF/ **/.ghc.environment.* ### Helm ### # Chart dependencies **/**/charts/*.tgz ### Homebrew ### **/Brewfile.lock.json ### Hugo ### # Generated files by hugo **/public/ **/resources/_gen/ **/hugo_stats.json # Executable may be added to repository **/hugo.exe **/hugo.darwin **/hugo.linux ### Images ### # JPEG **/*.jpg **/*.jpeg **/*.jpe **/*.jif **/*.jfif **/*.jfi # JPEG 2000 **/*.jp2 **/*.j2k **/*.jpf **/*.jpx **/*.jpm **/*.mj2 # JPEG XR **/*.jxr **/*.hdp **/*.wdp # Graphics Interchange Format **/*.gif # RAW **/*.raw # Web P **/*.webp # Portable Network Graphics **/*.png # Animated Portable Network Graphics **/*.apng # Multiple-image Network Graphics **/*.mng # Tagged Image File Format **/*.tiff **/*.tif # Scalable Vector Graphics **/*.svg **/*.svgz # Portable Document Format # X BitMap **/*.xbm # BMP **/*.bmp **/*.dib # ICO **/*.ico # 3D Images **/*.3dm **/*.max ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### Intellij+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 **/.idea/ # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 **/modules.xml **/.idea/misc.xml # Sonarlint plugin **/.idea/sonarlint ### Intellij+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### Java ### # Compiled class file # Log file # BlueJ files **/*.ctxt # Mobile Tools for Java (J2ME) **/.mtj.tmp/ # Package Files # **/*.nar # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml **/hs_err_pid* ### Java-Web ### ## ignoring target file ### JEnv ### # JEnv local Java version configuration file **/.java-version # Used by previous versions of JEnv **/.jenv-version ### JetBrains ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### JetBrains+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 # Sonarlint plugin ### JetBrains+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### JMeter ### # JMeter common ignore files # http://jmeter.apache.org/ # Ignore Summary/Aggregrate reports **/*.jtl # Ignore log files # Ignore customized user.properties **/user.properties ### Erlang ### **/.eunit **/*.beam **/*.plt **/erl_crash.dump **/.concrete/DEV_MODE # rebar 2.x **/.rebar **/rel/example_project **/ebin/*.beam **/deps # rebar 3 **/.rebar3 **/_build/ **/_checkouts/ ### JupyterNotebooks ### # gitignore template for Jupyter Notebooks # website: http://jupyter.org/ **/*/.ipynb_checkpoints/* # IPython # Remove previous ipynb_checkpoints # git rm -r .ipynb_checkpoints/ ### Kotlin ### # Compiled class file # Log file # BlueJ files # Mobile Tools for Java (J2ME) # Package Files # # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml ### LAMP ### # LAMP Stack Base ### LAMP.Linux Stack ### # temporary files which can be created if a process still has a handle open of a deleted file **/.fuse_hidden* # KDE directory preferences **/.directory # Linux trash folder which might appear on any partition or disk **/.Trash-* # .nfs files are created when an open file is removed but is still being accessed **/.nfs* ### LAMP.PHP Stack ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### LaTeX ### ## Core latex/pdflatex auxiliary files: **/*.lof **/*.lot **/*.fls **/*.toc **/*.fmt **/*.fot **/*.cb **/*.cb2 **/.*.lb ## Intermediate documents: **/*.dvi **/*.xdv **/*-converted-to.* # these rules might exclude image files for figures etc. # *.ps # *.eps # *.pdf ## Generated if empty string is given at "Please type another file name for output:" **/.pdf ## Bibliography auxiliary files (bibtex/biblatex/biber): **/*.bbl **/*.bcf **/*.blg **/*-blx.aux **/*-blx.bib **/*.run.xml ## Build tool auxiliary files: **/*.fdb_latexmk **/*.synctex **/*.synctex(busy) **/*.synctex.gz **/*.synctex.gz(busy) **/*.pdfsync ## Build tool directories for auxiliary files # latexrun **/latex.out/ ## Auxiliary and intermediate files from other packages: # algorithms **/*.alg **/*.loa # achemso **/acs-*.bib # amsthm **/*.thm # beamer **/*.nav **/*.pre **/*.snm **/*.vrb # changes **/*.soc # comment **/*.cut # cprotect # elsarticle (documentclass of Elsevier journals) **/*.spl # endnotes **/*.ent # fixme **/*.lox # feynmf/feynmp **/*.mf **/*.mp **/*.t[1-9] **/*.t[1-9][0-9] **/*.tfm #(r)(e)ledmac/(r)(e)ledpar **/*.end **/*.?end **/*.[1-9] **/*.[1-9][0-9] **/*.[1-9][0-9][0-9] **/*.[1-9]R **/*.[1-9][0-9]R **/*.[1-9][0-9][0-9]R **/*.eledsec[1-9] **/*.eledsec[1-9]R **/*.eledsec[1-9][0-9] **/*.eledsec[1-9][0-9]R **/*.eledsec[1-9][0-9][0-9] **/*.eledsec[1-9][0-9][0-9]R # glossaries **/*.acn **/*.acr **/*.glg **/*.glo **/*.gls **/*.glsdefs **/*.lzs # uncomment this for glossaries-extra (will ignore makeindex's style files!) # *.ist # gnuplottex **/*-gnuplottex-* # gregoriotex **/*.gaux **/*.gtex # htlatex **/*.4ct **/*.4tc **/*.idv **/*.lg **/*.trc **/*.xref # hyperref **/*.brf # knitr **/*-concordance.tex # TODO Comment the next line if you want to keep your tikz graphics files **/*.tikz **/*-tikzDictionary # listings **/*.lol # luatexja-ruby **/*.ltjruby # makeidx **/*.idx **/*.ilg **/*.ind # minitoc **/*.maf **/*.mlf **/*.mlt **/*.mtc **/*.mtc[0-9]* **/*.slf[0-9]* **/*.slt[0-9]* **/*.stc[0-9]* # minted **/_minted* **/*.pyg # morewrites **/*.mw # nomencl **/*.nlg **/*.nlo **/*.nls # pax **/*.pax # pdfpcnotes **/*.pdfpc # sagetex **/*.sagetex.sage **/*.sagetex.py **/*.sagetex.scmd # scrwfile **/*.wrt # sympy **/*.sout **/*.sympy **/sympy-plots-for-*.tex/ # pdfcomment **/*.upa **/*.upb # pythontex **/*.pytxcode **/pythontex-files-*/ # tcolorbox **/*.listing # thmtools **/*.loe # TikZ & PGF **/*.dpth **/*.md5 **/*.auxlock # todonotes **/*.tdo # vhistory **/*.hst **/*.ver # easy-todo **/*.lod # xcolor **/*.xcp # xmpincl **/*.xmpi # xindy **/*.xdy # xypic precompiled matrices and outlines **/*.xyc **/*.xyd # endfloat **/*.ttt **/*.fff # Latexian **/TSWLatexianTemp* ## Editors: # WinEdt **/*.sav # Texpad **/.texpadtmp # LyX **/*.lyx~ # Kile **/*.backup # gummi **/.*.swp # KBibTeX **/*~[0-9]* # TeXnicCenter **/*.tps # auto folder when using emacs and auctex **/./auto/* **/*.el # expex forward references with \gathertags **/*-tags.tex # standalone packages **/*.sta # Makeindex log files **/*.lpz # REVTeX puts footnotes in the bibliography by default, unless the nofootinbib # option is specified. Footnotes are the stored in a file with suffix Notes.bib. # Uncomment the next line to have this generated file ignored. #*Notes.bib ### LaTeX Patch ### # LIPIcs / OASIcs **/*.vtc # glossaries **/*.glstex ### Less ### **/*.less ### Linux ### # temporary files which can be created if a process still has a handle open of a deleted file # KDE directory preferences # Linux trash folder which might appear on any partition or disk # .nfs files are created when an open file is removed but is still being accessed ### Lua ### # Compiled Lua sources **/luac.out # luarocks build files **/*.src.rock # Object files **/*.os # Precompiled Headers # Libraries **/*.def # Shared objects (inc. Windows DLLs) # Executables ### macOS ### # General **/.DS_Store **/.AppleDouble **/.LSOverride # Icon must end with two \r **/Icon # Thumbnails **/._* # Files that might appear in the root of a volume **/.DocumentRevisions-V100 **/.fseventsd **/.Spotlight-V100 **/.TemporaryItems **/.Trashes **/.VolumeIcon.icns **/.com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share **/.AppleDB **/.AppleDesktop **/Network Trash Folder **/Temporary Items **/.apdisk ### MATLAB ### # Windows default autosave extension **/*.asv # OSX / *nix default autosave extension **/*.m~ # Compiled MEX binaries (all platforms) **/*.mex* # Packaged app and toolbox files **/*.mlappinstall **/*.mltbx # Generated helpsearch folders **/helpsearch*/ # Simulink code generation folders **/slprj/ **/sccprj/ # Matlab code generation folders **/codegen/ # Simulink autosave extension **/*.autosave # Simulink cache files **/*.slxc # Octave session info **/octave-workspace ### Maven ### **/pom.xml.tag **/pom.xml.releaseBackup **/pom.xml.versionsBackup **/pom.xml.next **/release.properties **/dependency-reduced-pom.xml **/buildNumber.properties **/.mvn/timing.properties # https://github.com/takari/maven-wrapper#usage-without-binary-jar **/.mvn/wrapper/maven-wrapper.jar ### Mercurial ### **/.hg/ **/.hgignore **/.hgsigs **/.hgsub **/.hgsubstate **/.hgtags ### MicrosoftOffice ### # Word temporary **/~$*.doc* # Word Auto Backup File **/Backup of *.doc* # Excel temporary **/~$*.xls* # Excel Backup File **/*.xlk # PowerPoint temporary **/~$*.ppt* # Visio autosave temporary files **/*.~vsd* ### Node ### # Logs **/logs **/npm-debug.log* **/yarn-debug.log* **/yarn-error.log* **/lerna-debug.log* # Diagnostic reports (https://nodejs.org/api/report.html) **/report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data **/pids **/*.pid **/*.seed **/*.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover **/lib-cov # Coverage directory used by tools like istanbul **/coverage **/*.lcov # nyc test coverage **/.nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) # Bower dependency directory (https://bower.io/) **/bower_components # node-waf configuration **/.lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) **/build/Release # Dependency directories **/node_modules/ **/jspm_packages/ # TypeScript v1 declaration files **/typings/ # TypeScript cache **/*.tsbuildinfo # Optional npm cache directory **/.npm # Optional eslint cache **/.eslintcache # Microbundle cache **/.rpt2_cache/ **/.rts2_cache_cjs/ **/.rts2_cache_es/ **/.rts2_cache_umd/ # Optional REPL history **/.node_repl_history # Output of 'npm pack' # Yarn Integrity file **/.yarn-integrity # dotenv environment variables file **/.env.test **/.env*.local # parcel-bundler cache (https://parceljs.org/) **/.parcel-cache # Next.js build output **/.next # Nuxt.js build / generate output **/.nuxt # Gatsby files **/.cache/ # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output **/.vuepress/dist # Serverless directories **/.serverless/ # FuseBox cache **/.fusebox/ # DynamoDB Local files **/.dynamodb/ # TernJS port file **/.tern-port # Stores VSCode versions used for testing VSCode extensions **/.vscode-test ### Octave ### # Windows default autosave extension # OSX / *nix default autosave extension # Compiled MEX binaries (all platforms) # Packaged app and toolbox files # Generated helpsearch folders # Simulink code generation folders # Matlab code generation folders # Simulink autosave extension # Simulink cache files # Octave session info ### OSX ### # General # Icon must end with two \r # Thumbnails # Files that might appear in the root of a volume # Directories potentially created on remote AFP share ### Packer ### # Cache objects **/packer_cache/ # Crash log **/crash.log # For built boxes **/*.box ### Patch ### ### Perl ### **/!Build/ **/.last_cover_stats **/META.yml **/META.json **/MYMETA.* **/*.pm.tdy **/*.bs # Devel::Cover **/cover_db/ # Devel::NYTProf **/nytprof.out # Dizt::Zilla **/.build/ # Module::Build **/Build **/Build.bat # Module::Install **/inc/ # ExtUtils::MakeMaker **/blib/ **/_eumm/ **/*.gz **/Makefile **/Makefile.old **/MANIFEST.bak **/pm_to_blib ### Perl6 ### # Gitignore for Perl 6 (http://www.perl6.org) # As part of https://github.com/github/gitignore # precompiled files **/.precomp **/lib/.precomp ### PHPUnit ### # Covers PHPUnit # Reference: https://phpunit.de/ # Generated files **/.phpunit.result.cache # PHPUnit **/app/phpunit.xml **/phpunit.xml # Build data **/build/ ### PowerShell ### # Exclude packaged modules # Exclude .NET assemblies from source ### Puppet ### # gitignore template for Puppet modules # website: https://forge.puppet.com/ # Built packages **/pkg/* # Should run on multiple platforms so don't check in **/Gemfile.lock # Tests **/spec/fixtures/* **/coverage/* # Third-party **/vendor/* ### PuTTY ### # Private key **/*.ppk ### PyCharm ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### PyCharm+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 # Sonarlint plugin ### PyCharm+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### pydev ### **/.pydevproject ### Python ### # Byte-compiled / optimized / DLL files # C extensions # Distribution / packaging # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. # Installer logs # Unit test / coverage reports # Translations # Django stuff: # Flask stuff: # Scrapy stuff: # Sphinx documentation # PyBuilder # Jupyter Notebook # IPython # pyenv # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. # PEP 582; used by e.g. github.com/David-OConnor/pyflow # Celery stuff # SageMath parsed files # Environments # Spyder project settings # Rope project settings # mkdocs documentation # mypy # Pyre type checker # pytype static type analyzer # profiling data ### R ### # History files **/.Rhistory **/.Rapp.history # Session Data files **/.RData # User-specific files **/.Ruserdata # Example code in package build process **/*-Ex.R # Output files from R CMD build **/*.tar.gz # Output files from R CMD check **/*.Rcheck/ # RStudio files **/.Rproj.user/ # produced vignettes **/vignettes/*.html **/vignettes/*.pdf # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 **/.httr-oauth # knitr and R markdown default cache directories **/*_cache/ **/cache/ # Temporary files created by R markdown **/*.utf8.md **/*.knit.md # R Environment Variables **/.Renviron ### R.Bookdown Stack ### # R package: bookdown caching files **/*_files/ ### Rails ### **/*.rbc **/capybara-*.html **/.rspec **/db/*.sqlite3 **/db/*.sqlite3-journal **/db/*.sqlite3-[0-9]* **/public/system **/coverage/ **/spec/tmp **/rerun.txt **/pickle-email-*.html # Ignore all logfiles and tempfiles. **/log/* **/tmp/* **/!/log/.keep **/!/tmp/.keep # TODO Comment out this rule if you are OK with secrets being uploaded to the repo **/config/initializers/secret_token.rb **/config/master.key # Only include if you have production secrets in this file, which is no longer a Rails default # config/secrets.yml # dotenv, dotenv-rails # TODO Comment out these rules if environment variables can be committed **/.env.* ## Environment normalization: **/.bundle **/vendor/bundle # these should all be checked in to normalize the environment: # Gemfile.lock, .ruby-version, .ruby-gemset # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: **/.rvmrc # if using bower-rails ignore default bower_components path bower.json files **/vendor/assets/bower_components **/*.bowerrc **/bower.json # Ignore pow environment settings **/.powenv # Ignore Byebug command history file. **/.byebug_history # Ignore node_modules # Ignore precompiled javascript packs **/public/packs **/public/packs-test **/public/assets # Ignore yarn files **/yarn-error.log # Ignore uploaded files in development **/storage/* **/!/storage/.keep ### react ### **/.DS_* **/**/*.backup.* **/**/*.back.* **/*.sublime* **/psd **/thumb **/sketch ### ReactNative ### # React Native Stack Base **/.expo **/__generated__ ### ReactNative.Android Stack ### # Built application files **/*.aar **/*.ap_ **/*.aab # Files for the ART/Dalvik VM **/*.dex # Java class files # Generated files **/gen/ # Uncomment the following line in case you need and you don't have the release build type files in your app # release/ # Gradle files # Local configuration file (sdk path, etc) # Proguard folder generated by Eclipse **/proguard/ # Log Files # Android Studio Navigation editor temp files **/.navigation/ # Android Studio captures folder **/captures/ # IntelliJ **/.idea/workspace.xml **/.idea/tasks.xml **/.idea/gradle.xml **/.idea/assetWizardSettings.xml **/.idea/dictionaries **/.idea/libraries # Android Studio 3 in .gitignore file. **/.idea/caches **/.idea/modules.xml # Comment next line if keeping position of elements in Navigation Editor is relevant for you **/.idea/navEditor.xml # Keystore files # Uncomment the following lines if you do not want to check your keystore files in. #*.jks #*.keystore # External native build folder generated in Android Studio 2.2 and later **/.externalNativeBuild **/.cxx/ # Google Services (e.g. APIs or Firebase) # google-services.json # Freeline **/freeline.py **/freeline/ **/freeline_project_description.json # fastlane **/fastlane/report.xml **/fastlane/Preview.html **/fastlane/screenshots **/fastlane/test_output **/fastlane/readme.md # Version control **/vcs.xml # lint **/lint/intermediates/ **/lint/generated/ **/lint/outputs/ **/lint/tmp/ # lint/reports/ ### ReactNative.Buck Stack ### **/buck-out/ **/.buckconfig.local **/.buckd/ **/.buckversion **/.fakebuckversion ### ReactNative.Gradle Stack ### **/.gradle # Ignore Gradle GUI config **/gradle-app.setting # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) **/!gradle-wrapper.jar # Cache of project **/.gradletasknamecache # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties ### ReactNative.Linux Stack ### # temporary files which can be created if a process still has a handle open of a deleted file # KDE directory preferences # Linux trash folder which might appear on any partition or disk # .nfs files are created when an open file is removed but is still being accessed ### ReactNative.Node Stack ### # Logs # Diagnostic reports (https://nodejs.org/api/report.html) # Runtime data # Directory for instrumented libs generated by jscoverage/JSCover # Coverage directory used by tools like istanbul # nyc test coverage # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) # Bower dependency directory (https://bower.io/) # node-waf configuration # Compiled binary addons (https://nodejs.org/api/addons.html) # Dependency directories # TypeScript v1 declaration files # TypeScript cache # Optional npm cache directory # Optional eslint cache # Microbundle cache # Optional REPL history # Output of 'npm pack' # Yarn Integrity file # dotenv environment variables file # parcel-bundler cache (https://parceljs.org/) # Next.js build output # Nuxt.js build / generate output # Gatsby files # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output # Serverless directories # FuseBox cache # DynamoDB Local files # TernJS port file # Stores VSCode versions used for testing VSCode extensions ### ReactNative.Xcode Stack ### # Xcode # # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore ## User settings **/xcuserdata/ ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) **/*.xcscmblueprint **/*.xccheckout ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) **/DerivedData/ **/*.moved-aside **/*.pbxuser **/!default.pbxuser **/*.mode1v3 **/!default.mode1v3 **/*.mode2v3 **/!default.mode2v3 **/*.perspectivev3 **/!default.perspectivev3 ## Gcc Patch **/*.gcno ### ReactNative.macOS Stack ### # General # Icon must end with two \r **/Icon # Thumbnails # Files that might appear in the root of a volume # Directories potentially created on remote AFP share ### Redis ### # Ignore redis binary dump (dump.rdb) files **/*.rdb ### ROOT ### # ROOT Home Page : https://root.cern.ch/ # ROOT Used by Experimental Physicists, not necessarily HEP # ROOT based on C++ # Files generated by ROOT, observed with v6.xy **/*.pcm ### Ruby ### **/.config **/InstalledFiles **/pkg/ **/spec/reports/ **/spec/examples.txt **/test/tmp/ **/test/version_tmp/ **/tmp/ # Used by dotenv library to load environment variables. # .env # Ignore Byebug command history file. ## Specific to RubyMotion: **/.dat* **/.repl_history **/*.bridgesupport **/build-iPhoneOS/ **/build-iPhoneSimulator/ ## Specific to RubyMotion (use of CocoaPods): # We recommend against adding the Pods directory to your .gitignore. However # you should judge for yourself, the pros and cons are mentioned at: # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control # vendor/Pods/ ## Documentation cache and generated files: **/.yardoc/ **/_yardoc/ **/doc/ **/rdoc/ **/.bundle/ **/lib/bundler/man/ # for a library or gem, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # Gemfile.lock # .ruby-version # .ruby-gemset # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: # Used by RuboCop. Remote config files pulled in from inherit_from directive. # .rubocop-https?--* ### Ruby Patch ### # Used by RuboCop. Remote config files pulled in from inherit_from directive. # .rubocop-https?--* ### Rust ### # Generated by Cargo # will have compiled files and executables # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html **/Cargo.lock ### SBT ### # Simple Build Tool # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control **/dist/* **/lib_managed/ **/src_managed/ **/project/boot/ **/project/plugins/project/ **/.history **/.lib/ ### Scala ### ### Serverless ### # Ignore build directory **/.serverless ### Sonar ### #Sonar generated dir **/.sonar/ ### SonarQube ### # SonarQube ignore files. # https://docs.sonarqube.org/display/SCAN/Analyzing+with+SonarQube+Scanner # Sonar Scanner working directories **/.sonar/ **/.scannerwork/ # http://www.sonarlint.org/commandline/ # SonarLint working directories, configuration files (including credentials) **/.sonarlint/ ### Spark ### **/*#*# **/*.#* **/*.pyc **/*.pyo **/.ensime **/.ensime_cache/ **/.ensime_lucene **/.generated-mima* **/R-unit-tests.log **/R/unit-tests.out **/R/cran-check.out **/R/pkg/vignettes/sparkr-vignettes.html **/R/pkg/tests/fulltests/Rplots.pdf **/build/*.jar **/build/apache-maven* **/build/scala* **/build/zinc* **/cache **/checkpoint **/conf/*.cmd **/conf/*.conf **/conf/*.properties **/conf/*.sh **/conf/*.xml **/conf/java-opts **/conf/slaves **/derby.log **/dev/create-release/*final **/dev/create-release/*txt **/dev/pr-deps/ **/docs/_site **/docs/api **/sql/docs **/sql/site **/lint-r-report.log **/log/ **/logs/ **/project/build/target/ **/project/plugins/lib_managed/ **/project/plugins/project/build.properties **/project/plugins/src_managed/ **/project/plugins/target/ **/python/lib/pyspark.zip **/python/deps **/python/test_coverage/coverage_data **/python/test_coverage/htmlcov **/python/pyspark/python **/reports/ **/scalastyle-on-compile.generated.xml **/scalastyle-output.xml **/scalastyle.txt **/spark-*-bin-*.tgz **/spark-tests.log **/streaming-tests.log **/unit-tests.log **/work/ **/docs/.jekyll-metadata # For Hive **/TempStatsStore/ **/metastore/ **/metastore_db/ **/sql/hive-thriftserver/test_warehouses **/warehouse/ **/spark-warehouse/ # For R session data **/.RHistory **/*.Rproj **/*.Rproj.* **/.Rproj.user # For SBT **/.jvmopts ### Splunk ### # gitignore template for Splunk apps # documentation: http://docs.splunk.com/Documentation/Splunk/6.2.3/admin/Defaultmetaconf # Splunk local meta file **/local.meta # Splunk local folder **/local ### Spreadsheet ### **/*.xlr **/*.xls **/*.xlsx ### SSH ### **/**/.ssh/id_* **/**/.ssh/*_id_* **/**/.ssh/known_hosts ### SublimeText ### # Cache files for Sublime Text **/*.tmlanguage.cache **/*.tmPreferences.cache **/*.stTheme.cache # Workspace files are user-specific **/*.sublime-workspace # Project files should be checked into the repository, unless a significant # proportion of contributors will probably not be using Sublime Text # *.sublime-project # SFTP configuration file **/sftp-config.json # Package control specific files **/Package Control.last-run **/Package Control.ca-list **/Package Control.ca-bundle **/Package Control.system-ca-bundle **/Package Control.cache/ **/Package Control.ca-certs/ **/Package Control.merged-ca-bundle **/Package Control.user-ca-bundle **/oscrypto-ca-bundle.crt **/bh_unicode_properties.cache # Sublime-github package stores a github token in this file # https://packagecontrol.io/packages/sublime-github **/GitHub.sublime-settings ### SVN ### **/.svn/ ### Terraform ### # Local .terraform directories **/**/.terraform/* # .tfstate files **/*.tfstate **/*.tfstate.* # Crash log files # Ignore any .tfvars files that are generated automatically for each Terraform run. Most # .tfvars files are managed as part of configuration and so should be included in # version control. # example.tfvars # Ignore override files as they are usually used to override resources locally and so # are not checked in **/override.tf **/override.tf.json **/*_override.tf **/*_override.tf.json # Include override files you do wish to add to version control using negated pattern # !example_override.tf # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan # example: *tfplan* ### Terragrunt ### # terragrunt cache directories **/**/.terragrunt-cache/* ### TortoiseGit ### # Project-level settings **/.tgitconfig ### Vagrant ### # General **/.vagrant/ # Log files (if you are creating logs in debug mode, uncomment this) # *.log ### Vagrant Patch ### ### venv ### # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ **/[Bb]in **/[Ii]nclude **/[Ll]ib **/[Ll]ib64 **/[Ll]ocal **/[Ss]cripts **/pyvenv.cfg **/pip-selfcheck.json ### VirtualEnv ### # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ ### Julia ### # Files generated by invoking Julia with --code-coverage **/*.jl.cov **/*.jl.*.cov # Files generated by invoking Julia with --track-allocation **/*.jl.mem # System-specific files and directories generated by the BinaryProvider and BinDeps packages # They contain absolute paths specific to the host computer, and so should not be committed **/deps/deps.jl **/deps/build.log **/deps/downloads/ **/deps/usr/ **/deps/src/ # Build artifacts for creating documentation generated by the Documenter package **/docs/build/ **/docs/site/ # File generated by Pkg, the package manager, based on a corresponding Project.toml # It records a fixed state of all packages used by the project. As such, it should not be # committed for packages, but should be committed for applications that require a static # environment. **/Manifest.toml ### VisualStudioCode ### ### VisualStudioCode Patch ### # Ignore all local history of files **/.ionide ### vs ### ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files **/*.rsuser **/*.suo **/*.user **/*.userosscache **/*.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) **/*.userprefs # Mono auto generated files **/mono_crash.* # Build results **/[Dd]ebug/ **/[Dd]ebugPublic/ **/[Rr]elease/ **/[Rr]eleases/ **/x64/ **/x86/ **/[Aa][Rr][Mm]/ **/[Aa][Rr][Mm]64/ **/bld/ **/[Bb]in/ **/[Oo]bj/ **/[Ll]og/ **/[Ll]ogs/ # Visual Studio 2015/2017 cache/options directory **/.vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files **/Generated\ Files/ # MSTest test Results **/[Tt]est[Rr]esult*/ **/[Bb]uild[Ll]og.* # NUnit **/*.VisualState.xml **/TestResult.xml **/nunit-*.xml # Build Results of an ATL Project **/[Dd]ebugPS/ **/[Rr]eleasePS/ **/dlldata.c # Benchmark Results **/BenchmarkDotNet.Artifacts/ # .NET Core **/project.lock.json **/project.fragment.lock.json **/artifacts/ # StyleCop **/StyleCopReport.xml # Files built by Visual Studio **/*_i.c **/*_p.c **/*_h.h **/*.meta **/*.iobj **/*.ipdb **/*.pgc **/*.pgd **/*.rsp **/*.sbr **/*.tlb **/*.tli **/*.tlh **/*.tmp_proj **/*_wpftmp.csproj **/*.vspscc **/*.vssscc **/.builds **/*.pidb **/*.svclog **/*.scc # Chutzpah Test files **/_Chutzpah* # Visual C++ cache files **/ipch/ **/*.aps **/*.ncb **/*.opendb **/*.opensdf **/*.cachefile **/*.VC.db **/*.VC.VC.opendb # Visual Studio profiler **/*.psess **/*.vsp **/*.vspx **/*.sap # Visual Studio Trace Files **/*.e2e # TFS 2012 Local Workspace **/$tf/ # Guidance Automation Toolkit **/*.gpState # ReSharper is a .NET coding add-in **/_ReSharper*/ **/*.[Rr]e[Ss]harper **/*.DotSettings.user # TeamCity is a build add-in **/_TeamCity* # DotCover is a Code Coverage Tool **/*.dotCover # AxoCover is a Code Coverage Tool **/.axoCover/* **/!.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool **/coverage*[.json, .xml, .info] # Visual Studio code coverage results **/*.coverage **/*.coveragexml # NCrunch **/_NCrunch_* **/.*crunch*.local.xml **/nCrunchTemp_* # MightyMoose **/*.mm.* **/AutoTest.Net/ # Web workbench (sass) **/.sass-cache/ # Installshield output folder **/[Ee]xpress/ # DocProject is a documentation generator add-in **/DocProject/buildhelp/ **/DocProject/Help/*.HxT **/DocProject/Help/*.HxC **/DocProject/Help/*.hhc **/DocProject/Help/*.hhk **/DocProject/Help/*.hhp **/DocProject/Help/Html2 **/DocProject/Help/html # Click-Once directory **/publish/ # Publish Web Output **/*.[Pp]ublish.xml **/*.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted **/*.pubxml **/*.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted **/PublishScripts/ # NuGet Packages **/*.nupkg # NuGet Symbol Packages **/*.snupkg # The packages folder can be ignored because of Package Restore **/**/[Pp]ackages/* # except build/, which is used as an MSBuild target. **/!**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files **/*.nuget.props **/*.nuget.targets # Microsoft Azure Build Output **/csx/ **/*.build.csdef # Microsoft Azure Emulator **/ecf/ **/rcf/ # Windows Store app package directories and files **/AppPackages/ **/BundleArtifacts/ **/Package.StoreAssociation.xml **/_pkginfo.txt **/*.appx **/*.appxbundle **/*.appxupload # Visual Studio cache files # files ending in .cache can be ignored **/*.[Cc]ache # but keep track of directories ending in .cache **/!?*.[Cc]ache/ # Others **/ClientBin/ **/~$* **/*.dbmdl **/*.dbproj.schemaview **/*.jfm **/*.pfx **/*.publishsettings **/orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects **/Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) **/_UpgradeReport_Files/ **/Backup*/ **/UpgradeLog*.XML **/UpgradeLog*.htm **/ServiceFabricBackup/ **/*.rptproj.bak # SQL Server files **/*.mdf **/*.ldf **/*.ndf # Business Intelligence projects **/*.rdl.data **/*.bim.layout **/*.bim_*.settings **/*.rptproj.rsuser **/*- [Bb]ackup.rdl **/*- [Bb]ackup ([0-9]).rdl **/*- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes **/FakesAssemblies/ # GhostDoc plugin setting file **/*.GhostDoc.xml # Node.js Tools for Visual Studio **/.ntvs_analysis.dat # Visual Studio 6 build log **/*.plg # Visual Studio 6 workspace options file **/*.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) **/*.vbw # Visual Studio LightSwitch build output **/**/*.HTMLClient/GeneratedArtifacts **/**/*.DesktopClient/GeneratedArtifacts **/**/*.DesktopClient/ModelManifest.xml **/**/*.Server/GeneratedArtifacts **/**/*.Server/ModelManifest.xml **/_Pvt_Extensions # Paket dependency manager **/.paket/paket.exe **/paket-files/ # FAKE - F# Make **/.fake/ # CodeRush personal settings **/.cr/personal # Python Tools for Visual Studio (PTVS) # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio **/*.tss # Telerik's JustMock configuration file **/*.jmconfig # BizTalk build output **/*.btp.cs **/*.btm.cs **/*.odx.cs **/*.xsd.cs # OpenCover UI analysis results **/OpenCover/ # Azure Stream Analytics local run output **/ASALocalRun/ # MSBuild Binary and Structured Log **/*.binlog # NVidia Nsight GPU debugger configuration file **/*.nvuser # MFractors (Xamarin productivity tool) working folder **/.mfractor/ # Local History for Visual Studio **/.localhistory/ # BeatPulse healthcheck temp database **/healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 **/MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder **/.ionide/ ### vscode ### ### Vue ### # gitignore template for Vue.js projects # Recommended template: Node.gitignore # TODO: where does this rule come from? **/docs/_book # TODO: where does this rule come from? **/test/ ### Vuejs ### # Recommended template: Node.gitignore **/npm-debug.log **/yarn-error.log ### Waf ### # For projects that use the Waf build system: https://waf.io/ # Dot-hidden on Unix-like systems **/.waf-*-*/ **/.waf3-*-*/ # Hidden directory on Windows (no dot) **/waf-*-*/ **/waf3-*-*/ # Lockfile **/.lock-waf_*_build ### Windows ### # Windows thumbnail cache files **/Thumbs.db **/Thumbs.db:encryptable **/ehthumbs.db **/ehthumbs_vista.db # Dump file **/*.stackdump # Folder config file **/[Dd]esktop.ini # Recycle Bin used on file shares **/$RECYCLE.BIN/ # Windows Installer files **/*.msix # Windows shortcuts **/*.lnk ### Xcode ### # Xcode # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore ### Xcode Patch ### **/*.xcodeproj/* **/!*.xcodeproj/project.pbxproj **/!*.xcodeproj/xcshareddata/ **/!*.xcworkspace/contents.xcworkspacedata **/**/xcshareddata/WorkspaceSettings.xcsettings ### XcodeInjection ### # Code Injection # After new code Injection tools there's a generated folder /iOSInjectionProject # https://github.com/johnno1962/injectionforxcode **/iOSInjectionProject/ ### Gradle ### # Ignore Gradle GUI config # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) # Cache of project # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties ### Gradle Patch ### **/**/build/ ### VisualStudio ### # User-specific files # User-specific files (MonoDevelop/Xamarin Studio) # Mono auto generated files # Build results # Visual Studio 2015/2017 cache/options directory # Uncomment if you have tasks that create the project's static files in wwwroot # Visual Studio 2017 auto generated files # MSTest test Results # NUnit # Build Results of an ATL Project # Benchmark Results # .NET Core # StyleCop # Files built by Visual Studio # Chutzpah Test files # Visual C++ cache files # Visual Studio profiler # Visual Studio Trace Files # TFS 2012 Local Workspace # Guidance Automation Toolkit # ReSharper is a .NET coding add-in # TeamCity is a build add-in # DotCover is a Code Coverage Tool # AxoCover is a Code Coverage Tool # Coverlet is a free, cross platform Code Coverage Tool # Visual Studio code coverage results # NCrunch # MightyMoose # Web workbench (sass) # Installshield output folder # DocProject is a documentation generator add-in # Click-Once directory # Publish Web Output # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted # NuGet Packages # NuGet Symbol Packages # The packages folder can be ignored because of Package Restore # except build/, which is used as an MSBuild target. # Uncomment if necessary however generally it will be regenerated when needed # NuGet v3's project.json files produces more ignorable files # Microsoft Azure Build Output # Microsoft Azure Emulator # Windows Store app package directories and files # Visual Studio cache files # files ending in .cache can be ignored # but keep track of directories ending in .cache # Others # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) # RIA/Silverlight projects # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) # SQL Server files # Business Intelligence projects # Microsoft Fakes # GhostDoc plugin setting file # Node.js Tools for Visual Studio # Visual Studio 6 build log # Visual Studio 6 workspace options file # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) # Visual Studio LightSwitch build output # Paket dependency manager # FAKE - F# Make # CodeRush personal settings # Python Tools for Visual Studio (PTVS) # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio # Telerik's JustMock configuration file # BizTalk build output # OpenCover UI analysis results # Azure Stream Analytics local run output # MSBuild Binary and Structured Log # NVidia Nsight GPU debugger configuration file # MFractors (Xamarin productivity tool) working folder # Local History for Visual Studio # BeatPulse healthcheck temp database # Backup folder for Package Reference Convert tool in Visual Studio 2017 # Ionide (cross platform F# VS Code tools) working folder # End of https://www.toptal.com/developers/gitignore/api/ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh ================================================ FILE: .drone.yml ================================================ --- # XXX: putting this separator further down with code causes a parsing bug in drone lint # # Author: Hari Sekhon # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # D r o n e C I # ============================================================================ # # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ # # brew install drone-cli # # cd to this directory # # drone exec [--pipeline default] [--include=thisstep] [--exclude=thatstep] kind: pipeline type: docker name: default steps: - name: build image: ubuntu:18.04 #environment: # DEBUG: 1 commands: - setup/ci_bootstrap.sh - make init - make ci - make test trigger: branch: - master ================================================ FILE: .editorconfig ================================================ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # http://EditorConfig.org # stop recursing upwards for other .editorconfig files root = true # Unix-style newlines with a newline ending every file [*] indent_size = 4 indent_style = space end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true [*.go] indent_size = 4 indent_style = tab end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true [Makefile] indent_size = 4 indent_style = tab end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true [{*.md,*.hcl,*.tf,*.tfvars}] indent_size = 2 indent_style = space end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true [*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true [.*] indent_size = 4 indent_style = space end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true # ============================================================================ # # Older Stuff, don't think I use this any more # ============================================================================ # # Matches multiple files with brace expansion notation # Set default charset #[*.{js,py}] #charset = utf-8 # Indentation override for all JS under lib directory #[lib/**.js] #indent_style = space #indent_size = 2 # Matches the exact files either package.json or .travis.yml #[{package.json,.travis.yml}] #indent_style = space #indent_size = 2 #[*.xml] #indent_style = space #indent_size = 2 ================================================ FILE: .envrc ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Mon Feb 22 17:42:01 2021 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # D i r E n v # ============================================================================ # # https://direnv.net/man/direnv-stdlib.1.html # See Also: # # .envrc-aws # .envrc-gcp # .envrc-kubernetes # direnv stdlib - loads .envrc from parent dir up to / # # useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. # # bypasses security authorization though - use with care #source_up # # source_up must be loaded before set -u otherwise gets this error: # # direnv: loading .envrc # /bin/bash: line 226: $1: unbound variable # # source_up causes this error is up .envrc is found in parent directories: # # direnv: No ancestor .envrc found set -euo pipefail [ -n "${DEBUG:-}" ] && set -x src="$(readlink -f "${BASH_SOURCE[0]}")" srcdir="$(cd "$(dirname "$src")" && pwd)" # ============================================================================ # # P r e - C o m m i t # ============================================================================ # # Automatically install Pre-Commit Git hooks if not already present if ! type -P pre-commit &>/dev/null; then if uname -s | grep -q Darwin && type -P brew &>/dev/null; then echo echo "Pre-commit is not installed - installing now using Homebrew..." echo brew install pre-commit echo elif type -P pip &>/dev/null; then echo echo "Pre-commit is not installed - installing now using Pip..." echo pip install pre-commit fi fi if [ -f .pre-commit-config.yaml ] && type -P pre-commit &>/dev/null && git rev-parse --is-inside-work-tree &>/dev/null; then hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" if [ -L "$hook" ]; then echo "Detected symlink hook: " echo ls -l "$hook" echo echo "Removing" rm -f "$hook" fi if ! [ -f "$hook" ]; then echo echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." echo pre-commit install fi fi # ============================================================================ # # D o c k e r C o m p o s e # ============================================================================ # export COMPOSE_PROJECT_NAME="bash-tools" # ============================================================================ # # G i t H u b # ============================================================================ # #export GITHUB_ORGANIZATION=HariSekhon # ============================================================================ # # A n s i b l e # ============================================================================ # # use the local repo's ansible.cfg rather than: # # $PWD/ansible.cfg # ~/.ansible.cfg # /etc/ansible/ansible.cfg # # set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used #export ANSIBLE_CONFIG="/path/to/ansible.cfg" # ============================================================================ # # C l o u d f l a r e # ============================================================================ # #export CLOUDFLARE_EMAIL=hari@... #export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens #export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module # export the variables for terraform #export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" #export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # ============================================================================ # # Load External Envrc Files If Present # ============================================================================ # # XXX: safer to bring all these external .envrc inline if you're worried about changes # to it bypassing 'direnv allow' authorization load_if_exists(){ # first arg is a path to a .envrc # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes # to pass the context name 'docker-desktop' to switch to local envrc="$1" shift if ! [[ "$envrc" =~ ^/ ]]; then envrc="$srcdir/$envrc" fi if [ -f "$envrc" ]; then # prevent looping on symlinks to this .envrc if given if [ "$(readlink "$envrc")" = "$src" ]; then return fi echo echo "Loading $envrc" # shellcheck disable=SC1090,SC1091 . "$envrc" "$@" fi } # don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc # (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses # to install to system Python) #load_if_exists ~/.envrc # ============================================================================ # # P y t h o n # ============================================================================ # #for envrc in \ # .envrc-aws \ # .envrc-gcp \ # .envrc-terraform \ # .envrc-python \ # ; do # load_if_exists "$envrc" #done load_if_exists .envrc-python # ============================================================================ # # J a v a # ============================================================================ # load_if_exists .envrc-java # ============================================================================ # # A W S # ============================================================================ # #if [[ "$PWD" =~ /aws/ ]]; then load_if_exists .envrc-aws #fi # ============================================================================ # # G C P # ============================================================================ # #if [[ "$PWD" =~ /gcp/ ]]; then load_if_exists .envrc-gcp #fi # ============================================================================ # # T e r r a f o r m # ============================================================================ # #if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then load_if_exists .envrc-terraform #fi # ============================================================================ # # K u b e r n e t e s # ============================================================================ # #if [ -f "$srcdir/.envrc-kubernetes" ]; then load_if_exists .envrc-kubernetes docker-desktop #fi # ============================================================================ # # . E n v # ============================================================================ # echo # read .env too #dotenv load_if_exists .envrc.local ================================================ FILE: .envrc-aws ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-07-27 12:42:32 +0100 (Tue, 27 Jul 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # A W S D i r E n v # ============================================================================ # # https://direnv.net/man/direnv-stdlib.1.html # See Also: # # .envrc # .envrc-gcp # .envrc-kubernetes # direnv stdlib - loads .envrc from parent dir up to / # # useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. # # bypasses security authorization though - use with care #source_up # # source_up must be loaded before set -u otherwise gets this error: # # direnv: loading .envrc # /bin/bash: line 226: $1: unbound variable set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -n "${CI:-}" ]; then exit 0 fi # XXX: Edit - crucial to set to the right environment, the rest of the inferred settings below depend on this if [ -z "${AWS_PROFILE:-}" ]; then exit 0 fi aws configure list 2>/dev/null || : echo # If not logged in: # # - and we know the AWS_PROFILE # - and AWS_NO_AUTOLOGIN is not set # - check for SSO key in config section for this profile # - if found then do an automatic 'aws sso login' # if ! aws sts get-caller-identity --output table; then if [ -n "${AWS_PROFILE:-}" ] && [ -z "${AWS_NO_AUTOLOGIN:-}" ]; then # assumes you're not putting a blank line until the next section block #if sed -n "/profile.*$AWS_PROFILE/,/^[[:space:]]*$/p" ~/.aws/config | grep -q sso_start_url; then # goes until the next [profile ...] section instead, should be more reliable if sed -n "/profile.*$AWS_PROFILE/,/^[[:space:]]*\[.+\]/p" ~/.aws/config | grep -q sso_start_url; then echo aws sso login fi fi fi echo # 'aws sts get-caller-identity --query Account' succeeds in returning the account id # from the ~/.aws/config even if 'aws sso login' has expired AWS_ACCOUNT_ID="$( aws sts get-caller-identity --query Account --output text || aws configure get sso_account_id || : )" echo "AWS Account ID: $AWS_ACCOUNT_ID" export AWS_ACCOUNT_ID echo # might not have permissions to the Organizations in which case this will error instead of return AWS_ACCOUNT="$(aws organizations describe-account --account-id "$AWS_ACCOUNT_ID" 2>/dev/null)" if [ -n "$AWS_ACCOUNT" ]; then echo "AWS Account: $AWS_ACCOUNT" export AWS_ACCOUNT echo fi AWS_DEFAULT_REGION="$(aws configure get region || :)" # use region configured in profile by default AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" # XXX: Edit default fallback region export AWS_DEFAULT_REGION echo "AWS Region: $AWS_DEFAULT_REGION" echo export AWS_DEFAULT_OUTPUT=json # XXX: Edit, or remove if only have 1 cluster in account, will auto-determine below export EKS_CLUSTER="mycluster" # safer but slower #eks_clusters=() #while IFS='' read -r line; do # eks_clusters+=("$line") ##done < <(aws eks list-clusters --output=json | jq -r '.clusters[]') #done < <(aws eks list-clusters --query 'clusters[]' --output text) #if [ "${#eks_clusters[@]}" -eq 1 ]; then # export EKS_CLUSTER="${eks_clusters[*]}" #fi eks_clusters="$( aws eks list-clusters --query 'clusters' --output text | tr '[:space:]' '\n' | sed '/^[[:space:]]*$/d' )" if [ -n "$eks_clusters" ]; then num_eks_clusters="$(grep -c . <<< "$eks_clusters")" echo "EKS Clusters ($num_eks_clusters):" echo echo "$eks_clusters" echo # If EKS_CLUSTER isn't set and there is only one EKS cluster in this account and region, then use it if [ -z "${EKS_CLUSTER:-}" ]; then if [ "$num_eks_clusters" = 1 ]; then EKS_CLUSTER="$eks_clusters" fi fi else num_eks_clusters=0 fi if [ -n "${EKS_CLUSTER:-}" ]; then # kubectl context is easily created by running adjacent aws_kube_creds.sh script first export EKS_CONTEXT="arn:aws:eks:$AWS_DEFAULT_REGION:$AWS_ACCOUNT_ID:cluster/$EKS_CLUSTER" if command -v kubectl &>/dev/null; then if ! kubectl config get-clusters | grep -Fxq "$EKS_CONTEXT"; then echo "EKS Cluster '$EKS_CLUSTER' not configured, configuring now" aws eks update-kubeconfig --name "$EKS_CLUSTER" echo fi fi # XXX: safer to inline .envrc-kubernetes if you're worried about changes to it bypassing 'direnv allow' authorization # shellcheck disable=SC1090,SC1091 . "$srcdir/.envrc-kubernetes" "$EKS_CONTEXT" ${EKS_NAMESPACE:+"$EKS_NAMESPACE"} fi if [ "$num_eks_clusters" = 1 ]; then if grep -q '^[[:space:]]*export[[:space:]]*EKS_CLUSTER' .envrc && ! grep -q "^export EKS_CLUSTER=$eks_clusters$" .envrc; then echo echo "Updating EKS_CLUSTER in .envrc from:" echo grep '^[[:space:]]*export[[:space:]]*EKS_CLUSTER' .envrc echo echo "to" echo echo "export EKS_CLUSTER=$eks_clusters" echo perl -pi -e "s/^\\s*export\s+EKS_CLUSTER=.*/export EKS_CLUSTER=$eks_clusters/" .envrc echo fi fi # better to load this dynamically from credentials, using functions in .bash.d/aws.sh #export AWS_ACCESS_KEY_ID=... #export AWS_SECRET_ACCESS_KEY=... #export AWS_SESSION_TOKEN=... #export AWS_CONFIG_FILE=~/.aws/config #export AWS_SHARED_CREDENTIALS_FILE=~/.aws/credentials #export AWS_MAX_ATTEMPTS=3 # to quickly export prefixed AWS environment keys if they exist for simple overrides, see examples below aws_access_key_env(){ env="$1" for key in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do varname="${env}_${key}" if [ -n "${!varname:-}" ]; then export "$key"="${!varname}" fi done } #aws_access_key_env "DEV" #aws_access_key_env "STAGING" #aws_access_key_env "PROD" #aws_access_key_env "MGMT" # pull the secret using this command whenever you need it: # # aws_secret_get.sh "$JENKINS_ADMIN_PASSWORD_AWS_SECRET" | copy_to_clipboard.sh # export JENKINS_ADMIN_PASSWORD_AWS_SECRET="jenkins-admin-password" ================================================ FILE: .envrc-gcp ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Mon Feb 22 17:42:01 2021 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G C P D i r E n v # ============================================================================ # # https://direnv.net/man/direnv-stdlib.1.html # See Also: # # .envrc # .envrc-aws # .envrc-kubernetes # direnv stdlib - loads .envrc from parent dir up to / # # useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. # # bypasses security authorization though - use with care #source_up # # source_up must be loaded before set -u otherwise gets this error: # # direnv: loading .envrc # /bin/bash: line 226: $1: unbound variable set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -n "${CI:-}" ]; then exit 0 fi # https://cloud.google.com/sdk/gcloud/reference/config # If using other services, infer the environment variables to put below by reading: # # gcloud topic configurations # or # gcloud config set --help gcloud_config(){ local config="${1:-}" if [ -z "$config" ]; then echo "no config passed to gcloud_config() function" >&2 return 1 fi if [ -z "${CI:-}" ]; then return fi # don't waste time if not using GCloud SDK, ie. not found in $PATH if type -P gcloud; then # protect from setting this if the config does exist as this can cause auth problems by unsetting the core.account if gcloud config configurations list --format='get(name)' | grep -q "^$config$"; then export CLOUDSDK_ACTIVE_CONFIG_NAME="$config" fi fi } #gcloud_config dev #gcloud_config staging #gcloud_config production # XXX: Edit export CLOUDSDK_CORE_PROJECT=myproject echo "CLOUDSDK_CORE_PROJECT=$CLOUDSDK_CORE_PROJECT" echo # XXX: Edit export CLOUDSDK_COMPUTE_REGION="${CLOUDSDK_COMPUTE_REGION:-eu-west-2}" echo "CLOUDSDK_COMPUTE_REGION=$CLOUDSDK_COMPUTE_REGION" echo REGION="$CLOUDSDK_COMPUTE_REGION" # you should probably not set CLOUDSDK_COMPUTE_ZONE # # 'gcloud compute ssh' will auto-determine the zone # # setting CLOUDSDK_COMPUTE_ZONE explicitly breaks the above command in 2/3 cases due to a VM being in a different zone: # # ERROR: (gcloud.compute.ssh) Could not fetch resource: # - The resource 'projects//zones//instances/' was not found # # gcp/gce_ssh.sh script in this repo can work around that if you do set this # #export CLOUDSDK_COMPUTE_ZONE="${REGION}-a" # or b or c export CLOUDSDK_AI_REGION="$REGION" export CLOUDSDK_AI_PLATFORM_REGION="$REGION" export CLOUDSDK_DATAPROC_REGION="$REGION" export CLOUDSDK_DEPLOY_REGION="$REGION" export CLOUDSDK_FILESTORE_REGION="$REGION" export CLOUDSDK_FUNCTIONS_REGION="$REGION" export CLOUDSDK_MEMCACHE_REGION="$REGION" export CLOUDSDK_REDIS_REGION="$REGION" export CLOUDSDK_RUN_REGION="$REGION" export CLOUDSDK_RUN_CLUSTER_LOCATION="$REGION" export CLOUDSDK_VMWARE_REGION="$REGION" # XXX: Edit export CLOUDSDK_RUN_PLATFORM=managed #export CLOUDSDK_RUN_PLATFORM=gke #export CLOUDSDK_RUN_PLATFORM=kubernetes #export CLOUDSDK_RUN_CLUSTER=mycluster export CLOUDSDK_GCLOUDIGNORE_ENABLED=True #export CLOUDSDK_BUILDS_USE_KANIKO=True # XXX: Edit, or remove if only have 1 cluster in project, will auto-determine below export CLOUDSDK_CONTAINER_CLUSTER=mycluster # GKE cluster name # safer but slower #gke_clusters=() #while IFS='' read -r line; do # gke_clusters+=("$line") #done < <(gcloud container clusters list --format='get(name)') #if [ "${#gke_clusters[@]}" -eq 1 ]; then # export CLOUDSDK_CONTAINER_CLUSTER="${gke_clusters[*]}" #fi gke_clusters="$( gcloud container clusters list --format='get(name)' | sed '/^[[:space:]]*$/d' )" if [ -n "$gke_clusters" ]; then num_gke_clusters="$(grep -c . <<< "$gke_clusters")" echo "GKE Clusters ($num_gke_clusters):" echo echo "$gke_clusters" echo # If GKE_CLUSTER isn't set and there is only one GKE cluster in this account and region, then use it if [ -z "${GKE_CLUSTER:-}" ]; then if [ "$num_gke_clusters" = 1 ]; then GKE_CLUSTER="$gke_clusters" fi fi else num_gke_clusters=0 fi # alternatively call gke_kube_context() function in .envrc-kubernetes which will do this # and comment out auto-running kube_context() on sourcing .envrc-kubernetes if [ -n "${CLOUDSDK_CONTAINER_CLUSTER:-}" ]; then echo "CLOUDSDK_CONTAINER_CLUSTER=$CLOUDSDK_CONTAINER_CLUSTER" echo # kubectl context is easily created by running adjacent aws_kube_creds.sh script first export GKE_CONTEXT="gke_${CLOUDSDK_CORE_PROJECT}_${CLOUDSDK_COMPUTE_REGION}_${CLOUDSDK_CONTAINER_CLUSTER}" # XXX: safer to inline .envrc-kubernetes if you're worried about changes to it bypassing 'direnv allow' authorization # shellcheck disable=SC1090,SC1091 . "$srcdir/.envrc-kubernetes" "$GKE_CONTEXT" ${GKE_NAMESPACE:+"$GKE_NAMESPACE"} fi # pull the secret using this command whenever you need it: # # gcp_secret_get.sh "$JENKINS_ADMIN_PASSWORD_GCP_SECRET" | copy_to_clipboard.sh # export JENKINS_ADMIN_PASSWORD_GCP_SECRET="jenkins-admin-password" ================================================ FILE: .envrc-java ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-03-20 01:55:20 +0800 (Thu, 20 Mar 2025) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # J a v a D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present # https://direnv.net/man/direnv-stdlib.1.html set -euo pipefail [ -n "${DEBUG:-}" ] && set -x #srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -f .sdkmanrc ]; then if ! [ -f ~/.sdkman/etc/config ] || ! grep -q '^[[:space:]]*sdkman_auto_env[[:space:]]*=[[:space:]]*true' ~/.sdkman/etc/config; then if [ -f ~/.sdkman/bin/sdkman-init.sh ]; then # shellcheck disable=SC1090 . ~/.sdkman/bin/sdkman-init.sh fi # it's not a binary but a function, so no type -P if type sdk &>/dev/null; then sdk env install fi fi fi # read .env too #dotenv ================================================ FILE: .envrc-kubernetes ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-02-25 10:10:53 +0000 (Thu, 25 Feb 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # K u b e r n e t e s D i r E n v # ============================================================================ # # https://direnv.net/man/direnv-stdlib.1.html # See Also: # # .envrc # .envrc-aws # .envrc-gcp # direnv stdlib - loads .envrc from parent dir up to / # # useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. # # bypasses security authorization though - use with care #source_up # # source_up must be loaded before set -u otherwise gets this error: # # direnv: loading .envrc # /bin/bash: line 226: $1: unbound variable set -euo pipefail [ -n "${DEBUG:-}" ] && set -x arg="${1:-}" if [ "${arg##*/}" = "${BASH_SOURCE[0]##*/}" ]; then shift fi # XXX: Edit this - hardcode for localized convenience CONTEXT="${1:-docker-desktop}" # if set will also set the namespace for extra convenience NAMESPACE="${2:-}" #NAMESPACE="jenkins" # function so can place in topdir .envrc and have subdirs 'source_up' or simply . ../.envrc to reuse this code among many .envrc environments kube_context(){ local context="$1" local namespace="${2:-}" if command -v kubectl &>/dev/null; then local tmpdir="/tmp/.kube" mkdir -pv "$tmpdir" local default_kubeconfig="${HOME:-$(cd ~ && pwd)}/.kube/config" local original_kubeconfig="${KUBECONFIG:-$default_kubeconfig}" # reload safety - do not source from new tmpdir - not necessary for direnv but useful for local sourcing tests #if [[ "$original_kubeconfig" =~ $tmpdir ]]; then # echo "ignoring \$KUBECONFIG=$original_kubeconfig, using default home location $default_kubeconfig" # original_kubeconfig="$default_kubeconfig" #fi # isolate the kubernetes context to avoid a race condition affecting any other shells or scripts # epoch is added because $$ and $PPID are direnv sub-processes and may be reused later, so using epoch to add uniqueness local epoch epoch="$(date +%s)" export KUBECONFIG="$tmpdir/config.${EUID:-${UID:-$(id -u)}}.$$.$epoch" # load your real kube config to isolated staging area to source the context info local src_kubeconfig="" local kubeconfig_source_locations=" $original_kubeconfig $default_kubeconfig $PWD/.kube/config /etc/rancher/k3s/k3s.yaml" for kubeconfig in $kubeconfig_source_locations; do if [ -f "$kubeconfig" ]; then src_kubeconfig="$kubeconfig" break fi done if [ -n "$src_kubeconfig" ]; then if [ "$src_kubeconfig" != "$KUBECONFIG" ]; then cp -f -- "$src_kubeconfig" "$KUBECONFIG" fi else if [[ "$PWD" =~ k8|kube ]]; then echo "WARNING: failed to find one of:" >&2 echo "$kubeconfig_source_locations" | sort -u >&2 echo >&2 fi fi # race condition - 'kubectl config get-contexts' fails to find the context and switch in many runs without this sleep context_found=0 local i for ((i=0; i < 5; i++)); do # surprisingly unreliable - if kubectl config get-contexts -o name | grep -Fxq "$context" can miss even after these succeed #if [ -s "$KUBECONFIG" ]; then #if cmp --quiet "$from_kubeconfig" "$KUBECONFIG"; then if kubectl config get-contexts -o name | grep -Fxq "$context"; then context_found=1 break fi sleep 0.1 done # this randomly misses the context, and not even 'sync; sync; sleep 1' is reliable to stop that happening in testing #if kubectl config get-contexts -o name 2>/dev/null | grep -Fxq "$context"; then if [ "$context_found" = 1 ]; then kubectl config use-context "$CONTEXT" echo if [ -n "${namespace:-}" ]; then kubectl config set-context "$context" --namespace "$namespace" echo fi fi fi } gke_kube_context(){ local CONTEXT for _ in CLOUDSDK_CORE_PROJECT CLOUDSDK_COMPUTE_REGION CLOUDSDK_CONTAINER_CLUSTER; do if [ -z "${!_}" ]; then echo "WARNING: \$$_ is not set" >&2 fi done # if CLOUDSDK_CONTAINER_CLUSTER and it's generated as a naming convention such as "${CLOUDSDK_CORE_PROJECT}-${CLOUDSDK_COMPUTE_REGION}" #export CLOUDSDK_CONTAINER_CLUSTER="${CLOUDSDK_CONTAINER_CLUSTER:-${CLOUDSDK_CORE_PROJECT}-${CLOUDSDK_COMPUTE_REGION}}" # the context naming convention for GKE clusters imported via: # # gcloud container clusters get-credentials "$cluster" --zone "$zone" # # use gke_kube_creds.sh to auto-populate this for all GKE clusters in the current project # and gcp_foreach_project.sh to do this for all GCP projects. Both scripts are found here: # # https://github.com/HariSekhon/DevOps-Bash-tools # # should be using a regional cluster CONTEXT="gke_${CLOUDSDK_CORE_PROJECT}_${CLOUDSDK_COMPUTE_REGION}_${CLOUDSDK_CONTAINER_CLUSTER}" # not a zonal cluster #CONTEXT="gke_${CLOUDSDK_CORE_PROJECT}_${CLOUDSDK_COMPUTE_ZONE}_${CLOUDSDK_CONTAINER_CLUSTER}" kube_context "$CONTEXT" "${NAMESPACE:-}" } kube_context "$CONTEXT" "$NAMESPACE" #export ARGOCD_SERVER="argocd.mycompany.com" #export ARGOCD_OPTS="${ARGOCD_OPTS:-} --grpc-web" #if [ -n "${ARGOCD_AUTH_TOKEN_MYCOMPANY_OR_ENV:-}" ]; then # export ARGOCD_AUTH_TOKEN="$ARGOCD_AUTH_TOKEN_MYCOMPANY_OR_ENV" #fi #export ARGOCD_APP="myapp" ================================================ FILE: .envrc-python ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Mon Feb 22 17:42:01 2021 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P y t h o n D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present # https://direnv.net/man/direnv-stdlib.1.html set -euo pipefail [ -n "${DEBUG:-}" ] && set -x #srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default for venv in "$PWD/venv" "$HOME/venv"; do if [ -f "$venv/bin/activate" ]; then echo echo "Virtualenv directory found in: $venv" echo echo "Activating Virtualenv inside the directory: $venv" # shellcheck disable=SC1091 source "$venv/bin/activate" break fi done # read .env too #dotenv ================================================ FILE: .envrc-terraform ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Mon Feb 22 17:42:01 2021 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T e r r a f o r m D i r E n v # ============================================================================ # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x #srcdir="$(dirname "${BASH_SOURCE[0]}")" # set this to the local repo's config instead of $HOME/.terraformrc #export TF_CLI_CONFIG_FILE="$PWD/configs/.terraformrc" # if not already set in $HOME/.terraformrc #export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" # XXX: beware that tfenv and tgswitch replace the terraform / terragrunt in the path and this is a race condition between different shells # it is not as good as the KUBECONFIG trick done in the .envrc-kubernetes # would auto-determine the version from the state file, but this commands seems to always return the version of your local binary pulling the state file, not the version from the actual terraform_version field in state file if you see when opening it up in the cloud bucket # terraform state pull | jq -r .terraform_version export TERRAFORM_VERSION=1.1.9 # overrides .terraform-version file to make this single source of truth export TFENV_TERRAFORM_VERSION="$TERRAFORM_VERSION" export TFENV_AUTO_INSTALL=true # Terragrunt export TG_VERSION=0.39.2 # Terragrunt Provide Cache prevents wasting tonnes of space &time re-downloading duplicate 600MB # Provider plugins for every Terragrunt module resulting in massive duplication see: # # https://terragrunt.gruntwork.io/docs/features/provider-cache-server/ # # Stores plugins in: # # $HOME/.cache/terragrunt/providers # or on Mac # $HOME/Library/Caches/terragrunt/providers # export TG_PROVIDER_CACHE=1 export TG_PROVIDER_CACHE_HOST=127.0.0.1 # #export TG_PROVIDER_CACHE_DIR="/new/path/to/cache/dir" # # to cache from registries other than registry.terraform.io, registry.opentofu.org # eg. if you have your own private registry #export TG_PROVIDER_CACHE_REGISTRY_NAMES="example1.com,example2.com" # if tgswitch is installed, trigger it to use the above TF_VERSION environment variable and switch to the correct version of Terragrunt # better than adding the ugly shell hook from the docs - https://github.com/warrensbox/tgswitch?tab=readme-ov-file#get-the-version-from-a-subdirectory if type -P tgswitch &>/dev/null; then tgswitch fi # XXX: set these or other variables for Terraform code to find export CLOUDFLARE_EMAIL=hari@... export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens #export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module # export the variables for terraform export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # GITHUB_* environment variables may interfere with GitHub provider, so unset them for env_var in $(env | awk -F= '$1 ~ /GITHUB/ {print $1}'); do unset "$env_var" done ================================================ FILE: .git-templates/git-secrets/hooks/commit-msg ================================================ #!/usr/bin/env bash git secrets --commit_msg_hook -- "$@" ================================================ FILE: .git-templates/git-secrets/hooks/pre-commit ================================================ #!/usr/bin/env bash git secrets --pre_commit_hook -- "$@" ================================================ FILE: .git-templates/git-secrets/hooks/prepare-commit-msg ================================================ #!/usr/bin/env bash git secrets --prepare_commit_msg_hook -- "$@" ================================================ FILE: .gitconfig ================================================ # # Author: Hari Sekhon # Date: 2012-01-31 14:08:42 +0000 (Tue, 31 Jan 2012) # # vim:ts=4:sts=4:sw=4:et # configure your user name and email in ~/.gitconfig.local #[user] # name = Hari Sekhon # email = harisekhon@gmail.com [include] # XXX: put your [user] section in $HOME/.gitconfig.local eg. # # [user] # name = Hari Sekhon # email = harisekhon@gmail.com # path = .gitconfig.local [core] # detects filemode changes filemode = true # defaults to $VISUAL or $EDITOR which is set in .bash.d/env.sh #editor = vim # global .gitignore excludesfile = ~/.gitignore # stops unicode chars coming out as \xxx and double quoted filenames in git status (used in .bash.d/git.sh git_rm_untracked function) quotePath = false #whitespace = trailing-space,space-before-tab [pull] rebase = false [push] # default = current default = simple [alias] name = config --get user.name email = config --get user.email who = !git config -l | grep -E '^user\\.(name|email)' whoami = who co = checkout ci = commit p = push st = status stq = !git_foreach_repo.sh git status | grep --color=no -e "=======" -e branch -e GitHub br = branch ba = branch -a bav = branch -a -vvv cp = cherry-pick ls = ls-files root = rev-parse --show-toplevel rem = remote -v remotes = remote -v tags = tag -l branches = branch -a prune-branches = ! git remote prune origin && git branch -vv | cut -c 3- | awk '$4 ~ /gone\\]/ {print $1}' | xargs git branch -d # the results with multi-origin remotes are not reliable without a git pull first, even when only pulling from the primary GitHub origin in = ! git pull && git log HEAD..FETCH_HEAD out = ! git pull && git log FETCH_HEAD..HEAD inp = ! git pull && git log -p HEAD..FETCH_HEAD outp = ! git pull && git log -p FETCH_HEAD..HEAD ind = ! git pull && git diff HEAD..FETCH_HEAD outd = ! git pull && git diff FETCH_HEAD..HEAD age = for-each-ref --format '%(authordate:iso) %(refname:short)' --sort=-authordate refs/remotes refs/heads unstage = reset HEAD -- last = log -1 HEAD # Show files ignored by git: ign = ls-files -o -i --exclude-standard ignored = !git clean -ndX | sed -e 's/^Would remove //' | sed 's/^Would skip repository //' untracked = ls-files --others --exclude-standard # how to use commands inside git aliases visual = !gitk df = diff dc = diff --cached lg = log -p lol = log --graph --decorate --pretty=oneline --abbrev-commit lola = log --graph --decorate --pretty=oneline --abbrev-commit --all # avoid diff-so-fancy so we can create patches patch = !git --no-pager diff --no-color #ffm = merge --ff-only #ffp = pull --ff-only #fp = fetch --prune #mp = merge --no-commit --no-ff #ma = merge --abort #dno = diff --name-only #gone = !git branch -vv | grep ': gone' #gd = !git branch -vv | awk '/: gone/ {print $1}' | xargs --no-run-if-empty -n1 git branch -D [help] # autocorrects git commands and executes the inferred command # dangerous this just autocorrected my git rename to git rebase, lucky it errored out... autocorrect = 0 [homebrew] donationmessage = false # ============================================================================ # # G i t C o l o r # ============================================================================ # # colors: normal, black, red, green, yellow, blue, magenta, cyan, white # effects: bold, dim, ul, blink and reverse # if 2 colours given - 1st is foreground, 2nd is background [color] ui = auto diff = auto grep = auto # do not set always here, use: # -c color.status=always # for specific overrides, otherwise may break gitci and related functions status = auto branch = auto [color "branch"] current = green ul local = yellow remote = red # default plain = white [color "diff"] new = green old = red # default plain = white whitespace = yellow reverse func = yellow #frag = cyan # default #meta = green bold # default # from diff-so-fancy meta = 11 frag = magenta bold [color "grep"] context = white filename = cyan function = yellow linenumber = white match = white magenta selected = green separator = white [color "status"] added = white blue changed = magenta untracked = cyan branch = magenta blink nobranch = red blink # ============================================================================ # # D i f f - s o - f a n c y # ============================================================================ # # detected if installed and set via $GIT_PAGER in .bash.d/git.sh #[core] # pager = diff-so-fancy | less --tabs=4 -RFX #[pager] # don't set --pattern, overrides -F and doesn't quit less automatically for short diffs #diff = diff-so-fancy | less --tabs=4 -RFX --pattern '^(Date|added|deleted|modified): ' #diff = diff-so-fancy | less --tabs=4 -RFX # truncate lines in less, only for 'git blame' #blame = less -S [color "diff-highlight"] #oldNormal = red bold oldHighlight = white red # black doesn't contrast well with red bg, use white #newNormal = green bold newHighlight = white magenta [diff-so-fancy] stripLeadingSymbols = false # diff-so-fancy but I prefer most of my old preferences #[color "diff"] # meta = 11 # frag = magenta bold # commit = yellow bold # old = red bold # new = green bold # whitespace = red reverse # ============================================================================ # #[difftool "sourcetree"] # cmd = opendiff \"$LOCAL\" \"$REMOTE\" # path = # #[mergetool "sourcetree"] # cmd = /Applications/SourceTree.app/Contents/Resources/opendiff-w.sh \"$LOCAL\" \"$REMOTE\" -ancestor \"$BASE\" -merge \"$MERGED\" # trustExitCode = true #[filter "media"] # clean = git media clean %f # smudge = git media smudge %f # required = true # ============================================================================ # # C r e d e n t i a l s H e l p e r s # ============================================================================ # # more specific credential addresses below take priority, so if you want to paste in a GitHub token # you will need to comment out the GitHub credential help sections below #[credential] # helper = store # When prompted, enter your username and PAT token, not password, otherwise you'll get this error: # # remote: Support for password authentication was removed on August 13, 2021. # remote: Please see https://docs.github.com/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication. # See credential helpers: # # git config --list --show-origin | grep credential # generated by git/git_remotes_set_https_creds_helpers.sh [credential "https://github.com"] # without the "sleep 1" the Git command may miss catching the output and hang instead helper = "!f() { sleep 1; echo \"username=${GITHUB_USER}\"; echo \"password=${GH_TOKEN:-${GITHUB_TOKEN}}\"; }; f" [credential "https://gist.github.com"] helper = "!f() { sleep 1; echo \"username=${GITHUB_USER}\"; echo \"password=${GH_TOKEN:-${GITHUB_TOKEN}}\"; }; f" [credential "https://gitlab.com"] helper = "!f() { sleep 1; echo \"password=${GITLAB_TOKEN}\"; }; f" # XXX: Bitbucket actually needs https://@ regardless otherwise results in 403 errors [credential "https://bitbucket.org"] #helper = "!f() { sleep 1; echo \"password=${BITBUCKET_TOKEN}\"; }; f" helper = "!f() { sleep 1; echo \"username=${BITBUCKET_USER}\"; echo \"password=${BITBUCKET_APP_PASSWORD:-${BITBUCKET_TOKEN}}\"; }; f" [credential "https://dev.azure.com"] helper = "!f() { sleep 1; echo \"username=${AZURE_DEVOPS_USER}\"; echo \"password=${AZURE_DEVOPS_TOKEN}\"; }; f" # ============================================================================ # # A W S S e c r e t s # ============================================================================ # # AWS Secrets prevents committing secrets in to Git # install git-secrets hooks in any repo initialized or cloned to prevent credential leak [init] templateDir = ~/.git-templates/git-secrets [secrets] providers = git secrets --aws-provider patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16} patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')? patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')? # doesn't work, doesn't support (?! ) negative lookahead regex #patterns = (\bhari|sekhon\b)(!.*@gmail.com) # only applies to contents, not metadata to prevent wrong author commits #patterns = hari|sekhon # These are sample keys so ignore false positives from scanning tools # trivy:ignore:aws-access-key-id allowed = AKIAIOSFODNN7EXAMPLE # trivy:ignore:aws-access-key-id allowed = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY # better to add to private repos .git/config only #allowed = AWS_ACCOUNT_ID # ============================================================================ # # G C P S o u r c e R e p o s i t o r i e s # ============================================================================ # # GCloud SDK command clone and sets up the repo auth: # # gcloud source repos clone "$repo" --project="$project" # # Remote origin: # # https://source.developers.google.com/p/$project/r/$repo # # GCloud SDK adds this to .git/config in a repo cloned via: # # # having a blank helper before the real help prevents this error when pushing: # # bad input: .......... #[credential "https://source.developers.google.com/"] # helper = # helper = !gcloud auth git-helper --account=hari@.iam.gserviceaccount.com --ignore-unknown $@ [filter "lfs"] clean = git-lfs clean -- %f smudge = git-lfs smudge -- %f process = git-lfs filter-process required = true ================================================ FILE: .gitconfig.local ================================================ # # Author: Hari Sekhon # Date: 2012-01-31 14:08:42 +0000 (Tue, 31 Jan 2012) # [user] name = Hari Sekhon email = harisekhon@gmail.com ================================================ FILE: .github/CODEOWNERS ================================================ # # Author: Hari Sekhon # Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021) # # vim:ts=4:sts=4:sw=4:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam # Tips: # # * includes changes under .github/ # dir/* only matches first level file changes but doesn't recurse # dir/ recurses # # - CODEOWNERS in base branch of PR determines review request # - paths are case sensitive # - last match wins, use * at top for overall owner then override with more specific teams #* @harisekhon # username or email address #* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access #* @myorg/devops #k8s @myorg/devops @myorg/sre-team #apps/ @myorg/developers #apps/dir2 # ignores dir2 as no owner/team specified on this line #src/ @myorg/developers #docs/ docs@example.com #.github/workflows @ci-cd-team ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ Please be specific about your issue and include debug output from running after setting `export DEBUG=1` in your shell. You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. ================================================ FILE: .github/workflows/actions-allowed.txt ================================================ # # Author: Hari Sekhon # Date: 2022-01-26 19:04:03 +0000 (Wed, 26 Jan 2022) # # vim:ts=4:sts=4:sw=4:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # 3rd Party GitHub Actions Allow List # ============================================================================ # # technically should fix to a SHA, but I trust these authors, if I didn't, I wouldn't run their actions at all checkmarx/kics-action@* fairwindsops/pluto/github-action@* megalinter/megalinter@v5 returntocorp/semgrep-action@v1 ================================================ FILE: .github/workflows/alpine.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Alpine on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/alpine.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest caches: apk pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/alpine_3.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Alpine 3 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/alpine_3.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 caches: apk pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/centos.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: CentOS on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/centos.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest caches: yum pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/centos7.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: CentOS 7 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/centos7.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 caches: yum pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/centos8.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: CentOS 8 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/centos8.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 caches: yum pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/checkov.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C h e c k o v G i t H u b W o r k f l o w # ============================================================================ # # Static analysis of Terraform code - publishes report to GitHub Security tab # https://github.com/bridgecrewio/checkov-action --- name: Checkov on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write jobs: checkov: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/codeowners.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # C o d e O w n e r s # ============================================================================ # --- name: CodeOwners on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - CODEOWNERS - .github/CODEOWNERS pull_request: branches: - master - main paths: - CODEOWNERS - .github/CODEOWNERS workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: validate: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/commit_adjacent_repos.sh ================================================ #!/usr/bin/env bash git_foreach_repo.sh 'gitu .github/workflows ||:' ================================================ FILE: .github/workflows/debian.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_10.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 10 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_10.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_11.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 11 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_11.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:11 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_12.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 12 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_12.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:12 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_6.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 6 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_6.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 # causes nodejs errors #caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_7.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 7 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_7.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 # causes nodejs errors #caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_8.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 8 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_8.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/debian_9.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Debian 9 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/debian_9.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/docker_bash_alpine.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Docker Build (Alpine) on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' - .github/ - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml pull_request: branches: - master - main paths-ignore: - '**/*.md' - .github/ workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: # github.event.repository context not available in scheduled workflows if: | github.repository_owner == 'HariSekhon' && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: # GitHub Actions doesn't yet support referencing env context in uses repo_tags: | harisekhon/bash-tools:alpine ghcr.io/harisekhon/bash-tools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-alpine debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} ================================================ FILE: .github/workflows/docker_bash_centos.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Docker Build (CentOS) on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' - .github/ - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml pull_request: branches: - master - main paths-ignore: - '**/*.md' - .github/ workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: # github.event.repository context not available in scheduled workflows if: | github.repository_owner == 'HariSekhon' && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: # GitHub Actions doesn't yet support referencing env context in uses repo_tags: | harisekhon/bash-tools:centos ghcr.io/harisekhon/bash-tools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-centos debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} ================================================ FILE: .github/workflows/docker_bash_debian.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Docker Build (Debian) on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' - .github/ - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml pull_request: branches: - master - main paths-ignore: - '**/*.md' - .github/ workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: # github.event.repository context not available in scheduled workflows if: | github.repository_owner == 'HariSekhon' && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: # GitHub Actions doesn't yet support referencing env context in uses repo_tags: | harisekhon/bash-tools:debian ghcr.io/harisekhon/bash-tools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-debian debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} ================================================ FILE: .github/workflows/docker_bash_fedora.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Docker Build (Fedora) on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' - .github/ - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml pull_request: branches: - master - main paths-ignore: - '**/*.md' - .github/ workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: # github.event.repository context not available in scheduled workflows if: | github.repository_owner == 'HariSekhon' && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: # GitHub Actions doesn't yet support referencing env context in uses repo_tags: | harisekhon/bash-tools:fedora ghcr.io/harisekhon/bash-tools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-fedora debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} ================================================ FILE: .github/workflows/docker_bash_ubuntu.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Docker Build (Ubuntu) on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' - .github/ - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml pull_request: branches: - master - main paths-ignore: - '**/*.md' - .github/ workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: # github.event.repository context not available in scheduled workflows if: | github.repository_owner == 'HariSekhon' && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: # GitHub Actions doesn't yet support referencing env context in uses repo_tags: | harisekhon/bash-tools:latest harisekhon/bash-tools:ubuntu ghcr.io/harisekhon/bash-tools:latest ghcr.io/harisekhon/bash-tools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-ubuntu debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} ================================================ FILE: .github/workflows/dockerhub_status_alpine.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: DockerHub Alpine #env: # DEBUG: 1 on: push: branches: - master schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' jobs: bash_tools_alpine: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/bash-tools --tag alpine tools_alpine: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/tools --tag alpine alpine-github: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/alpine-github github_alpine: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/github --tag alpine ================================================ FILE: .github/workflows/dockerhub_status_centos.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: DockerHub CentOS #env: # DEBUG: 1 on: push: branches: - master schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' jobs: bash_tools_centos: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/bash-tools --tag centos tools_centos: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/tools --tag centos centos-github: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/centos-github github_centos: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/github --tag centos ================================================ FILE: .github/workflows/dockerhub_status_debian.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: DockerHub Debian #env: # DEBUG: 1 on: push: branches: - master schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' jobs: bash_tools_debian: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/bash-tools --tag debian tools_debian: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/tools --tag debian debian-github: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/debian-github github_debian: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/github --tag debian ================================================ FILE: .github/workflows/dockerhub_status_fedora.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: DockerHub Fedora #env: # DEBUG: 1 on: push: branches: - master schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' jobs: bash_tools_fedora: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/bash-tools --tag fedora tools_fedora: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/tools --tag fedora fedora-github: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/fedora-github github_fedora: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/github --tag fedora ================================================ FILE: .github/workflows/dockerhub_status_ubuntu.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: DockerHub Ubuntu #env: # DEBUG: 1 on: push: branches: - master schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' jobs: bash_tools_ubuntu: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/bash-tools --tag ubuntu tools_ubuntu: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/tools --tag ubuntu ubuntu-github: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/ubuntu-github github_ubuntu: name: check timeout-minutes: 1 runs-on: ubuntu-latest container: harisekhon/nagios-plugins steps: - name: check run: check_dockerhub_repo_build_status.py --repo harisekhon/github --tag ubuntu ================================================ FILE: .github/workflows/fedora.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Fedora on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/fedora.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora caches: yum pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/fork-sync.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # F o r k S y n c # ============================================================================ # # For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch --- name: Fork Sync on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 */3 * * *' permissions: contents: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: fork_sync: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/fork-update-pr.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # F o r k U p d a t e P R # ============================================================================ # # For a fork of the original repo, activate to keep its branches up to date via Pull Requests # # To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date --- name: Fork Update PR on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 10 * * 1' permissions: contents: write pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: fork_update_pr: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ghcr_bash_ubuntu.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: GHCR Build (Ubuntu) on: push: branches: - master - main paths-ignore: - **/*.md workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docker_build: if: github.event.repository.fork == false && github.ref_type == 'branch' && ( github.ref_name == github.event.repository.default_branch || github.ref_name == 'docker' ) name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master with: image: bash-tools tags: ubuntu latest dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-bash-tools-ubuntu ================================================ FILE: .github/workflows/grype.yaml ================================================ # # Author: Hari Sekhon # Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G r y p e # ============================================================================ # --- name: Grype on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: Grype: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Grype uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/json.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # J S O N # ============================================================================ # # Validate any JSON files found in the repo --- name: JSON on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.json' - .github/workflows/json.yaml pull_request: branches: - master - main paths: - '**/*.json' - .github/workflows/json.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read jobs: check_json: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/kics.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # K i c s # ============================================================================ # --- name: Kics on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: kics: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Kics uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/mac.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Mac on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/alpine.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-latest caches: brew pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/mac_11.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Mac 11 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/alpine.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-11 caches: brew pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/mac_12.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Mac 12 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/alpine.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-12 caches: brew pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/markdown.yaml ================================================ # # Author: Hari Sekhon # Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # M a r k D o w n # ============================================================================ # --- name: Markdown on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.md' - .mdlrc - .mdl.rb - .markdownlint.rb - .github/workflows/markdown.yaml pull_request: branches: - master - main paths: - '**/*.md' - .mdlrc - .mdl.rb - .markdownlint.rb - .github/workflows/markdown.yaml workflow_dispatch: permissions: contents: read pull-requests: read jobs: Markdown: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Markdown uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master ================================================ FILE: .github/workflows/push_all_repos.sh ================================================ #!/usr/bin/env bash git_foreach_repo.sh 'push' ================================================ FILE: .github/workflows/pypy2.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: PyPy 2 on: push: branches: - master - main paths: - '**/*.py' pull_request: branches: - master - main paths: - '**/*.py' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy2 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/pypy3.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: PyPy 3 on: push: branches: - master - main paths: - '**/*.py' pull_request: branches: - master - main paths: - '**/*.py' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy3 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python2.7.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 2.7 on: push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python2.7.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python2.7.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 2.7 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.10.yaml ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.10 on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.10.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.10.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: "3.10" caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.11.yaml ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.11 on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.11.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.11.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.11 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: "3.11" caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.6.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.6 on: push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.6.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.6.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.6 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.7.yaml ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.7 on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.7.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.7.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.7 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.8.yaml ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.8 on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.8.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.8.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.8 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/python3.9.yaml ================================================ # # Author: Hari Sekhon # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Python 3.9 on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.9.yaml pull_request: branches: - master - main paths: - '**/*.py' - pylib - requirements.txt - .github/workflows/python3.9.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Python 3.9 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.9 caches: apt pip debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/self_hosted.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2020-04-10 13:31:46 +0100 (Fri, 10 Apr 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # name: Self Hosted #env: # DEBUG: 1 on: # [push] push: branches: - master paths-ignore: - *.md schedule: # * is a special character in YAML so you have to quote this string - cron: '0 7 * * *' permissions: contents: read jobs: build: #name: build timeout-minutes: 60 runs-on: self-hosted steps: - uses: actions/checkout@v2 - uses: actions/cache@v4 with: path: ~/.cache/pip key: pip-${{ runner.os }}-${{ runner.arch }} # -${{ hashFiles('**/requirements.txt') }} restore-keys: | pip-${{ runner.os }}-${{ runner.arch }} - uses: actions/cache@v4 with: path: ~/.cpanm key: cpanm-${{ runner.os }}-${{ runner.arch }} # -${{ hashFiles('**/cpan-requirements.txt') }} restore-keys: | cpanm-${{ runner.os }}-${{ runner.arch }} - name: init run: make init - name: build run: make ci - name: test run: make test ================================================ FILE: .github/workflows/semgrep-cloud.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S e m g r e p C l o u d W o r k f l o w # ============================================================================ # # Logs results to https://semgrep.dev/ --- name: Semgrep Cloud on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: semgrep: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/semgrep.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S e m g r e p G i t H u b W o r k f l o w # ============================================================================ # # Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts # https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions --- name: Semgrep on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: semgrep: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/shellcheck.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S h e l l C h e c k # ============================================================================ # # Validate any shell scripts found in the repo --- name: ShellCheck on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.sh' - .github/workflows/shellcheck.yaml pull_request: branches: - master - main paths: - '**/*.sh' - .github/workflows/shellcheck.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: shellcheck: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: ShellCheck uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/sonarcloud.yaml ================================================ # # Author: Hari Sekhon # Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S o n a r C l o u d # ============================================================================ # --- name: SonarCloud on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: permissions: contents: read pull-requests: read jobs: SonarCloud: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: SonarCloud uses: HariSekhon/GitHub-Actions/.github/workflows/sonarcloud.yaml@master secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} ================================================ FILE: .github/workflows/sync_to_adjacent_repos.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-02-13 15:36:35 +0000 (Thu, 13 Feb 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir" sync_file(){ local filename="$1" local repo="$2" local dir="${3:-}" if [ -z "$dir" ]; then dir="$repo" fi dir="$(tr '[:upper:]' '[:lower:]' <<< "$dir")" if ! [ -d "../../../$dir" ]; then echo "WARNING: repo dir $dir not found, skipping..." return 0 fi target="../../../$dir/.github/workflows/$filename" if [ -f "$target.disabled" ]; then target="$target.disabled" fi if [ -f "$target" ] || [ -n "${NEW:-}" ]; then targetdir="${target%/*}" mkdir -p -v "$targetdir" echo "syncing $filename -> $target" perl -p -e "s/(DevOps-)?Bash-tools/$repo/i" "$filename" > "$target" #if [[ "$repo" =~ nagios-plugins ]]; then # timeout=240 # perl -pi -e "s/(^\\s*timeout-minutes:).*/\\1 $timeout/" "$target" # perl -pi -e 's/(^[[:space:]]+make$)/\1 build zookeeper/' "$target" #fi fi } sed 's/#.*//; s/:/ /' ../../setup/repos.txt | grep -v -e bash-tools \ -e github-actions \ -e actions \ -e '^[[:space:]]*$' | while read -r repo dir; do if [ $# -gt 0 ]; then for filename in "$@"; do sync_file "$filename" "$repo" "$dir" done else for filename in *.yaml; do sync_file "$filename" "$repo" "$dir" done fi done ================================================ FILE: .github/workflows/trivy.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T r i v y # ============================================================================ # # Scan files in the local repo --- name: Trivy on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: trivy: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Trivy uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/trivy_image.yaml ================================================ # # Author: Hari Sekhon # Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # T r i v y D o c k e r I m a g e S c a n # ============================================================================ # # Scan the docker image for this repo --- name: Trivy Docker Image Scan on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: actions: read contents: read security-events: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: trivy: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Trivy Docker Image Scan uses: HariSekhon/GitHub-Actions/.github/workflows/trivy_image.yaml@master with: image: harisekhon/bash-tools severity: '' ================================================ FILE: .github/workflows/ubuntu.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_14.04.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu 14.04 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_14.04.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_16.04.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu 16.04 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_16.04.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_18.04.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu 18.04 on: push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_18.04.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_20.04.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu 20.04 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_20.04.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_22.04.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: Ubuntu 22.04 on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_22.04.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:22.04 caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/ubuntu_github.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: GitHub Actions Ubuntu on: # yamllint disable-line rule:truthy push: branches: - master paths-ignore: - '**/*.md' - '.github/workflows/*' - '!.github/workflows/ubuntu_github.yaml' - '**/Dockerfile' - '**/Jenkinsfile' - '**/.envrc*' - .checkov.yaml - .dockerignore - .gcloudignore - .editorconfig - '.gitconfig*' - .gitignore - .grype.yaml - .hound.yml - .terraformignore - Jenkinsfile - .appveyor.yml - .buildkite/pipeline.yml - .circleci/config.yml - .cirrus.yml - .concourse.yml - .drone.yml - .gitlab-ci.yml - .gocd.yml - .scrutinizer.yml - .semaphore/semaphore.yml - .travis.yml - .werckerignore - azure-pipelines.yml - bitbucket-pipelines.yml - buddy.yml - buildspec.yml - cloudbuild.yaml - codefresh.yml - codeship.yml - shippable.yml - wercker.yml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 7 * * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: caches: apt pip cpanm debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/url_links.yaml.disabled ================================================ # # Author: Hari Sekhon # Date: 2022-01-26 14:10:12 +0000 (Wed, 26 Jan 2022) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # --- name: URL Links on: push: branches: - master - main pull_request: branches: - master - main workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: url_links: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: URL Links uses: HariSekhon/GitHub-Actions/.github/workflows/url_links.yaml@master with: ignore_urls_without_dots: 'true' # any value enables this url_links_ignored: | https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv somerepository.com mycompany https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK ================================================ FILE: .github/workflows/validate.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # V a l i d a t i o n # ============================================================================ # # Run all custom validations against files in the repo --- name: Validation on: # yamllint disable-line rule:truthy push: branches: - master - main paths-ignore: - '**/*.md' pull_request: branches: - master - main paths-ignore: - '**/*.md' workflow_dispatch: inputs: debug: type: boolean required: false default: false #schedule: # - cron: '0 0 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: validate: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .github/workflows/xml.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # X M L # ============================================================================ # # Validate any XML files found in the repo --- name: XML on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.xml' - .github/workflows/xml.yaml pull_request: branches: - master - main paths: - '**/*.xml' - .github/workflows/xml.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read jobs: check_xml: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Check XML uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master ================================================ FILE: .github/workflows/yaml.yaml ================================================ # # Author: Hari Sekhon # Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Y A M L # ============================================================================ # # Validate any YAML files found in the repo --- name: YAML on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.yml' - '**/*.yaml' - .github/workflows/yaml.yaml pull_request: branches: - master - main paths: - '**/*.yml' - '**/*.yaml' - .github/workflows/yaml.yaml workflow_dispatch: inputs: debug: type: boolean required: false default: false schedule: - cron: '0 0 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: check_yaml: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master with: debug: ${{ github.event.inputs.debug }} ================================================ FILE: .gitignore ================================================ # # Author: Hari Sekhon # Date: 2012-06-24 22:03:34 +0000 (Sun, 24 Jun 2012) # # vim: filetype=conf # # ============================================================================ # # . g i t i g n o r e # ============================================================================ # # https://git-scm.com/docs/gitignore # =================================================================== # Ignores any file in any subdirectory with matching basename or path # # (can use **/filename but not necessary) # =================================================================== # Once in a while you should check which files have been ignored via: # # git status --ignored # # to check that there aren't some legit files you need to commit # ================================================================= # To find out which line is causing a given file to be ignored, do: # # git check-ignore -v $filename *#*# *.a *.avi *.bak *.bak.* *.bin *.bkp *.bz2 *.class *.dump *.flv *.gif *.gpg *.gz *.img *.jar *.jks *.jpeg *.jpg *.kdb *.lock *.log *.lzo *.macports-saved_* *.mp3 *.mp4 *.mpeg *.mpg *.o *.orig *.out *.p12 *.part *.pyc *.pyo *.stderr *.stdout *.swo *.swp *.tar *.tbz2 *.tgz *.tmp *.wmv *.zip *~ ~* .a .activator/ .adobe .agent.env .aliaslists .android/ .anyconnect .ApacheDirectoryStudio/ .atftp_history .atom/ .audacious/.thumbs .audacious/accels .audacious/log .audacious/playlist.xspf .awless/ # AWS Access + Secret keys are stored in here .aws/credentials .aws/config .aws/token # SSO Access + Secret Keys & STS are stored in .json files in here .aws/cli/cache/ .aws/shell/ .bashrc_dynamichosts .bashrc_dynamichosts.src .bash_history .bash_sessions .bash_vars .boot2docker/ .cache .cassandra/ .cassandra/cqlshrc .cassandra/cqlsh_history .cassandra/nodetool_history .cbq_history .ccm/ .CFUserTextEncoding # Codefresh config contains API Key .cfconfig .circleci/build_agent_settings.json # Circle CI API Token stored in here .circleci/cli.yml .circleci/update_check.yml .Codefresh/agent/ .compiz .conda/ #.config # contains repo_token / COVERALLS_REPO_TOKEN .coveralls.yml .cpan/build/ .cpan/FTPstats.yml .cpan/histfile .cpan/Metadata .cpan/sources/ .cpanm/ .cups/ .data .dbshell .dbus .dcos/ .DCOPserver_* .devcenter/.metadata/ .devcenter/DevCenter/.default/ .devcenter/DevCenter/.metadata/ .devcenter/logging/ .docker_vars .docker/machine/ .dropbox/ .DS_Store .dvdcss/ # contains things like auth tokens #.envrc* .erlang.cookie .evolution .fluxbox/backgrounds .fluxbox/BEST .fluxbox/best-styles/styles-backup .fluxbox/DIVISION2 .fluxbox/fbrun_history .fluxbox/lastwallpaper .fluxbox/menu .fluxbox/NOBG .fluxbox/OTHERTHEMES .fluxbox/startup.log # contains Concourse bearer auth token .flyrc # contains GCP credentials for Ruby Fog library (like Boto) .fog .fontconfig .fseventsd .gaim/accels .gaim/icons .gaim/status.xml .gconf .gconfd .gem .gimp-* .github_actions_runner/ .gitk .gmvault/ .gnome .gnome2 .gnome2_private .gnupg/gpg-agent-info-* .gnupg/private-keys-* .gnuplot_history .gpg-agent.env .gradle .groovy .gstreamer-* .gvfs .hg #.htoprc .ICEauthority .IdeaIC* .idea .inkscape-etc/ .ion3 .ipython/ .irb-save-history .irb_history # contains creds .iredisrc .ivy2/ .jline-jython.history .kde .kodos .kube/ .ldapvi_history .lesshst .links2 .local .m2/ .macports/ .macromedia .matplotlib/ .mcop .mcoprc .minikube/ .minishift/ .minishift.env .mozilla .mtools/ .mysql_history .nbprofiler/ .neo4j_shell_history .npm/ .octave_hist .openoffice.org .openoffice.org2 .oracle_jre_usage/ .ovftool.ssldb .Qsync/ .parallel/ # PostgreSQL password file .pgpass # stores credentials .pig_history .pki .pentaho/ .psql_history .pulse .puppet/ .pwm3 .PyCharm*/ .pylint.d .python-eggs/ .python_history .qt .qicon .qnicon # contains usernames and passwords .rabbitmqadmin.conf .RData .recently-used .recently-used.xbel .rediscli_history .Rhistory .rnd .rstudio-desktop/ .rbenv/ # 'rbenv local' $PWD version file #.ruby-version .sbt/[[:digit:]].[[:digit:]]*/ .sbt/boot/ .sbt/preloaded/ .sbt/repositories .sbt/*/plugins/project/target/ .sbt/*/plugins/target/ .scala_history .sdkman/archives/ .sdkman/bin/ .sdkman/candidates/ .sdkman/etc/config .sdkman/ext/ .sdkman/src/ .sdkman/tmp/ .sdkman/var/ # Semaphore CI - contains auth token .sem.yaml .serverauth.* # $PWD/.serverless local service artifacts .serverless/ # $HOME/.serverless installation .serverless/bin/ .sh_history # contains client id and secret for Shpotify .shpotify.cfg .Skype .snowsql # Snowflake password stored in plaintext in here .snowsql/config .spark_history .Spotlight-* .spumux/ .sqlite_history .sqlline/history .ssh/known_hosts .ssh-agent.env .subversion/auth .svn .swatch_script.* .TemporaryItems .terraform.d/ .terragrunt .themes .thumbnails .tilda/locks/* .tmux/ .tomboy .tomboy.log .Trash .Trashes .travis/ .vagrant .vagrant.d/ .vboxclient-autoresize.pid .vboxclient-clipboard.pid .vboxclient-seamless.pid .vim/ .viminfo .vnc/ .wapi .wget-hsts .wine .wireshark-etc/ .wireshark/ .wmii-* .Xauthority .xine .xmms .xsession-errors .zenmap-etc/pango/pangorc # GCP credentials application_default_credentials.json *keyfile.json *credentials.json abs-guide.pdf android-sdks/ Applications/ AT.postflight.* # contains OAuth token auth.json bin/altfirewalls bin/altnames bin/contrib/* bin/desktops bin/firewalls bin/servers #bitbucket/ #bitbucket/* boxes/ Box Documents/ # contains webhook URL which should not be committed publicly buildkite-pipeline*.json .buildkite-pipelines/ c chinook.*sql* chinook.psql Calibre Library/ cli_junkie.gif debs/ Desktop/ Documents/ Downloads/ drive/ Dropbox/ eclipse/ enterprise fatpacks/ fatlib/ fluxbox-themes fluxbox/debian007.jpg # internal repos #git/ #git/* # public repos - clashes with TeamCity-CI repo's .teamcity/GitHub #github/ #github/* gitolite-admin/ gitroot/ GNUstep Google*Drive go/bin/ go/src/github.com/ google-cloud-sdk/ hadoop-sources/ hgroot/ IdeaProjects/ Images infrastructure/puppetinfrastructure/ #jython* Library mbox mcollective-plugins/ Movies Music norev NS-GUISettings *OneDrive*/ ops/ original-tars originals_tars otherpics Qsync/ perl5/ Pictures Public puppet.git/ PycharmProjects/ pytools_checks/ rpms/ setup/mac_settings/ shm/ siege.log Sites # golang libraries src/github.com src/golang.org subversion/ svnroot/ testing/ tmp.* # common vagrant stuff vagrant/boxes/ vagrant/data/ vagrant/gems/ vagrant/insecure_private_key vagrant/rgloader/ vagrant/setup_version vagrant/tmp/ # custom stuff found in vagrant vagrant/basho_bench/ vagrant/id_rsa.pub vagrant/jce_policy-6 vagrant/jython vagrant/kibana/ vagrant/lib vagrant/mrepo vagrant/mx4j-* vagrant/mysql-connector-* vagrant/thrift-* venv/ VirtualBox VMs/ VirtualBoxShared visualvm* windows/bin2 wordlists/*.lower wordlists/hosts.large wordlists/hosts.medium wordlists/hosts.small wordlists/pw.medium wordlists/pw.small wordlists/pw.tiny wordlists/snmp wordlists/ultimate.* wordlists/users.large wordlists/users.medium wordlists/users.small wordlists/users.tiny wordlists/users.weighted *.doc *.docx *.xls *.xlsx *.log *.msg *.pages *.rtf *.wpd *.wps # ============================================================================ # # M o b i l e B u i l d s # ============================================================================ # # SwiftPackageManager Packages # Obj-C/Swift specific *.hmap # App packaging *.ipa *.dSYM.zip *.dSYM # Playgrounds timeline.xctimeline playground.xcworkspace .build/ # Fastlane fastlane/report.xml fastlane/Preview.html fastlane/screenshots/**/*.png fastlane/screenshots/**/*.jp*g fastlane/test_output fastlane/.env.default iOSInjectionProject/ # ixGuard ixguard-license.txt mapping.yml protectionreport.html statistics.yml telemetry_dump.json # ============================================================================ # # # Regenerate all sections below in to a single arg for API call via: # # grep '[C]reated by https://' .gitignore | sed 's,.*/,,' | tr ',' '\n' | sort -u | tr '\n' ',' | sed 's/,$//' | xargs echo gitignore.io_api.sh # # eg. # # gitignore.io_api.sh ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh >> .gitignore # # Find new or missing tags you aren't using yet: # # grep '[C]reated by https://' .gitignore | sed 's,.*/,,' | tr ',' '\n' | sort -u | tr '\n' ',' | sed 's/,$//' | gitignore.io_api.sh missing # # ============================================================================ # # Created by https://www.toptal.com/developers/gitignore/api/ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh # Edit at https://www.toptal.com/developers/gitignore?templates=ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh ### Ansible ### *.retry ### ApacheHadoop ### *.iml *.ipr *.iws *.orig *.rej .idea .svn .classpath .project .settings target hadoop-common-project/hadoop-kms/downloads/ hadoop-hdfs-project/hadoop-hdfs/downloads hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml ### AppCode ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff .idea/**/workspace.xml .idea/**/tasks.xml .idea/**/usage.statistics.xml .idea/**/dictionaries .idea/**/shelf # Generated files .idea/**/contentModel.xml # Sensitive or high-churn files .idea/**/dataSources/ .idea/**/dataSources.ids .idea/**/dataSources.local.xml .idea/**/sqlDataSources.xml .idea/**/dynamic.xml .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml # Gradle .idea/**/gradle.xml .idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake cmake-build-*/ # Mongo Explorer plugin .idea/**/mongoSettings.xml # File-based project format # IntelliJ out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Cursive Clojure plugin .idea/replstate.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties # Editor-based Rest Client .idea/httpRequests # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser ### AppCode Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint .idea/**/sonarlint/ # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin .idea/**/sonarIssues.xml # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced .idea/**/markdown-navigator.xml .idea/**/markdown-navigator-enh.xml .idea/**/markdown-navigator/ # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 .idea/$CACHE_FILE$ # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream .idea/codestream.xml ### AppEngine ### # Google App Engine generated folder appengine-generated/ ### Archive ### ### Mostly from https://en.wikipedia.org/wiki/List_of_archive_formats ## Archiving only # The traditional archive format on Unix-like systems, now used mainly for the creation of static libraries. *.a *.ar # RPM files consist of metadata concatenated with (usually) a cpio archive. Newer RPM systems also support other archives, as cpio is becoming obsolete. cpio is also used with initramfs. *.cpio # A self-extracting archive that uses the Bourne shell (sh). *.shar # A system for storing multiple files. LBR archives typically contained files processed by SQ, or the archive itself was compressed with SQ. LBR archives that were compressed with SQ ended with the extension .LQR *.LBR # An archive format originally used mainly for archiving and distribution of the exact, nearly-exact, or custom-modified contents of an optical storage medium such as a CD-ROM or DVD-ROM. However, it can be used to archive the contents of other storage media, selected partitions, folders, and/or files. The resulting archive is typically optimized for convenient rendering to (re-)writable CD or DVD media. *.iso # A library format used primarily on the Commodore 64 and 128 lines of computers. This bears no resemblance to the DOS LBR format. While library files were quick to implement (a number of programs exist to work with them) they are crippled in that they cannot grow with use: once a file has been created it cannot be amended (files added, changed or deleted) without recreating the entire file. *.lbr # An archive format used by Mozilla for storing binary diffs. Used in conjunction with bzip2. *.mar # A common archive format used on Unix-like systems. Generally used in conjunction with compressors such as gzip, bzip2, compress or xz to create .tar.gz, .tar.bz2, .tar.Z or tar.xz files. *.tar # Package managers # Red Hat Package Manager *.rpm # Debian package *.deb # MicroSoft Installer *.msi *.msm *.msp # Mozilla package installer *.xpi # Ruby Package *.gem ### Archives ### # It's better to unpack these files and commit the raw source because # git has its own built in compression methods. *.7z *.jar *.rar *.zip *.gz *.gzip *.tgz *.bzip *.bzip2 *.bz2 *.xz *.lzma *.cab *.xar # Packing-only formats # Package management formats *.dmg *.egg *.txz ### ArchLinuxPackages ### *.tar.* *.exe *.log *.log.* *.sig pkg/ # XXX: might conflict with standard java golang src/ directory structure #src/ ### Audio ### *.aif *.iff *.m3u *.m4a *.mid *.mp3 *.mpa *.ra *.wav *.wma *.ogg *.flac ### Autotools ### # http://www.gnu.org/software/automake # XXX: I track a master Makefile.in in DevOps-Bash-tools to inherit in other projects #Makefile.in /ar-lib /mdate-sh /py-compile /test-driver /ylwrap .deps/ # http://www.gnu.org/software/autoconf autom4te.cache /autoscan.log /autoscan-*.log /aclocal.m4 /compile /config.guess /config.h.in /config.log /config.status /config.sub /configure /configure.scan /depcomp /install-sh /missing /stamp-h1 # https://www.gnu.org/software/libtool/ /ltmain.sh # http://www.gnu.org/software/texinfo /texinfo.tex # http://www.gnu.org/software/m4/ m4/libtool.m4 m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 m4/lt~obsolete.m4 # Generated Makefile # (meta build system like autotools, # can automatically generate from config.status script # (which is called by configure script)) # # XXX: always want to commit Makefiles #Makefile ### Autotools Patch ### ### Backup ### *.bak *.gho *.ori *.tmp ### Basic ### # Apples Build *.build *.apples # Initialized files #*.ini # used by Ansible for inventory.ini *.basic ### BitTorrent ### *.torrent ### C ### # Prerequisites # XXX: clashes with .bash.d/* and .conf.d/* type stuff #*.d # Object files *.o *.ko *.obj *.elf # Linker output *.ilk *.map *.exp # Precompiled Headers *.gch *.pch # Libraries *.lib *.la *.lo # Shared objects (inc. Windows DLLs) *.dll *.so *.so.* *.dylib # Executables *.out *.app *.i*86 *.x86_64 *.hex # Debug files *.dSYM/ *.su *.idb *.pdb # Kernel Module Compile Results # XXX: *.mod* changed to avoid clashing with Golang's new module system go.mod *.mod?* .tmp_versions/ modules.order Module.symvers Mkfile.old dkms.conf ### Zsh ### # Zsh compiled script + zrecompile backup *.zwc *.zwc.old # Zsh completion-optimization dumpfile *zcompdump* # Zsh zcalc history .zcalc_history # A popular plugin manager's files ._zplugin .zplugin_lstupd # zdharma/zshelldoc tool's files zsdoc/data # robbyrussell/oh-my-zsh/plugins/per-directory-history plugin's files # (when set-up to store the history in the local directory) .directory_history # MichaelAquilina/zsh-autoswitch-virtualenv plugin's files # (for Zsh plugins using Python) .venv # Zunit tests' output /tests/_output/* !/tests/_output/.gitkeep ### C++ ### # Prerequisites # Compiled Object files *.slo # Precompiled Headers # Compiled Dynamic libraries # Fortran module files # XXX: would conflict with Golang's go.mod #*.mod *.smod # Compiled Static libraries *.lai # Executables ### certificates ### *.pem *.key *.crt *.cer *.priv ### Cloud9 ### # Cloud9 IDE - http://c9.io .c9revisions .c9 ### ChefCookbook ### .vagrant /cookbooks # Bundler # XXX: would prevent tracking ~/bin scripts #bin/* .bundle/* .kitchen/ .kitchen.local.yml .kitchen.*.local.yml kitchen.local.yml kitchen.*.local.yml ### Code ### .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json *.code-workspace ### Code-Java ### # Language Support for Java(TM) by Red Hat extension for Visual Studio Code - https://marketplace.visualstudio.com/items?itemName=redhat.java factoryConfiguration.json ### Clojure ### # XXX: always want to commit pom.xml #pom.xml pom.xml.asc *.class # XXX: would break lib submodule updates #/lib/ /classes/ /target/ /checkouts/ .lein-deps-sum .lein-repl-history .lein-plugins/ .lein-failures .nrepl-port .cpcache/ ### Compressed ### *.pkg *.sit *.sitx *.zipx ### CompressedArchive ### ## Archiving and compression # Open source file format. Used by 7-Zip. # Mac OS X, restoration on different platforms is possible although not immediate Yes Based on 7z. Preserves Spotlight metadata, resource forks, owner/group information, dates and other data which would be otherwise lost with compression. *.s7z # Old archive versions only Proprietary format *.ace # A format that compresses and doubly encrypt the data (AES256 and CAS256) avoiding brute force attacks, also hide files in an AFA file. It has two ways to safeguard data integrity and subsequent repair of the file if has an error (repair with AstroA2P (online) or Astrotite (offline)). *.afa # A mainly Korean format designed for very large archives. *.alz # Android application package (variant of JAR file format). *.apk # ?? *.arc # Originally DOS, now multiple *.arj # Open archive format, used by B1 Free Archiver (http://dev.b1.org/standard/archive-format.html) *.b1 # Binary Archive with external header *.ba # Proprietary format from the ZipTV Compression Components *.bh # The Microsoft Windows native archive format, which is also used by many commercial installers such as InstallShield and WISE. # Originally DOS, now DOS and Windows Created by Yaakov Gringeler; released last in 2003 (Compressia 1.0.0.1 beta), now apparently defunct. Free trial of 30 days lets user create and extract archives; after that it is possible to extract, but not to create. *.car # Open source file format. *.cfs # Compact Pro archive, a common archiver used on Mac platforms until about Mac OS 7.5.x. Competed with StuffIt; now obsolete. *.cpt # Windows, Unix-like, Mac OS X Open source file format. Files are compressed individually with either gzip, bzip2 or lzo. *.dar # DiskDoubler Mac OS obsolete *.dd # ?? *.dgc # Apple Disk Image upports "Internet-enabled" disk images, which, once downloaded, are automatically decompressed, mounted, have the contents extracted, and thrown away. Currently, Safari is the only browser that supports this form of extraction; however, the images can be manually extracted as well. This format can also be password-protected or encrypted with 128-bit or 256-bit AES encryption. # Enterprise Java Archive archive *.ear # ETSoft compressed archive # The predecessor of DGCA. *.gca # Originally DOS Yes, but may be covered by patents DOS era format; uses arithmetic/Markov coding *.ha # MS Windows HKI *.hki # Produced by ICEOWS program. Excels at text file compression. *.ice # Java archive, compatible with ZIP files # Open sourced archiver with compression using the PAQ family of algorithms and optional encryption. *.kgb # Originally DOS, now multiple Multiple Yes The standard format on Amiga. *.lzh *.lha # Archiver originally used on The Amiga. Now copied by Microsoft to use in their .cab and .chm files. *.lzx # file format from NoGate Consultings, a rival from ARC-Compressor. *.pak # A disk image archive format that supports several compression methods as well as splitting the archive into smaller pieces. *.partimg # An experimental open source packager (http://mattmahoney.net/dc) *.paq* # Open source archiver supporting authenticated encryption, volume spanning, customizable object level and volume level integrity checks (form CRCs to SHA-512 and Whirlpool hashes), fast deflate based compression *.pea # The format from the PIM - a freeware compression tool by Ilia Muraviev. It uses an LZP-based compression algorithm with set of filters for executable, image and audio files. *.pim # PackIt Mac OS obsolete *.pit # Used for data in games written using the Quadruple D library for Delphi. Uses byte pair compression. *.qda # A proprietary archive format, second in popularity to .zip files. # The format from a commercial archiving package. Odd among commercial packages in that they focus on incorporating experimental algorithms with the highest possible compression (at the expense of speed and memory), such as PAQ, PPMD and PPMZ (PPMD with unlimited-length strings), as well as a proprietary algorithms. *.rk # Self Dissolving ARChive Commodore 64, Commodore 128 Commodore 64, Commodore 128 Yes SDAs refer to Self Dissolving ARC files, and are based on the Commodore 64 and Commodore 128 versions of ARC, originally written by Chris Smeets. While the files share the same extension, they are not compatible between platforms. That is, an SDA created on a Commodore 64 but run on a Commodore 128 in Commodore 128 mode will crash the machine, and vice versa. The intended successor to SDA is SFX. *.sda # A pre-Mac OS X Self-Extracting Archive format. StuffIt, Compact Pro, Disk Doubler and others could create .sea files, though the StuffIt versions were the most common. *.sea # Scifer Archive with internal header *.sen # Commodore 64, Commodore 128 SFX is a Self Extracting Archive which uses the LHArc compression algorithm. It was originally developed by Chris Smeets on the Commodore platform, and runs primarily using the CS-DOS extension for the Commodore 128. Unlike its predecessor SDA, SFX files will run on both the Commodore 64 and Commodore 128 regardless of which machine they were created on. *.sfx # An archive format designed for the Apple II series of computers. The canonical implementation is ShrinkIt, which can operate on disk images as well as files. Preferred compression algorithm is a combination of RLE and 12-bit LZW. Archives can be manipulated with the command-line NuLib tool, or the Windows-based CiderPress. *.shk # A compression format common on Apple Macintosh computers. The free StuffIt Expander is available for Windows and OS X. # The replacement for the .sit format that supports more compression methods, UNIX file permissions, long file names, very large files, more encryption options, data specific compressors (JPEG, Zip, PDF, 24-bit image, MP3). The free StuffIt Expander is available for Windows and OS X. # A royalty-free compressing format *.sqx # The "tarball" format combines tar archives with a file-based compression scheme (usually gzip). Commonly used for source and binary distribution on Unix-like platforms, widely available elsewhere. *.tar.gz *.tar.Z *.tar.bz2 *.tbz2 *.tar.lzma *.tlz # UltraCompressor 2.3 was developed to act as an alternative to the then popular PKZIP application. The main feature of the application is its ability to create large archives. This means that compressed archives with the UC2 file extension can hold almost 1 million files. *.uc *.uc0 *.uc2 *.ucn *.ur2 *.ue2 # Based on PAQ, RZM, CSC, CCM, and 7zip. The format consists of a PAQ, RZM, CSC, or CCM compressed file and a manifest with compression settings stored in a 7z archive. *.uca # A high compression rate archive format originally for DOS. *.uha # Web Application archive (Java-based web app) *.war # File-based disk image format developed to deploy Microsoft Windows. *.wim # XAR # Native format of the Open Source KiriKiri Visual Novel engine. Uses combination of block splitting and zlib compression. The filenames and pathes are stored in UTF-16 format. For integrity check, the Adler-32 hashsum is used. For many commercial games, the files are encrypted (and decoded on runtime) via so-called "cxdec" module, which implements xor-based encryption. *.xp3 # Yamazaki zipper archive. Compression format used in DeepFreezer archiver utility created by Yamazaki Satoshi. Read and write support exists in TUGZip, IZArc and ZipZag *.yz1 # The most widely used compression format on Microsoft Windows. Commonly used on Macintosh and Unix systems as well. # application/x-zoo zoo Multiple Multiple Yes *.zoo # Journaling (append-only) archive format with rollback capability. Supports deduplication and incremental update based on last-modified dates. Multi-threaded. Compresses in LZ77, BWT, and context mixing formats. Open source. *.zpaq # Archiver with a compression algorithm based on the Burrows-Wheeler transform method. *.zz ### Compression ### ### From https://en.wikipedia.org/wiki/List_of_archive_formats ## Compression only # An open source, patent- and royalty-free compression format. The compression algorithm is a Burrows-Wheeler transform followed by a move-to-front transform and finally Huffman coding # Old compressor for QNX4 OS. The compression algorithm is a modified LZSS, with an adaptive Huffman coding. *.F # GNU Zip, the primary compression format used by Unix-like systems. The compression algorithm is DEFLATE. # An alternate LZMA algorithm implementation, with support for checksums and ident bytes. *.lz # The LZMA compression algorithm as used by 7-Zip # An implementation of the LZO data compression algorithm *.lzo # A compression program designed to do particularly well on very large files containing long distance redundancy. *.rz # Windows compress/decompress- Linux and Mac OS X decompress only A compression program designed to do high compression on SF2 files (SoundFont) *.sfark # A compression format invented by Google and open-sourced in 2011. Snappy aims for very high speeds, reasonable compression, and maximum stability rather than maximum compression or compatibility with any other compression library. *.sz # Squeeze: A program which compressed files. A file which was "squeezed" had the middle initial of the name changed to "Q", so that a squeezed text file would end with .TQT, a squeezed executable would end with .CQM or .EQE. Typically used with .LBR archives, either by storing the squeezed files in the archive, or by storing the files decompressed and then compressing the archive, which would have a name ending in ".LQR". # XXX: this prevents .sql files from being tracked #*.?Q? # A compression program written by Steven Greenberg implementing the LZW algorithm. For several years in the CP/M world when no implementation was available of ARC, CRUNCHed files stored in .LBR archives were very popular. CRUNCH's implementation of LZW had a somewhat unique feature of modifying and occasionally clearing the code table in memory when it became full, resulting in a few percent better compression on many files. *.?Z? # A compression format using LZMA2 to yield very high compression ratios. # The traditional Huffman coding compression format. *.z # The traditional LZW compression format. *.Z # Joke compression program, actually increasing file size *.infl # Compression format(s) used by some DOS and Windows install programs. MS-DOS includes expand.exe to decompress its install files. The compressed files are created with a matching compress.exe command. The compression algorithm is LZSS. *.??_ ### CMake ### CMakeLists.txt.user CMakeCache.txt CMakeFiles CMakeScripts Testing cmake_install.cmake install_manifest.txt compile_commands.json CTestTestfile.cmake _deps ### CMake Patch ### # External projects *-prefix/ ### Database ### *.accdb *.db *.dbf *.mdb *.sqlite3 ### DataRecovery ### ## Data recovery # File format used by dvdisaster to be used for data recovery when discs become damaged or partially unreadable. *.ecc # File format used in conjunction with any archive format to provide redundancy and data recovery, most often in newsgroup distribution of binary files. *.par *.par2 ### Diff ### *.patch *.diff ### direnv ### .direnv #.envrc ### CodeBlocks ### # specific to CodeBlocks IDE *.layout *.depend # generated directories # XXX: would prevent tracking ~/bin scripts #bin/ obj/ ### DocFx ### .cache /**/_site/ ### Docpress ### # docpress documentation generator: https://docpress.github.io/index.html _docpress/ ### Docz ### .docz ### dotenv ### .env ### DotfilesSh ### local-patch patched-src ### DotSettings ### *.DotSettings ### Dropbox ### # Dropbox settings and caches .dropbox .dropbox.attr .dropbox.cache ### Eclipse ### .metadata tmp/ *.swp *~.nib local.properties .settings/ .loadpath .recommenders # External tool builders .externalToolBuilders/ # Locally stored "Eclipse launch configurations" *.launch # PyDev specific (Python IDE for Eclipse) *.pydevproject # CDT-specific (C/C++ Development Tooling) .cproject # CDT- autotools .autotools # Java annotation processor (APT) .factorypath # PDT-specific (PHP Development Tools) .buildpath # sbteclipse plugin .target # Tern plugin .tern-project # TeXlipse plugin .texlipse # STS (Spring Tool Suite) .springBeans # Code Recommenders .recommenders/ # Annotation Processing .apt_generated/ .apt_generated_test/ # Scala IDE specific (Scala & Java development for Eclipse) .cache-main .scala_dependencies .worksheet # Uncomment this line if you wish to ignore the project description file. # Typically, this file would be tracked if it contains build/dependency configurations: #.project ### Eclipse Patch ### # Spring Boot Tooling .sts4-cache/ ### Data ### # XXX: would prevent tracking sample test csv data #*.csv *.dat *.efx *.gbr *.pps *.ppt *.pptx *.sdf *.tax2010 *.vcf # XXX: would prevent tracking pom.xml #*.xml ### Erlang ### .eunit *.beam *.plt erl_crash.dump .concrete/DEV_MODE # rebar 2.x .rebar rel/example_project ebin/*.beam deps # rebar 3 .rebar3 _build/ _checkouts/ ### Executable ### # XXX: .cgi scripts are old but we might want to track them #*.cgi *.com *.gadget *.pif # XXX: would prevent tracking Windows Scripting files #*.vb #*.wsf ### Firebase ### **/node_modules/* **/.firebaserc ### Firebase Patch ### .runtimeconfig.json .firebase/ ### Flask ### instance/* !instance/.gitignore .webassets-cache ### Flask.Python Stack ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ # XXX: breaks lib submodule updates #lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ pytestdebug.log # Translations *.mo *.pot # Django stuff: local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ doc/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments env/ venv/ ENV/ env.bak/ venv.bak/ pythonenv* # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # profiling data .prof ### Git ### # Created by git for backups. To disable backups in Git: # $ git config --global mergetool.keepBackup false # Created by git when using merge tools for conflicts *.BACKUP.* *.BASE.* *.LOCAL.* *.REMOTE.* *_BACKUP_*.txt *_BASE_*.txt *_LOCAL_*.txt *_REMOTE_*.txt ### GitBook ### # Node rules: ## Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt ## Dependency directory ## Commenting this out is preferred by some people, see ## https://docs.npmjs.com/misc/faq#should-i-check-my-node_modules-folder-into-git node_modules # Book build output _book # eBook build output *.epub *.mobi *.pdf ### Go ### # Binaries for programs and plugins *.exe~ # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE # Dependency directories (remove the comment below to include it) # vendor/ ### Go Patch ### /vendor/ /Godeps/ ### GPG ### secring.* ### DiskImage ### *.toast *.vcd ### Grails ### # .gitignore for Grails 1.2 and 1.3 # Although this should work for most versions of grails, it is # suggested that you use the "grails integrate-with --git" command # to generate your .gitignore file. # web application files /web-app/WEB-INF/classes # default HSQL database files for production mode /prodDb.* # general HSQL database files *Db.properties *Db.script # logs /stacktrace.log /test/reports /logs # project release file /*.war # plugin release files /*.zip /plugin.xml # older plugin install locations /plugins /web-app/plugins # "temporary" build files /target ### Groovy ### # .gitignore created from Groovy contributors in https://github.com/apache/groovy/blob/master/.gitignore user.gradle .gradle/ *.DS_Store *~ .shelf ### grunt ### # Grunt usually compiles files inside this directory # Grunt usually preprocesses files such as coffeescript, compass... inside the .tmp directory .tmp/ ### Haskell ### dist dist-* cabal-dev *.hi *.hie *.chi *.chs.h *.dyn_o *.dyn_hi .hpc .hsenv .cabal-sandbox/ cabal.sandbox.config *.prof *.aux *.hp *.eventlog .stack-work/ cabal.project.local cabal.project.local~ .HTF/ .ghc.environment.* ### Helm ### # Chart dependencies **/charts/*.tgz ### Homebrew ### Brewfile.lock.json ### Hugo ### # Generated files by hugo # XXX: might interfere with projects code structure #/public/ /resources/_gen/ hugo_stats.json # Executable may be added to repository hugo.exe hugo.darwin hugo.linux ### Images ### # JPEG *.jpg *.jpeg *.jpe *.jif *.jfif *.jfi # JPEG 2000 *.jp2 *.j2k *.jpf *.jpx *.jpm *.mj2 # JPEG XR *.jxr *.hdp *.wdp # Graphics Interchange Format *.gif # RAW *.raw # Web P *.webp # Portable Network Graphics *.png # Animated Portable Network Graphics *.apng # Multiple-image Network Graphics *.mng # Tagged Image File Format *.tiff *.tif # Scalable Vector Graphics *.svg *.svgz # Portable Document Format # X BitMap *.xbm # BMP *.bmp *.dib # ICO *.ico # 3D Images *.3dm *.max ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### Intellij+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 .idea/ # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 modules.xml .idea/misc.xml # Sonarlint plugin .idea/sonarlint ### Intellij+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### Intellij+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### Java ### # Compiled class file # Log file # BlueJ files *.ctxt # Mobile Tools for Java (J2ME) .mtj.tmp/ # Package Files # *.nar # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* ### Java-Web ### ## ignoring target file ### JEnv ### # JEnv local Java version configuration file .java-version # Used by previous versions of JEnv .jenv-version ### JetBrains ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### JetBrains+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 # Sonarlint plugin ### JetBrains+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### JetBrains+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### JMeter ### # JMeter common ignore files # http://jmeter.apache.org/ # Ignore Summary/Aggregrate reports *.jtl # Ignore log files # Ignore customized user.properties user.properties ### Emacs ### # -*- mode: gitignore; -*- \#*\# /.emacs.desktop /.emacs.desktop.lock *.elc auto-save-list tramp .\#* # Org-mode .org-id-locations *_archive # flymake-mode *_flymake.* # eshell files /eshell/history /eshell/lastdir # elpa packages /elpa/ # reftex files *.rel # AUCTeX auto folder /auto/ # cask packages .cask/ # Flycheck flycheck_*.el # server auth directory /server/ # projectiles files .projectile # directory configuration .dir-locals.el # network security /network-security.data ### JupyterNotebooks ### # gitignore template for Jupyter Notebooks # website: http://jupyter.org/ */.ipynb_checkpoints/* # IPython # Remove previous ipynb_checkpoints # git rm -r .ipynb_checkpoints/ ### Kotlin ### # Compiled class file # Log file # BlueJ files # Mobile Tools for Java (J2ME) # Package Files # # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml ### LAMP ### # LAMP Stack Base ### LAMP.Linux Stack ### # temporary files which can be created if a process still has a handle open of a deleted file .fuse_hidden* # KDE directory preferences .directory # Linux trash folder which might appear on any partition or disk .Trash-* # .nfs files are created when an open file is removed but is still being accessed .nfs* ### LAMP.PHP Stack ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### LaTeX ### ## Core latex/pdflatex auxiliary files: *.lof *.lot *.fls *.toc *.fmt *.fot *.cb *.cb2 .*.lb ## Intermediate documents: *.dvi *.xdv *-converted-to.* # these rules might exclude image files for figures etc. # *.ps # *.eps # *.pdf ## Generated if empty string is given at "Please type another file name for output:" .pdf ## Bibliography auxiliary files (bibtex/biblatex/biber): *.bbl *.bcf *.blg *-blx.aux *-blx.bib *.run.xml ## Build tool auxiliary files: *.fdb_latexmk *.synctex *.synctex(busy) *.synctex.gz *.synctex.gz(busy) *.pdfsync ## Build tool directories for auxiliary files # latexrun latex.out/ ## Auxiliary and intermediate files from other packages: # algorithms *.alg *.loa # achemso acs-*.bib # amsthm *.thm # beamer *.nav *.pre *.snm *.vrb # changes *.soc # comment *.cut # cprotect # elsarticle (documentclass of Elsevier journals) *.spl # endnotes *.ent # fixme *.lox # feynmf/feynmp *.mf *.mp *.t[1-9] *.t[1-9][0-9] *.tfm #(r)(e)ledmac/(r)(e)ledpar *.end *.?end *.[1-9] *.[1-9][0-9] *.[1-9][0-9][0-9] *.[1-9]R *.[1-9][0-9]R *.[1-9][0-9][0-9]R *.eledsec[1-9] *.eledsec[1-9]R *.eledsec[1-9][0-9] *.eledsec[1-9][0-9]R *.eledsec[1-9][0-9][0-9] *.eledsec[1-9][0-9][0-9]R # glossaries *.acn *.acr *.glg *.glo *.gls *.glsdefs *.lzs # uncomment this for glossaries-extra (will ignore makeindex's style files!) # *.ist # gnuplottex *-gnuplottex-* # gregoriotex *.gaux *.gtex # htlatex *.4ct *.4tc *.idv *.lg *.trc *.xref # hyperref *.brf # knitr *-concordance.tex # TODO Comment the next line if you want to keep your tikz graphics files *.tikz *-tikzDictionary # listings *.lol # luatexja-ruby *.ltjruby # makeidx *.idx *.ilg *.ind # minitoc *.maf *.mlf *.mlt *.mtc *.mtc[0-9]* *.slf[0-9]* *.slt[0-9]* *.stc[0-9]* # minted _minted* *.pyg # morewrites *.mw # nomencl *.nlg *.nlo *.nls # pax *.pax # pdfpcnotes *.pdfpc # sagetex *.sagetex.sage *.sagetex.py *.sagetex.scmd # scrwfile *.wrt # sympy *.sout *.sympy sympy-plots-for-*.tex/ # pdfcomment *.upa *.upb # pythontex *.pytxcode pythontex-files-*/ # tcolorbox *.listing # thmtools *.loe # TikZ & PGF *.dpth *.md5 *.auxlock # todonotes *.tdo # vhistory *.hst *.ver # easy-todo *.lod # xcolor *.xcp # xmpincl *.xmpi # xindy *.xdy # xypic precompiled matrices and outlines *.xyc *.xyd # endfloat *.ttt *.fff # Latexian TSWLatexianTemp* ## Editors: # WinEdt *.sav # Texpad .texpadtmp # LyX *.lyx~ # Kile *.backup # gummi .*.swp # KBibTeX *~[0-9]* # TeXnicCenter *.tps # auto folder when using emacs and auctex ./auto/* *.el # expex forward references with \gathertags *-tags.tex # standalone packages *.sta # Makeindex log files *.lpz # REVTeX puts footnotes in the bibliography by default, unless the nofootinbib # option is specified. Footnotes are the stored in a file with suffix Notes.bib. # Uncomment the next line to have this generated file ignored. #*Notes.bib ### LaTeX Patch ### # LIPIcs / OASIcs *.vtc # glossaries *.glstex ### Less ### *.less ### Linux ### # temporary files which can be created if a process still has a handle open of a deleted file # KDE directory preferences # Linux trash folder which might appear on any partition or disk # .nfs files are created when an open file is removed but is still being accessed ### Lua ### # Compiled Lua sources luac.out # luarocks build files *.src.rock # Object files *.os # Precompiled Headers # Libraries *.def # Shared objects (inc. Windows DLLs) # Executables ### macOS ### # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk ### MATLAB ### # Windows default autosave extension *.asv # OSX / *nix default autosave extension *.m~ # Compiled MEX binaries (all platforms) *.mex* # Packaged app and toolbox files *.mlappinstall *.mltbx # Generated helpsearch folders helpsearch*/ # Simulink code generation folders slprj/ sccprj/ # Matlab code generation folders codegen/ # Simulink autosave extension *.autosave # Simulink cache files *.slxc # Octave session info octave-workspace ### Maven ### pom.xml.tag pom.xml.releaseBackup pom.xml.versionsBackup pom.xml.next release.properties dependency-reduced-pom.xml buildNumber.properties .mvn/timing.properties # https://github.com/takari/maven-wrapper#usage-without-binary-jar .mvn/wrapper/maven-wrapper.jar ### Mercurial ### .hg/ .hgignore .hgsigs .hgsub .hgsubstate .hgtags ### MicrosoftOffice ### # Word temporary ~$*.doc* # Word Auto Backup File Backup of *.doc* # Excel temporary ~$*.xls* # Excel Backup File *.xlk # PowerPoint temporary ~$*.ppt* # Visio autosave temporary files *.~vsd* ### Node ### # Logs logs npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage *.lcov # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # TypeScript cache *.tsbuildinfo # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ .rts2_cache_es/ .rts2_cache_umd/ # Optional REPL history .node_repl_history # Output of 'npm pack' # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env.test .env*.local # parcel-bundler cache (https://parceljs.org/) .parcel-cache # Next.js build output .next # Nuxt.js build / generate output .nuxt # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output .vuepress/dist # Serverless directories .serverless/ # FuseBox cache .fusebox/ # DynamoDB Local files .dynamodb/ # TernJS port file .tern-port # Stores VSCode versions used for testing VSCode extensions .vscode-test ### Octave ### # Windows default autosave extension # OSX / *nix default autosave extension # Compiled MEX binaries (all platforms) # Packaged app and toolbox files # Generated helpsearch folders # Simulink code generation folders # Matlab code generation folders # Simulink autosave extension # Simulink cache files # Octave session info ### OSX ### # General # Icon must end with two \r # Thumbnails # Files that might appear in the root of a volume # Directories potentially created on remote AFP share ### Packer ### # Cache objects packer_cache/ # Crash log crash.log # For built boxes *.box ### Patch ### ### Perl ### !Build/ .last_cover_stats /META.yml /META.json /MYMETA.* *.pm.tdy *.bs # Devel::Cover cover_db/ # Devel::NYTProf nytprof.out # Dizt::Zilla /.build/ # Module::Build Build Build.bat # Module::Install inc/ # ExtUtils::MakeMaker /blib/ /_eumm/ /*.gz # XXX: always want to commit Makefiles #/Makefile /Makefile.old /MANIFEST.bak /pm_to_blib ### Perl6 ### # Gitignore for Perl 6 (http://www.perl6.org) # As part of https://github.com/github/gitignore # precompiled files .precomp lib/.precomp ### PHPUnit ### # Covers PHPUnit # Reference: https://phpunit.de/ # Generated files .phpunit.result.cache # PHPUnit /app/phpunit.xml /phpunit.xml # Build data /build/ ### PowerShell ### # Exclude packaged modules # Exclude .NET assemblies from source ### Puppet ### # gitignore template for Puppet modules # website: https://forge.puppet.com/ # Built packages pkg/* # Should run on multiple platforms so don't check in Gemfile.lock # Tests spec/fixtures/* coverage/* # Third-party vendor/* ### PuTTY ### # Private key *.ppk ### PyCharm ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream ### PyCharm+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm+all Patch ### # Ignores the whole .idea folder and all .iml files # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 # Sonarlint plugin ### PyCharm+iml ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff # Generated files # Sensitive or high-churn files # Gradle # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake # Mongo Explorer plugin # File-based project format # IntelliJ # mpeltonen/sbt-idea plugin # JIRA plugin # Cursive Clojure plugin # Crashlytics plugin (for Android Studio and IntelliJ) # Editor-based Rest Client # Android studio 3.1+ serialized cache file ### PyCharm+iml Patch ### # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 ### pydev ### .pydevproject ### Python ### # Byte-compiled / optimized / DLL files # C extensions # Distribution / packaging # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. # Installer logs # Unit test / coverage reports # Translations # Django stuff: # Flask stuff: # Scrapy stuff: # Sphinx documentation # PyBuilder # Jupyter Notebook # IPython # pyenv # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. # PEP 582; used by e.g. github.com/David-OConnor/pyflow # Celery stuff # SageMath parsed files # Environments # Spyder project settings # Rope project settings # mkdocs documentation # mypy # Pyre type checker # pytype static type analyzer # profiling data ### R ### # History files .Rhistory .Rapp.history # Session Data files .RData # User-specific files .Ruserdata # Example code in package build process *-Ex.R # Output files from R CMD build /*.tar.gz # Output files from R CMD check /*.Rcheck/ # RStudio files .Rproj.user/ # produced vignettes vignettes/*.html vignettes/*.pdf # OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 .httr-oauth # knitr and R markdown default cache directories *_cache/ /cache/ # Temporary files created by R markdown *.utf8.md *.knit.md # R Environment Variables .Renviron ### R.Bookdown Stack ### # R package: bookdown caching files /*_files/ ### Rails ### *.rbc capybara-*.html .rspec /db/*.sqlite3 /db/*.sqlite3-journal /db/*.sqlite3-[0-9]* /public/system /coverage/ /spec/tmp rerun.txt pickle-email-*.html # Ignore all logfiles and tempfiles. /log/* /tmp/* !/log/.keep !/tmp/.keep # TODO Comment out this rule if you are OK with secrets being uploaded to the repo config/initializers/secret_token.rb config/master.key # Only include if you have production secrets in this file, which is no longer a Rails default # config/secrets.yml # dotenv, dotenv-rails # TODO Comment out these rules if environment variables can be committed .env.* ## Environment normalization: /.bundle /vendor/bundle # these should all be checked in to normalize the environment: # Gemfile.lock, .ruby-version, .ruby-gemset # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: .rvmrc # if using bower-rails ignore default bower_components path bower.json files /vendor/assets/bower_components *.bowerrc bower.json # Ignore pow environment settings .powenv # Ignore Byebug command history file. .byebug_history # Ignore node_modules # Ignore precompiled javascript packs /public/packs /public/packs-test /public/assets # Ignore yarn files /yarn-error.log # Ignore uploaded files in development /storage/* !/storage/.keep ### react ### .DS_* **/*.backup.* **/*.back.* *.sublime* psd thumb sketch ### ReactNative ### # React Native Stack Base .expo __generated__ ### ReactNative.Android Stack ### # Built application files *.aar *.ap_ *.aab # Files for the ART/Dalvik VM *.dex # Java class files # Generated files gen/ # Uncomment the following line in case you need and you don't have the release build type files in your app # release/ # Gradle files # Local configuration file (sdk path, etc) # Proguard folder generated by Eclipse proguard/ # Log Files # Android Studio Navigation editor temp files .navigation/ # Android Studio captures folder captures/ # IntelliJ .idea/workspace.xml .idea/tasks.xml .idea/gradle.xml .idea/assetWizardSettings.xml .idea/dictionaries .idea/libraries # Android Studio 3 in .gitignore file. .idea/caches .idea/modules.xml # Comment next line if keeping position of elements in Navigation Editor is relevant for you .idea/navEditor.xml # Keystore files # Uncomment the following lines if you do not want to check your keystore files in. #*.jks #*.keystore # External native build folder generated in Android Studio 2.2 and later .externalNativeBuild .cxx/ # Google Services (e.g. APIs or Firebase) # google-services.json # Freeline freeline.py freeline/ freeline_project_description.json # fastlane fastlane/report.xml fastlane/Preview.html fastlane/screenshots fastlane/test_output #fastlane/readme.md # Version control vcs.xml # lint lint/intermediates/ lint/generated/ lint/outputs/ lint/tmp/ # lint/reports/ ### ReactNative.Buck Stack ### buck-out/ .buckconfig.local .buckd/ .buckversion .fakebuckversion ### ReactNative.Gradle Stack ### .gradle # Ignore Gradle GUI config gradle-app.setting # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) !gradle-wrapper.jar # Cache of project .gradletasknamecache # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties ### ReactNative.Linux Stack ### # temporary files which can be created if a process still has a handle open of a deleted file # KDE directory preferences # Linux trash folder which might appear on any partition or disk # .nfs files are created when an open file is removed but is still being accessed ### ReactNative.Node Stack ### # Logs # Diagnostic reports (https://nodejs.org/api/report.html) # Runtime data # Directory for instrumented libs generated by jscoverage/JSCover # Coverage directory used by tools like istanbul # nyc test coverage # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) # Bower dependency directory (https://bower.io/) # node-waf configuration # Compiled binary addons (https://nodejs.org/api/addons.html) # Dependency directories # TypeScript v1 declaration files # TypeScript cache # Optional npm cache directory # Optional eslint cache # Microbundle cache # Optional REPL history # Output of 'npm pack' # Yarn Integrity file # dotenv environment variables file # parcel-bundler cache (https://parceljs.org/) # Next.js build output # Nuxt.js build / generate output # Gatsby files # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output # Serverless directories # FuseBox cache # DynamoDB Local files # TernJS port file # Stores VSCode versions used for testing VSCode extensions ### ReactNative.Xcode Stack ### # Xcode # # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore ## User settings xcuserdata/ ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) *.xcscmblueprint *.xccheckout ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) DerivedData/ *.moved-aside *.pbxuser !default.pbxuser *.mode1v3 !default.mode1v3 *.mode2v3 !default.mode2v3 *.perspectivev3 !default.perspectivev3 ## Gcc Patch /*.gcno ### ReactNative.macOS Stack ### # General # Icon must end with two \r Icon # Thumbnails # Files that might appear in the root of a volume # Directories potentially created on remote AFP share ### Redis ### # Ignore redis binary dump (dump.rdb) files *.rdb ### ROOT ### # ROOT Home Page : https://root.cern.ch/ # ROOT Used by Experimental Physicists, not necessarily HEP # ROOT based on C++ # Files generated by ROOT, observed with v6.xy *.pcm ### Ruby ### /.config /InstalledFiles /pkg/ /spec/reports/ /spec/examples.txt /test/tmp/ /test/version_tmp/ /tmp/ # Used by dotenv library to load environment variables. # .env # Ignore Byebug command history file. ## Specific to RubyMotion: .dat* .repl_history *.bridgesupport build-iPhoneOS/ build-iPhoneSimulator/ ## Specific to RubyMotion (use of CocoaPods): # We recommend against adding the Pods directory to your .gitignore. However # you should judge for yourself, the pros and cons are mentioned at: # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control # vendor/Pods/ ## Documentation cache and generated files: /.yardoc/ /_yardoc/ /doc/ /rdoc/ /.bundle/ /lib/bundler/man/ # for a library or gem, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # Gemfile.lock # .ruby-version # .ruby-gemset # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: # Used by RuboCop. Remote config files pulled in from inherit_from directive. # .rubocop-https?--* ### Ruby Patch ### # Used by RuboCop. Remote config files pulled in from inherit_from directive. # .rubocop-https?--* ### Rust ### # Generated by Cargo # will have compiled files and executables # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock ### SBT ### # Simple Build Tool # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control dist/* lib_managed/ src_managed/ project/boot/ project/plugins/project/ .history .lib/ ### Scala ### ### Serverless ### # Ignore build directory .serverless ### Sonar ### #Sonar generated dir /.sonar/ ### SonarQube ### # SonarQube ignore files. # https://docs.sonarqube.org/display/SCAN/Analyzing+with+SonarQube+Scanner # Sonar Scanner working directories .sonar/ .scannerwork/ # http://www.sonarlint.org/commandline/ # SonarLint working directories, configuration files (including credentials) #.sonarlint/ ### Spark ### *#*# *.#* *.pyc *.pyo .ensime .ensime_cache/ .ensime_lucene .generated-mima* R-unit-tests.log R/unit-tests.out R/cran-check.out R/pkg/vignettes/sparkr-vignettes.html R/pkg/tests/fulltests/Rplots.pdf build/*.jar build/apache-maven* build/scala* build/zinc* cache checkpoint conf/*.cmd conf/*.conf conf/*.properties conf/*.sh conf/*.xml conf/java-opts conf/slaves derby.log dev/create-release/*final dev/create-release/*txt dev/pr-deps/ docs/_site docs/api sql/docs sql/site lint-r-report.log log/ logs/ project/build/target/ project/plugins/lib_managed/ project/plugins/project/build.properties project/plugins/src_managed/ project/plugins/target/ python/lib/pyspark.zip python/deps python/test_coverage/coverage_data python/test_coverage/htmlcov python/pyspark/python reports/ scalastyle-on-compile.generated.xml scalastyle-output.xml scalastyle.txt spark-*-bin-*.tgz spark-tests.log streaming-tests.log unit-tests.log work/ docs/.jekyll-metadata # For Hive TempStatsStore/ metastore/ metastore_db/ sql/hive-thriftserver/test_warehouses warehouse/ spark-warehouse/ # For R session data .RHistory *.Rproj *.Rproj.* .Rproj.user # For SBT .jvmopts ### Splunk ### # gitignore template for Splunk apps # documentation: http://docs.splunk.com/Documentation/Splunk/6.2.3/admin/Defaultmetaconf # Splunk local meta file local.meta # Splunk local folder local ### Spreadsheet ### *.xlr *.xls *.xlsx ### SSH ### **/.ssh/id_* **/.ssh/*_id_* **/.ssh/known_hosts ### SublimeText ### # Cache files for Sublime Text *.tmlanguage.cache *.tmPreferences.cache *.stTheme.cache # Workspace files are user-specific *.sublime-workspace # Project files should be checked into the repository, unless a significant # proportion of contributors will probably not be using Sublime Text # *.sublime-project # SFTP configuration file sftp-config.json # Package control specific files Package Control.last-run Package Control.ca-list Package Control.ca-bundle Package Control.system-ca-bundle Package Control.cache/ Package Control.ca-certs/ Package Control.merged-ca-bundle Package Control.user-ca-bundle oscrypto-ca-bundle.crt bh_unicode_properties.cache # Sublime-github package stores a github token in this file # https://packagecontrol.io/packages/sublime-github GitHub.sublime-settings ### SVN ### .svn/ ### Terraform ### # Local .terraform directories **/.terraform/* # .tfstate files *.tfstate *.tfstate.* # Crash log files # Ignore any .tfvars files that are generated automatically for each Terraform run. Most # .tfvars files are managed as part of configuration and so should be included in # version control. # example.tfvars # Ignore override files as they are usually used to override resources locally and so # are not checked in override.tf override.tf.json *_override.tf *_override.tf.json # Include override files you do wish to add to version control using negated pattern # !example_override.tf # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan # example: *tfplan* ### Terragrunt ### # terragrunt cache directories **/.terragrunt-cache/* ### TortoiseGit ### # Project-level settings /.tgitconfig ### Vagrant ### # General .vagrant/ # Log files (if you are creating logs in debug mode, uncomment this) # *.log ### Vagrant Patch ### ### venv ### # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ # XXX: prevents committing scripts at ~/bin #[Bb]in [Ii]nclude # XXX: prevents committing submodule lib #[Ll]ib [Ll]ib64 [Ll]ocal #[Ss]cripts pyvenv.cfg pip-selfcheck.json ### VirtualEnv ### # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ ### Julia ### # Files generated by invoking Julia with --code-coverage *.jl.cov *.jl.*.cov # Files generated by invoking Julia with --track-allocation *.jl.mem # System-specific files and directories generated by the BinaryProvider and BinDeps packages # They contain absolute paths specific to the host computer, and so should not be committed deps/deps.jl deps/build.log deps/downloads/ deps/usr/ deps/src/ # Build artifacts for creating documentation generated by the Documenter package docs/build/ docs/site/ # File generated by Pkg, the package manager, based on a corresponding Project.toml # It records a fixed state of all packages used by the project. As such, it should not be # committed for packages, but should be committed for applications that require a static # environment. Manifest.toml ### VisualStudioCode ### ### VisualStudioCode Patch ### # Ignore all local history of files .ionide ### vs ### ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ # XXX: prevents committing scripts at ~/bin #[Bb]in/ [Oo]bj/ [Ll]og/ [Ll]ogs/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUnit *.VisualState.xml TestResult.xml nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.meta *.iobj *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp_proj *_wpftmp.csproj *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool coverage*[.json, .xml, .info] # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # NuGet Symbol Packages *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- [Bb]ackup.rdl *- [Bb]ackup ([0-9]).rdl *- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ ### vscode ### ### Vue ### # gitignore template for Vue.js projects # Recommended template: Node.gitignore # TODO: where does this rule come from? docs/_book # TODO: where does this rule come from? # XXX: covers up standard Python unit test path # test/ ### Vuejs ### # Recommended template: Node.gitignore npm-debug.log yarn-error.log ### Waf ### # For projects that use the Waf build system: https://waf.io/ # Dot-hidden on Unix-like systems .waf-*-*/ .waf3-*-*/ # Hidden directory on Windows (no dot) waf-*-*/ waf3-*-*/ # Lockfile .lock-waf_*_build ### Windows ### # Windows thumbnail cache files Thumbs.db Thumbs.db:encryptable ehthumbs.db ehthumbs_vista.db # Dump file *.stackdump # Folder config file [Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ # Windows Installer files *.msix # Windows shortcuts *.lnk ### Xcode ### # Xcode # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore ### Xcode Patch ### *.xcodeproj/* !*.xcodeproj/project.pbxproj !*.xcodeproj/xcshareddata/ !*.xcworkspace/contents.xcworkspacedata **/xcshareddata/WorkspaceSettings.xcsettings ### XcodeInjection ### # Code Injection # After new code Injection tools there's a generated folder /iOSInjectionProject # https://github.com/johnno1962/injectionforxcode iOSInjectionProject/ ### Gradle ### # Ignore Gradle GUI config # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) # Cache of project # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties ### Gradle Patch ### **/build/ ### VisualStudio ### # User-specific files # User-specific files (MonoDevelop/Xamarin Studio) # Mono auto generated files # Build results # Visual Studio 2015/2017 cache/options directory # Uncomment if you have tasks that create the project's static files in wwwroot # Visual Studio 2017 auto generated files # MSTest test Results # NUnit # Build Results of an ATL Project # Benchmark Results # .NET Core # StyleCop # Files built by Visual Studio # Chutzpah Test files # Visual C++ cache files # Visual Studio profiler # Visual Studio Trace Files # TFS 2012 Local Workspace # Guidance Automation Toolkit # ReSharper is a .NET coding add-in # TeamCity is a build add-in # DotCover is a Code Coverage Tool # AxoCover is a Code Coverage Tool # Coverlet is a free, cross platform Code Coverage Tool # Visual Studio code coverage results # NCrunch # MightyMoose # Web workbench (sass) # Installshield output folder # DocProject is a documentation generator add-in # Click-Once directory # Publish Web Output # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted # NuGet Packages # NuGet Symbol Packages # The packages folder can be ignored because of Package Restore # except build/, which is used as an MSBuild target. # Uncomment if necessary however generally it will be regenerated when needed # NuGet v3's project.json files produces more ignorable files # Microsoft Azure Build Output # Microsoft Azure Emulator # Windows Store app package directories and files # Visual Studio cache files # files ending in .cache can be ignored # but keep track of directories ending in .cache # Others # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) # RIA/Silverlight projects # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) # SQL Server files # Business Intelligence projects # Microsoft Fakes # GhostDoc plugin setting file # Node.js Tools for Visual Studio # Visual Studio 6 build log # Visual Studio 6 workspace options file # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) # Visual Studio LightSwitch build output # Paket dependency manager # FAKE - F# Make # CodeRush personal settings # Python Tools for Visual Studio (PTVS) # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio # Telerik's JustMock configuration file # BizTalk build output # OpenCover UI analysis results # Azure Stream Analytics local run output # MSBuild Binary and Structured Log # NVidia Nsight GPU debugger configuration file # MFractors (Xamarin productivity tool) working folder # Local History for Visual Studio # BeatPulse healthcheck temp database # Backup folder for Package Reference Convert tool in Visual Studio 2017 # Ionide (cross platform F# VS Code tools) working folder # End of https://www.toptal.com/developers/gitignore/api/ansible,apachehadoop,appcode,appengine,archive,archives,archlinuxpackages,audio,autotools,backup,basic,batch,bittorrent,c,c++,certificates,chefcookbook,clojure,cloud9,cmake,code,code-java,codeblocks,compressed,compressedarchive,compression,data,database,datarecovery,diff,direnv,diskimage,docfx,docpress,docz,dotenv,dotfilessh,dotsettings,dropbox,eclipse,emacs,erlang,executable,firebase,flask,git,gitbook,go,gpg,gradle,grails,groovy,grunt,haskell,helm,homebrew,hugo,images,intellij,intellij+all,intellij+iml,java,java-web,jenv,jetbrains,jetbrains+all,jetbrains+iml,jmeter,julia,jupyternotebooks,kotlin,lamp,latex,less,linux,lua,macos,matlab,maven,mercurial,microsoftoffice,node,octave,osx,packer,patch,perl,perl6,phpunit,powershell,puppet,putty,pycharm,pycharm+all,pycharm+iml,pydev,python,r,rails,react,reactnative,redis,root,ruby,rust,sbt,scala,serverless,sonar,sonarqube,spark,splunk,spreadsheet,ssh,sublimetext,svn,terraform,terragrunt,tortoisegit,vagrant,venv,virtualenv,visualstudio,visualstudiocode,vs,vscode,vue,vuejs,waf,windows,xcode,xcodeinjection,zsh # for DevOps-Bash-tools packages/*.sh scripts ## needed to unmask directory ignore above otherwise !packages/*packages*.sh isn't respected !packages/ !packages/*.sh ================================================ FILE: .gitlab-ci.yml ================================================ # vim:ts=2:sts=2:sw=2:et # # Author: Hari Sekhon # Date: Sun Feb 23 19:02:10 2020 +0000 # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G i t L a b C I # ============================================================================ # # https://docs.gitlab.com/ee/ci/yaml/README.html #include: '.gitlab/*.y*ml' image: ubuntu:18.04 job: before_script: - setup/ci_bootstrap.sh script: - make init && make ci test ================================================ FILE: .gitmodules ================================================ ================================================ FILE: .hound.yml ================================================ # # Author: Hari Sekhon # Date: 2020-09-23 10:28:21 +0100 (Wed, 23 Sep 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # --- # https://intercom.help/hound/en/articles/2138537-flake8 flake8: enabled: true #config_file: .flake8 # http://help.houndci.com/en/articles/2138564-shellcheck shellcheck: enabled: true #config_file: .shellcheck.yml # http://help.houndci.com/en/articles/2138524-golint golint: enabled: false #fail_on_violations: true ================================================ FILE: .mdl.rb ================================================ #!/usr/bin/env ruby # vim:ts=4:sts=4:sw=4:et:filetype=ruby # # Author: Hari Sekhon # Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # all #exclude_rule 'MD001' #exclude_rule 'MD003' #exclude_rule 'MD005' exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af #exclude_rule 'MD012' exclude_rule 'MD013' # long lines cannot be split if they are URLs #exclude_rule 'MD022' exclude_rule 'MD024' # Multiple headers with the same content #exclude_rule 'MD025' exclude_rule 'MD026' # Trailing punctuation in header - sometimes I want to do etc. or ... at the end of a heading #exclude_rule 'MD031' #exclude_rule 'MD032' exclude_rule 'MD033' # inline HTML is important for formatting exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines #exclude_rule 'MD039' #exclude_rule 'MD056' ================================================ FILE: .mdlrc ================================================ mdlrc_dir = File.expand_path('..', __FILE__) style_file = File.join(mdlrc_dir, '.mdl.rb') style style_file ================================================ FILE: .pre-commit-config.yaml ================================================ # # Author: Hari Sekhon # Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) # # vim:ts=2:sts=2:sw=2:et # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P r e - C o m m i t # ============================================================================ # --- fail_fast: false #exclude: *.tmp$ repos: # will accept anything that 'git clone' understands # this means you can set this to a local git repo to develop your own hook repos interactively - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: - id: check-yaml # Common errors #- id: end-of-file-fixer # ruins .gitignore Icon\r - id: trailing-whitespace args: [--markdown-linebreak-ext=md] # Git style - id: check-added-large-files - id: check-merge-conflict - id: check-vcs-permalinks #- id: forbid-new-submodules # Cross platform - id: check-case-conflict - id: mixed-line-ending args: [--fix=lf] # Security - id: detect-aws-credentials args: ['--allow-missing-credentials'] # rewrites python files with useless changes like changing single quotes to double quotes #- repo: https://github.com/psf/black # rev: 24.8.0 # hooks: # - id: black # Git secrets Leaks - repo: https://github.com/awslabs/git-secrets.git # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: # # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file # rev: 5357e18 hooks: - id: git-secrets - repo: https://github.com/markdownlint/markdownlint rev: v0.12.0 hooks: - id: markdownlint name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl args: [-s, .mdl.rb] language: ruby files: \.(md|mdown|markdown)$ ================================================ FILE: .pylintrc ================================================ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # P y L i n t C o n f i g # ============================================================================ # # pylint --generate-rcfile >> .pylintrc [MAIN] # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no # Clear in-memory caches upon conclusion of linting. Useful if running pylint # in a server-like mode. clear-cache-post-run=no # Load and enable all available extensions. Use --list-extensions to see a list # all available extensions. #enable-all-extensions= # In error mode, messages with a category besides ERROR or FATAL are # suppressed, and no reports are done by default. Error mode is compatible with # disabling specific errors. #errors-only= # Always return a 0 (non-error) status code, even if lint errors are found. # This is primarily useful in continuous integration scripts. #exit-zero= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. extension-pkg-allow-list= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) extension-pkg-whitelist= # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages # specified are enabled, while categories only check already-enabled messages. fail-on= # Specify a score threshold under which the program will exit with error. fail-under=10 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. #from-stdin= # Files or directories to be skipped. They should be base names, not paths. ignore=CVS # Add files or directories matching the regular expressions patterns to the # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, # it can't be used as an escape character. ignore-paths= # Files or directories matching the regular expression patterns are skipped. # The regex matches against base names, not paths. The default value ignores # Emacs file locks ignore-patterns=^\.# # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use, and will cap the count on Windows to # avoid hangs. jobs=1 # Control the amount of potential inferred values when inferring a single # object. This can help the performance when dealing with large functions or # complex, nested conditions. limit-inference-results=100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= # Pickle collected data for later comparisons. persistent=yes # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. py-version=3.11 # Discover python modules and packages in the file system subtree. recursive=no # Add paths to the list of the source roots. Supports globbing patterns. The # source root is an absolute path or a path relative to the current working # directory used to determine a package namespace for modules located under the # source root. source-roots= # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no # In verbose mode, extra non-checker-related info will be displayed. #verbose= [BASIC] # Naming style matching correct argument names. argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- # naming-style. If left empty, argument names will be checked with the set # naming style. #argument-rgx= # Naming style matching correct attribute names. attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- # style. If left empty, attribute names will be checked with the set naming # style. #attr-rgx= # Bad variable names which should always be refused, separated by a comma. bad-names=foo, bar, baz, toto, tutu, tata # Bad variable names regexes, separated by a comma. If names match any regex, # they will always be refused bad-names-rgxs= # Naming style matching correct class attribute names. class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- # attribute-naming-style. If left empty, class attribute names will be checked # with the set naming style. #class-attribute-rgx= # Naming style matching correct class constant names. class-const-naming-style=UPPER_CASE # Regular expression matching correct class constant names. Overrides class- # const-naming-style. If left empty, class constant names will be checked with # the set naming style. #class-const-rgx= # Naming style matching correct class names. class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming- # style. If left empty, class names will be checked with the set naming style. #class-rgx= # Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- # style. If left empty, constant names will be checked with the set naming # style. #const-rgx= # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 # Naming style matching correct function names. function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- # naming-style. If left empty, function names will be checked with the set # naming style. #function-rgx= # Good variable names which should always be accepted, separated by a comma. good-names=i, j, k, ex, Run, _ # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted good-names-rgxs= # Include a hint for the correct naming format with invalid-name. include-naming-hint=no # Naming style matching correct inline iteration names. inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides # inlinevar-naming-style. If left empty, inline iteration names will be checked # with the set naming style. #inlinevar-rgx= # Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- # style. If left empty, method names will be checked with the set naming style. #method-rgx= # Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- # style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=^_ # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. # These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty # Regular expression matching correct type alias names. If left empty, type # alias names will be checked with the set naming style. #typealias-rgx= # Regular expression matching correct type variable names. If left empty, type # variable names will be checked with the set naming style. #typevar-rgx= # Naming style matching correct variable names. variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- # naming-style. If left empty, variable names will be checked with the set # naming style. #variable-rgx= [CLASSES] # Warn about protected attribute access inside special methods check-protected-access-in-special-methods=no # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp, asyncSetUp, __post_init__ # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs [DESIGN] # List of regular expressions of class ancestor names to ignore when counting # public methods (see R0903) exclude-too-few-public-methods= # List of qualified class names to ignore when counting class parents (see # R0901) ignored-parents= # Maximum number of arguments for function / method. max-args=5 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr=5 # Maximum number of branch for function / method body. max-branches=12 # Maximum number of locals for function / method body. max-locals=15 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of public methods for a class (see R0904). max-public-methods=20 # Maximum number of return / yield for function / method body. max-returns=6 # Maximum number of statements in function / method body. max-statements=50 # Minimum number of public methods for a class (see R0903). min-public-methods=2 [EXCEPTIONS] # Exceptions that will emit a warning when caught. overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Maximum number of characters on a single line. max-line-length=120 # Maximum number of lines in a module. max-module-lines=1000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no [IMPORTS] # List of modules that can be imported at any level, not just the top level # one. allow-any-import-level= # Allow explicit reexports by alias from a package __init__. allow-reexport-from-package=no # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no # Deprecated modules which should not be used, separated by a comma. deprecated-modules= # Output a graph (.gv or any supported image format) of external dependencies # to the given file (report RP0402 must not be disabled). ext-import-graph= # Output a graph (.gv or any supported image format) of all (i.e. internal and # external) dependencies to the given file (report RP0402 must not be # disabled). import-graph= # Output a graph (.gv or any supported image format) of internal dependencies # to the given file (report RP0402 must not be disabled). int-import-graph= # Force import order to recognize a module as part of the standard # compatibility libraries. known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant # Couples of modules and preferred modules, separated by a comma. preferred-modules= [LOGGING] # The type of string formatting that logging methods do. `old` means using % # formatting, `new` is for `{}` formatting. logging-format-style=old # Logging modules to check that the string format arguments are in logging # function parameter format. logging-modules=logging [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, # UNDEFINED. confidence=HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once). You can also use "--disable=all" to # disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, suppressed-message, useless-suppression, deprecated-pragma, use-symbolic-message-instead, missing-class-docstring, missing-function-docstring, super-with-arguments, consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. enable=c-extension-no-member [METHOD_ARGS] # List of qualified names (i.e., library.method) which require a timeout # parameter e.g. 'requests.api.get,requests.api.post' timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME, XXX, TODO # Regular expression of note tags to take in consideration. notes-rgx= [REFACTORING] # Maximum number of nested blocks for function / method body max-nested-blocks=5 # Complete name of functions that never returns. When checking for # inconsistent-return-statements if a never returning function is called then # it will be considered as an explicit return statement and no message will be # printed. never-returning-functions=sys.exit,argparse.parse_error [REPORTS] # Python expression which should return a score less than or equal to 10. You # have access to the variables 'fatal', 'error', 'warning', 'refactor', # 'convention', and 'info' which contain the number of messages in each # category, as well as 'statement' which is the total number of statements # analyzed. This score is used by the global evaluation report (RP0004). evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details. msg-template= # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio). You can also give a reporter class, e.g. # mypackage.mymodule.MyReporterClass. #output-format= # Tells whether to display a full report or only the messages. reports=no # Activate the evaluation score. score=yes [SIMILARITIES] # Comments are removed from the similarity computation ignore-comments=yes # Docstrings are removed from the similarity computation ignore-docstrings=yes # Imports are removed from the similarity computation ignore-imports=yes # Signatures are removed from the similarity computation ignore-signatures=yes # Minimum lines number of a similarity. min-similarity-lines=4 [SPELLING] # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 # Spelling dictionary name. No available dictionaries : You need to install # both the python package and the system dependency for enchant to work.. spelling-dict= # List of comma separated words that should be considered directives if they # appear at the beginning of a comment and should not be checked. spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains the private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to the private dictionary (see the # --spelling-private-dict-file option) instead of raising a message. spelling-store-unknown-words=no [STRING] # This flag controls whether inconsistent-quotes generates a warning when the # character used as a quote delimiter is used inconsistently within a module. check-quote-consistency=no # This flag controls whether the implicit-str-concat should generate a warning # on implicit string concatenation in sequences defined over several lines. check-str-concat-over-line-jumps=no [TYPECHECK] # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members= # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. ignore-none=yes # This flag controls whether pylint should warn about no-member and similar # checks whenever an opaque object is returned when inferring. The inference # can return multiple potential results while evaluating a Python object, but # some branches might not be evaluated, which results in partial inference. In # that case, it might be useful to still emit no-member and other checks for # the rest of the inferred objects. ignore-on-opaque-inference=yes # List of symbolic message names to ignore for Mixin members. ignored-checks-for-mixins=no-member, not-async-context-manager, not-context-manager, attribute-defined-outside-init # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. missing-member-hint=yes # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 # The total number of similar names that should be taken in consideration when # showing a hint for a missing member. missing-member-max-choices=1 # Regex pattern to define which classes are considered mixins. mixin-class-rgx=.*[Mm]ixin # List of decorators that change the signature of a decorated function. signature-mutators= [VARIABLES] # List of additional names supposed to be defined in builtins. Remember that # you should avoid defining new builtins when possible. additional-builtins= # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes # List of names allowed to shadow builtins allowed-redefined-builtins= # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_, _cb # A regular expression matching the name of dummy variables (i.e. expected to # not be used). dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ # Argument names that match this expression will be ignored. ignored-argument-names=_.*|^ignored_|^unused_ # Tells whether we should check for unused import in __init__ files. init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io ================================================ FILE: .scrutinizer.yml ================================================ # # Author: Hari Sekhon # Date: 2020-03-17 11:41:13 +0000 (Tue, 17 Mar 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # build: image: default-bionic nodes: auto: commands: - repo="${SCRUTINIZER_PROJECT#*/}"; git clone "https://github.com/$repo" build - cd ~/build - pwd - ls -l - make init - make ci test ================================================ FILE: .semaphore/semaphore.yml ================================================ # # Author: Hari Sekhon # Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) # # vim:ts=2:sts=2:sw=2:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # S e m a p h o r e C I # ============================================================================ # # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 name: DevOps-Bash-tools agent: # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: - name: Linux build run: when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})" #execution_time_limit: # hours: 2 task: #env_vars: # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python #- name: PYTHON # value: python3 #- name: PIP # value: pip3 prologue: commands: - cache restore # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server # # causes error: # # Not replacing deleted config file /etc/ssh/sshd_config # #- sudo rm -f /etc/ssh/sshd_config - export DEBIAN_FRONTEND=noninteractive - sudo -E apt-get update - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" - sudo dpkg --configure -a --force-confmiss --force-confnew #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections #- sudo dpkg --configure -a --force-confdef --force-confold #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build commands: - checkout - setup/ci_bootstrap.sh - make init - make ci - make test epilogue: commands: - cache store - name: Mac build run: when: "branch = 'master'" task: # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) #env_vars: # to match /usr/local/bin/pip version from $PATH #- name: PYTHON # value: python3 # must be quoted to force string, otherwise pipeline fails to run with this parsing error: # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] #- name: DEBUG # value: "1" agent: # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 os_image: macos-xcode15 prologue: commands: - cache restore # fix for: # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available. - brew install openssl - brew reinstall python - brew reinstall wget # avoid Mac SSL errors: # # ERROR: Loading command: install (LoadError) # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle # ERROR: While executing gem ... (NoMethodError) # undefined method `invoke_with_build_args' for nil:NilClass# # - rbenv global system # also considered this: # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done # # fix for python vs pip version mismatch - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python jobs: - name: build commands: - checkout - make init - make ci - make test epilogue: commands: - cache store ================================================ FILE: .sonarcloud.properties ================================================ sonar.host.url=https://sonarcloud.io ================================================ FILE: .sonarlint/connectedMode.json ================================================ { "sonarCloudOrganization": "harisekhon", "projectKey": "HariSekhon_DevOps-Bash-tools" } ================================================ FILE: .terraformignore ================================================ # # Author: Hari Sekhon # Date: 2020-09-24 17:08:01 +0100 (Thu, 24 Sep 2020) # # vim:ts=4:sts=4:sw=4:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Requires Terraform 0.12.11+ # # Prevents upload of paths to Terraform Cloud # # Same format as .gitignore # - directories must end with forward slash / # - negate matches using ! # # Only works at the root of the config directory # https://www.terraform.io/docs/backends/types/remote.html#excluding-files-from-upload-with-terraformignore # defaults .git/ .terraform/ # custom .hg/ .svn/ .ssh/ github/ gitroot/ mercurial/ hg/ hgroot/ svn/ svnroot/ # exclude all hidden dot files .* ================================================ FILE: .trivyignore ================================================ #aws-access-key-id #aws-account-id gcp-service-account ================================================ FILE: .zlogin ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-13 18:58:03 +0000 (Fri, 13 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Z S H L o g i n # ============================================================================ # ================================================ FILE: .zlogout ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-13 18:58:03 +0000 (Fri, 13 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Z S H L o g o u t # ============================================================================ # ================================================ FILE: .zprofile ================================================ # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # (forked from .bash_profile) # ============================================================================ # # Z S H P r o f i l e # ============================================================================ # # goes horribly wrong - too much advanced bash #if [[ -e ~/.profile ]]; then # emulate sh -c 'source ~/.profile' #fi ================================================ FILE: .zshenv ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-13 18:58:03 +0000 (Fri, 13 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Z S H E n v # ============================================================================ # # sourced by both interactive shells and scripts # # be careful with you put in here ================================================ FILE: .zshrc ================================================ #!/usr/bin/env bash # shellcheck disable=SC1091 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) # (forked from .bashrc) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Z S H # ============================================================================ # # https://wiki.archlinux.org/index.php/Zsh # goes horribly wrong - too much advanced bash #if [[ -e ~/.bashrc ]]; then # emulate sh -c 'source ~/.bashrc' #fi autoload -Uz compinit promptinit compinit # completes ssh/scp/sftp hostnames as long as HashKnownHosts not set in ~/.ssh/config promptinit # prompt -l - list themes # prompt -p - preview themes #prompt suse # install Oh-My-ZSH # sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" # custom themes: # mkdir ~/.zprompts # fpath=("$HOME/.zprompts" "$fpath[@]") # uniq all items in $PATH and $path array typeset -U PATH path path=("$HOME/.local/bin" "$HOME/bin" "$path[@]") export PATH # autocompletion with an arrow-key driven interface - tab twice to enable zstyle ':completion:*' menu select # autocompletions with sudo # allows zsh completion scripts run commands with sudo privileges - do not enable if using untrusted autocompletion scripts!! #zstyle ':completion::complete:*' gain-privileges 1 # ============================================================================ # # S e t t i n g s # ============================================================================ # # compatible style with other shells #set -o AUTO_CD # casse insensitive, underscores stripped setopt AUTO_CD setopt COMPLETE_ALIASES setopt CORRECT export SPROMPT="Correct %R to %r? [Yes, No, Abort, Edit] " autoload U colors && colors # expand wilcard expansion on unquoted variables like Bash setopt GLOB_SUBST export PATH="$PATH:/opt/homebrew/bin/" # ============================================================================ # # Oh-My-ZSH # ============================================================================ # # If you come from bash you might have to change your $PATH. # export PATH=$HOME/bin:/usr/local/bin:$PATH # Path to your oh-my-zsh installation. export ZSH="/Users/hari.sekhon/.oh-my-zsh" # Set name of the theme to load --- if set to "random", it will # load a random theme each time oh-my-zsh is loaded, in which case, # to know which specific one was loaded, run: echo $RANDOM_THEME # See https://github.com/ohmyzsh/ohmyzsh/wiki/Themes ZSH_THEME="robbyrussell" # also messed up #ZSH_THEME="agnoster" # messes up both Terminal and iTerm2 from both brew and git cloned installations #if [ -f /usr/local/opt/powerlevel9k/powerlevel9k.zsh-theme ]; then # source /usr/local/opt/powerlevel9k/powerlevel9k.zsh-theme #fi # # Oh-My-ZSH ~/.oh-my-zsh/custom/themes/powerlevel9k #ZSH_THEME="powerlevel9k/powerlevel9k" # Set list of themes to pick from when loading at random # Setting this variable when ZSH_THEME=random will cause zsh to load # a theme from this variable instead of looking in ~/.oh-my-zsh/themes/ # If set to an empty array, this variable will have no effect. # ZSH_THEME_RANDOM_CANDIDATES=( "robbyrussell" "agnoster" ) # Uncomment the following line to use case-sensitive completion. # CASE_SENSITIVE="true" # Uncomment the following line to use hyphen-insensitive completion. # Case-sensitive completion must be off. _ and - will be interchangeable. # HYPHEN_INSENSITIVE="true" # Uncomment the following line to disable bi-weekly auto-update checks. # DISABLE_AUTO_UPDATE="true" # Uncomment the following line to automatically update without prompting. # DISABLE_UPDATE_PROMPT="true" # Uncomment the following line to change how often to auto-update (in days). # export UPDATE_ZSH_DAYS=13 # Uncomment the following line if pasting URLs and other text is messed up. # DISABLE_MAGIC_FUNCTIONS=true # Uncomment the following line to disable colors in ls. # DISABLE_LS_COLORS="true" # Uncomment the following line to disable auto-setting terminal title. # DISABLE_AUTO_TITLE="true" # Uncomment the following line to enable command auto-correction. # ENABLE_CORRECTION="true" # Uncomment the following line to display red dots whilst waiting for completion. # COMPLETION_WAITING_DOTS="true" # Uncomment the following line if you want to disable marking untracked files # under VCS as dirty. This makes repository status check for large repositories # much, much faster. # DISABLE_UNTRACKED_FILES_DIRTY="true" # Uncomment the following line if you want to change the command execution time # stamp shown in the history command output. # You can set one of the optional three formats: # "mm/dd/yyyy"|"dd.mm.yyyy"|"yyyy-mm-dd" # or set a custom format using the strftime function format specifications, # see 'man strftime' for details. # HIST_STAMPS="mm/dd/yyyy" # Would you like to use another custom folder than $ZSH/custom? # ZSH_CUSTOM=/path/to/new-custom-folder # Which plugins would you like to load? # Standard plugins can be found in ~/.oh-my-zsh/plugins/* # Custom plugins may be added to ~/.oh-my-zsh/custom/plugins/ # Example format: plugins=(rails git textmate ruby lighthouse) # Add wisely, as too many plugins slow down shell startup. plugins=(git) source $ZSH/oh-my-zsh.sh # User configuration # export MANPATH="/usr/local/man:$MANPATH" # You may need to manually set your language environment # export LANG=en_US.UTF-8 # Preferred editor for local and remote sessions # if [[ -n $SSH_CONNECTION ]]; then # export EDITOR='vim' # else # export EDITOR='mvim' # fi # Compilation flags # export ARCHFLAGS="-arch x86_64" # Set personal aliases, overriding those provided by oh-my-zsh libs, # plugins, and themes. Aliases can be placed here, though oh-my-zsh # users are encouraged to define aliases within the ZSH_CUSTOM folder. # For a full list of active aliases, run `alias`. # # Example aliases # alias zshconfig="mate ~/.zshrc" # alias ohmyzsh="mate ~/.oh-my-zsh" # ============================================================================ # # Lines configured by zsh-newuser-install HISTFILE=~/.histfile HISTSIZE=1000 SAVEHIST=1000 setopt appendhistory autocd extendedglob nomatch notify unsetopt beep bindkey -e # End of lines configured by zsh-newuser-install # The following lines were added by compinstall zstyle :compinstall filename '/home/hari/.zshrc' autoload -Uz compinit compinit # End of lines added by compinstall # added by travis gem - Travis is legacy now, don't bother with this #[ -f /Users/hari/.travis/travis.sh ] && source /Users/hari/.travis/travis.sh if type -P direnv &>/dev/null; then eval "$(direnv hook zsh)" fi autoload -U +X bashcompinit && bashcompinit complete -o nospace -C /Users/hari/bin/terraform terraform complete -o nospace -C /Users/hari/bin/terraform tf complete -o nospace -C /usr/local/bin/terragrunt terragrunt #THIS MUST BE AT THE END OF THE FILE FOR SDKMAN TO WORK!!! export SDKMAN_DIR="/Users/hari/.sdkman" [[ -s "/Users/hari/.sdkman/bin/sdkman-init.sh" ]] && source "/Users/hari/.sdkman/bin/sdkman-init.sh" ================================================ FILE: DOCKER_STATUS.md ================================================ # Docker Status Page generated by `docker_generate_status_page.sh` in [HariSekhon/DevOps-Bash-tools](https://github.com/HariSekhon/DevOps-Bash-tools) This page relies on shields.io which is slow so a lot of it may not load properly the first time so you may need to do one or more page reloads to get all the badges to load. 50 docker repos - `:latest` tag build status: [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/alluxio.svg)](https://hub.docker.com/r/harisekhon/alluxio/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/alluxio.svg)](https://hub.docker.com/r/harisekhon/alluxio) - [harisekhon/alluxio](https://hub.docker.com/r/harisekhon/alluxio) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/alpine-dev.svg)](https://hub.docker.com/r/harisekhon/alpine-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/alpine-dev.svg)](https://hub.docker.com/r/harisekhon/alpine-dev) - [harisekhon/alpine-dev](https://hub.docker.com/r/harisekhon/alpine-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/alpine-github.svg)](https://hub.docker.com/r/harisekhon/alpine-github/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/alpine-github.svg)](https://hub.docker.com/r/harisekhon/alpine-github) - [harisekhon/alpine-github](https://hub.docker.com/r/harisekhon/alpine-github) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/apache-drill.svg)](https://hub.docker.com/r/harisekhon/apache-drill/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/apache-drill.svg)](https://hub.docker.com/r/harisekhon/apache-drill) - [harisekhon/apache-drill](https://hub.docker.com/r/harisekhon/apache-drill) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/cassandra-dev.svg)](https://hub.docker.com/r/harisekhon/cassandra-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/cassandra-dev.svg)](https://hub.docker.com/r/harisekhon/cassandra-dev) - [harisekhon/cassandra-dev](https://hub.docker.com/r/harisekhon/cassandra-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/centos-dev.svg)](https://hub.docker.com/r/harisekhon/centos-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-dev.svg)](https://hub.docker.com/r/harisekhon/centos-dev) - [harisekhon/centos-dev](https://hub.docker.com/r/harisekhon/centos-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/centos-github.svg)](https://hub.docker.com/r/harisekhon/centos-github/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github.svg)](https://hub.docker.com/r/harisekhon/centos-github) - [harisekhon/centos-github](https://hub.docker.com/r/harisekhon/centos-github) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/centos-java.svg)](https://hub.docker.com/r/harisekhon/centos-java/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-java.svg)](https://hub.docker.com/r/harisekhon/centos-java) - [harisekhon/centos-java](https://hub.docker.com/r/harisekhon/centos-java) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/centos-scala.svg)](https://hub.docker.com/r/harisekhon/centos-scala/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-scala.svg)](https://hub.docker.com/r/harisekhon/centos-scala) - [harisekhon/centos-scala](https://hub.docker.com/r/harisekhon/centos-scala) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/ci_intentionally_broken_test_do_not_use.svg)](https://hub.docker.com/r/harisekhon/ci_intentionally_broken_test_do_not_use/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/ci_intentionally_broken_test_do_not_use.svg)](https://hub.docker.com/r/harisekhon/ci_intentionally_broken_test_do_not_use) - [harisekhon/ci_intentionally_broken_test_do_not_use](https://hub.docker.com/r/harisekhon/ci_intentionally_broken_test_do_not_use) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/collectd.svg)](https://hub.docker.com/r/harisekhon/collectd/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/collectd.svg)](https://hub.docker.com/r/harisekhon/collectd) - [harisekhon/collectd](https://hub.docker.com/r/harisekhon/collectd) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/consul.svg)](https://hub.docker.com/r/harisekhon/consul/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/consul.svg)](https://hub.docker.com/r/harisekhon/consul) - [harisekhon/consul](https://hub.docker.com/r/harisekhon/consul) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/consul-dev.svg)](https://hub.docker.com/r/harisekhon/consul-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/consul-dev.svg)](https://hub.docker.com/r/harisekhon/consul-dev) - [harisekhon/consul-dev](https://hub.docker.com/r/harisekhon/consul-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/debian-dev.svg)](https://hub.docker.com/r/harisekhon/debian-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/debian-dev.svg)](https://hub.docker.com/r/harisekhon/debian-dev) - [harisekhon/debian-dev](https://hub.docker.com/r/harisekhon/debian-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/debian-github.svg)](https://hub.docker.com/r/harisekhon/debian-github/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/debian-github.svg)](https://hub.docker.com/r/harisekhon/debian-github) - [harisekhon/debian-github](https://hub.docker.com/r/harisekhon/debian-github) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/debian-java.svg)](https://hub.docker.com/r/harisekhon/debian-java/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/debian-java.svg)](https://hub.docker.com/r/harisekhon/debian-java) - [harisekhon/debian-java](https://hub.docker.com/r/harisekhon/debian-java) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/h2o.svg)](https://hub.docker.com/r/harisekhon/h2o/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/h2o.svg)](https://hub.docker.com/r/harisekhon/h2o) - [harisekhon/h2o](https://hub.docker.com/r/harisekhon/h2o) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/hadoop.svg)](https://hub.docker.com/r/harisekhon/hadoop/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/hadoop.svg)](https://hub.docker.com/r/harisekhon/hadoop) - [harisekhon/hadoop](https://hub.docker.com/r/harisekhon/hadoop) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/hadoop-dev.svg)](https://hub.docker.com/r/harisekhon/hadoop-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/hadoop-dev.svg)](https://hub.docker.com/r/harisekhon/hadoop-dev) - [harisekhon/hadoop-dev](https://hub.docker.com/r/harisekhon/hadoop-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/hbase.svg)](https://hub.docker.com/r/harisekhon/hbase/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/hbase.svg)](https://hub.docker.com/r/harisekhon/hbase) - [harisekhon/hbase](https://hub.docker.com/r/harisekhon/hbase) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/hbase-dev.svg)](https://hub.docker.com/r/harisekhon/hbase-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/hbase-dev.svg)](https://hub.docker.com/r/harisekhon/hbase-dev) - [harisekhon/hbase-dev](https://hub.docker.com/r/harisekhon/hbase-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/jython.svg)](https://hub.docker.com/r/harisekhon/jython/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/jython.svg)](https://hub.docker.com/r/harisekhon/jython) - [harisekhon/jython](https://hub.docker.com/r/harisekhon/jython) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/kafka.svg)](https://hub.docker.com/r/harisekhon/kafka/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/kafka.svg)](https://hub.docker.com/r/harisekhon/kafka) - [harisekhon/kafka](https://hub.docker.com/r/harisekhon/kafka) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/mesos.svg)](https://hub.docker.com/r/harisekhon/mesos/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/mesos.svg)](https://hub.docker.com/r/harisekhon/mesos) - [harisekhon/mesos](https://hub.docker.com/r/harisekhon/mesos) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/nagios-plugin-kafka.svg)](https://hub.docker.com/r/harisekhon/nagios-plugin-kafka/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/nagios-plugin-kafka.svg)](https://hub.docker.com/r/harisekhon/nagios-plugin-kafka) - [harisekhon/nagios-plugin-kafka](https://hub.docker.com/r/harisekhon/nagios-plugin-kafka) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/nagios-plugins.svg)](https://hub.docker.com/r/harisekhon/nagios-plugins/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/nagios-plugins.svg)](https://hub.docker.com/r/harisekhon/nagios-plugins) - [harisekhon/nagios-plugins](https://hub.docker.com/r/harisekhon/nagios-plugins) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/nifi.svg)](https://hub.docker.com/r/harisekhon/nifi/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/nifi.svg)](https://hub.docker.com/r/harisekhon/nifi) - [harisekhon/nifi](https://hub.docker.com/r/harisekhon/nifi) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/presto.svg)](https://hub.docker.com/r/harisekhon/presto/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/presto.svg)](https://hub.docker.com/r/harisekhon/presto) - [harisekhon/presto](https://hub.docker.com/r/harisekhon/presto) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/presto-cli.svg)](https://hub.docker.com/r/harisekhon/presto-cli/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/presto-cli.svg)](https://hub.docker.com/r/harisekhon/presto-cli) - [harisekhon/presto-cli](https://hub.docker.com/r/harisekhon/presto-cli) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/presto-cli-dev.svg)](https://hub.docker.com/r/harisekhon/presto-cli-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/presto-cli-dev.svg)](https://hub.docker.com/r/harisekhon/presto-cli-dev) - [harisekhon/presto-cli-dev](https://hub.docker.com/r/harisekhon/presto-cli-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/presto-dev.svg)](https://hub.docker.com/r/harisekhon/presto-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/presto-dev.svg)](https://hub.docker.com/r/harisekhon/presto-dev) - [harisekhon/presto-dev](https://hub.docker.com/r/harisekhon/presto-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools) - [harisekhon/pytools](https://hub.docker.com/r/harisekhon/pytools) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/rabbitmq-cluster.svg)](https://hub.docker.com/r/harisekhon/rabbitmq-cluster/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/rabbitmq-cluster.svg)](https://hub.docker.com/r/harisekhon/rabbitmq-cluster) - [harisekhon/rabbitmq-cluster](https://hub.docker.com/r/harisekhon/rabbitmq-cluster) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/riak.svg)](https://hub.docker.com/r/harisekhon/riak/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/riak.svg)](https://hub.docker.com/r/harisekhon/riak) - [harisekhon/riak](https://hub.docker.com/r/harisekhon/riak) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/riak-dev.svg)](https://hub.docker.com/r/harisekhon/riak-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/riak-dev.svg)](https://hub.docker.com/r/harisekhon/riak-dev) - [harisekhon/riak-dev](https://hub.docker.com/r/harisekhon/riak-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/serf.svg)](https://hub.docker.com/r/harisekhon/serf/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/serf.svg)](https://hub.docker.com/r/harisekhon/serf) - [harisekhon/serf](https://hub.docker.com/r/harisekhon/serf) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/solr.svg)](https://hub.docker.com/r/harisekhon/solr/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/solr.svg)](https://hub.docker.com/r/harisekhon/solr) - [harisekhon/solr](https://hub.docker.com/r/harisekhon/solr) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/solrcloud.svg)](https://hub.docker.com/r/harisekhon/solrcloud/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/solrcloud.svg)](https://hub.docker.com/r/harisekhon/solrcloud) - [harisekhon/solrcloud](https://hub.docker.com/r/harisekhon/solrcloud) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/solrcloud-dev.svg)](https://hub.docker.com/r/harisekhon/solrcloud-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/solrcloud-dev.svg)](https://hub.docker.com/r/harisekhon/solrcloud-dev) - [harisekhon/solrcloud-dev](https://hub.docker.com/r/harisekhon/solrcloud-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/spark.svg)](https://hub.docker.com/r/harisekhon/spark/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/spark.svg)](https://hub.docker.com/r/harisekhon/spark) - [harisekhon/spark](https://hub.docker.com/r/harisekhon/spark) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/spotify-tools.svg)](https://hub.docker.com/r/harisekhon/spotify-tools/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/spotify-tools.svg)](https://hub.docker.com/r/harisekhon/spotify-tools) - [harisekhon/spotify-tools](https://hub.docker.com/r/harisekhon/spotify-tools) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/superset.svg)](https://hub.docker.com/r/harisekhon/superset/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/superset.svg)](https://hub.docker.com/r/harisekhon/superset) - [harisekhon/superset](https://hub.docker.com/r/harisekhon/superset) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/tachyon.svg)](https://hub.docker.com/r/harisekhon/tachyon/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/tachyon.svg)](https://hub.docker.com/r/harisekhon/tachyon) - [harisekhon/tachyon](https://hub.docker.com/r/harisekhon/tachyon) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/tcollector.svg)](https://hub.docker.com/r/harisekhon/tcollector/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/tcollector.svg)](https://hub.docker.com/r/harisekhon/tcollector) - [harisekhon/tcollector](https://hub.docker.com/r/harisekhon/tcollector) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/tools.svg)](https://hub.docker.com/r/harisekhon/tools/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/tools.svg)](https://hub.docker.com/r/harisekhon/tools) - [harisekhon/tools](https://hub.docker.com/r/harisekhon/tools) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/ubuntu-dev.svg)](https://hub.docker.com/r/harisekhon/ubuntu-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/ubuntu-dev.svg)](https://hub.docker.com/r/harisekhon/ubuntu-dev) - [harisekhon/ubuntu-dev](https://hub.docker.com/r/harisekhon/ubuntu-dev) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/ubuntu-github.svg)](https://hub.docker.com/r/harisekhon/ubuntu-github/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/ubuntu-github.svg)](https://hub.docker.com/r/harisekhon/ubuntu-github) - [harisekhon/ubuntu-github](https://hub.docker.com/r/harisekhon/ubuntu-github) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/ubuntu-java.svg)](https://hub.docker.com/r/harisekhon/ubuntu-java/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/ubuntu-java.svg)](https://hub.docker.com/r/harisekhon/ubuntu-java) - [harisekhon/ubuntu-java](https://hub.docker.com/r/harisekhon/ubuntu-java) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/zookeeper.svg)](https://hub.docker.com/r/harisekhon/zookeeper/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/zookeeper.svg)](https://hub.docker.com/r/harisekhon/zookeeper) - [harisekhon/zookeeper](https://hub.docker.com/r/harisekhon/zookeeper) [![Docker Build Status](https://img.shields.io/docker/cloud/build/harisekhon/zookeeper-dev.svg)](https://hub.docker.com/r/harisekhon/zookeeper-dev/builds) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/zookeeper-dev.svg)](https://hub.docker.com/r/harisekhon/zookeeper-dev) - [harisekhon/zookeeper-dev](https://hub.docker.com/r/harisekhon/zookeeper-dev) ================================================ FILE: Gemfile ================================================ # # Author: Hari Sekhon # Date: 2022-05-13 15:25:18 +0100 (Fri, 13 May 2022) # # vim:ts=4:sts=4:sw=4:et # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # G e m f i l e # ============================================================================ # # https://bundler.io/gemfile.html # This isn't automatically installed since ruby code is not much used in this repo # see also: # # setup/gem-packages.txt # setup/gem-packages-desktop.txt # optional for desktop use source 'https://rubygems.org' gem 'cfn-nag' gem 'json' gem 'gitlab' ================================================ FILE: Jenkinsfile ================================================ // vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy // // Author: Hari Sekhon // Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) // // https://github.com/HariSekhon/DevOps-Bash-tools // // License: see accompanying Hari Sekhon LICENSE file // // If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish // // https://www.linkedin.com/in/HariSekhon // // ========================================================================== // // J e n k i n s P i p e l i n e // ========================================================================== // // Epic Jenkinsfile template: // // https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile // Official Documentation: // // https://jenkins.io/doc/book/pipeline/syntax/ // // https://www.jenkins.io/doc/pipeline/steps/ // // https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/ pipeline { // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top agent any options { timestamps() timeout(time: 2, unit: 'HOURS') } triggers { cron('H 10 * * 1-5') pollSCM('H/2 * * * *') } stages { stage ('Checkout') { steps { checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Bash-tools']]]) } } stage('Build') { steps { echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}" echo 'Building...' timeout(time: 10, unit: 'MINUTES') { retry(3) { // sh 'apt update -q' // sh 'apt install -qy make' // sh 'make init' sh """ setup/ci_bootstrap.sh && make init """ } } timeout(time: 180, unit: 'MINUTES') { sh 'make ci' } } } stage('Test') { options { retry(2) } steps { echo 'Testing...' timeout(time: 120, unit: 'MINUTES') { sh 'make test' } } } } } ================================================ FILE: LICENSE ================================================ Copyright 2016 Hari Sekhon Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ # # Author: Hari Sekhon # Date: 2016-01-17 12:56:53 +0000 (Sun, 17 Jan 2016) # # vim:ts=4:sts=4:sw=4:noet # # https://github.com/HariSekhon/DevOps-Bash-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # include Makefile.in REPO := HariSekhon/DevOps-Bash-tools CONF_FILES := $(shell sed "s/\#.*//; /^[[:space:]]*$$/d" setup/files.txt) #CODE_FILES := $(shell find . -type f -name '*.sh' -o -type f -name '.bash*' | sort) #CODE_FILES := $(shell git ls-files | grep -E -e '\.sh$$' -e '\.bash[^/]*$$' -e '\.groovy$$' | sort) CODE_FILES := $(shell \ if type git >/dev/null 2>&1; then \ git ls-files | \ grep -E -e '\.sh$$' -e '\.bash[^/]*$$' -e '\.groovy$$' | \ sort | \ while read -r filepath; do \ test -f "$$filepath" || continue; \ test -d "$$filepath" && continue; \ test -L "$$filepath" && continue; \ echo "$$filepath"; \ done; \ else \ find . -type f; \ fi \ ) BASH_PROFILE_FILES := $(shell echo .bashrc .bash_profile .bash.d/*.sh) #.PHONY: * CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) TRUNK_BRANCH := $(shell git symbolic-ref refs/remotes/origin/HEAD | sed 's|.*/||') DEFAULT_TITLE := [GD-00] - merge $(CURRENT_BRANCH) to $(TRUNK_BRANCH) title ?= $(DEFAULT_TITLE) # =================== define MAKEFILE_USAGE Repo specific options: make install builds all script dependencies, installs AWS CLI, GitHub CLI, symlinks all config files to $$HOME and adds sourcing of bash profile make link symlinks all config files to $$HOME and adds sourcing of bash profile make unlink removes all symlinks pointing to this repo's config files and removes the sourcing lines from .bashrc and .bash_profile make python-desktop installs all Python Pip packages for desktop workstation listed in setup/pip-packages-desktop.txt make perl-desktop installs all Perl CPAN packages for desktop workstation listed in setup/cpan-packages-desktop.txt make ruby-desktop installs all Ruby Gem packages for desktop workstation listed in setup/gem-packages-desktop.txt make golang-desktop installs all Golang packages for desktop workstation listed in setup/go-packages-desktop.txt make nodejs-desktop installs all NodeJS packages for desktop workstation listed in setup/npm-packages-desktop.txt make desktop installs all of the above + many desktop OS packages listed in setup/ make mac-desktop all of the above + installs a bunch of major common workstation software packages like Ansible, Terraform, MiniKube, MiniShift, SDKman, Travis CI, CCMenu, Parquet tools etc. make linux-desktop make ls-scripts print list of scripts in this project, ignoring code libraries in lib/ and .bash.d/ make github-cli installs GitHub CLI make kubernetes installs Kubernetes kubectl and kustomize to ~/bin/ make terraform installs Terraform to ~/bin/ make vim installs Vundle and plugins make tmux installs TMUX TPM and plugin for kubernetes context make ccmenu installs and (re)configures CCMenu to watch this and all other major HariSekhon GitHub repos make status open the Github Status page of all my repos build statuses across all CI platforms make aws installs AWS CLI tools make azure installs Azure CLI make gcp installs Google Cloud SDK make aws-shell sets up AWS Cloud Shell: installs core packages and links configs (maintains itself across future Cloud Shells via .aws_customize_environment hook) make gcp-shell sets up GCP Cloud Shell: installs core packages and links configs (maintains itself across future Cloud Shells via .customize_environment hook) make azure-shell sets up Azure Cloud Shell (limited compared to gcp-shell, doesn't install OS packages since there is no sudo) endef # not including azure here because it requires interactive prompt and hangs automatic testing of make docker-* .PHONY: build build: @echo ================ @echo Bash Tools Build @echo ================ @$(MAKE) git-summary @$(MAKE) init @$(MAKE) system-packages @$(MAKE) aws github-cli .PHONY: init init: git @echo "running init:" git submodule update --init --recursive @echo .PHONY: install install: build @$(MAKE) link @$(MAKE) aws @$(MAKE) gcp @$(MAKE) github-cli @$(MAKE) pip .PHONY: uninstall uninstall: unlink @echo "Not removing any system packages for safety" .PHONY: bash bash: link @: .PHONY: link link: @setup/shell_link.sh .PHONY: unlink unlink: @setup/shell_unlink.sh .PHONY: mac-desktop mac-desktop: desktop @setup/mac_desktop.sh .PHONY: mac mac: mac-desktop @: .PHONY: linux-desktop linux-desktop: desktop @setup/linux_desktop.sh .PHONY: linux linux: linux-desktop @: .PHONY: ccmenu: @setup/ccmenu_setup.sh .PHONY: desktop desktop: install @if [ -x /sbin/apk ]; then $(MAKE) apk-packages-desktop; fi @if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-desktop; fi @if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages-desktop; fi @if [ `uname` = Darwin ]; then \ if type brew >/dev/null 2>/dev/null; then \ $(MAKE) homebrew-packages-desktop; \ fi; \ fi @# do these late so that we have the above system packages installed first to take priority and not install from source where we don't need to @$(MAKE) perl-desktop @$(MAKE) golang-desktop @$(MAKE) nodejs-desktop @$(MAKE) ruby-desktop @# no packages any more since jgrep is no longer found @#$(MAKE) ruby-desktop .PHONY: apk-packages-desktop apk-packages-desktop: system-packages @echo "Alpine desktop not supported at this time" @exit 1 .PHONY: apt-packages-desktop apt-packages-desktop: system-packages NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/packages/apt_install_packages.sh setup/deb-packages-desktop.txt .PHONY: yum-packages-desktop yum-packages-desktop: system-packages NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/packages/yum_install_packages.sh setup/rpm-packages-desktop.txt .PHONY: homebrew-packages-desktop homebrew-packages-desktop: system-packages homebrew @: .PHONY: brew-packages-desktop brew-packages-desktop: homebrew-packages-desktop @: .PHONY: homebrew homebrew: system-packages brew @: .PHONY: brew brew: which -a brew || install/install_homebrew.sh which -a wget || brew install wget NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/packages/brew_install_packages_if_absent.sh setup/brew-packages-desktop.txt NO_FAIL=1 NO_UPDATE=1 CASK=1 $(BASH_TOOLS)/packages/brew_install_packages_if_absent.sh setup/brew-packages-desktop-casks.txt @# doesn't pass the packages correctly yet @#NO_FAIL=1 NO_UPDATE=1 TAP=1 $(BASH_TOOLS)/packages/brew_install_packages.sh setup/brew-packages-desktop-taps.txt NO_FAIL=1 NO_UPDATE=1 TAP=1 $(BASH_TOOLS)/packages/brew_install_packages.sh setup/brew-packages-desktop-taps.txt .PHONY: perl-desktop perl-desktop: system-packages cpan-desktop @: .PHONY: cpan-desktop cpan-desktop: cpan NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/perl/perl_cpanm_install_if_absent.sh setup/cpan-packages-desktop.txt .PHONY: golang-desktop golang-desktop: system-packages go-desktop @: .PHONY: go-desktop go-desktop: system-packages go @: .PHONY: go go: NO_FAIL=1 $(BASH_TOOLS)/packages/golang_install_if_absent.sh setup/go-packages-desktop.txt .PHONY: ruby-desktop ruby-desktop: system-packages gem-desktop @: .PHONY: gem-desktop gem-desktop: gem NO_FAIL=1 $(BASH_TOOLS)/packages/ruby_gem_install_if_absent.sh setup/gem-packages-desktop.txt .PHONY: python-desktop python-desktop: system-packages pip-desktop .PHONY: pip pip-desktop: pip PIP=$(PIP) ./python/python_pip_install_if_absent.sh setup/pip-packages-desktop.txt if uname -s | grep -q Darwin; then \ PIP=$(PIP) ./python/python_pip_install_if_absent.sh setup/pip-packages-mac.txt; \ fi .PHONY: nodejs-desktop nodejs-desktop: system-packages npm-desktop .PHONY: npm-desktop npm-desktop: npm $(BASH_TOOLS)/packages/nodejs_npm_install_if_absent.sh $(BASH_TOOLS)/setup/npm-packages-desktop.txt .PHONY: aws aws: system-packages python-version @if ! command -v aws; then install/install_aws_cli.sh; fi # @$(MAKE) codecommit # #.PHONY: codecommit #codecommit: @# needed for github_mirror_repos_to_aws_codecommit.sh and dependent GitHub Actions workflows @if uname -s | grep -q Darwin; then \ xargs(){ \ gxargs "$$@"; \ }; \ fi; \ grep '^git-remote-codecommit' requirements.txt | \ PIP=$(PIP) xargs --no-run-if-empty ./python/python_pip_install_if_absent.sh || : .PHONY: aws-shell aws-shell: @if [ "${AWS_EXECUTION_ENV:-}" != "CloudShell" ]; then echo "Not running inside AWS Cloud Shell"; exit 1; fi @$(MAKE) system-packages aws link .PHONY: azure azure: system-packages @install/install_azure_cli.sh .PHONY: azure-shell azure-shell: link : .PHONY: gcp gcp: system-packages @./install/install_gcloud_sdk.sh @./install/install_cloud_sql_proxy.sh .PHONY: gcp-shell gcp-shell: @if [ -z "${DEVSHELL_PROJECT_ID:-}" ]; then echo "Not running inside Google Cloud Shell"; exit 1; fi @$(MAKE) system-packages link .PHONY: github-cli github-cli: ~/bin/gh @: ~/bin/gh: install/install_github_cli.sh .PHONY: digital-ocean: ~/bin/doctl @: ~/bin/doctl: install/install_doctl.sh .PHONY: kubernetes kubernetes: kubectl kustomize @: .PHONY: k8s k8s: kubernetes @: .PHONY: kubectl kubectl: ~/bin/kubectl @: ~/bin/kubectl: install/install_kubectl.sh .PHONY: kustomize kustomize: ~/bin/kustomize @: ~/bin/kustomize: install/install_kustomize.sh .PHONY: vim vim: ~/.vim/bundle/Vundle.vim @: ~/.vim/bundle/Vundle.vim: install/install_vundle.sh .PHONY: tmux tmux: ~/.tmux/plugins/tpm ~/.tmux/plugins/kube.tmux @: ~/.tmux/plugins/tpm: git clone https://github.com/tmux-plugins/tpm ~/.tmux/plugins/tpm ~/.tmux/plugins/kube.tmux: wget -O ~/.tmux/plugins/kube.tmux https://raw.githubusercontent.com/jonmosco/kube-tmux/master/kube.tmux .PHONY: test test: ./checks/check_all.sh .PHONY: clean clean: @rm -fv -- setup/terraform.zip .PHONY: ls-scripts ls-scripts: @$(MAKE) ls | grep -v -e 'lib/' -e '\.bash' .PHONY: ls-scripts2 ls-scripts2: @$(MAKE) ls | grep -v -e 'lib/' -e '\.bash' -e 'setup/' .PHONY: wcbashrc wcbashrc: @wc $(BASH_PROFILE_FILES) @printf "Total Bash Profile files: " @ls $(BASH_PROFILE_FILES) | wc -l .PHONY: wcbash wcbash: wcbashrc @: .PHONY: wcbashrc2 wcbashrc2: @printf "Total Bash Profile files: " @ls $(BASH_PROFILE_FILES) | wc -l @printf "Total line count without # comments: " @ls $(BASH_PROFILE_FILES) | xargs sed 's/#.*//;/^[[:space:]]*$$/d' | wc -l .PHONY: wcbash2 wcbash2: wcbashrc2 @: .PHONY: pipreqs-mapping pipreqs-mapping: #wget -O resources/pipreqs_mapping.txt https://raw.githubusercontent.com/HariSekhon/pipreqs/mysql-python/pipreqs/mapping wget -O resources/pipreqs_mapping.txt https://raw.githubusercontent.com/bndr/pipreqs/master/pipreqs/mapping .PHONY: pip-mapping pip-mapping: pipreqs-mapping @: .PHONY: status-page status-page: ./cicd/generate_status_page.sh; . .bash.d/git.sh; gitu STATUS.md .PHONY: dialog-install dialog-install: install/install_packages.sh dialog # Raise Pull Requests from the command line like this: # # You need GitHub CLI installed ('make' installs it for you) and authenticated eg.: # # gh auth login # # # https://cli.github.com/manual/gh_auth_login # # Example: # # make pr title="Hari code to avoid clicking" # .PHONY: pr pr: dialog-install git push --set-upstream origin "$(CURRENT_BRANCH)" if [ -z "$$GITHUB_PULL_REQUEST_TITLE" ]; then \ if [ "$(title)" = "$(DEFAULT_TITLE)" ]; then \ GITHUB_PULL_REQUEST_TITLE="$$(dialog --inputbox "Pull Request Title:" 8 40 "$(DEFAULT_TITLE)" 3>&1 1>&2 2>&3)"; \ else \ GITHUB_PULL_REQUEST_TITLE="$(title)"; \ fi; \ fi; \ export GITHUB_PULL_REQUEST_TITLE; \ github_pull_request_create.sh \ "$(REPO)" \ "$(CURRENT_BRANCH)" \ "$(TRUNK_BRANCH)" # raise a PR in one command with Auto-Merge enabled - use this for trivial PRs of low / no impact like MkDocs updates .PHONY: auto-pr auto-pr: update @# - if GITHUB_PULL_REQUEST_AUTO_MERGE=true then marks the PR for auto-merge once it is approved and passes pre-requisite checks @# - if GITHUB_PULL_REQUEST_SQUASH=true while GITHUB_PULL_REQUEST_AUTO_MERGE=true then it marks @# the PR's auto-merge to be done using a squash commit to avoid any CLI prompt for how to merge it GITHUB_PULL_REQUEST_AUTO_MERGE=true \ GITHUB_PULL_REQUEST_SQUASH=true \ $(MAKE) pr # Example: # # make autopr title="Documented something" # .PHONY: autopr autopr: auto-pr @: .PHONY: sync sync: sync_configs_to_adjacent_repos.sh ================================================ FILE: Makefile.in ================================================ # # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # ifneq ("$(wildcard bash-tools)", "") BASH_TOOLS := bash-tools else BASH_TOOLS := . endif # would fail bootstrapping on Alpine #SHELL := /usr/bin/env bash export PATH := $(PATH):/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/homebrew/bin # Python breaking backwards compatibility as usual - set this to use historic behaviour and treat the user like an adult # this way if the install is run for --user instead of in a virtualenv it'll still work export PIP_BREAK_SYSTEM_PACKAGES := 1 DOCKER_IMAGE := harisekhon/github ifneq ("$(wildcard /.dockerenv)", "") INSIDE_DOCKER := 1 else INSIDE_DOCKER := endif CODE_FILES := $(shell \ if type git >/dev/null 2>&1; then \ git ls-files | \ while read filepath; do \ test -f "$$filepath" || continue; \ test -d "$$filepath" & continue; \ test -L "$$filepath" & continue; \ echo "$$filepath"; \ done; \ fi \ ) CPANM := cpanm export PIP := pip3 export PYTHON := python3 FATPACKS_DIR := fatpacks SUDO := sudo SUDO_PIP := sudo -H SUDO_PERL := sudo PYTHON_VIRTUALENV := ifdef PERLBREW_PERL # can't put this here, nor @commented, otherwise gets error - "commands commence before first target. Stop." #echo "Perlbrew environment detected, not calling sudo" SUDO_PERL = else PERLBREW_PERL := endif # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules locally to non-system python otherwise they're not found by programs. # Perms not set correctly on custom python install in Travis perl build so workaround is done to chown to travis user in .travis.yml # Better than modifying $PATH to put /usr/bin first which is likely to affect many other things including potentially not finding the perlbrew installation first # Looks like Perl travis builds are now using system Python - do not use TRAVIS env ifdef VIRTUAL_ENV #echo "Virtual Env / Conda detected, not calling sudo" SUDO_PIP := PYTHON_VIRTUALENV := 1 endif ifdef CONDA_DEFAULT_ENV SUDO_PIP := PYTHON_VIRTUALENV := 1 endif # must come after to reset SUDO_PERL/SUDO_PIP to blank if root # EUID / UID not exported in Make # USER not populated in Docker ifeq '$(shell id -u)' '0' #echo "root UID detected, not calling sudo" SUDO := SUDO_PERL := SUDO_PIP := endif # placeholders to silence check_makefile.sh warnings - should be set in client Makefiles after sourcing ifndef REPO REPO := NOTSET endif ifndef ARGS ARGS := NOTSET endif ifndef CONF_FILES CONF_FILES := NOTSET endif define MAKEFILE_USAGE_COMMON Usage: Common Options: make help show this message make build installs all dependencies - OS packages and any language libraries via native tools eg. pip, cpanm, gem, go etc that are not available via OS packages make build-retry retries 'make build' x 3 until success to try to mitigate temporary upstream repo failures triggering false alerts in CI systems make ci prints env, then runs 'build-retry' for more resilient CI builds with debugging make printenv prints environment variables, CPU cores, OS release, $$PWD, Git branch, hashref etc. Useful for CI debugging make system-packages installs OS packages only (detects OS via whichever package manager is available) make test run tests make clean removes compiled / generated files, downloaded tarballs, temporary files etc. make submodules initialize and update submodules to the right release (done automatically by build / system-packages) make init same as above, often useful to do in CI systems to get access to additional submodule provided targets such as 'make ci' make cpan install any modules listed in any cpan-requirements.txt files if not already installed make gem install any modules listed in any gem-requirements.txt files if not already installed make npm install any modules listed in any npm-requirements.txt files if not already installed make pip install any modules listed in any requirements.txt files if not already installed make python-compile compile any python files found in the current directory and 1 level of subdirectory make pycompile make github open browser at github project make readme open browser at github's README make github-url print github url and copy to clipboard make status open browser at Github CI Builds overview Status page for all projects make ls print list of code files in project make wc show counts of files and lines endef #make ${VENV} make a virtualenv in the base directory (see VENV) #make pip-install install python packages in requirements.txt #make git-config set local git configuration export MAKEFILE_USAGE_COMMON export MAKEFILE_USAGE # doesn't seem to work #.DEFAULT: build # @echo running default # $(MAKE) build # won't be run the first time - will default to first target which will only then initialize submodules .PHONY: default default: git printenv @$(MAKE) main .PHONY: printenv printenv: git @ printf "CPU Cores: "; nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null; : @ # $$USER not always set in sh @ # printf "Git hashref: "; git rev-parse HEAD @ # printf "Git hashref: "; git log --pretty=format:'%H' -n 1 @ # printf "Git branch: "; git branch --show-current # doesn't work on Alpine @ # printf "Git branch: "; git show-branch --current # prints too many branches @ # sort --ignore-case switch not available on Alpine, must use sort -f which is available on both Mac and all Linux distros @ . $(BASH_TOOLS)/lib/ci.sh || : ; \ if is_CI || test -f /.dockerenv; then \ echo; \ echo "USER = `whoami`"; \ echo "PWD = $$PWD"; \ echo; \ printf "Git branch: "; git rev-parse --abbrev-ref HEAD; \ printf "Git commit: "; git log --pretty=format:"%ai %cn %H %s" -n 1; echo; \ echo; \ if [ -f /.dockerenv ]; then \ echo "Running inside Docker:"; \ ls -l /.dockerenv 2>/dev/null; \ fi; \ echo; \ echo "OS RELEASE:"; \ echo; \ uname -a || : ; \ echo; \ cat /etc/*release || : ; \ echo; \ unset MAKEFILE_USAGE; \ unset MAKEFILE_USAGE_COMMON; \ unset TERMCAP; \ echo; \ echo "CI ENVIRONMENT:"; \ echo; \ env | grep -vi -e PASS -e TOKEN -e KEY -e SECRET | sort -f; \ echo; \ echo; \ if which java 2>/dev/null; then \ which java; \ java -version; \ echo; \ fi; \ echo "PATH:"; echo "$$PATH" | tr ':' '\n'; \ echo; \ else \ env | grep -E 'BUILD|PIPELINE|JOB|STAGE|\' tests because the exit 3 doesn't actually get called and leads make to think there is a matching target, which then fail to execute # catchall - any unrecognized target will print usage #%:: # @# don't use less, it will make target tests hang # @echo Unrecognized option $@; \ # echo; \ # $(MAKE) usage; .PHONY: quick quick: QUICK=1 $(MAKE) build .PHONY: git git: @# not using install_packages_if_absent.sh as we don't need a package on Mac, it comes with XCode type git 2>/dev/null || $(BASH_TOOLS)/packages/install_packages.sh git .PHONY: submodules submodules: git @echo "checking out any git submodules:" git submodule update --init --recursive @echo .PHONY: git-clean git-clean: git @git clean -n -d @printf "\n\n%s" "If you're happy with this list, run:" @printf "\n\n%s\n\n" "git clean -f -d" .PHONY: gitignore gitignore: $(BASH_TOOLS)/git/update_gitignore.io.sh .PHONY: btest btest: bash-test @: .PHONY: bash-test bash-test: $(BASH_TOOLS)/checks/check_all.sh .PHONY: test #test: precommit test: bash-test @: precommit: pre-commit @: pre-commit: pre-commit run --all-files .PHONY: push push: test git push .PHONY: system-packages system-packages: submodules if [ -x /sbin/apk ]; then $(MAKE) apk-packages; fi if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages; fi @# /usr/bin/yum is a symlink to dnf-3 on newer RHEL systems, so fails -x /usr/bin/yum if [ -e /usr/bin/yum ]; then $(MAKE) yum-packages; fi @# /usr/local/bin/brew on older macOS @# /opt/homebrew/bin/brew on newer macOS if which -a brew && [ `uname` = Darwin ]; then $(MAKE) homebrew-packages; fi .PHONY: system-packages-perl system-packages-perl: system-packages if [ -x /sbin/apk ]; then $(MAKE) apk-packages-perl; fi if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-perl; fi @# /usr/bin/yum is a symlink to dnf-3 on newer RHEL systems, so fails -x /usr/bin/yum if [ -e /usr/bin/yum ]; then $(MAKE) yum-packages-perl; fi .PHONY: system-packages-python system-packages-python: system-packages if [ -x /sbin/apk ]; then $(MAKE) apk-packages-python; fi if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-python; fi @# /usr/bin/yum is a symlink to dnf-3 on newer RHEL systems, so fails -x /usr/bin/yum if [ -e /usr/bin/yum ]; then $(MAKE) yum-packages-python; fi .PHONY: apk-packages apk-packages: # not portable in Alpine sh #for x in apk-packages{,-perl,-python}{,-dev}.txt; do \ for x in apk-packages.txt apk-packages-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" #for x in apk-packages-{optional,cpan,pip}.txt; do \ for x in apk-packages-optional.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" .PHONY: apk-packages-perl apk-packages-perl: for x in apk-packages-perl.txt apk-packages-perl-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" #for x in apk-packages-{optional,cpan,pip}.txt; do \ # don't put comments inside the for loop, breaks syntax expecting 'done' # no point installing system cpan packages if using perlbrew as they won't be found inside perlbrew for x in apk-packages-cpan.txt; do \ if [ -z "$(PERLBREW_PERL)" ]; then \ find . -maxdepth 3 -path "*/setup/$$x"; \ fi; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" .PHONY: apk-packages-python apk-packages-python: for x in apk-packages-python.txt apk-packages-python-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" # no point installing system pip packages when they won't be found in virtualenv and will need to be pip installed anyway for x in apk-packages-pip.txt; do \ if [ -z "$(PYTHON_VIRTUALENV)" ]; then \ find . -maxdepth 3 -path "*/setup/$$x"; \ fi; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apk_install_packages.sh" .PHONY: apt-packages apt-packages: #for x in deb-packages{,-perl,-python}{,-dev}.txt; do \ for x in deb-packages.txt deb-packages-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" #for x in deb-packages-{optional,cpan,pip}.txt; do \ for x in deb-packages-optional.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" .PHONY: apt-packages-perl apt-packages-perl: for x in deb-packages-perl.txt deb-packages-perl-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" for x in deb-packages-cpan.txt; do \ if [ -z "$(PERLBREW_PERL)" ] && \ [ -z "$(GOOGLE_CLOUD_SHELL)" ]; then \ find . -maxdepth 3 -path "*/setup/$$x"; \ fi; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" .PHONY: apt-packages-python apt-packages-python: for x in deb-packages-python.txt deb-packages-python-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" for x in deb-packages-pip.txt; do \ if [ -z "$(PYTHON_VIRTUALENV)" ] && \ [ -z "$(GOOGLE_CLOUD_SHELL)" ]; then \ find . -maxdepth 3 -path "*/setup/$$x"; \ fi; \ done | NO_FAIL=1 NO_UPDATE=1 xargs "$(BASH_TOOLS)/packages/apt_install_packages.sh" .PHONY: yum-packages yum-packages: # needed for Fedora to have find and xargs to use below "$(BASH_TOOLS)/packages/yum_install_packages.sh" findutils # if on Amazon Linux 2 install epel this way if type -P amazon-linux-extras; then \ $(SUDO) amazon-linux-extras install epel -y; \ fi $(BASH_TOOLS)/install/install_epel_repo.sh # installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package for x in rpm-packages.txt rpm-packages-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh" for x in rpm-packages-optional.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh" .PHONY: yum-packages-perl yum-packages-perl: # installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package for x in rpm-packages-perl.txt rpm-packages-perl-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh" for x in rpm-packages-cpan.txt; do \ if [ -z "$(PERLBREW_PERL)" ]; then \ find . -maxdepth 3 -path "*/setup/$$x"; \ fi; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh" .PHONY: yum-packages-python yum-packages-python: # installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package for x in rpm-packages-python.txt rpm-packages-python-dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh" . "$(BASH_TOOLS)/lib/python.sh"; \ set +o pipefail || : ; \ if ! inside_virtualenv; then \ for x in rpm-packages-pip.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/yum_install_packages.sh"; \ fi .PHONY: homebrew-packages homebrew-packages: # Fails if any of the packages are already installed, ignore and continue - if it's a problem the latest build steps will fail with missing headers for x in brew-packages.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/brew_install_packages.sh" @# fix for OpenSSL 1.0 -> 1.1 library linkage breaking python -c 'import hashlib', which break pips, eg: @# https://stackoverflow.com/questions/20399331/error-importing-hashlib-with-python-2-7-but-not-with-2-6 $(BASH_TOOLS)/setup/brew_fix_openssl_dependencies.sh .PHONY: system-packages-remove system-packages-remove: if [ -x /sbin/apk ]; then $(MAKE) apk-packages-remove; fi if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-remove; fi if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages-remove; fi .PHONY: apk-packages-remove apk-packages-remove: for x in apk-packages-{,perl-,python-}dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/apk_remove_packages.sh" $(SUDO) rm -fr -- /var/cache/apk/* .PHONY: apt-packages-remove apt-packages-remove: for x in deb-packages-{,perl-,python-}dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/apt_remove_packages.sh" .PHONY: yum-packages-remove yum-packages-remove: for x in rpm-packages-{,perl-,python-}dev.txt; do \ find . -maxdepth 3 -path "*/setup/$$x"; \ done | NO_FAIL=1 xargs "$(BASH_TOOLS)/packages/yum_remove_packages.sh" .PHONY: cpan cpan:: find . -maxdepth 3 -path '*/setup/cpan-requirements*.txt' | grep -v cpan-requirements-optional.txt | xargs --no-run-if-empty "$(BASH_TOOLS)/perl/perl_cpanm_install_if_absent.sh" @$(MAKE) cpan-optional .PHONY: cpan-optional cpan-optional:: find . -maxdepth 3 -path '*/setup/cpan-requirements-optional.txt' | NO_FAIL=1 xargs --no-run-if-empty "$(BASH_TOOLS)/perl/perl_cpanm_install_if_absent.sh" .PHONY: gems gems:: gem @: .PHONY: gem gem:: find . -maxdepth 3 -path '*/setup/gem-packages.txt' | xargs --no-run-if-empty "$(BASH_TOOLS)/packages/ruby_gem_install_if_absent.sh" @$(MAKE) gem-optional .PHONY: gem-optional gem-optional:: find . -maxdepth 3 -path '*/setup/gem-packages-optional.txt' | NO_FAIL=1 PIP="$(PIP)" xargs --no-run-if-empty "$(BASH_TOOLS)/packages/ruby_gem_install_if_absent.sh" .PHONY: npm npm:: find . -maxdepth 3 -path '*/setup/npm-requirements.txt' -o -path '*/setup/npm-packages.txt' | xargs --no-run-if-empty "$(BASH_TOOLS)/packages/nodejs_npm_install_if_absent.sh" @$(MAKE) npm-optional .PHONY: npm-optional npm-optional:: find . -maxdepth 3 -path '*/setup/npm-requirements-optional.txt' | NO_FAIL=1 xargs --no-run-if-empty "$(BASH_TOOLS)/packages/nodejs_npm_install_if_absent.sh" .PHONY: pip pip:: find . -maxdepth 3 -path '*/requirements.txt' | PIP="$(PIP)" xargs --no-run-if-empty "$(BASH_TOOLS)/python/python_pip_install_if_absent.sh" @$(MAKE) pip-optional .PHONY: pip-optional pip-optional:: find . -maxdepth 3 -path '*/requirements-optional.txt' | NO_FAIL=1 PIP="$(PIP)" xargs --no-run-if-empty "$(BASH_TOOLS)/python/python_pip_install_if_absent.sh" .PHONY: pip-user pip-user:: PYTHON_USER_INSTALL=1 $(MAKE) pip .PHONY: fatpacks fatpacks: $(BASH_TOOLS)/perl/perl_generate_fatpacks.sh *.pl @echo @if [ -d lib/resources ]; then \ cp -av -- lib/resources fatpacks/; \ fi @if $(MAKE) -n fatpacks-local >/dev/null 2>&1; then \ echo; \ echo "fatpacks-local target detected, running:"; \ $(MAKE) fatpacks-local; \ fi @echo @if [ -n "`ls "$(FATPACKS_DIR)"`" ]; then \ tar czvf fatpacks.tar.gz "$(FATPACKS_DIR)"; \ echo; \ echo "Generated fatpacks.tar.gz containing $(FATPACKS_DIR)/ directory of perl scripts with all dependencies bundled"; \ fi .PHONY: fatpack fatpack: fatpacks @: .PHONY: python-compile python-compile: $(BASH_TOOLS)/python/python_compile.sh .PHONY: pycompile pycompile: python-compile @: .PHONY: python-version python-version: $(BASH_TOOLS)/setup/which_python_installed.sh .PHONY: golang-version golang-version: @echo && \ which go && \ ls -l `which go` && \ echo && \ go version || : ; \ echo .PHONY: go-version go-version: golang-version @: .PHONY: golang-clean golang-clean: @$(BASH_TOOLS)/packages/golang_rm_binaries.sh .PHONY: go-clean go-clean: golang-clean @: # ======================= # Nice tricks for pure Python projects # - borrowed from https://gist.github.com/bsmith89/c6811893c1cbd2a72cc1d144a197bef2#file-makefile #VENV = .venv #export VIRTUAL_ENV := $(abspath ${VENV}) # putting the venv/bin at the start of the path means that the venv python will be called # and the venv libraries used automatically, so no need to 'source .venv/bin/activate' first # although it misses the hash flush 'hash -r' that the venv activate script does #export PATH := ${VIRTUAL_ENV}/bin:${PATH} #${VENV}: # python3 -m venv "$@" #pip-install: requirements.txt | ${VENV} # pip install --upgrade -r requirements.txt # ======================= .PHONY: sonar sonar: sonar-scanner .PHONY: update update: update2 @# putting this here instead of inline dep because otherwise check_makefile.sh will fail the target as build target doesn't exist in this Makefile.in @$(MAKE) build .PHONY: update2 update2: update-no-recompile @: .PHONY: update-no-recompile update-no-recompile: git pull --no-edit $(MAKE) submodules .PHONY: update-submodules update-submodules: git submodule update --init --remote .PHONY: updatem updatem: update-submodules @: .PHONY: docker-run docker-run: docker run -ti --rm ${DOCKER_IMAGE} ${ARGS} .PHONY: run run: docker-run @: .PHONY: concourse concourse: $(BASH_TOOLS)/cicd/concourse.sh .PHONY: fly fly: concourse @: .PHONY: docker-mount docker-mount: # --privileged=true is needed to be able to: # mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE} bash -c "cd /code; exec bash" .PHONY: docker-mount-alpine docker-mount-alpine: # --privileged=true is needed to be able to: # mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:alpine bash -c "cd /code; exec bash" .PHONY: docker-mount-debian docker-mount-debian: # --privileged=true is needed to be able to: # mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:debian bash -c "cd /code; exec bash" .PHONY: docker-mount-centos docker-mount-centos: # --privileged=true is needed to be able to: # mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:centos bash -c "cd /code; exec bash" .PHONY: docker-mount-ubuntu docker-mount-ubuntu: # --privileged=true is needed to be able to: # mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:ubuntu bash -c "cd /code; exec bash" .PHONY: mount mount: docker-mount @: .PHONY: mount-alpine mount-alpine: docker-mount-alpine @: .PHONY: mount-debian mount-debian: docker-mount-debian @: .PHONY: mount-centos mount-centos: docker-mount-centos @: .PHONY: mount-ubuntu mount-ubuntu: docker-mount-ubuntu @: # checks dockerhub build status for this repo - needs check_dockerhub_repo_build_status.py from Advanced Nagios Plugins Collection to be in $PATH .PHONY: dockerhub-status dockerhub-status: check_dockerhub_repo_build_status.py -r "$(DOCKER_IMAGE)" # For quick testing only - for actual Dockerfile builds see https://hub.docker.com/u/harisekhon and Dockerfiles source repo https://github.com/HariSekhon/Dockerfiles .PHONY: docker-alpine docker-alpine: $(BASH_TOOLS)/docker/docker_mount_build_exec.sh alpine .PHONY: docker-debian docker-debian: $(BASH_TOOLS)/docker/docker_mount_build_exec.sh debian .PHONY: docker-centos docker-centos: $(BASH_TOOLS)/docker/docker_mount_build_exec.sh centos .PHONY: docker-fedora docker-fedora: $(BASH_TOOLS)/docker/docker_mount_build_exec.sh fedora .PHONY: docker-ubuntu docker-ubuntu: $(BASH_TOOLS)/docker/docker_mount_build_exec.sh ubuntu .PHONY: travis travis: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://travis-ci.org/$(REPO)" .PHONY: travis-log travis-log: travis_last_log.py --failed $(REPO) .PHONY: travis-debug travis-debug: travis_debug_session.py $(REPO) .PHONY: browse browse: github @: .PHONY: commitcount commitcount: @# interestingly, even on 10,000 commit repos, there are no duplicate short hashes shown from: @# git log --all --pretty=format:"%h" | sort | uniq -d @git log --all --pretty=format:"%h" | wc -l .PHONY: github github: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)" .PHONY: github-url github-url: @. $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://github.com/$(REPO)" | tee /dev/stderr | tr -d '\n' | paste_clipboard .PHONY: gitlab gitlab: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://gitlab.com/$(REPO)" .PHONY: gitlab-url gitlab-url: @. $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://gitlab.com/$(REPO)" | tee /dev/stderr | tr -d '\n' | paste_clipboard .PHONY: bitbucket bitbucket: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://bitbucket.org/$(REPO)/src/master/" .PHONY: bitbucket-url bitbucket-url: @. $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://bitbucket.org/$(REPO)/src/master/" | tee /dev/stderr | tr -d '\n' | paste_clipboard .PHONY: status status: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://bitbucket.org/HariSekhon/DevOps-Bash-tools/src/master/STATUS.md" .PHONY: readme readme: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)/blob/master/README.md" .PHONY: issues issues: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)/issues" .PHONY: github dockerhub: @. $(BASH_TOOLS)/.bash.d/network.sh; browser "https://hub.docker.com/u/harisekhon" .PHONY: dockerhub-url dockerhub-url: @. $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://hub.docker.com/u/harisekhon" | tee /dev/stderr | tr -d '\n' | paste_clipboard .PHONY: startrack startrack: @echo "Don't run this too much, you will hit an API limit against your IP" @. $(BASH_TOOLS)/.bash.d/network.sh; \ browser "https://seladb.github.io/StarTrack-js/?\ u=$$(sed 's/\/.*//' <<< "$(REPO)")\ &r=$$(sed 's/.*\///' <<< "$(REPO)")" .PHONY: star star: startrack @: .PHONY: allstars allstars: @echo "Takes a while, don't run this all the time or you will hit an API limit against your IP" @REPOS="Nagios-Plugins Dockerfiles DevOps-Python-tools DevOps-Perl-tools DevOps-Bash-Tools Nagios-Plugin-Kafka HAProxy-configs"; \ . $(BASH_TOOLS)/.bash.d/network.sh; \ browser "https://seladb.github.io/StarTrack-js/#/preload?\ $$(\ for repo in $$REPOS; do \ printf "%s" "&r=HariSekhon,$$repo"; \ done | \ sed 's/\&//'\ )" .PHONY: ls ls: @echo $(CODE_FILES) | tr ' ' '\n' | sort .PHONY: wc wc: if [ -x wc.sh ]; then ./wc.sh; exit 1; fi @# CODE_FILES := definitions in Makefiles must not be quoted or will get wc error 'open: File name too long' @wc -l $(CODE_FILES) @printf 'Total Lines:\t\t\t' @cat $(CODE_FILES) | wc -l | sed 's/[[:space:]]//g' @printf 'Total Lines without # comments:\t' @sed 's/#.*//;/^[[:space:]]*$$/d' $(CODE_FILES) | wc -l | sed 's/[[:space:]]//g' @printf 'Total Files:\t\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | wc -l | sed 's/[[:space:]]//g' @printf 'of which not the following:\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -Ev -e '\.bash' \ -e lib/ \ -e install/ \ -e setup/ \ -e 'tests*/' \ -e vagrant/ \ | wc -l | sed 's/[[:space:]]//g' @printf 'of which .bash*:\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -c '\.bash' @printf 'of which lib/:\t\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -c lib/ @printf 'of which install/:\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -c install/ @printf 'of which setup/:\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -c setup/ @printf 'of which test(s)/:\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -Ec 'tests*/' @printf 'of which vagrant/:\t\t' @tr ' ' '\n' <<< "$(CODE_FILES)" | grep -c vagrant/ mdl: @echo "Checking Markdown for issues" @echo @if .mdl.rb; then \ mdl -s .mdl.rb *.md; \ else \ mdl *.md; \ fi # finds .swp, would need to port out code lists #.PHONY: wcall #wcall: # find . -type f --not -path '*.git*' -exec cat {} \; | wc -l # #.PHONY: wcall #wcall: # find . -type f -not -path '*.git*' -exec sed 's/#.*//;/^[[:space:]]*$$/d' {} \; | wc -l .PHONY: repos repos: @if ! grep -q "OTHER_REPOS_START" README.md || \ ! grep -q "OTHER_REPOS_END" README.md || \ ! grep -q "More Core Repos" README.md; then \ echo "Adding More Core Repos section to README.md"; \ printf '\n## More Core Repos\n\n\n\n\n' >> README.md; \ fi; \ markdown_replace_repos.sh mac-exclude-backups: @for path in $(MAC_TIME_MACHINE_BACKUP_EXCLUDE_PATHS); do \ if [[ "$${path:0:1}" != / ]]; then \ path="$$PWD/$$path"; \ fi; \ echo "Excluding: $$path"; \ sudo tmutil addexclusion -p "$$path"; \ done nobackup: @if uname | grep -q Darwin; then \ $(MAKE) mac-exclude-backups; \ fi ================================================ FILE: README.md ================================================ # Hari Sekhon - DevOps Bash Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-bash-tools?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-bash-tools?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/network) [![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Bash-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) [![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Bash-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/badge/license-MIT-green)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/LICENSE) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Bash-tools?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/commits/master) [![Codacy](https://app.codacy.com/project/badge/Grade/dffc1bfd13404c95b5a0ab97fd47974e)](https://www.codacy.com/gh/HariSekhon/DevOps-Bash-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/devops-bash-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/devops-bash-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Bash-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Bash-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Bash-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Bash-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Bash-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Bash-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Bash-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Bash-tools) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Bash-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Bash-tools) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](#hari-sekhon---devops-bash-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](#hari-sekhon---devops-bash-tools) [![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/bash-tools) [![Dockerfile](https://img.shields.io/badge/repo-Dockerfiles-blue?logo=docker&logoColor=white)](https://github.com/HariSekhon/Dockerfiles) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/bash-tools?label=DockerHub%20pulls&logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/bash-tools) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts) [![StarCharts](https://img.shields.io/badge/Star-Charts-blue?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md) [![Mac Homebrew](https://img.shields.io/badge/Mac-Homebrew-999999?logo=apple&logoColor=white)](https://brew.sh/) [![Alpine](https://img.shields.io/badge/Linux-Alpine-0D597F?logo=alpine%20linux)](https://alpinelinux.org/) [![CentOS](https://img.shields.io/badge/Linux-CentOS-262577?logo=centos&logoColor=white)](https://www.centos.org/) [![Debian](https://img.shields.io/badge/Linux-Debian-A81D33?logo=debian)](https://www.debian.org/) [![Fedora](https://img.shields.io/badge/Linux-Fedora-294172?logo=fedora&logoColor=white)](https://getfedora.org/) [![Redhat](https://img.shields.io/badge/Linux-Redhat-EE0000?logo=red%20hat)](https://www.redhat.com/en) [![Rocky](https://img.shields.io/badge/Linux-Rocky-10B981?logo=rockylinux&logoColor=white)](https://rockylinux.org/) [![Ubuntu](https://img.shields.io/badge/Linux-Ubuntu-E95420?logo=ubuntu&logoColor=white)](https://ubuntu.com/) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse&logoColor=white)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/cicd/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go&logoColor=white)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/cicd/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Bash-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Bash-tools) [![BuildKite](https://img.shields.io/buildkite/f11bdd9690a9bac9a8edc6094dc2f2b9af3218a7a15d4ec17d/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-bash-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/devops-bash-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/devops-bash-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Bash-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Bash-tools) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Bash-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e53eaeea284e010982eaa6e&pipeline=DevOps-Bash-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Bash-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Bash-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Bash-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Bash-tools) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/travis/.travis.yml) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed%20by-Hound-8E64B0.svg)](https://houndci.com) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Bash-tools) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Bash-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Bash-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Bash-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=1&branchName=master) [![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Bash-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-bash-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/cicd/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/cicd/cloudbuild.yaml) [![ShellCheck](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/xml.yaml) [![Markdown](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/kics.yaml) [![Grype](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/semgrep-cloud.yaml) [![Trivy](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_alpine.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/docker_bash_ubuntu.yaml) [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac.yaml) [![Mac 11](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Ubuntu%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Debian%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Debian+10%22) [![Debian 11](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Debian+11%22) [![Debian 12](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Debian%2012/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Debian+12%22) [![Fedora](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Python+3.10%22) [![Python 3.11](https://github.com/HariSekhon/DevOps-Bash-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Bash-tools/actions?query=workflow%3A%22Python+3.11%22) [git.io/bash-tools](https://git.io/bash-tools) 1000+ DevOps Shell Scripts and Advanced Bash environment. Fast, Advanced Systems Engineering, Automation, APIs, shorter CLIs, etc. Heavily used in many [GitHub repos](https://github.com/search?o=desc&q=user%3Aharisekhon+type%3Arepository&type=Repositories), dozens of [DockerHub builds](https://hub.docker.com/r/harisekhon) ([Dockerfiles](https://github.com/HariSekhon/Dockerfiles)) and 600+ [CI builds](https://harisekhon.github.io/CI-CD/). ## Summary - Scripts for many popular DevOps technologies, see [Index](#index) below for more details - Advanced configs for common tools like [Git](https://git-scm.com/), [vim](https://www.vim.org/), [screen](https://www.gnu.org/software/screen/), [tmux](https://github.com/tmux/tmux/wiki), [PostgreSQL psql](https://www.postgresql.org/) etc... - CI configs for most major Continuous Integration products (see [CI builds](https://harisekhon.github.io/CI-CD/) page) - CI scripts for a drop-in framework of standard checks to run in all [CI builds](https://harisekhon.github.io/CI-CD/), CI detection, accounting for installation differences across CI environments, root vs user, virtualenvs etc. - API scripts auto-handling authentication, tokens and other details to quickly query popular APIs with a few keystrokes just supplying the `/path/endpoint` - Advanced Bash environment - `.bashrc` + `.bash.d/*.sh` - aliases, functions, colouring, dynamic Git & shell behaviour enhancements, automatic pathing for installations and major languages like Python, Perl, Ruby, NodeJS, Golang across Linux distributions and Mac. See [.bash.d/README.md](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.bash.d/README.md) - Installs the best systems packages - [AWS CLI](https://aws.amazon.com/cli/), [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/?view=azure-cli-latest), [GCloud SDK](https://cloud.google.com/sdk), [Digital Ocean CLI](https://docs.digitalocean.com/reference/doctl/), [Terraform](https://www.terraform.io/), [Terragrunt](https://terragrunt.gruntwork.io/), [GitHub CLI](https://github.com/cli/cli), [Kubernetes](https://kubernetes.io/) [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/) & [kustomize](https://kustomize.io/), [Helm](https://helm.sh/), [eksctl](https://eksctl.io/), [Docker-Compose](https://docs.docker.com/compose/), [jq](https://stedolan.github.io/jq/) and many others... extensive package lists for servers and desktops for most major Linux distributions package managers and Mac - `install/` - contains many installation scripts for popular open source software and direct binary downloads from GitHub releases - `configs/` - contains many dot configs for common technologies like ViM, top, Screen, Tmux, MySQL, PostgreSQL etc. - `setup/` - contains setup scripts, package lists, extra configs, Mac OS X settings etc. - Utility Libraries used by many hundreds of scripts and [builds](https://harisekhon.github.io/CI-CD/) across [repos](https://github.com/search?o=desc&q=user%3Aharisekhon+type%3Arepository&type=Repositories): - `.bash.d/` - interactive library - `lib/` - scripting and CI library - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ scripts for [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/), [AWS Athena](https://aws.amazon.com/athena/) + [CloudTrail](https://aws.amazon.com/cloudtrail/), [Google BigQuery](https://cloud.google.com/bigquery) - [Templates](https://github.com/HariSekhon/Templates) - templates for common programming languages and build configs - [Kubernetes Configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs for most common scenarios, including Production Best Practices, Tips & Tricks See Also: [similar DevOps repos](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/README.md#see-also) in other languages Hari Sekhon Cloud & Big Data Contractor, United Kingdom (ex-Cloudera, former Hortonworks Consultant) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/)
*(you're welcome to connect with me on LinkedIn)* ### Quick Setup To bootstrap, install packages and link in to your shell profile to inherit all configs, do: ```bash curl -L https://git.io/bash-bootstrap | sh ``` - Adds sourcing to `.bashrc`/`.bash_profile` to automatically inherit all `.bash.d/*.sh` environment enhancements for all technologies (see [Inventory](#index) below) - Symlinks `.*` config dotfiles to `$HOME` for [git](https://git-scm.com/), [vim](https://www.vim.org/), top, [htop](https://hisham.hm/htop/), [screen](https://www.gnu.org/software/screen/), [tmux](https://github.com/tmux/tmux/wiki), [editorconfig](https://editorconfig.org/), [Ansible](https://www.ansible.com/), [PostgreSQL](https://www.postgresql.org/) `.psqlrc` etc. (only when they don't already exist so there is no conflict with your own configs) - Installs OS package dependencies for all scripts (detects the OS and installs the right RPMs, Debs, Apk or Mac HomeBrew packages) - Installs Python packages - Installs [AWS CLI](https://aws.amazon.com/cli/) To only install package dependencies to run scripts, simply `cd` to the git clone directory and run `make`: ```shell git clone https://github.com/HariSekhon/DevOps-Bash-tools bash-tools cd bash-tools make ``` `make install` sets your shell profile to source this repo. See [Individual Setup Parts](#individual-setup-parts) below for more install/uninstall options. ## Index - [Dot Configs](#dot-configs) - `.gitconfig`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, `.gitignore`... - [Bash Environment & Libraries](#bash-environment--libraries) - `.bashrc`, `.bash.d/` interactive library, `lib/` scripting library - [Installation Scripts](#installation-scripts) for many popular open source technologies - [Linux & Mac](#linux--mac) - curl OAuth / JWT, LDAP, find duplicate files, SSL certificate get/validate, URL encoding/decoding, Vagrant - [Mac & AppleScript](#mac--applescript) - Mac settings and UI automation scripts, send keystrokes, mouse clicks, detect foreground app, switch app, detect locked screen or screensaver, activate screensaver, Hammerspoon system event handlers such as automatically switching audio to be able to Shazam while watching on AirPods - [Monitoring](#monitoring) - Grafana, Prometheus, Node Exporter, scripted collection of common Linux & Mac cli monitoring stats and log locations for quick generation of vendor support tarball bundles both locally and over SSH - [AWS - Amazon Web Services](#aws---amazon-web-services) - AWS account summary, lots of IAM reports, CIS Benchmark config hardening, EC2, ECR, EKS, Spot termination, S3 access logging, KMS key rotation info, SSM, CloudTrail, CloudWatch billing alarm with SNS notification topic and subscription for email alerts - [GCP - Google Cloud Platform](#gcp---google-cloud-platform) - massive GCP auto-inventory, scripts for GCE, GKE, GCR, Secret Manager, BigQuery, Cloud SQL, Cloud Scheduler, Terraform service account creation - [Kubernetes](#kubernetes) - massive Kubernetes auto-inventory, cluster management scripts & tricks - [Docker](#docker) - Docker API, Dockerhub API, Quay.io API scripts - [Databases](#databases) - fast CLI wrappers, instant Docker sandboxes (PostgreSQL, MySQL, MariaDB, SQLite), [SQL scripts](https://github.com/HariSekhon/SQL-scripts), SQL script testers against all versions of a DB, advanced `.psqlrc` - [Data](#data) - data tools, converters and format validators for Avro, Parquet, CSV, JSON, INI / Properties files (Java), LDAP LDIF, XML, YAML - [Big Data & NoSQL](#big-data--nosql) - Kafka, Hadoop, HDFS, Hive, Impala, ZooKeeper, Cloudera Manager API & Cloudera Navigator API scripts - [Git - GitHub, GitLab, Bitbucket, Azure DevOps](#git---github-gitlab-bitbucket-azure-devops) - scripts for Git local & mirror management, GitHub, GitLab & BitBucket APIs - [Markdown](#markdown) - generate Markdown indexes and debug `mdl` issues like MD005 inconsistent list indentation in large `README.md` files - [CI/CD - Continuous Integration / Continuous Delivery](#cicd---continuous-integration--continuous-deployment) - API scripts & build pipeline configs for most major CI systems: - Jenkins, Concourse, GoCD, TeamCity - one-touch boot & build - Azure DevOps Pipelines, GitHub Actions Workflows, GitLab CI, BitBucket Pipelines, AppVeyor, BuildKite, Travis CI, Circle CI, Codefresh, CodeShip, Drone.io, Semaphore CI, Shippable ... - Terraform Cloud, Octopus Deploy - Checkov / Bridgecrew Cloud - [AI & IPaaS](#ai--ipaas) - OpenAI (ChatGPT), Make.com - [Internet Services](#internet-services) - Google Maps, Cloudflare, DataDog, Digital Ocean, Kong API Gateway, GitGuardian, Jira, NGrok, Traefik, Pingdom, Wordpress and various pastebins and file upload sites - [Java](#java) - Java utilies to debug running Java programs or decompile Java JAR code for deeper debugging - [Python](#python) - Python utilities & library management - [Perl](#perl) - Perl utilities & library management - [Golang](#golang) - Golang utilities - [Diagrams](#diagrams) - scripts to generate diagrams from D2lang, MermaidJS and Python Mingrammer source code used in my [HariSekhon/Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) repo - [Media](#media) - video downloaders & converts, MP3 metadata editing, grouping and ordering of albums and audiobooks, mkv/avi to mp4 converters, 720p video downscaler for posting to social media, download YouTube videos or even entire channels and videos from other social media sites like Twitter / X or Facebook, terminal gif capture - [Spotify](#spotify) - 40+ Spotify API scripts for backups, managing playlists, track deduplication, URI conversion, search, add/delete, liked tracks, followed artists, top artists, top tracks etc. - [More Linux & Mac](#more-linux--mac) - more systems administration scripts, package installation automation - [Builds, Languages & Linting](#builds-languages--linting) - programming language, build system & CI linting - [Templates](https://github.com/HariSekhon/Templates) - Templates for AWS, GCP, Terraform, Docker, Jenkins, Cloud Build, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml etc. - [Kubernetes Configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs for most common scenarios, including Production Best Practices, Tips & Tricks ### Dot Configs Top-level dotfiles and `configs/` directory: - `.*` - dot conf files for lots of common software eg. advanced `.vimrc`, `.gitconfig`, massive `.gitignore`, `.editorconfig`, `.screenrc`, `.tmux.conf` etc. - `.vimrc` - contains many awesome [vim](https://www.vim.org/) tweaks, plus hotkeys for linting lots of different file types in place, including Python, Perl, Bash / Shell, Dockerfiles, JSON, YAML, XML, CSV, INI / Properties files, LDAP LDIF etc without leaving the editor! - `.screenrc` - fancy [screen](https://www.gnu.org/software/screen/) configuration including advanced colour bar, large history, hotkey reloading, auto-blanking etc. - `.tmux.conf` - fancy [tmux](https://github.com/tmux/tmux/wiki) configuration include advanced colour bar and plugins, settings, hotkey reloading etc. - [Git](https://git-scm.com/): - `.gitconfig` - advanced Git configuration - `.gitignore` - extensive Git ignore of trivial files you shouldn't commit - enhanced Git diffs - protections against committing AWS secret keys or merge conflict unresolved files ### Bash Environment & Libraries Top-level `.bashrc` and `.bash.d/` directory: - `.bashrc` - shell tuning and sourcing of `.bash.d/*.sh` - `.bash.d/*.sh` - thousands of lines of advanced bashrc code, aliases, functions and environment variables for: - [Linux](https://en.wikipedia.org/wiki/Linux) & [Mac](https://en.wikipedia.org/wiki/MacOS) - SCM - [Git](https://git-scm.com/), [Mercurial](https://www.mercurial-scm.org/), [Svn](https://subversion.apache.org) - [AWS](https://aws.amazon.com/) - [GCP](https://cloud.google.com/) - [Docker](https://www.docker.com/) - [Kubernetes](https://kubernetes.io/) - [Kafka](http://kafka.apache.org/) - [Vagrant](https://www.vagrantup.com/) - automatic GPG and SSH agent handling for handling encrypted private keys without re-entering passwords, and lazy evaluation to only prompt key load the first time SSH is called - and lots more - see [.bash.d/README](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.bash.d/README.md) for a more detailed list - run `make bash` to link `.bashrc`/`.bash_profile` and the `.*` dot config files to your `$HOME` directory to auto-inherit everything - `lib/*.sh` - Bash utility libraries full of functions for [Docker](https://www.docker.com/), environment, CI detection ([Travis CI](https://travis-ci.org/), [Jenkins](https://jenkins.io/) etc), port and HTTP url availability content checks etc. Sourced from all my other [GitHub repos](https://github.com/harisekhon) to make setting up Dockerized tests easier. ### Installation Scripts - `install/install_*.sh` - various simple to use installation scripts for common technologies like: - [AWS CLI](https://aws.amazon.com/cli/) - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/?view=azure-cli-latest) - [GCloud SDK](https://cloud.google.com/sdk) - [GitHub CLI](https://cli.github.com/) - [Terraform](https://www.terraform.io/) - [Terragrunt](https://terragrunt.gruntwork.io/) - [Direnv](https://direnv.net/) - [Ansible](https://www.ansible.com/) - [K3s](https://k3s.io) - [MiniKube](https://kubernetes.io/docs/setup/learning-environment/minikube/) (Kubernetes) - [MiniShift](https://www.okd.io/minishift/) ([Redhat OpenShift](https://www.openshift.com/) / [OKD](https://www.okd.io/) dev VMs) - [Maven](https://maven.apache.org/) - [Gradle](https://gradle.org/) - [SBT](https://www.scala-sbt.org/) - [EPEL](https://fedoraproject.org/wiki/EPEL) - [RPMforge](http://repoforge.org/) - [Homebrew](https://brew.sh/) - [Travis CI](https://travis-ci.org/) - [Circle CI](https://circleci.com/) - [AppVeyor](https://www.appveyor.com/) - [BuildKite](https://buildkite.com) - [Avro Tools](https://avro.apache.org/) - [Parquet Tools](https://github.com/apache/parquet-mr/tree/master/parquet-tools) - [Prometheus](https://prometheus.io/) - various JDKs and RDBMS JDBC connector jars - and many more... ### Linux & Mac `bin/` directory: - `login.sh` - logs to major Cloud platforms if their credentials are found in the environment, CLIs such as AWS, GCP, Azure, GitHub... Docker registries: DockerHub, GHCR, ECR, GCR, GAR, ACR, Gitlab, Quay... - `clean_caches.sh` - cleans out OS package and programming language caches - useful to save space or reduce Docker image size - `crypto_dice_rolls.sh` - generates 100 random dice rolls to test a new crypto hardware wallet's fidelity (do not use this for your real crypto seed as your machine could be infected with malware which steals your seed phrase) - `delete_duplicate_files.sh` - deletes duplicate files with (N) suffixes, commonly caused by web browser downloads, in the given or current directory. Checks they're exact duplicates of a matching basename file without the (N) suffix with the exact same checksum for safety. Prompts to delete per file. To auto-accept deletions, do `yes | delete_duplicate_files.sh`. This is a fast way of cleaning up your `~/Downloads` directory and can be put your user crontab - `disk_speed_read_sequential_dd.sh` - runs a sequential read speed test from the given file using dd and bypassing filesystem cache for a more accurate test - `disk_speed_read_random_dd.sh` - runs a random I/O read speed test from the given file using dd and bypassing filesystem cache for a more accurate test - `disk_speed_write_sequential_dd.sh` - runs a sequential write speed test to a file in the given or current directory using dd and bypassing filesystem cache for a more accurate test - `disk_speed_read_sequential_fio.sh` - runs a sequential read speed test in the current or given directory using fio - `disk_speed_read_random_fio.sh` - runs a random I/O read test in the current or given directory using fio - `disk_speed_write_sequential_fio.sh` - runs a sequential write speed test to the current or given directory using fio - `disk_speed_write_random_fio.sh` - runs a sequential write speed test to the current or given directory using fio - `download_url_file.sh` - downloads a file from a URL using wget with no clobber and continue support, or curl with atomic replacement to avoid race conditions. Used by `github/github_download_release_file.sh`, `github_download_release_jar.sh`, and `install/download_*_jar.sh` - `curl_auth.sh` - shortens `curl` command by auto-loading your OAuth2 / JWT API token or username & password from environment variables or interactive starred password prompt through a ram file descriptor to avoid placing them on the command line (which would expose your credentials in the process list or OS audit log files). Used by many other adjacent API querying scripts - `curl_with_cookies.sh` - extracts cookies for a given URL from your `\$BROWSER`'s cookie jar and passes them to the `curl` command along with the rest of the args (workaround for older curl builds and Homebrew builds that don't have the newer `--cookies-from-browser functionality) - `find_duplicate_files*.sh` - finds duplicate files by size and/or checksum in given directory trees. Checksums are only done on files that already have matching byte counts for efficiency - `find_broken_links.sh` - find broken links with delays to avoid tripping defenses - `find_broken_symlinks.sh` - find broken symlinks pointing to non-existent files/directories - `find_lock.sh` - tries to find if a lockfile is used in the given or current working directory by taking snapshots of the file list before and after a prompt in which you should open/close an application - `foreach_path_bin.sh` - runs each binary of the given name found in `$PATH` with the args given. Useful to find all the installed versions of a program in different paths eg. `~/bin/` vs `/usr/local/bin/` eg. `foreach_path_bin.sh terraform --version` - `http_duplicate_urls.sh` - find duplicate URLs in a given web page - `htmldecode.sh` - decodes HTML encoding. Detects available tools such as Perl, Python or xmlstarlet and uses whatever is available - `ldapsearch.sh` - shortens `ldapsearch` command by inferring switches from environment variables - `ldap_user_recurse.sh` / `ldap_group_recurse.sh` - recurse Active Directory LDAP users upwards to find all parent groups, or groups downwards to find all nested users (useful for debugging LDAP integration and group-based permissions) - `linux_distro_versions.sh` - quickly returns the list of major versions for a given Linux distro - `diff_line_threshold.sh` - compares two files vs a line count diff threshold to determine if they are radically different. Used to avoid overwriting files which are not mere updates but completely different files - `mv.sh` - moves directory trees resumably and removes the source files as they're copied over. Useful to migrate data from one disk to another, optionally with checksums. Uses rsync and shows the overall % of files transferred and the MB/s data transfer rate - `network_gateway.sh` - get the network gateway IP address on Linux or Mac - `open.sh` - opens given arg, file or URL using whatever default system opener is available for Linux or Mac - `organize_downloads.sh` - moves files of well-known extensions in the `$HOME/Downloads` directory older than 1 week to capitalized subdirectories of their type to keep the `$HOME/Downloads/` directory tidy - `copy_to_clipboard.sh` - copies stdin or string arg to system clipboard on Linux or Mac - `paste_from_clipboard.sh` - pastes from system clipboard to stdout on Linux or Mac - `paste_from_clipboard_upon_changes.sh` - pastes from system clipboard to stdout on Linux or Mac whenever the clipboard changes - `paste_diff_settings.sh` - takes snapshots of before and after clipboard changes and diffs them to show config changes - `processes_ram_sum.sh` - sums the RAM usage of all processes matching a given regex in GB to one decimal place - `pldd.sh` - parses `/proc` on Linux to show the runtime `.so` loaded dynamic shared libraries a program pid is using. Runtime equivalent of the classic static `ldd` command and because the system `pldd` command often fails to attach to a process - `random_select.sh` - selects one of given args at random. Useful for sampling, running randomized subsets of large test suites etc. - `random_number.sh` - prints a random integer between two integer arguments (inclusive) - `random_string.sh` - prints a random alphanumeric string of a given length - `screen_terminal_to_stdout.sh` - dumps the GNU Screen terminal output to stdout - `screen_terminal_to_clipboard.sh` - dumps the GNU Screen terminal output to a temp file and copies to clipboard for sharing & debugging purposes - `shields_embed_logo.sh` - base64 encodes a given icon file or url and prints the `logo=...` url parameter you need to add the [shields.io](https://shields.io/) badge url - `shorten_text_selection.sh` - shortens the selected text in the prior window. Replaces `and` with `&` and crushes out multiple blank lines. I use this for LinkedIn comments due to the short 1250 character limit - `shred_file.sh` - overwrites a file 7 times to DoD standards before deleting it to prevent recovery of sensitive information - `shred_free_space.sh` - overwrites free space to prevent recovery of sensitive information for files that have already been deleted - `split.sh` - split large files into N parts (defaults to the number of your CPU cores) to parallelize operations on them - `ssl_get_cert.sh` - gets a remote `host:port` server's SSL cert in a format you can pipe, save and use locally, for example in Java truststores - `ssl_verify_cert.sh` - verifies a remote SSL certificate (battle tested more feature-rich version `check_ssl_cert.pl` exists in the [Advanced Nagios Plugins](https://github.com/HariSekhon/Nagios-Plugins) repo) - `ssl_verify_cert_by_ip.sh` - verifies SSL certificates on specific IP addresses, useful to test SSL source addresses for CDNs, such as Cloudflare Proxied sources before enabling SSL Full-Strict Mode for end-to-end, or Kubernetes ingresses (see also `curl_k8s_ingress.sh`) - `text_filter_ending_substrings.sh` - for a given patterns file of substring endings, print all lines that match in the following files. Uses awk to safely handle all characters as literals, unlike grep, while also maintaining end anchoring which you cannot do using `grep -F`. Optimized awk code uses a bucketing hash for performance to not attempt matching lines which are shorter than patterns, reducing the number of match attempts - `tmux_vertical.sh` - launches tmux with N-way vertical shell split or commands given as args in equally balanced vertical panes. Fast way to launch a bunch of shell or commands in an easily reviewable side-by-side way - `tmux_horizontal.sh` - same as above but split horizontally - `tmux_square.sh` - same as above but with 4 panes in a square tiled view - `urlencode.sh` / `urldecode.sh` - URL encode/decode quickly on the command line, in pipes etc. - `urlextract.sh` - extracts the URLs from a given string arg, file or standard input - `url_extract_redirects.sh` - extracts the URLs from a given string arg, file or standard input, queries each one and outputs the redirected urls instead to stdout - `url_replace_redirects.sh` - extracts the URLs from a given string arg, file or standard input, queries each one and outputs the entire contents to stdout with the urls replaced by the redirected urls - `urlopen.sh` - opens the URL given as an arg, or first URL found from stdin or a given file. Uses the system's default browser - `vagrant_hosts.sh` - generate `/etc/hosts` output from a `Vagrantfile` - `vagrant_total_mb.sh` - calculate the RAM committed to VMs in a `Vagrantfile` See also [Knowledge Base notes for Linux](https://github.com/HariSekhon/Knowledge-Base/blob/main/linux.md) and [Mac](https://github.com/HariSekhon/Knowledge-Base/blob/main/mac.md). ### Mac & AppleScript Mac automation scripts to automate the Mac UI and settings `bin/` directory: - `mac_diff_settings.sh` - takes before and after snapshots of UI setting changes and diffs them to make it easy to find `defaults` keys to add to `setup/mac_settings.sh` to save settings - `mac_restore_file.sh` - checks all the backup mount points for the latest backup that has a given file and then restores it - `mac_backup_du_in_progress.sh` - find large files in the currently in-progress Time Machine backup to find out what is taking so long and racking up so many more GB of changes than you expect. This helps discover large but unnecessary files that you might want to exclude using the adjacent script `mac_backup_exclude_paths.sh` - `mac_backup_exclude_paths.sh` - excludes many common large caches, docker and VM paths from macOS Time Machine backups - `mac_backup_find_excluded_paths.sh` - does a deep search for macOS Time Machine excluded backup paths on file/folder attributes. See [HariSekhon/Knowledge-Base Mac page](https://github.com/HariSekhon/Knowledge-Base/blob/main/mac.md#time-machine) for why - `mac_rmdir.sh` - safely delete a directory on Mac only if it is empty of actual data, by first removing macOS hidden metadata files and dirs such as `.fseventsd/`, `.Spotlight-V100/` and `.DS_Store` - straight `rmdir` fails otherwise - `mac_iso_to_usb.sh` - converts a given ISO file to a USB bootable image and burns it onto a given or detected inserted USB drive - `mac_ramdisk.sh` - creates a mac ramdisk of given MB size. Useful for performance, or even testing disk write scripts such as `disk_speed_write_*.sh` without wearing out your SSD - `mac_delete_local_snapshots.sh` - deletes local macOS snapshots to free up disk space. When there is a substantial discrepancy between what the `df -h` command and the Finder UI shows, this is often the cause - `copy_to_clipboard.sh` - copies stdin or string arg to system clipboard on Linux or Mac - `paste_from_clipboard.sh` - pastes from system clipboard to stdout on Linux or Mac - `paste_from_clipboard_upon_changes.sh` - pastes from system clipboard to stdout on Linux or Mac whenever the clipboard changes - `paste_diff_settings.sh` - Takes snapshots of before and after clipboard changes and diffs them to show config changes `applescript/` directory: - `keystrokes.sh` - send N keystroke combinations - `mouse_clicks.sh` - send N mouse click combinations to sequence of screen coordinates - `get_mouse_coordinates.sh` - print the current mouse coordinates - to know what to pass to above script - `mouse_clicks_remote_desktop.sh` - switches to Microsoft Remote Desktop, waits 10 seconds and then clicks the mouse once a minute to prevent the screensaver from coming on. Workaround to Active Directory Group Policies that don't let you disable the screensaver. Point your mouse to an area that will have no mouse click effect, the Cmd-Tab to Terminal and run this - `get_frontmost_process_title.scpt` - detect the frontmost window - to detect if you should send keystrokes / mouse clicks) - `set_frontmost_process.scpt` - switch to bring the given app to the foreground to send keystrokes / mouse clicks to it - `browser_get_default.scpt` - get the default configured browser in format passable to Applescript (for above script) - `is_screen_locked.py` - detect if the screen is locked to stop sending keystrokes or mouse clicks - `is_screensaver_running.scpt` - detect if the screensaver is running to stop sending keystrokes or mouse clicks - `reopen_app.sh` - relaunch a given app (used to reload Shazam to detect DB changes after removing tracks programmatically from its DB) - `spotify_app_search.sh` - runs a search in the Spotify App on Mac using Applescript - `shazam_app_dump_tracks.sh` - dumps `artist - track` one per line from the Shazam local sqlite DB - `shazam_app_delete_track.sh` - deletes a given `"artist" "track"` from the Shazam local sqlite DB - `shazam_search_spotify_then_delete_track.sh` - searches for each Shazam'd track in the local Spotify desktop app, then prompts to delete each track from the local Shazam DB once you've saved it in Spotify. Useful to migrate Shazam'd tracks to Spotify after Apple removed the integration - `screensaver_activate.scpt` - activate screensaver - `shorten_text_selection.scpt` - shortens the selected text in the prior window. Replaces `and` with `&` and crushes out multiple blank lines. I use this for LinkedIn comments due to the short 1250 character limit - `start_app_at_login.sh` - adds an app to the Login items to auto-start Hammerspoon code has been moved to its own repo: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) See also [Mac](https://github.com/HariSekhon/Knowledge-Base/blob/main/mac.md) page in [HariSekhon/Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base). ### Monitoring `monitoring/` directory: - `dump_stats.sh` - dumps common command outputs to text files in a local tarball. Useful to collect support information for vendor support cases - `grafana_api.sh` - queries the [Grafana](https://grafana.com/) API with authentication - `log_timestamp_large_intervals.sh` - finds log lines whose timestamp intervals exceed the given number of seconds and outputs those log lines with the difference between the last and current timestamps. Useful to find actions that are taking a long time from log files such as CI/CD logs - `prometheus.sh` - starts [Prometheus](https://prometheus.io/) locally, downloading it if not found in `$PATH` - `prometheus_docker.sh` - starts [Prometheus](https://prometheus.io/) in Docker using `docker-compose` - `prometheus_node_exporter.sh` - starts Prometheus `node_exporter` locally, downloading it if not found in `$PATH` - `ssh_dump_stats.sh` - uses SSH and `dump_stats.sh` to dump common command outputs from remote servers to a local tarball. Useful for vendor support cases - `ssh_dump_logs.sh` - Uses SSH to dump logs from server to local text files for uploading to vendor support cases See doc pages in [HariSekhon/Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) on Grafana, Prometheus, OpenTSDB, InfluxDB etc. ### Databases `mysql/`, `postgres/`, `sql/` and `bin/` directories: - [sql/](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL scripts for [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/), [Google BigQuery](https://cloud.google.com/bigquery) and [AWS Athena](https://aws.amazon.com/athena/) [CloudTrail](https://aws.amazon.com/cloudtrail/) logs integration - `sqlite.sh` - one-touch [SQLite](https://www.sqlite.org/index.html), starts sqlite3 shell with sample 'chinook' database loaded - `mysql*.sh` - [MySQL](https://www.mysql.com/) scripts: - `mysql.sh` - shortens `mysql` command to connect to [MySQL](https://www.mysql.com/) by auto-populating switches from both standard environment variables like `$MYSQL_TCP_PORT`, `$DBI_USER`, `$MYSQL_PWD` (see [doc](https://dev.mysql.com/doc/refman/8.0/en/environment-variables.html)) and other common environment variables like `$MYSQL_HOST` / `$HOST`, `$MYSQL_USER` / `$USER`, `$MYSQL_PASSWORD` / `$PASSWORD`, `$MYSQL_DATABASE` / `$DATABASE` - `mysql_foreach_table.sh` - executes a SQL query against every table, replacing `{db}` and `{table}` in each iteration eg. `select count(*) from {table}` - `mysql_*.sh` - various scripts using `mysql.sh` for row counts, iterating each table, or outputting clean lists of databases and tables for quick scripting - `mysqld.sh` - one-touch [MySQL](https://www.mysql.com/), boots docker container + drops in to `mysql` shell, with `/sql` scripts mounted in container for easy sourcing eg. `source /sql/.sql`. Optionally loads sample 'chinook' database - see also the [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) repo for many more straight MySQL SQL scripts - `mariadb.sh` - one-touch [MariaDB](https://mariadb.org/), boots docker container + drops in to `mysql` shell, with `/sql` scripts mounted in container for easy sourcing eg. `source /sql/.sql`. Optionally loads sample 'chinook' database - `postgres*.sh` / `psql.sh` - [PostgreSQL](https://www.postgresql.org/) scripts: - `postgres.sh` - one-touch [PostgreSQL](https://www.postgresql.org/), boots docker container + drops in to `psql` shell, with `/sql` scripts mounted in container for easy sourcing eg. `\i /sql/.sql`. Optionally loads sample 'chinook' database - `psql.sh` - shortens `psql` command to connect to [PostreSQL](https://www.postgresql.org/) by auto-populating switches from environment variables, using both standard postgres supported environment variables like `$PG*` (see [doc](https://www.postgresql.org/docs/12/libpq-envars.html)) as well as other common environment variables like `$POSTGRESQL_HOST` / `$POSTGRES_HOST` / `$HOST`, `$POSTGRESQL_USER` / `$POSTGRES_USER` / `$USER`, `$POSTGRESQL_PASSWORD` / `$POSTGRES_PASSWORD` / `$PASSWORD`, `$POSTGRESQL_DATABASE` / `$POSTGRES_DATABASE` / `$DATABASE` - `postgres_foreach_table.sh` - executes a SQL query against every table, replacing `{db}`, `{schema}` and `{table}` in each iteration eg. `select count(*) from {table}` - `postgres_*.sh` - various scripts using `psql.sh` for row counts, iterating each table, or outputting clean lists of databases, schemas and tables for quick scripting - `checks/check_sqlfluff.sh` - recursively iterates all SQL code files found in the given or current directory and runs SQLFluff linter against them, inferring the different SQL dialects from each path/filename/extension ### AWS - Amazon Web Services `aws/` directory: - [AWS](https://aws.amazon.com/) scripts - `aws_*.sh`: - `aws_profile.sh` - switches to an AWS Profile selected from a convenient interactive menu list of AWS profiles from `$AWS_CONFIG_FILE` - useful when you have lots of AWS work profiles - see also [HariSekhon/Environments](https://github.com/HariSekhon/Environments) for automated switching using direnv when `cd`ing into relevant directories - `aws_cli_create_credential.sh` - creates an AWS service account user for CI/CD or CLI with Admin permissions (or other group or policy), creates an AWS Access Key, saves a credentials CSV and even prints the shell export commands and aws credentials file config to configure your environment to start using it. Useful trick to avoid CLI reauth to `aws sso login` every day. - `aws_terraform_create_credential.sh` - creates a AWS terraform service account with Administrator permissions for Terraform Cloud or other CI/CD systems to run Terraform plan and apply, since no CI/CD systems can work with AWS SSO workflows. Stores the access key as both CSV and prints shell export commands and credentials file config as above - `.envrc-aws` - copy to `.envrc` for [direnv](https://direnv.net/) to auto-load AWS configuration settings such as AWS Profile, Compute Region, EKS cluster kubectl context etc. - calls `.envrc-kubernetes` to set the `kubectl` context isolated to current shell to prevent race conditions between shells and scripts caused by otherwise naively changing the global `~/.kube/config` context - `aws_sso_ssh.sh` - launches local AWS SSO authentication pop-up (if not already authenticated), then scp's the latest resultant `~/.aws/sso/cache/` file to the remote server and SSH's there so that you can use AWS CLI or kubectl to EKS remotely on that server easily, without having to copy and paste the token from remote aws sso login to your local web browser - `aws_terraform_create_s3_bucket.sh` - creates a Terraform S3 bucket for storing the backend state, locks out public access, enables versioning, encryption, and locks out Power Users role and optionally any given user/group/role ARNs via a bucket policy for safety - `aws_terraform_create_dynamodb_table.sh` - creates a Terraform locking table in DynamoDB for use with the S3 backend, plus custom IAM policy which can be applied to less privileged accounts - `aws_terraform_create_all.sh` - runs all of the above, plus also applies the custom DynamoDB IAM policy to the user to ensure if the account is less privileged it can still get the Terraform lock (useful for GitHub Actions environment secret for a read only user to generate Terraform Plans in Pull Request without needing approval) - `aws_terraform_iam_grant_s3_dynamodb.sh` - creates IAM policies to access any S3 buckets and DynamoDB tables with `terraform-state` or `tf-state` in their names, and attaches them to the given user. Useful for limited permissions CI/CD accounts that run Terraform Plan eg. in GitHub Actions pull requests - `aws_account_summary.sh` - prints AWS account summary in `key = value` pairs for easy viewing / grepping of things like `AccountMFAEnabled`, `AccountAccessKeysPresent`, useful for checking whether the root account has MFA enabled and no access keys, comparing number of users vs number of MFA devices etc. (see also `check_aws_root_account.py` in [Advanced Nagios Plugins](https://github.com/HariSekhon/Nagios-Plugins)) - `aws_billing_alarm.sh` - creates a [CloudWatch](https://aws.amazon.com/cloudwatch/) billing alarm and [SNS](https://aws.amazon.com/sns/) topic with subscription to email you when you incur charges above a given threshold. This is often the first thing you want to do on an account - `aws_budget_alarm.sh` - creates an [AWS Budgets](https://aws.amazon.com/cloudwatch/) billing alarm and [SNS](https://aws.amazon.com/sns/) topic with subscription to email you when both when you start incurring forecasted charges of over 80% of your budget, and 90% actual usage. This is often the first thing you want to do on an account - `aws_batch_stale_jobs.sh` - lists [AWS Batch](https://aws.amazon.com/batch/) jobs that are older than N hours in a given queue - `aws_batch_kill_stale_jobs.sh` - finds and kills [AWS Batch](https://aws.amazon.com/batch/) jobs that are older than N hours in a given queue - `aws_cloudfront_distribution_for_origin.sh` - returns the AWS CloudFront ARN of the distribution which serves origins containing a given substring. Useful for quickly finding the CloudFront ARN needed to give permissions to a private S3 bucket exposed via CloudFront - `aws_cloudtrails_cloudwatch.sh` - lists [Cloud Trails](https://aws.amazon.com/cloudtrail/) and their last delivery to [CloudWatch](https://aws.amazon.com/cloudwatch/features/) Logs (should be recent) - `aws_cloudtrails_event_selectors.sh` - lists [Cloud Trails](https://aws.amazon.com/cloudtrail/) and their event selectors to check each one has at least one event selector - `aws_cloudtrails_s3_accesslogging.sh` - lists [Cloud Trails](https://aws.amazon.com/cloudtrail/) buckets and their Access Logging prefix and target bucket. Checks [S3 access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html) is enabled - `aws_cloudtrails_s3_kms.sh` - lists [Cloud Trails](https://aws.amazon.com/cloudtrail/) and whether their [S3](https://aws.amazon.com/s3/) buckets are [KMS](https://aws.amazon.com/kms/) secured - `aws_cloudtrails_status.sh` - lists [Cloud Trails](https://aws.amazon.com/cloudtrail/) status - if logging, multi-region and log file validation enabled - `aws_config_all_types.sh` - lists [AWS Config](https://aws.amazon.com/config/) recorders, checking all resource types are supported (should be true) and includes global resources (should be true) - `aws_config_recording.sh` - lists [AWS Config](https://aws.amazon.com/config/) recorders, their recording status (should be true) and their last status (should be success) - `aws_csv_creds.sh` - prints AWS credentials from a CSV file as shell export statements. Useful to quickly switch your shell to some exported credentials from a service account for testing permissions or pipe to upload to a CI/CD system via an API (eg. `jenkins_cred_add*.sh`, `github_actions_repo*_set_secret.sh`, `gitlab_*_set_env_vars.sh`, `circleci_*_set_env_vars.sh`, `bitbucket_*_set_env_vars.sh`, `terraform_cloud_*_set_vars.sh`, `kubectl_kv_to_secret.sh`). Supports new user and new access key csv file formats. - `aws_codecommit_csv_creds.sh` - prints AWS [CodeCommit](https://aws.amazon.com/codecommit/) Git credentials from a CSV file as shell export statements. Similar use case and chaining as above - `aws_ec2_*.sh` - AWS [EC2](https://aws.amazon.com/ec2/) scripts: - `aws_ec2_instance_*.sh` - AWS EC2 [Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Instances.html) scripts: - `aws_ec2_instance_name_to_id.sh` - looks up an EC2 instance ID from an instance name with extra safety checks that only a single instance ID is returned and a reverse lookup on that instance ID to re-verify it matches the name. This level of safety is important when wanting to terminate an EC2 instance by name. If an instance ID is passed, returns it as is for convenience. Used by adjacent scripts - `aws_ec2_instances.sh` - lists AWS EC2 instances, their DNS names and States in an easy to read table output - `aws_ec2_instance_ip.sh` - determines an EC2 instance IP address, trying first for a public IP, or failing that a private IP - `aws_ec2_instance_clone.sh` - clones an AWS EC2 instance by creating an AMI from the original and then booting a new instance from the AMI with the same settings as the original instance. Useful to testing risky things on a separate EC2 instance, such as Server Administrator recovery of Tableau - `aws_ec2_instance_wait_for_ready.sh` - polls an AWS EC2 instance and waits for it to finish initializing to a ready state. Used by adjacent scripts - `aws_ec2_instance_terminate_by_name.sh` - terminate an AWS EC2 instance by name for convenience, resolves its instance ID, verifies unique and then terminates by ID - `aws_ec2_ami*.sh` - AWS EC2 [AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) scripts: - `aws_ec2_amis.sh` - list AWS EC2 AMIs belonging to your account in an easy to read table output - `aws_ec2_ami_ids.sh` - lists AWS EC2 AMI IDs only, one per line, to be used in adjacent scripts that creating mapping tables and translate AMI IDs to names in inventory scripts `aws_info_ec2*.sh` - `aws_ec2_ami_name_to_id.sh` - looks up an EC2 AMI ID from a name with extra safety checks that only a single AMI ID is returned and a reverse lookup on that AMI ID to re-verify it matches the name - `aws_ec2_ami_boot.sh` - boots a personal EC2 instance of a given AMI for testing - `aws_ec2_ami_boot_ssh.sh` - boots a personal EC2 instance of a given AMI, determines the public or private IP, and drops you into an SSH shell - `aws_ec2_ami_create_from_instance.sh` - creates an AWS EC2 AMI from an EC2 instance and waits for it to become available for use - `aws_ec2_ami_share_to_account.sh` - shares an AMI with another AWS account. Can specify AMI by name or id - `aws_ec2_ebs_*.sh` - AWS EC2 [EBS](https://aws.amazon.com/ebs/) scripts: - `aws_ec2_ebs_volumes.sh` - list EC2 instances and their EBS volumes in the current region - `aws_ec2_ebs_create_snapshot_and_wait.sh - creates a snapshot of a given EBS volume ID and waits for it to complete with exponential backoff - `aws_ec2_ebs_resize_and_wait.sh - resizes an EBS volume and waits for it to complete modifying and optionally optimizing with exponential backoff - `aws_ec2_ebs_volumes_unattached.sh` - list an unattached EBS volumes in a table format - `aws_ec2_launch_templates_ami_id.sh` - for each Launch Template lists the AMI ID of the latest version. Useful to check EKS upgrades of node groups via Terragrunt have taken effect - `aws_ecr_*.sh` - AWS [ECR](https://aws.amazon.com/ecr/) docker image management scripts: - `aws_ecr_docker_login.sh` - authenticates Docker to AWS ECR, inferring the ECR registry from the current AWS Account ID and Region - `aws_ecr_docker_build_push.sh` - builds a docker image and pushes it to ECR with not just the `latest` docker tag but also the current Git hashref and Git tags - `aws_ecr_list_repos.sh` - lists ECR repos, and their docker image mutability and whether image scanning is enabled - `aws_ecr_list_tags.sh` - lists all the tags for a given ECR docker image - `aws_ecr_newest_image_tags.sh` - lists the tags for the given ECR docker image with the newest creation date (can use this to determine which image version to tag as `latest`) - `aws_ecr_alternate_tags.sh` - lists all the tags for a given ECR docker `image:tag` (use arg `:latest` to see what version / build hashref / date tag has been tagged as `latest`) - `aws_ecr_tag_image.sh` - tags an ECR image with another tag without pulling and pushing it - `aws_ecr_tag_image_by_digest.sh` - same as above but tags an ECR image found via digest (more accurate as reference by existing tag can be a moving target). Useful to recover images that have become untagged - `aws_ecr_tag_latest.sh` - tags a given ECR docker `image:tag` as `latest` without pulling or pushing the docker image - `aws_ecr_tag_branch.sh` - tags a given ECR `image:tag` with the current Git branch without pulling or pushing the docker image - `aws_ecr_tag_datetime.sh` - tags a given ECR docker image with its creation date and UTC timestamp (when it was uploaded to ECR) without pulling or pushing the docker image - `aws_ecr_tag_newest_image_as_latest.sh` - finds and tags the newest build of a given ECR docker image as `latest` without pulling or pushing the docker image - `aws_ecr_tags_timestamps.sh` - lists all the tags and their timestamps for a given ECR docker image - `aws_ecr_tags_old.sh` - lists tags older than N days for a given ECR docker image - `aws_ecr_delete_old_tags.sh` - deletes tags older than N days for a given ECR docker image. Lists the image:tags to be deleted and prompts for confirmation safety - `aws_emr_clusters_last_steps.sh` - shows the last N steps executed on each EMR cluster and their EndTime to find idle clusters that should be removed. Also checks CloudWatch for number of steps running within the last few months to catch directly submitted jobs such as Spark, Hive, Glue or Athena which won't show up in the native steps list - `aws_foreach_profile.sh` - executes a templated command across all AWS named profiles configured in AWS CLIv2, replacing `{profile}` in each iteration. Combine with other scripts for powerful functionality, auditing, setup etc. eg. `aws_kube_creds.sh` to configure `kubectl` config to all EKS clusters in all environments - `aws_foreach_region.sh` - executes a templated command against each AWS region enabled for the current account, replacing `{region}` in each iteration. Combine with AWS CLI or scripts to find resources across regions - `aws_iam_*.sh` - AWS [IAM](https://aws.amazon.com/iam/) scripts: - `aws_iam_password_policy.sh` - prints [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) in `key = value` pairs for easy viewing / grepping (used by `aws_harden_password_policy.sh` before and after to show the differences) - `aws_iam_harden_password_policy.sh` - strengthens [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) according to [CIS Foundations Benchmark](https://d1.awsstatic.com/whitepapers/compliance/AWS_CIS_Foundations_Benchmark.pdf) recommendations - `aws_iam_replace_access_key.sh` - replaces the non-current IAM access key (Inactive, Not Used, longer time since used, or an explicitly given key), outputting the new key as shell export statements (useful for piping to the same tools listed for `aws_csv_creds.sh` above) - `aws_iam_policies_attached_to_users.sh` - finds [AWS IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage.html) directly attached to users (anti-best practice) instead of groups - `aws_iam_policies_granting_full_access.sh` - finds [AWS IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage.html) granting full access (anti-best practice) - `aws_iam_policies_unattached.sh` - lists unattached [AWS IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage.html) - `aws_iam_policy_attachments.sh` - finds all users, groups and roles where a given IAM policy is attached, so that you can remove all these references in your Terraform code and avoid this error `Error: error deleting IAM policy arn:aws:iam::***:policy/mypolicy: DeleteConflict: Cannot delete a policy attached to entities.` - `aws_iam_policy_delete.sh` - deletes an IAM policy, by first handling all prerequisite steps of deleting all prior versions and all detaching all users, groups and roles - `aws_iam_generate_credentials_report_wait.sh` - generates an AWS IAM [credentials report](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html) - `aws_iam_users.sh` - list your IAM users - `aws_iam_users_access_key_age.sh` - prints AWS users [access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) status and age (see also `aws_users_access_key_age.py` in [DevOps Python tools](https://github.com/HariSekhon/DevOps-Python-tools) which can filter by age and status) - `aws_iam_users_access_key_age_report.sh` - prints AWS users [access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) status and age using a bulk credentials report (faster for many users) - `aws_iam_users_access_key_last_used.sh` - prints AWS users [access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) last used date - `aws_iam_users_access_key_last_used_report.sh` - same as above using bulk credentials report (faster for many users) - `aws_iam_users_last_used_report.sh` - lists AWS users password/access keys last used dates - `aws_iam_users_mfa_active_report.sh` - lists AWS users password enabled and [MFA](https://aws.amazon.com/iam/features/mfa/) enabled status - `aws_iam_users_without_mfa.sh` - lists AWS users with password enabled but no MFA - `aws_iam_users_mfa_serials.sh` - lists AWS users [MFA](https://aws.amazon.com/iam/features/mfa/) serial numbers (differentiates Virtual vs Hardware MFAs) - `aws_iam_users_pw_last_used.sh` - lists AWS users and their password last used date - `aws_ip_ranges.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range API - `aws_info*.sh`: - `aws_info_all_profiles.sh` - calls `aws_info.sh` for all AWS profiles using `aws_foreach_profile.sh` - `aws_info.sh` - lists AWS deployed resources in the current or specified AWS account profile - `aws_info_ec2.sh` - lists AWS EC2 Instances resources deployed in the current AWS account - `aws_info_ec2_csv.sh` - lists AWS EC2 Instances in quoted CSV format in the current AWS account - `aws_info_ec2_all_profiles_csv.sh` - lists AWS EC2 Instances in quoted CSV format across all configured AWS profiles for their configured region - `aws_eks_cloudwatch_logs.sh` - enables and fetches AWS EKS Master logs via CloudWatch - `aws_eks_ssh_dump_logs.sh` - fetch system logs from EKS Worker Nodes EC2 VMs (eg. for support debug requests by vendors) - `aws_eks_cluster_versions.sh` - iterates EKS clusters to list each AWS EKS cluster name and version in the current account. Combine with `aws_foreach_profile.sh` and `aws_foreach_region.sh` to audit your EKS cluster versions across accounts and regions - `aws_eks_addon_versions.sh` - lists the EKS addon versions available for the given cluster by checking its version before checking addons - `aws_eks_available_ips.sh` - lists the number of available IP addresses in the EKS subnets for the given cluster (5 required for an EKS upgrade) - `aws_eks_ami_create.sh` - creates a custom EKS AMI quickly off the base EKS template and then running a shell script in it before saving it to a new AMI. See also [HariSekhon/Packer](https://github.com/HariSekhon/Packer) for more advanced build - `aws_kms_key_rotation_enabled.sh` - lists [AWS KMS](https://aws.amazon.com/kms/) keys and whether they have key rotation enabled - `aws_kube_creds.sh` - auto-loads all [AWS EKS](https://aws.amazon.com/eks/) clusters credentials in the current --profile and --region so your kubectl is ready to rock on AWS - `aws_kubectl.sh` - runs kubectl commands safely fixed to a given [AWS EKS](https://aws.amazon.com/eks/) cluster using config isolation to avoid concurrency race conditions - `aws_logs_*.sh` - some useful log queries in last N hours (24 hours by default): - `aws_logs_batch_jobs.sh` - lists AWS Batch job submission requests and their callers - `aws_logs_ec2_spot.sh` - lists AWS EC2 Spot fleet creation requests, their caller and first tag value for origin hint - `aws_logs_ecs_tasks.sh` - lists AWS ECS task run requests, their callers and job definitions - `aws_meta.sh` - AWS [EC2 Metadata API](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) query shortcut. See also the official [ec2-metadata](https://aws.amazon.com/code/ec2-instance-metadata-query-tool/) shell script with more features - `aws_nat_gateways_public_ips.sh` - lists the public IPs of all NAT gateways. Useful to give to clients to permit through firewalls for webhooks or similar calls - `aws_rds_list.sh` - list RDS instances with select fields - Name, Status, Engine, AZ, Instance Type, Storage - `aws_rds_open_port_to_my_ip.sh` - adds a security group to an RDS DB instance to open its native database SQL port to your public IP address - `aws_rds_get_version.sh` - quickly retrieve the version of an RDS database to know which JDBC jar version to download using `install/download_*_jdbc.sh` when setting up connections - `aws_route53_check_ns_records.sh` - checks AWS [Route 53](https://aws.amazon.com/route53/) public hosted zones NS servers are delegated in the public DNS hierarchy and that there are no rogue NS servers delegated not matching the Route 53 zone configuration - `aws_sso_accounts.sh` - lists all AWS SSO accounts the current SSO user has access to - `aws_sso_configs.sh` - generates AWS SSO configs for all AWS SSO accounts the currently logged in SSO user has access to - `aws_sso_configs_save.sh` - saves AWS SSO configs generated by `aws_sso_configs.sh` to `~/.aws/config` if they're not already found - `aws_sso_account_id_names.sh` - parses AWS config for AWS SSO and outputs Account IDs and Profile names - `aws_sso_config_duplicate_sections.sh` - lists duplicate AWS SSO config sections that are using the same sso_account_id. Useful to deduplicate configs containing a mix of hand crafted and automatically generated `aws_sso_configs.sh` - `aws_sso_config_duplicate_profile_names.sh` - lists duplicate AWS SSO config profile names that are using the same sso_account_id - `aws_accounts_missing_from_config.sh` - for a list of AWS Account IDs in stdin or files, finds those missing from AWS config - `aws_sso_accounts_missing_from_list.sh` - for a list of AWS Account IDs in stdin or files, finds AWS SSO accounts in AWS config missing from the provided list - `aws_sso_env_creds.sh` - retrieves AWS SSO session credentials in the format of environment export commands for copying to other systems like Terraform Cloud - `aws_sso_role_arn.sh` - determines the currently authenticated AWS SSO user's base role ARN in IAM policy usable format - `aws_sso_role_arns.sh` - prints all AWS SSO role ARNs in IAM policy usable format - `aws_profile_config_add_if_missing.sh` - reads AWS profile config blocks from stdin and appends them to the `~/.aws/config` file if the profile section is not found - `aws_profile_generate_direnvs.sh` - generates subdirectories containing the `config.ini` and `.envrc` for every AWS profile found in the given file or `$AWS_CONFIG_FILE` or `~/.aws/config`. Useful to take a large generated AWS `config.ini` from `aws_sso_configs.sh` and then split it into subdirectories for direnvs - `aws_s3_bucket.sh` - creates an S3 bucket, blocks public access, enables versioning, encryption, and optionally locks out any given user/group/role ARNs via a bucket policy for safety (eg. to stop Power Users accessing a sensitive bucket like Terraform state) - `aws_s3_buckets_block_public_access.sh` - blocks public access to one or more given S3 buckets or files containing bucket names, one per line - `aws_s3_account_block_public_access.sh` - blocks S3 public access at the AWS account level - `aws_s3_check_buckets_public_blocked.sh` - iterates each S3 bucket and checks it has public access fully blocked via policy. Parallelized for speedup - `aws_s3_check_account_public_blocked.sh` - checks S3 public access is blocked at the AWS account level - `aws_s3_sync.sh` - syncs multiple AWS S3 URLs from file lists. Validates S3 URLs, source and destination list lengths matches, and optionally that path suffixes match, to prevent off-by-one human errors spraying data all over the wrong destination paths - `aws_s3_access_logging.sh` - lists [AWS S3](https://aws.amazon.com/s3/) buckets and their [access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html) status - `aws_s3_delete_bucket_with_versions.sh` - deletes a bucket including all versions. Use with caution! - `aws_spot_when_terminated.sh` - executes commands when the [AWS EC2](https://aws.amazon.com/ec2/) instance running this script is notified of [Spot Termination](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-spot-instances.html), acts as a latch mechanism that can be set any time after boot - `aws_sqs_check.sh` - sends a test message to an [AWS SQS](https://aws.amazon.com/sqs/) queue, retrieves it to check and then deletes it via the receipt handle id - `aws_sqs_delete_message.sh` - deletes 1-10 messages from a given [AWS SQS](https://aws.amazon.com/sqs/) queue (to help clear out test messages) - `aws_ssm_put_param.sh` - reads a value from a command line argument or non-echo prompt and saves it to AWS [Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/what-is-systems-manager.html). Useful for uploading a password without exposing it on your screen - `aws_secret*.sh` - AWS [Secrets Manager](https://aws.amazon.com/secrets-manager/) scripts: - `aws_secret_list.sh` - returns the list of secrets, one per line - `aws_secret_add.sh` - reads a value from a command line argument or non-echo prompt and saves it to Secrets Manager. Useful for uploading a password without exposing it on your screen - `aws_secret_add_binary.sh` - base64 encodes a given file's contents and saves it to Secrets Manager as a binary secret. Useful for uploading things like QR code screenshots for sharing MFA to recovery admin accounts - `aws_secret_update.sh` - reads a value from a command line argument or non-echo prompt and updates a given Secrets Manager secret. Useful for updating a password without exposing it on your screen - `aws_secret_update_binary.sh` - base64 encodes a given file's contents and updates a given Secrets Manager secret. Useful for updating a QR code screenshot for a root account - `aws_secret_get.sh` - gets a secret value for a given secret from Secrets Manager, retrieving either a secure string or secure binary depending on which is available - `eksctl_cluster.sh` - downloads [eksctl](https://eksctl.io/) and creates an [AWS EKS](https://aws.amazon.com/eks/) Kubernetes cluster See also [Knowledge Base notes for AWS](https://github.com/HariSekhon/Knowledge-Base/blob/main/aws.md). ### GCP - Google Cloud Platform `gcp/` directory: - [Google Cloud](https://cloud.google.com/) scripts - `gcp_*.sh` / `gce_*.sh` / `gke_*.sh` / `gcr_*.sh` / `bigquery_*.sh`: - `.envrc-gcp` - copy to `.envrc` for [direnv](https://direnv.net/) to auto-load GCP configuration settings such as Project, Region, Zone, GKE cluster kubectl context or any other GCloud SDK settings to shorten `gcloud` commands. Applies to the local shell environment only to avoid race conditions caused by naively changing the global gcloud config at `~/.config/gcloud/active_config` - calls `.envrc-kubernetes` to set the `kubectl` context isolated to current shell to prevent race conditions between shells and scripts caused by otherwise naively changing the global `~/.kube/config` context - `gcp_terraform_create_credential.sh` - creates a service account for [Terraform](https://www.terraform.io/) with full permissions, creates and downloads a credential key json and even prints the `export GOOGLE_CREDENTIALS` command to configure your environment to start using Terraform immediately. Run once for each project and combine with [direnv](https://direnv.net/) for fast easy management of multiple GCP projects - `gcp_ansible_create_credential.sh` - creates an [Ansible](https://www.ansible.com/) service account with permissions on the current project, creates and downloads a credential key json and prints the environment variable to immediately use it - `gcp_cli_create_credential.sh` - creates a GCloud SDK CLI service account with full owner permissions to all projects, creates and downloads a credential key json and even prints the `export GOOGLE_CREDENTIALS` command to configure your environment to start using it. Avoids having to reauth to `gcloud auth login` every day. - `gcp_spinnaker_create_credential.sh` - creates a [Spinnaker](https://spinnaker.io/) service account with permissions on the current project, creates and downloads a credential key json and even prints the Halyard CLI configuration commands to use it - `gcp_info.sh` - huge [Google Cloud](https://cloud.google.com/) inventory of deployed resources within the current project - Cloud SDK info plus all of the following (detects which services are enabled to query): - `gcp_info_compute.sh` - [GCE](https://cloud.google.com/compute/) Virtual Machine instances, [App Engine](https://cloud.google.com/appengine) instances, [Cloud Functions](https://cloud.google.com/functions), [GKE](https://cloud.google.com/kubernetes-engine) clusters, all [Kubernetes](https://kubernetes.io/) objects across all GKE clusters (see `kubernetes_info.sh` below for more details) - `gcp_info_storage.sh` - [Cloud SQL](https://cloud.google.com/sql) info below, plus: [Cloud Storage](https://cloud.google.com/storage) Buckets, [Cloud Filestore](https://cloud.google.com/filestore), [Cloud Memorystore Redis](https://cloud.google.com/memorystore), [BigTable](https://cloud.google.com/bigtable) clusters and instances, [Datastore](https://cloud.google.com/datastore) indexes - `gcp_info_cloud_sql.sh` - [Cloud SQL](https://cloud.google.com/sql) instances, whether their backups are enabled, and all databases on each instance - `gcp_info_cloud_sql_databases.sh` - lists databases inside each [Cloud SQL](https://cloud.google.com/sql) instance. Included in `gcp_info_cloud_sql.sh` - `gcp_info_cloud_sql_backups.sh` - lists backups for each [Cloud SQL](https://cloud.google.com/sql) instance with their dates and status. Not included in `gcp_info_cloud_sql.sh` for brevity. See also `gcp_sql_export.sh` further down for more durable backups to [GCS](https://cloud.google.com/storage) - `gcp_info_cloud_sql_users.sh` - lists users for each running [Cloud SQL](https://cloud.google.com/sql) instance. Not included in `gcp_info_cloud_sql.sh` for brevity but useful to audit users - `gcp_info_networking.sh` - VPC Networks, Addresses, Proxies, Subnets, Routers, Routes, VPN Gateways, VPN Tunnels, Reservations, Firewall rules, Forwarding rules, [Cloud DNS](https://cloud.google.com/dns) managed zones and verified domains - `gcp_info_bigdata.sh` - [Dataproc](https://cloud.google.com/dataproc) clusters and jobs in all regions, [Dataflow](https://cloud.google.com/dataflow) jobs in all regions, [PubSub](https://cloud.google.com/pubsub) messaging topics, [Cloud IOT](https://cloud.google.com/iot-core) registries in all regions - `gcp_info_tools.sh` - [Cloud Source Repositories](https://cloud.google.com/source-repositories), [Cloud Builds](https://cloud.google.com/cloud-build), [Container Registry](https://cloud.google.com/container-registry) images across all major repos (`gcr.io`, `us.gcr.io`, `eu.gcr.io`, `asia.gcr.io`), [Deployment Manager](https://cloud.google.com/deployment-manager) deployments - `gcp_info_auth_config.sh` - Auth Configurations, Organizations & Current Config - `gcp_info_projects.sh` - Projects names and IDs - `gcp_info_services.sh` - Services & APIs enabled - `gcp_service_apis.sh` - lists all available [GCP](https://cloud.google.com/) Services, APIs and their states (enabled/disabled), and provides `is_service_enabled()` function used throughout the adjacent scripts to avoid errors and only show relevant enabled services - `gcp_info_accounts_secrets.sh` - [IAM](https://cloud.google.com/iam) Service Accounts, [Secret Manager](https://cloud.google.com/secret-manager) secrets - `gcp_info_all_projects.sh` - same as above but for all detected projects - `gcp_foreach_project.sh` - executes a templated command across all GCP projects, replacing `{project_id}` and `{project_name}` in each iteration (used by `gcp_info_all_projects.sh` to call `gcp_info.sh`) - `gcp_find_orphaned_disks.sh` - lists orphaned disks across one or more GCP projects (not attached to any compute instance) - `gcp_secret*.sh` - Google [Secret Manager](https://cloud.google.com/secret-manager) scripts: - `gcp_secret_add.sh` - reads a value from a command line argument or non-echo prompt and saves it to GCP Secrets Manager. Useful for uploading a password without exposing it on your screen - `gcp_secret_add_binary.sh` - uploads a binary file to GCP Secrets Manager by base64 encoding it first. Useful for uploading QR code screenshots. Useful for uploading things like QR code screenshots for sharing MFA to recovery admin accounts - `gcp_secret_update.sh` - reads a value from a command line argument or non-echo prompt and updates a given GCP Secrets Manager secret. Useful for uploading a password without exposing it on your screen - `gcp_secret_get.sh` - finds the latest version of a given GCP Secret Manager secret and returns its value. Used by adjacent scripts - `gcp_secret_label_k8s.sh` - labels a given existing GCP secret with the current kubectl cluster name and namespace for later use by `gcp_secrets_to_kubernetes.sh` - `gcp_secrets_to_kubernetes.sh` - loads GCP secrets to Kubernetes secrets in a 1-to-1 mapping. Can specify a list of secrets or auto-loads all GCP secrets with labels `kubernetes-cluster` and `kubernetes-namespace` matching the current `kubectl` context (`kcd` to the right namespace first, see `.bash.d/kubernetes`). See also `kubernetes_get_secret_values.sh` to debug the actual values that got loaded. See also [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets) / [External Secrets](https://external-secrets.io/) in my [Kubernetes repo](https://github.com/HariSekhon/Kubernetes-configs) - `gcp_secrets_to_kubernetes_multipart.sh` - creates a Kubernetes secret from multiple GCP secrets (used to put `private.pem` and `public.pem` into the same secret to appear as files on volume mounts for apps in pods to use). See also [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets) / [External Secrets](https://external-secrets.io/) in my [Kubernetes repo](https://github.com/HariSekhon/Kubernetes-configs) - `gcp_secrets_labels.sh` - lists GCP Secrets and their labels, one per line suitable for quick views or shell pipelines - `gcp_secrets_update_lable.sh` - updates all GCP secrets in current project matching label key=value with a new label value - `gcp_service_account_credential_to_secret.sh` - creates GCP service account and exports a credential key to GCP Secret Manager (useful to stage or combine with `gcp_secrets_to_kubernetes.sh`) - `gke_*.sh` - Google [Kubernetes Engine](https://cloud.google.com/kubernetes-engine) scripts - `gke_kube_creds.sh` - auto-loads all GKE clusters credentials in the current / given / all projects so your kubectl is ready to rock on GCP - `gke_kubectl.sh` - runs kubectl commands safely fixed to a given GKE cluster using config isolation to avoid concurrency race conditions - `gke_firewall_rule_cert_manager.sh` - creates a GCP firewall rule for a given GKE cluster's masters to access [Cert Manager](https://cert-manager.io/) admission webhook (auto-determines the master cidr, network and target tags) - `gke_firewall_rule_kubeseal.sh` - creates a GCP firewall rule for a given GKE cluster's masters to access [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets) controller for `kubeseal` to work (auto-determines the master cidr, network and target tags) - `gke_nodepool_nodes.sh` - lists all nodes in a given nodepool on the current GKE cluster via kubectl labels (fast) - `gke_nodepool_nodes2.sh` - same as above via GCloud SDK (slow, iterates instance groups) - `gke_nodepool_taint.sh` - taints/untaints all nodes in a given GKE nodepool on the current cluster (see `kubectl_node_taints.sh` for a quick way to see taints) - `gke_nodepool_drain.sh` - drains all nodes in a given nodepool (to decommission or rebuild the node pool, for example with different taints) - `gke_persistent_volumes_disk_mappings.sh` - lists GKE kubernetes persistent volumes to GCP persistent disk names, along with PVC and namespace, useful when investigating, resizing PVs etc. - `gcr_*.sh` - Google [Container Registry](https://cloud.google.com/container-registry) scripts: - `gcr_list_tags.sh` - lists all the tags for a given GCR docker image - `gcr_newest_image_tags.sh` - lists the tags for the given GCR docker image with the newest creation date (can use this to determine which image version to tag as `latest`) - `gcr_alternate_tags.sh` - lists all the tags for a given GCR docker `image:tag` (use arg `:latest` to see what version / build hashref / date tag has been tagged as `latest`) - `gcr_tag_latest.sh` - tags a given GCR docker `image:tag` as `latest` without pulling or pushing the docker image - `gcr_tag_branch.sh` - tags a given GCR docker `image:tag` with the current Git branch without pulling or pushing the docker image - `gcr_tag_datetime.sh` - tags a given GCR docker image with its creation date and UTC timestamp (when it was uploaded or created by [Google Cloud Build](https://cloud.google.com/cloud-build)) without pulling or pushing the docker image - `gcr_tag_newest_image_as_latest.sh` - finds and tags the newest build of a given GCR docker image as `latest` without pulling or pushing the docker image - `gcr_tags_timestamps.sh` - lists all the tags and their timestamps for a given GCR docker image - `gcr_tags_old.sh` - lists tags older than N days for a given GCR docker image - `gcr_delete_old_tags.sh` - deletes tags older than N days for a given GCR docker image. Lists the image:tags to be deleted and prompts for confirmation safety - see also [cloudbuild.yaml](https://github.com/HariSekhon/Templates/blob/master/cloudbuild.yaml) in the [Templates](https://github.com/HariSekhon/Templates) repo - CI/CD on GCP - trigger Google Cloud Build and GKE Kubernetes deployments from orthogonal CI/CD systems like Jenkins / TeamCity: - `gcp_ci_build.sh` - script template for CI/CD to trigger Google Cloud Build to build docker container image with extra datetime and latest tagging - `gcp_ci_deploy_k8s.sh` - script template for CI/CD to deploy GCR docker image to GKE Kubernetes using Kustomize - `gce_*.sh` - Google [Compute Engine](https://cloud.google.com/compute/) scripts: - `gce_foreach_vm.sh` - run a command for each GCP VM instance matching the given name/ip regex in the current GCP project - `gce_host_ips.sh` - prints the IPs and hostnames of all or a regex match of GCE VMs for use in /etc/hosts - `gce_ssh.sh` - Runs `gcloud compute ssh` to a VM while auto-determining its zone first to override any inherited zone config and make it easier to script iterating through VMs - `gcs_ssh_keyscan.sh` - SSH keyscans all the GCE VMs returned from the above `gce_host_ips.sh` script and adds them to `~/.ssh/known_hosts` - `gce_meta.sh` - simple script to query the GCE metadata API from within Virtual Machines - `gce_when_preempted.sh` - GCE VM preemption latch script - can be executed any time to set one or more commands to execute upon preemption - `gce_is_preempted.sh` - GCE VM return true/false if preempted, callable from other scripts - `gce_instance_service_accounts.sh` - lists GCE VM instance names and their service accounts - `gcp_firewall_disable_default_rules.sh` - disables those lax GCP default network "allow all" firewall rules - `gcp_firewall_risky_rules.sh` - lists risky GCP firewall rules that are enabled and allow traffic from 0.0.0.0/0 - `gcp_sql_*.sh` - [Cloud SQL](https://cloud.google.com/sql) scripts: - `gcp_sql_backup.sh` - creates Cloud SQL backups - `gcp_sql_export.sh` - creates Cloud SQL exports to [GCS](https://cloud.google.com/storage) - `gcp_sql_enable_automated_backups.sh` - enable automated daily Cloud SQL backups - `gcp_sql_enable_point_in_time_recovery.sh` - enable point-in-time recovery with write-ahead logs - `gcp_sql_proxy.sh` - boots a [Cloud SQL Proxy](https://cloud.google.com/sql/docs/postgres/sql-proxy) to all Cloud SQL instances for fast convenient direct `psql` / `mysql` access via local sockets. Installs Cloud SQL Proxy if necessary - `gcp_sql_running_primaries.sh` - lists primary running Cloud SQL instances - `gcp_sql_service_accounts.sh` - lists Cloud SQL instance service accounts. Useful for copying to [IAM](https://cloud.google.com/iam) to grant permissions (eg. Storage Object Creator for SQL export backups to GCS) - `gcp_sql_create_readonly_service_account.sh` - creates a service account with read-only permissions to Cloud SQL eg. to run export backups to GCS - `gcp_sql_grant_instances_gcs_object_creator.sh` - grants minimal GCS objectCreator permission on a bucket to primary Cloud SQL instances for exports - `gcp_cloud_schedule_sql_exports.sh` - creates Google [Cloud Scheduler](https://cloud.google.com/scheduler) jobs to trigger a [Cloud Function](https://cloud.google.com/functions) via [PubSub](https://cloud.google.com/pubsub) to run [Cloud SQL](https://cloud.google.com/sql) exports to [GCS](https://cloud.google.com/storage) for all [Cloud SQL](https://cloud.google.com/sql) instances in the current GCP project - the Python [GCF](https://cloud.google.com/functions) function is in the [DevOps Python tools](https://github.com/HariSekhon/DevOps-Python-tools) repo - `bigquery_*.sh` - [BigQuery](https://cloud.google.com/bigquery) scripts: - `bigquery_list_datasets.sh` - lists BigQuery datasets in the current GCP project - `bigquery_list_tables.sh` - lists BigQuery tables in a given dataset - `bigquery_list_tables_all_datasets.sh` - lists tables for all datasets in the current GCP project - `bigquery_foreach_dataset.sh` - executes a templated command for each dataset - `bigquery_foreach_table.sh` - executes a templated command for each table in a given dataset - `bigquery_foreach_table_all_datasets.sh` - executes a templated command for each table in each dataset in the current GCP project - `bigquery_table_row_count.sh` - gets the row count for a given table - `bigquery_tables_row_counts.sh` - gets the row counts for all tables in a given dataset - `bigquery_tables_row_counts_all_datasets.sh` - gets the row counts for all tables in all datasets in the current GCP project - `bigquery_generate_query_biggest_tables_across_datasets_by_row_count.sh` - generates a BigQuery SQL query to find the top 10 biggest tables by row count - `bigquery_generate_query_biggest_tables_across_datasets_by_size.sh` - generates a BigQuery SQL query to find the top 10 biggest tables by size - see also the [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) repo for many more straight BigQuery SQL scripts - GCP [IAM](https://cloud.google.com/iam) scripts: - `gcp_service_account*.sh`: - `gcp_service_account_credential_to_secret.sh` - creates GCP service account and exports a credential key to GCP Secret Manager (useful to stage or combine with `gcp_secrets_to_kubernetes.sh`) - `gcp_service_accounts_credential_keys.sh` - lists all service account credential keys and expiry dates, can `grep 9999-12-31T23:59:59Z` to find non-expiring keys - `gcp_service_accounts_credential_keys_age.sh` - lists all service account credential keys age in days - `gcp_service_accounts_credential_keys_expired.sh` - lists expired service account credential keys that should be removed and recreated if needed - `gcp_service_account_members.sh` - lists all members and roles authorized to use any service accounts. Useful for finding GKE Workload Identity mappings - `gcp_iam_*.sh`: - `gcp_iam_roles_in_use.sh` - lists GCP IAM roles in use in the current or all projects - `gcp_iam_identities_in_use.sh` - lists GCP IAM identities (users/groups/serviceAccounts) in use in the current or all projects - `gcp_iam_roles_granted_to_identity.sh` - lists GCP IAM roles granted to identities matching the regex (users/groups/serviceAccounts) in the current or all projects - `gcp_iam_roles_granted_too_widely.sh` - lists GCP IAM roles which have been granted to allAuthenticatedUsers or even worse allUsers (unauthenticated) in one or all projects - `gcp_iam_roles_with_direct_user_grants.sh` - lists GCP IAM roles which have been granted directly to users in violation of best-practice group-based management - `gcp_iam_serviceaccount_members.sh` - lists members with permissions to use each GCP service account - `gcp_iam_serviceaccounts_without_permissions.sh` - finds service accounts without IAM permissionns, useful to detect obsolete service accounts after a 90 day unused permissions clean out - `gcp_iam_workload_identities.sh` - lists GKE Workload Identity integrations, uses `gcp_iam_serviceaccount_members.sh` - `gcp_iam_users_granted_directly.sh` - lists GCP IAM users which have been granted roles directly in violation of best-practice group-based management - `gcs_bucket_project.sh` - finds the GCP project that a given bucket belongs to using the GCP Storage API - `gcs_curl_file.sh` - retrieves a GCS file's contents from a given bucket and path using the GCP Storage API. Useful for starting shell pipelines or being called from other scripts - `firebase_foreach_project.sh` - executes a templated command across all Firebase projects, replacing `{project_id}`, `{project_number}` and `{project_name}` in each iteration See also [Knowledge Base notes for GCP](https://github.com/HariSekhon/Knowledge-Base/blob/main/gcp.md). ### Kubernetes `kubernetes/` directory: - `.envrc-kubernetes` - copy to `.envrc` for [direnv](https://direnv.net/) to auto-load the right Kubernetes `kubectl` context isolated to current shell to prevent race conditions between shells and scripts caused by otherwise naively changing the global `~/.kube/config` context - `aws/eksctl_cluster.sh` - quickly spins up an [AWS EKS](https://aws.amazon.com/eks/) cluster using `eksctl` with some sensible defaults - `kubernetes_info.sh` - huge [Kubernetes](https://kubernetes.io/) inventory listing of deployed resources across all namespaces in the current cluster / kube context: - cluster-info - master component statuses - nodes - namespaces - deployments, replicasets, replication controllers, statefulsets, daemonsets, horizontal pod autoscalers - storage classes, persistent volumes, persistent volume claims - service accounts, resource quotas, network policies, pod security policies - container images running - container images running counts descending - pods (might be too much detail if you have high replica counts, so done last, comment if you're sure nobody has deployed pods outside deployments) - `kubectl.sh` - runs kubectl commands safely fixed to a given context using config isolation to avoid concurrency race conditions - `kubectl_diff_apply.sh` - generates a kubectl diff and prompts to apply - `kustomize_diff_apply.sh` - runs Kustomize build, precreates any namespaces, shows a kubectl diff of the proposed changes, and prompts to apply - `kustomize_diff_branch.sh` - runs Kustomize build against the current and target base branch for current or all given directories, then shows the diff for each directory. Useful to detect differences when refactoring, such as switching to tagged bases - `kubectl_create_namespaces.sh` - creates any namespaces in yaml files or stdin, a prerequisite for a diff on a blank install, used by adjacent scripts for safety - `kubernetes_check_objects_namespaced.sh` - checks Kubernetes yaml(s) for objects which aren't explicitly namespaced, which can easily result in deployments to the wrong namespace. Reads the API resources from your current Kubernetes cluster and if successful excludes cluster-wide objects - `kustomize_check_objects_namespaced.sh` - checks Kustomize build yaml output for objects which aren't explicitly namespaced (uses above script) - `kubectl_deployment_pods.sh` - gets the pod names with their unpredictable suffixes for a given deployment by querying the deployment's selector labels and then querying pods that match those labels - `kubectl_get_all.sh` - finds all namespaced Kubernetes objects and requests them for the current or given namespace. Useful because `kubectl get all` misses a lof of object types - `kubectl_get_annotation.sh` - find a type of object with a given annotation - `kubectl_restart.sh` - restarts all or filtered deployments/statefulsets in the current or given namespace. Useful when debugging or clearing application problems - `kubectl_logs.sh` - tails all containers in all pods or filtered pods in the current or given namespace. Useful when debugging a distributed set of pods in live testing - `kubectl_kv_to_secret.sh` - creates a Kuberbetes secret from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `kubectl_secret_values.sh` - prints the keys and base64 decoded values within a given Kubernetes secret for quick debugging of Kubernetes secrets. See also: `gcp_secrets_to_kubernetes.sh` - `kubectl_secrets_download.sh` - downloads all secrets in current or given namespace to local files of the same name, useful as a backup before migrating to Sealed Secrets - `kubernetes_secrets_compare_gcp_secret_manager.sh` - compares each Kubernetes secret to the corresponding secret in GCP Secret Manager. Useful to safety check GCP Secret Manager values align before enabling [External Secrets](https://external-secrets.io/latest/) to replace them - `kubernetes_secret_to_external_secret.sh` - generates an [External Secret](https://external-secrets.io/latest/) from an existing Kubernetes secret - `kubernetes_secrets_to_external_secrets.sh` - generates [External Secrets](https://external-secrets.io/latest/) from all existing Kubernetes secrets found in the current or given namespace - `kubernetes_secret_to_sealed_secret.sh` - generates a [Bitnami Sealed Secret](https://github.com/bitnami-labs/sealed-secrets) from an existing Kubernetes secret - `kubernetes_secrets_to_sealed_secrets.sh` - generates [Bitnami Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets) from all existing Kubernetes secrets found in the current or given namespace - `kubectl_secrets_annotate_to_be_sealed.sh` - annotates secrets in current or given namespace to allow being overwritten by Sealed Secrets (useful to sync ArgoCD health) - `kubectl_secrets_not_sealed.sh` - finds secrets with no SealedSecret ownerReferences - `kubectl_secrets_to_be_sealed.sh` - finds secrets pending overwrite by Sealed Secrets with the managed annotation - `kubernetes_yaml_strip_live_fields.sh` - strips live fields from Kubernetes yaml object dumps. Useful so you can do `kubectl diff` or `kubectl apply` without hitting annoying errors about immutable fields left in exports from `kubectl get ... -o yaml` - `kubernetes_foreach_context.sh` - executes a command across all kubectl contexts, replacing `{context}` in each iteration (skips lab contexts `docker` / `minikube` / `minishift` to avoid hangs since they're often offline) - `kubernetes_foreach_namespace.sh` - executes a command across all kubernetes namespaces in the current cluster context, replacing `{namespace}` in each iteration - Can be chained with `kubernetes_foreach_context.sh` and useful when combined with `gcp_secrets_to_kubernetes.sh` to load all secrets from GCP to Kubernetes for the current cluster, or combined with `gke_kube_creds.sh` and `kubernetes_foreach_context.sh` for all clusters! - `kubernetes_api.sh` - finds Kubernetes API and runs your curl arguments against it, auto-getting authorization token and auto-populating OAuth authentication header - `kubernetes_autoscaler_release.sh` - finds the latest Kubernetes Autoscaler release that matches your local Kubernetes cluster version using kubectl and the GitHub API. Useful for quickly finding the image override version for `eks-cluster-autoscaler-kustomization.yaml` in the [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) repo - `kubernetes_etcd_backup.sh` - creates a timestamped backup of the Kubernetes Etcd database for a kubeadm cluster - `kubernetes_delete_stuck_namespace.sh` - to forcibly delete those pesky kubernetes namespaces of 3rd party apps like Knative that get stuck and hang indefinitely on the finalizers during deletion - `kubeadm_join_cmd.sh` - outputs `kubeadm join` command (generates new token) to join an existing Kubernetes cluster (used in [vagrant kubernetes](https://github.com/HariSekhon/DevOps-Bash-tools/tree/master/vagrant/kubernetes) provisioning scripts) - `kubeadm_join_cmd2.sh` - outputs `kubeadm join` command manually (calculates cert hash + generates new token) to join an existing Kubernetes cluster - `kubernetes_nodes_ssh_dump_logs.sh` - fetch logs from Kubernetes nodes (eg. for support debug requests by vendors) - `kubectl_exec.sh` - finds and execs to the first Kubernetes pod matching the given name regex, optionally specifying the container name regex to exec to, and shows the full generated `kubectl exec` command line for clarity - `kubectl_exec2.sh` - finds and execs to the first Kubernetes pod matching given pod filters, optionally specifying the container to exec to, and shows the full generated `kubectl exec` command line for clarity - `kubectl_pods_per_node.sh` - lists number of pods per node sorted descending - `kubectl_pods_important.sh` - lists important pods and their nodes to check on scheduling - `kubectl_pods_colocated.sh` - lists pods from deployments/statefulsets that are colocated on the same node - `kubectl_node_labels.sh` - lists nodes and their labels, one per line, easier to read visually or pipe in scripting - `kubectl_pods_running_with_labels.sh` - lists running pods with labels matching key=value pair arguments - `kubectl_node_taints.sh` - lists nodes and their taints - `kubectl_jobs_stuck.sh` - finds Kubernetes jobs stuck for hours or days with no completions - `kubectl_jobs_delete_stuck.sh` - prompts for confirmation to delete stuck Kubernetes jobs found by script above - `kubectl_images.sh` - lists Kubernetes container images running on the current cluster - `kubectl_image_counts.sh` - lists Kubernetes container images running counts sorted descending - `kubectl_image_deployments.sh` - lists which deployments, statefulsets or daemonsets container images belong to. Useful to find which deployment, statefulset or daemonset to upgrade to replace a container image eg. when replacing deprecated the k8s.gcr.io registry with registry.k8s.io - `kubectl_pod_count.sh` - lists Kubernetes pods total running count - `kubectl_pod_labels.sh` - lists Kubernetes pods and their labels, one label per line for easier shell script piping for further actions - `kubectl_pod_ips.sh` - lists Kubernetes pods and their pod IP addresses - `kubectl_container_count.sh` - lists Kubernetes containers total running count - `kubectl_container_counts.sh` - lists Kubernetes containers running counts by name sorted descending - `kubectl_pods_dump_*.sh` - dump stats / logs / jstacks from all pods matching a given regex and namespace to txt files for support debugging - `kubectl_pods_dump_stats.sh` - dump stats - `kubectl_pods_dump_logs.sh` - dump logs - `kubectl_pods_dump_jstacks.sh` - dump Java jstacks - `kubectl_pods_dump_all.sh` - calls the above `kubectl_pods_dump_*.sh` scripts for N iterations with a given interval - `kubectl_empty_namespaces.sh` - finds namespaces without any of the usual objects using `kubectl get all` - `kubectl_delete_empty_namespaces.sh` - removes empty namespaces, uses `kubectl_empty_namespaces.sh` - `kubectl_.sh` - quick launch one-off pods for interactive debuggging in Kubernetes - `kubectl_alpine.sh` - `kubectl_busybox.sh` - `kubectl_curl.sh` - `kubectl_dnsutils.sh` - `kubectl_gcloud_sdk.sh` - `kubectl_run_sa.sh` - launch a quick pod with the given service account to test private repo pull & other permissions - `kubectl_port_forward.sh` - launches `kubectl port-forward` to a given pod's port with an optional label or name filter. If more than one pod is found, prompts with an interactive dialogue to choose one. Optionally automatically opens the forwarded localhost URL in the default browser - `kubectl_port_forward_spark.sh` - does the above for Spark UI - `helm_template.sh` - templates a Helm chart for Kustomize deployments - `kustomize_parse_helm_charts.sh` - parses the [Helm](https://helm.sh/) charts from one or more `kustomization.yaml` files into TSV format for further shell pipe processing - `kustomize_install_helm_charts.sh` - installs the [Helm](https://helm.sh/) charts from one or more `kustomization.yaml` files the old fashioned Helm CLI way so that tools like [Nova](https://github.com/FairwindsOps/nova) can be used to detect outdated charts (used in [Kubernetes-configs](https://github.com/HariSekhon/Kubernetes-configs) repo's [CI](https://github.com/HariSekhon/Kubernetes-configs/actions/workflows/nova.yaml)) - `kustomize_update_helm_chart_versions.sh` - updates one or more `kustomization.yaml` files to the latest versions of any charts they contain - `kustomize_materialize.sh` - recursively materializes all `kustomization.yaml` to `kustomization.materialized.yaml` in the same directories for scanning with tools like [Pluto](https://github.com/FairwindsOps/pluto) to detect deprecated API objects inherited from embedded Helm charts. Parallelized for performance - ArgoCD: - `argocd_auto_sync.sh` - toggle Auto-sync on/off to allow repairs and maintenance operation for a given app and also disables / re-enables the App-of-Apps base apps to stop then re-enabling the app - `argocd_apps_sync.sh` - sync's all [ArgoCD](https://argo-cd.readthedocs.io/en/stable/) apps matching an optional ERE regex filter on their names using the ArgoCD CLI - `argocd_apps_wait_sync.sh` - sync's all [ArgoCD](https://argo-cd.readthedocs.io/en/stable/) apps matching an optional ERE regex filter on their names using the ArgoCD CLI's while also checking their health and operation - `argocd_generate_resource_whitelist.sh` - generates a yaml cluster and namespace resource whitelist for ArgoCD project config. If given an existing yaml, will merge in its original whitelists, dedupe, and write them back into the file using an in-place edit. Useful because ArgoCD 2.2+ doesn't show resources that aren't explicitly allowed, such as ReplicaSets and Pods - Pluto: - `pluto_detect_helm_materialize.sh` - recursively materializes all helm `Chart.yaml` and runs [Pluto](https://github.com/FairwindsOps/pluto) on each directory to work around [this issue](https://github.com/FairwindsOps/pluto/issues/444) - `pluto_detect_kustomize_materialize.sh` - recursively materializes all `kustomization.yaml` and runs [Pluto](https://github.com/FairwindsOps/pluto) on each directory to work around [this issue](https://github.com/FairwindsOps/pluto/issues/444) - `pluto_detect_kubectl_dump_objects.sh` - dumps all live Kubernetes objects to /tmp and runs [Pluto](https://github.com/FairwindsOps/pluto) to detect deprecated API objects on the cluster from any source - Rancher: - `rancher_api.sh` - queries the Rancher API with authentication - `rancher_kube_creds.sh` - downloads all Rancher clusters credentials into subdirectories matching cluster names, with `.envrc` in each, so a quick `cd` into one and your kubectl is ready to rock - see also Google Kubernetes Engine scripts in the [GCP - Google Cloud Platform](https://github.com/HariSekhon/DevOps-Bash-tools/#gcp---google-cloud-platform) section above - see also the [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) repo See also [Knowledge Base notes for Kubernetes](https://github.com/HariSekhon/Knowledge-Base/blob/main/kubernetes.md). ### Docker `docker/` directory: - `docker_*.sh` / `dockerhub_*.sh` - [Docker](https://www.docker.com/) / [DockerHub](https://hub.docker.com/) API scripts: - `dockerhub_api.sh` - queries DockerHub API v2 with or without authentication (`$DOCKERHUB_USER` & `$DOCKERHUB_PASSWORD` / `$DOCKERHUB_TOKEN`) - `docker_api.sh` - queries a Docker Registry with optional basic authentication if `$DOCKER_USER` & `$DOCKER_PASSWORD` are set - `docker_build_hashref.sh` - runs `docker build` and auto-generates docker image name and tag from relative Git path and commit short SHA hashref and a dirty sha suffix if git contents are modified. Useful to compare docker image sizes between your clean and modified versions of `Dockerfile` or contents - `docker_package_check.sh` - runs package installs on major versions of a docker image to check given packages are available before adding them and breaking builds across linux distro versions - `docker_registry_list_images.sh` - lists images in a given private Docker Registry - `docker_registry_list_tags.sh` - lists tags for a given image in a private Docker Registry - `docker_registry_get_image_manifest.sh` - gets a given image:tag manifest from a private Docker Registry - `docker_registry_tag_image.sh` - tags a given image with a new tag in a private Docker Registry via the API without pulling and pushing the image data (must faster and more efficient) - `dockerhub_list_tags.sh` - lists tags for a given DockerHub repo. See also [dockerhub_show_tags.py](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/dockerhub_show_tags.py) in the [DevOps Python tools](https://github.com/HariSekhon/DevOps-Python-tools) repo. - `dockerhub_list_tags_by_last_updated.sh` - lists tags for a given DockerHub repo sorted by last updated timestamp descending - `dockerhub_search.sh` - searches with a configurable number of returned items (older docker cli was limited to 25 results) - `clean_caches.sh` - cleans out OS package and programming language caches, call near end of `Dockerfile` to reduce Docker image size - see also the [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo - `quay_api.sh` - queries the [Quay.io](https://quay.io/) API with OAuth2 authentication token `$QUAY_TOKEN` See also [Knowledge Base notes for Docker](https://github.com/HariSekhon/Knowledge-Base/blob/main/docker.md). ### Data `data/` directory: - `avro_tools.sh` - runs Avro Tools jar, downloading it if not already present (determines latest version when downloading) - `parquet_tools.sh` - runs Parquet Tools jar, downloading it if not already present (determines latest version when downloading) - `csv_header_indices.sh` - list CSV headers with their zero indexed numbers, useful reference when coding against column positions - `ini_config_add_if_missing.sh` - reads INI config blocks from stdin and appends them to the specified file if the section is not found. Used by `aws_profile_config_add_if_missing.sh` - `ini_config_duplicate_sections.sh` - lists duplicate INI config sections that are using the same value for a given key in the given .ini file - `ini_config_duplicate_section_names.sh` - lists duplicate INI config section names that are using the same value for a given key in the given .ini file - `ini_grep_section.sh` - prints the named section from a given .ini file to stdout - `wordcount.sh` - counts and ranks words by their frequency in file(s) or stdin - Data format validation `validate_*.py` from [DevOps Python Tools repo](https://github.com/HariSekhon/DevOps-Python-tools): - CSV - JSON - [Avro](https://avro.apache.org/) - [Parquet](https://parquet.apache.org/) - INI / Properties files (Java) - LDAP LDIF - XML - YAML - `json2yaml.sh` - converts JSON to YAML - `yaml2json.sh` - converts YAML to JSON - needed for some APIs like GitLab CI linting (see [Gitlab](#git---github-gitlab-bitbucket-azure-devops) section above) ### Big Data & NoSQL `bigdata/` and `kafka/` directories: - `kafka_*.sh` - scripts to make [Kafka](http://kafka.apache.org/) CLI usage easier including auto-setting Kerberos to source TGT from environment and auto-populating broker and zookeeper addresses. These are auto-added to the `$PATH` when `.bashrc` is sourced. For something similar for [Solr](https://lucene.apache.org/solr/), see `solr_cli.pl` in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) repo. - `zookeeper*.sh` - [Apache ZooKeeper](https://zookeeper.apache.org/) scripts: - `zookeeper_client.sh` - shortens `zookeeper-client` command by auto-populating the zookeeper quorum from the environment variable `$ZOOKEEPERS` or else parsing the zookeeper quorum from `/etc/**/*-site.xml` to make it faster and easier to connect - `zookeeper_shell.sh` - shortens Kafka's `zookeeper-shell` command by auto-populating the zookeeper quorum from the environment variable `$KAFKA_ZOOKEEPERS` and optionally `$KAFKA_ZOOKEEPER_ROOT` to make it faster and easier to connect - `hive_*.sh` / `beeline*.sh` - [Apache Hive](https://hive.apache.org/) scripts: - `beeline.sh` - shortens `beeline` command to connect to [HiveServer2](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Overview) by auto-populating Kerberos and SSL settings, zookeepers for HiveServer2 HA discovery if the environment variable `$HIVE_HA` is set or using the `$HIVESERVER_HOST` environment variable so you can connect with no arguments (prompts for HiveServer2 address if you haven't set `$HIVESERVER_HOST` or `$HIVE_HA`) - `beeline_zk.sh` - same as above for [HiveServer2](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Overview) HA by auto-populating SSL and ZooKeeper service discovery settings (specify `$HIVE_ZOOKEEPERS` environment variable to override). Automatically called by `beeline.sh` if either `$HIVE_ZOOKEEPERS` or `$HIVE_HA` is set (the latter parses `hive-site.xml` for the ZooKeeper addresses) - `hive_foreach_table.sh` - executes a SQL query against every table, replacing `{db}` and `{table}` in each iteration eg. `select count(*) from {table}` - `hive_list_databases.sh` - list Hive databases, one per line, suitable for scripting pipelines - `hive_list_tables.sh` - list Hive tables, one per line, suitable for scripting pipelines - `hive_tables_metadata.sh` - lists a given DDL metadata field for each Hive table (to compare tables) - `hive_tables_location.sh` - lists the data location per Hive table (eg. compare external table locations) - `hive_tables_row_counts.sh` - lists the row count per Hive table - `hive_tables_column_counts.sh` - lists the column count per Hive table - ` impala*.sh` - [Apache Impala](https://impala.apache.org/) scripts: - `impala_shell.sh` - shortens `impala-shell` command to connect to [Impala](https://impala.apache.org/) by parsing the Hadoop topology map and selecting a random datanode to connect to its Impalad, acting as a cheap CLI load balancer. For a real load balancer see [HAProxy config for Impala](https://github.com/HariSekhon/HAProxy-configs) (and many other Big Data & NoSQL technologies). Optional environment variables `$IMPALA_HOST` (eg. point to an explicit node or an HAProxy load balancer) and `IMPALA_SSL=1` (or use regular impala-shell `--ssl` argument pass through) - `impala_foreach_table.sh` - executes a SQL query against every table, replacing `{db}` and `{table}` in each iteration eg. `select count(*) from {table}` - `impala_list_databases.sh` - list Impala databases, one per line, suitable for scripting pipelines - `impala_list_tables.sh` - list Impala tables, one per line, suitable for scripting pipelines - `impala_tables_metadata.sh` - lists a given DDL metadata field for each Impala table (to compare tables) - `impala_tables_location.sh` - lists the data location per Impala table (eg. compare external table locations) - `impala_tables_row_counts.sh` - lists the row count per Impala table - `impala_tables_column_counts.sh` - lists the column count per Impala table - `hdfs_*.sh` - Hadoop [HDFS](https://en.wikipedia.org/wiki/Apache_Hadoop#Hadoop_distributed_file_system) scripts: - `hdfs_checksum*.sh` - walks an HDFS directory tree and outputs HDFS native checksums (faster) or portable externally comparable CRC32, in serial or in parallel to save time - `hdfs_find_replication_factor_1.sh` / `hdfs_set_replication_factor_3.sh` - finds HDFS files with replication factor 1 / sets HDFS files with replication factor <=2 to replication factor 3 to repair replication safety and avoid no replica alarms during maintenance operations (see also Python API version in the [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) repo) - `hdfs_file_size.sh` / `hdfs_file_size_including_replicas.sh` - quickly differentiate HDFS files raw size vs total replicated size - `hadoop_random_node.sh` - picks a random Hadoop cluster worker node, like a cheap CLI load balancer, useful in scripts when you want to connect to any worker etc. See also the read [HAProxy Load Balancer configurations](https://github.com/HariSekhon/HAProxy-configs) which focuses on master nodes - `cloudera_*.sh` - [Cloudera](https://www.cloudera.com/) scripts: - `cloudera_manager_api.sh` - script to simplify querying [Cloudera Manager](https://www.cloudera.com/products/product-components/cloudera-manager.html) API using environment variables, prompts, authentication and sensible defaults. Built on top of `curl_auth.sh` - `cloudera_manager_impala_queries*.sh` - queries [Cloudera Manager](https://www.cloudera.com/products/product-components/cloudera-manager.html) for recent [Impala](https://impala.apache.org/) queries, failed queries, exceptions, DDL statements, metadata stale errors, metadata refresh calls etc. Built on top of `cloudera_manager_api.sh` - `cloudera_manager_yarn_apps.sh` - queries [Cloudera Manager](https://www.cloudera.com/products/product-components/cloudera-manager.html) for recent [Yarn](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) apps. Built on top of `cloudera_manager_api.sh` - `cloudera_navigator_api.sh` - script to simplify querying [Cloudera Navigator](https://www.cloudera.com/products/product-components/cloudera-navigator.html) API using environment variables, prompts, authentication and sensible defaults. Built on top of `curl_auth.sh` - `cloudera_navigator_audit_logs.sh` - fetches [Cloudera Navigator](https://www.cloudera.com/products/product-components/cloudera-navigator.html) audit logs for given service eg. hive/impala/hdfs via the API, simplifying date handling, authentication and common settings. Built on top of `cloudera_navigator_api.sh` - `cloudera_navigator_audit_logs_download.sh` - downloads [Cloudera Navigator](https://www.cloudera.com/products/product-components/cloudera-navigator.html) audit logs for each service by year. Skips existing logs, deletes partially downloaded logs on failure, generally retry safe (while true, Control-C, not `kill -9` obviously). Built on top of `cloudera_navigator_audit_logs.sh` See also [Knowledge Base notes for Hadoop](https://github.com/HariSekhon/Knowledge-Base/blob/main/hadoop.md). ### Git - GitHub, GitLab, Bitbucket, Azure DevOps `git/`, `github/`, `gitlab/`, `bitbucket/` and `azure_devops/` directories: - `git/*.sh` - [Git](https://git-scm.com/) scripts: - `precommit_run_changed_files.sh` - runs pre-commit on all files changed on the current branch vs the default branch. Useful to reproduce `pre-commit` checks that are failing in pull requests to get your PRs to pass - `git_diff_commit.sh` - quickly commits added or updated files to Git, showing a diff and easy enter prompt for each file. Super convenient for fast commits on the command line, and in vim and IDEs via hotkeys - `git_review_push.sh` - shows diff of what would be pushed upstream and prompts to push. Convenient for fast reviewed pushes via vim or IDEs hotkeys - `git_branch_delete_squash_merged.sh` - carefully detects if a squash merged branch you want to delete has no changes with the default trunk branch before deleting it. See [Squash Merges](https://github.com/HariSekhon/Knowledge-Base/blob/main/git.md#squash-merges-require-force-deleting-branches) in knowledge-base about why this is necessary. - `git_tag_release.sh` - creates a Git tag, auto-incrementing a `.N` suffix on the year/month/day date format if no exact version given - `git_foreach_branch.sh` - executes a command on all branches (useful in heavily version branched repos like in my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo) - `git_foreach_repo.sh` - executes a command against all adjacent repos from a given repolist (used heavily by many adjacent scripts) - `git_foreach_modified.sh` - executes a command against each file with git modified status - `git_foreach_repo_replace_readme_actions.sh` - updates the `README.md` badges for GitHub Actions to match the local repo name. Useful to bulk fix copied badges quickly and easily - `git_foreach_repo_update_readme.sh` - git-diff-commits the `README.md` for each Git repo checkout using adjacent `git_foreach_repo.sh` and `git_diff_commit.sh` scripts. Useful to quickly bulk update `README.md` in all your projects, such as when references need updating - `git_push_stats.sh` - shows the Git push stats to the remote origin for the current branch - number of commits and lines of diff, using the following `git_origin_*.sh` scripts: - `git_origin_log_to_push.sh` - shows the Git log in local branch that would be pushed to remote origin - `git_origin_files_to_push.sh` - shows the Git files in local branch that would be pushed to remote origin - `git_origin_diff_to_push.sh` - shows the Git diff of lines in local branch that would be pushed to remote origin - `git_origin_commit_count_to_push.sh` - shows the number of Git commits in local branch that would be pushed to remote origin - `git_origin_line_count_to_push.sh` - shows the Git number of lines changed in local branch that would be pushed to remote origin. These are lines actually added / changed / removed without surrounding context lines - `git_merge_all.sh` / `git_merge_master.sh` / `git_merge_master_pull.sh` - merges updates from master branch to all other branches to avoid drift on longer lived feature branches / version branches (eg. [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo) - `git_remotes_add_origin_providers.sh` - auto-creates remotes for the 4 major public repositories ([GitHub](https://github.com/)/[GitLab](https://gitlab.com/)/[Bitbucket](https://bitbucket.org)/[Azure DevOps](https://dev.azure.com/)), useful for `git pull -all` to fetch and merge updates from all providers in one command - `git_remotes_set_multi_origin.sh` - sets up multi-remote origin for unified push to automatically keep the 4 major public repositories in sync (especially useful for [Bitbucket](https://bitbucket.org) and [Azure DevOps](https://dev.azure.com/) which don't have [GitLab](https://gitlab.com/)'s auto-mirroring from [GitHub](https://github.com/) feature) - `git_remotes_set_https_to_ssh.sh` - converts local repo's remote URLs from https to ssh (more convenient with SSH keys instead of https auth tokens, especially since Azure DevOps expires personal access tokens every year) - `git_remotes_set_ssh_to_https.sh` - converts local repo's remote URLs from ssh to https (to get through corporate firewalls or hotels if you travel a lot) - `git_remotes_set_https_creds_helpers.sh` - adds Git credential helpers configuration to the local git repo to use http API tokens dynamically from environment variables if they're set - `git_repos_pull.sh` - pull multiple repos based on a source file mapping list - useful for easily sync'ing lots of Git repos among computers - `git_repos_update.sh` - same as above but also runs the `make update` build to install the latest dependencies, leverages the above script - `git_grep_env_vars.sh` - find environment variables in the current git repo's code base in the format `SOME_VAR` (useful to find undocumented environment variables in internal or open source projects such as ArgoCD eg. [argoproj/argocd-cd #8680](https://github.com/argoproj/argo-cd/pull/8680)) - `git_log_empty_commits.sh` - find empty commits in git history (eg. if a `git filter-branch` was run but `--prune-empty` was forgotten, leaking metadata like subjects containing file names or other sensitive info) - `git_log_me.sh` - shows only commits in the Git log done by you. Useful to remind yourself what parts the current Git repo you've been working on for periodic reviews, reports or even updating your CV! - `git_log_me_added.sh` - same as above but only file addition commits - `git_graph_commit_history_gnuplot.sh` - generates GNUplot graphs of Git commits per year and per month for the entire history of the local Git repo checkout - `git_graph_commit_history_mermaidjs.sh` - generates MermaidJS graphs of Git commits per year and per month for the entire history of the local Git repo checkout - `git_graph_commit_times_gnuplot.sh` - generates a GNUplot graph of Git commit times from the current Git repo checkout's `git log` - `git_graph_commit_times_mermaidjs.sh` - generates a MermaidJS graph of Git commit times from the current Git repo checkout's `git log` - `git_graph_commit_times_gnuplot_all_repos.sh` - generates GNUplot graph of the GitHub commit times from all local adjacent Git repo checkouts listed in `setup/repos.txt` using Git log in each checkout - `git_graph_commit_times_mermaidjs_all_repos.sh` - generates MermaidJS graph of the GitHub commit times from all local adjacent Git repo checkouts listed in `setup/repos.txt` using Git log in each checkout - `github_public_lines_of_code.sh` - checks out all public original source GitHub repos for the current or given user and then counts all lines of code for them with breakdowns of languages, files, code, comments and blanks - `git_revert_line.sh` - reverts the first line that matches a given regex from the Git head commit's version of the same line number. Useful to revert some changes caused by over zealous sed'ing scripts, where you want to cherry-pick revert a single line change - `git_files_no_uncommitted_changes.sh` - returns zero if given file(s) don't have uncommitted changes to Git, either staged or unstaged. Useful to be able to iterate over git files with in-place edits only if safe to do so without other uncommitted changes that would be at risk of being lost - `git_files_in_history.sh` - finds all filename / file paths in the git log history, useful for prepping for `git filter-branch` - `git_filter_branch_fix_author.sh` - rewrites Git history to replace author/committer name & email references (useful to replace default account commits). Powerful, read `--help` and `man git-filter-branch` carefully. Should only be used by Git Experts - `git_filter_repo_replace_text.sh` - rewrites Git history to replace a given text to scrub a credential or other sensitive token from history. Refuses to operate on tokens less than 8 chars for safety - `git_submodules_update.sh` - updates all submodules in the local git repo to the latest commit of their detected default trunk branch - `git_submodules_update_repos.sh` - updates submodules for all repos given as args or saved in the `setup/repos.txt` file - `git_askpass.sh` - credential helper script to use environment variables for git authentication - `github/*.sh` - [GitHub](https://github.com/) API / CLI scripts: - `github_api.sh` - queries the GitHub [API](https://docs.github.com/en/rest/reference). Can infer GitHub user, repo and authentication token from local checkout or environment (`$GITHUB_USER`, `$GITHUB_TOKEN`) - `github_install_binary.sh` - installs a binary from GitHub releases into $HOME/bin or /usr/local/bin. Auto-determines the latest release if no version specified, detects and unpacks any tarball or zip files - `github_foreach_repo.sh` - executes a templated command for each non-fork GitHub repo, replacing the `{owner}`/`{name}` or `{repo}` placeholders in each iteration - `github_graph_commit_times_gnuplot.sh` - generates GNUplot graph of GitHub commit times from all public GitHub repos for a given user. Fetches the commit data via the GitHub API - `github_graph_commit_times_mermaidjs.sh` - generates MermaidJS graph of the GitHub commit times from all public GitHub repos for a given user. Fetches the commit data via the GitHub API - `github_clone_or_pull_all_repos.sh` - git clones or pulls all repos for a user or organization into directories of the same name under the current directory - `github_download_release_file.sh` - downloads a file from GitHub Releases, optionally determining the latest version, uses `bin/download_url_file.sh` - `github_download_release_jar.sh` - downloads a JAR file from GitHub Releases (used by `install/download_*_jar.sh` for things like [JDBC](https://github.com/HariSekhon/Knowledge-Base/blob/main/jdbc.md) drivers or [Java](#java) [decompilers](https://github.com/HariSekhon/Knowledge-Base/blob/main/java.md#java-decompilers)), optionally determines latest version to download, and finally validates the downloaded file's format - `github_invitations.sh` - lists / accepts repo invitations. Useful to accept a large number of invites to repos generated by automation - `github_mirror_repos_to_gitlab.sh` - creates/syncs GitHub repos to GitLab for migrations or to cron fast free Disaster Recovery, including all branches and tags, plus the repo descriptions. Note this doesn't include PRs/wikis/releases - `github_mirror_repos_to_bitbucket.sh` - creates/syncs GitHub repos to BitBucket for migrations or to cron fast free Disaster Recovery, including all branches and tags, plus the repo descriptions. Note this doesn't include PRs/wikis/releases - `github_mirror_repos_to_aws_codecommit.sh` - creates/syncs GitHub repos to AWS CodeCommit for migrations or to cron fast almost free Disaster Recovery (close to $0 compared to $100-400+ per month for [Rewind BackHub](https://rewind.com/products/backups/github/)), including all branches and tags, plus the repo descriptions. Note this doesn't include PRs/wikis/releases - `github_mirror_repos_to_gcp_source_repos.sh` - creates/syncs GitHub repos to GCP Source Repos for migrations or to cron fast almost free Disaster Recovery (close to $0 compared to $100-400+ per month for [Rewind BackHub](https://rewind.com/products/backups/github/)), including all branches and tags. Note this doesn't include repo description/PRs/wikis/releases - `github_pull_request_create.sh` - creates a Pull Request idempotently by first checking for an existing PR between the branches, and also checking if there are the necessary commits between the branches, to avoid common errors from blindly raising PRs. Useful to automate code promotion across environment branches. Also works across repo forks and is used by `github_repo_fork_update.sh`. Even populates github pull request template and does Jira ticket number replacement from branch prefix - `github_pull_request_preview.sh` - opens a GitHub Pull Request preview page from the current local branch to the given or default branch - `github_push_pr_preview.sh` - pushes to GitHub origin, sets upstream branch, then open a Pull Request preview from current branch to the given or default trunk branch in your browser - `github_push_pr.sh` - pushes to GitHub origin, sets upstream branch, then idemopotently creates a Pull Request from current branch to the given or default trunk branch and opens the generated PR in your browser for review - `github_merge_branch.sh` - merges one branch into another branch via a Pull Request for full audit tracking all changes. Useful to automate feature PRs, code promotion across environment branches, or backport hotfixes from Production or Staging to trunk branches such as master, main, dev or develop - `github_remote_set_upstream.sh` - in a forked GitHub repo's checkout, determine the origin of the fork using GitHub CLI and configure a git remote to the upstream. Useful to be able to easily pull updates from the original source repo - `github_pull_merge_trunk.sh` - pulls the origin or fork upstream repo's trunk branch and merges it into the local branch, In a forked GitHub repo's checkout, determines the origin of the fork using GitHub CLI, configures a git remote to the upstream, pulls the default branch and if on a branch other than the default then merges the default branch to the local current branch. Simplifies and automates keeping your checkout or forked repo up to date with the original source repo to quickly resolve merge conflicts locally and submit updated Pull Requests - `github_forked_add_remote.sh` - quickly adds a forked repo as a remote from an interactive men list of forked repos - `github_forked_checkout_branch.sh` - quickly check out a forked repo's branch from an interactive menu lists of forked repos and their branches - `github_tag_hashref.sh` - Returns the GitHub commit hashref for a given GitHub Actions `owner/repo@tag` or `https://github.com/owner/repo@tag`. Useful for pinning 3rd party GitHub Actions to hashref instead of tag to follow [GitHub Actions Best Practices](https://github.com/HariSekhon/Knowledge-Base/blob/main/github-actions.md#github-actions-best-practices) - `github_actions_foreach_workflow.sh` - executes a templated command for each workflow in a given GitHub repo, replacing `{name}`, `{id}` and `{state}` in each iteration - `github_actions_aws_create_load_credential.sh` - creates an AWS user with group/policy, generates and downloads access keys, and uploads them to the given repo - `github_actions_in_use.sh` - lists GitHub Actions directly referenced in the .github/workflows in the current local repo checkout - `github_actions_in_use_repo.sh` - lists GitHub Actions for a given repo via the API, including following imported reusable workflows - `github_actions_in_use_across_repos.sh` - lists GitHub Actions in use across all your repos - `github_actions_repos_lockdown.sh` - secures GitHub Actions settings across all user repos to only GitHub, verified partners and selected 3rd party actions - `github_actions_repo_set_secret.sh` - sets a secret in the given repo from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `github_actions_repo_env_set_secret.sh` - sets a secret in the given repo and environment from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `github_actions_repo_secrets_overriding_org.sh` - finds any secrets for a repo that are overriding organization level secrets. Useful to combine with `github_foreach_repo.sh` for auditing - `github_actions_repo_restrict_actions.sh` - restricts GitHub Actions in the given repo to only running actions from GitHub and verfied partner companies (.eg AWS, Docker) - `github_actions_repo_actions_allow.sh` - allows select 3rd party GitHub Actions in the given repo - `github_actions_runner.sh` - generates a [GitHub Actions](https://github.com/features/actions) self-hosted runner token for a given Repo or Organization via the GitHub API and then runs a dockerized GitHub Actions runner with the appropriate configuration - `github_actions_runner_local.sh` - downloads, configures and runs a local GitHub Actions Runner for Linux or Mac - `github_actions_runner_token.sh` - generates a GitHub Actions runner token to register a new self-hosted runner - `github_actions_runners.sh` - lists GitHub Actions self-hosted runners for a given Repo or Organization - `github_actions_delete_offline_runners.sh` - deletes offline GitHub Actions self-hosted runners. Useful to clean up short-lived runners eg. Docker, Kubernetes - `github_actions_workflows.sh` - lists GitHub Actions workflows for a given repo (or auto-infers local repository) - `github_actions_workflow_runs.sh` - lists GitHub Actions workflow runs for a given workflow id or name - `github_actions_workflows_status.sh` - lists all GitHub Actions workflows and their statuses for a given repo - `github_actions_workflows_state.sh` - lists GitHub Actions workflows enabled/disabled states (GitHub now disables workflows after 6 months without a commit) - `github_actions_workflows_disabled.sh` - lists GitHub Actions workflows that are disabled. Combine with `github_foreach_repo.sh` to scan all repos to find disabled workflows - `github_actions_workflow_enable.sh` - enables a given GitHub Actions workflow - `github_actions_workflows_enable_all.sh` - enables all GitHub Actions workflows in a given repo. Useful to undo GitHub disabling all workflows in a repo after 6 months without a commit - `github_actions_workflows_trigger_all.sh` - triggers all workflows for the given repo - `github_actions_workflows_cancel_all_runs.sh` - cancels all workflow runs for the given repo - `github_actions_workflows_cancel_waiting_runs.sh` - cancels workflow runs that are in waiting state, eg. waiting for old deployment approvals - `github_actions_log.sh` - outputs the text log for a given GitHub Actions workflow run to the terminal. Fetches the last 10 runs and drops you into an interactive menu to hit enter on the one you want. Useful when the logs are too big for the UI and you have to open it in another tab which is very slow in browser - `github_actions_latest_log.sh` - same as above, but just fetches the latest workflow run log without any prompting - `github_ssh_get_user_public_keys.sh` - fetches a given GitHub user's public SSH keys via the API for piping to `~/.ssh/authorized_keys` or adjacent tools - `github_ssh_get_public_keys.sh` - fetches the currently authenticated GitHub user's public SSH keys via the API, similar to above but authenticated to get identifying key comments - `github_ssh_add_public_keys.sh` - uploads SSH keys from local files or standard input to the currently authenticated GitHub account. Specify pubkey files (default: `~/.ssh/id_rsa.pub`) or read from standard input for piping from adjacent tools - `github_ssh_delete_public_keys.sh` - deletes given SSH keys from the currently authenticated GitHub account by key id or title regex match - `github_gpg_get_user_public_keys.sh` - fetches a given GitHub user's public GPG keys via the API - `github_generate_status_page.sh` - generates a [STATUS.md](https://harisekhon.github.io/CI-CD/) page by merging all the README.md headers for all of a user's non-forked GitHub repos or a given list of any repos etc. - `github_purge_camo_cache.sh` - send HTTP Purge requests to all camo urls (badge caches) for the current or given GitHub repo's landing/README.md page - `github_ip_ranges.sh` - returns GitHub's IP ranges, either all by default or for a select given service such as hooks or actions - `github_sync_repo_descriptions.sh` - syncs GitHub repo descriptions to GitLab & BitBucket repos - `github_release.sh` - creates a GitHub Release, auto-incrementing a `.N` suffix on the year/month/day date format if no exact version given - `github_repo_check_pat_token.sh` - checks the given PAT token can access the given GitHub repo. Useful to test a PAT token used for integrations like ArgoCD - `github_repo_description.sh` - fetches the given repo's description (used by `github_sync_repo_descriptions.sh`) - `github_repo_find_files.sh` - finds files matching a regex in the current or given GitHub repo via the GitHub API - `github_repo_latest_release.sh` - returns the latest release tag for a given GitHub repo via the GitHub API - `github_repo_latest_release_filter.sh` - returns the latest release tag matching a given regex filter for a given GitHub repo via the GitHub API. Useful for getting the latest version of things like Kustomize which has other releases for kyaml - `github_repo_stars.sh` - fetches the stars, forks and watcher counts for a given repo - `github_repo_teams.sh` - fetches the GitHub Enterprise teams and their role permisions for a given repo. Combine with `github_foreach_repo.sh` to audit your all your personal or GitHub organization's repos - `github_repo_collaborators.sh` - fetches a repo's granted users and outside invited collaborators as well as their role permisions for a given repo. Combine with `github_foreach_repo.sh` to audit your all your personal or GitHub organization's repos - `github_repo_protect_branches.sh` - enables branch protections on the given repo. Can specify one or more branches to protect, otherwise finds and applies to any of `master`, `main`, `develop`, `dev`, `staging`, `production` - `github_repos_find_files.sh` - finds files matching a regex across all repos in the current GitHub organization or user account - `github_repo_fork_sync.sh` - sync's current or given fork, then runs `github_repo_fork_update.sh` to cascade changes to major branches via Pull Requests for auditability - `github_repo_fork_update.sh` - updates a forked repo by creating pull requests for full audit tracking and auto-merges PRs for non-production branches - `github_repos_public.sh` - lists public repos for a user or organization. Useful to periodically scan and account for any public repos - `github_repos_disable_wiki.sh` - disables the Wiki on one or more given repos to prevent documentation fragmentation and make people use the centralized documentation tool eg. Confluence or Slite - `github_repos_with_few_users.sh` - finds repos with few or no users (default: 1), which in Enterprises is a sign that a user has created a repo without assigning team privileges - `github_repos_with_few_teams.sh` - finds repos with few or no teams (default: 0), which in Enterprises is a sign that a user has created a repo without assigning team privileges - `github_repos_without_branch_protections.sh` - finds repos without any branch protection rules (use `github_repo_protect_branches.sh` on such repos) - `github_repos_not_in_terraform.sh` - finds all non-fork repos for current or given user/organization which are not found in `$PWD/*.tf` Terraform code - `github_teams_not_in_terraform.sh` - finds all teams for given organization which are not found in `$PWD/*.tf` Terraform code - `github_repos_sync_status.sh` - determines whether each GitHub repo's mirrors on GitLab / BitBucket / Azure DevOps are up to date with the latest commits, by querying all 3 APIs and comparing master branch hashrefs - `github_teams_not_idp_synced.sh` - finds GitHub teams that aren't sync'd from an IdP like Azure AD. These should usually be migrated or removed - `github_user_repos_stars.sh` - fetches the total number of stars for all original source public repos for a given user - `github_user_repos_forks.sh` - fetches the total number of forks for all original source public repos for a given user - `github_user_repos_count.sh` - fetches the total number of original source public repos for a given username - `github_user_followers.sh` - fetches the number of followers for a given username - `github_url_clipboard.sh` - copies a GitHub URL file's contents to the clipboard, converting the URL to a raw GitHub content URL where necessary - `gitlab/*.sh` - [GitLab](https://gitlab.com/) API scripts: - `gitlab_api.sh` - queries the GitLab [API](https://docs.gitlab.com/ee/api/api_resources.html). Can infer GitLab user, repo and authentication token from local checkout or environment (`$GITLAB_USER`, `$GITLAB_TOKEN`) - `gitlab_install_binary.sh` - installs a binary from GitLab releases into $HOME/bin or /usr/local/bin. Auto-determines the latest release if no version specified, detects and unpacks any tarball or zip files - `gitlab_push_mr_preview.sh` - pushes to GitLab origin, sets upstream branch, then open a Merge Request preview from current to default branch - `github_push_mr.sh` - pushes to GitLab origin, sets upstream branch, then idemopotently creates a Merge Request from current branch to the given or default trunk branch and opens the generated MR in your browser for review - `gitlab_foreach_repo.sh` - executes a templated command for each GitLab project/repo, replacing the `{user}` and `{project}` in each iteration - `gitlab_project_latest_release.sh` - returns the latest release tag for a given GitLab project (repo) via the GitLab API - `gitlab_project_set_description.sh` - sets the description for one or more projects using the GitLab API - `gitlab_project_set_env_vars.sh` - adds / updates GitLab project-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `gitlab_group_set_env_vars.sh` - adds / updates GitLab group-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `gitlab_project_create_import.sh` - creates a GitLab repo as an import from a given URL, and mirrors if on GitLab Premium (can only manually configure for public repos on free tier, API doesn't support configuring even public repos on free) - `gitlab_project_protect_branches.sh` - enables branch protections on the given project. Can specify one or more branches to protect, otherwise finds and applies to any of `master`, `main`, `develop`, `dev`, `staging`, `production` - `gitlab_project_mirrors.sh` - lists each GitLab repo and whether it is a mirror or not - `gitlab_pull_mirror.sh` - trigger a GitLab pull mirroring for a given project's repo, or auto-infers project name from the local git repo - `gitlab_ssh_get_user_public_keys.sh` - fetches a given GitLab user's public SSH keys via the API, with identifying comments, for piping to `~/.ssh/authorized_keys` or adjacent tools - `gitlab_ssh_get_public_keys.sh` - fetches the currently authenticated GitLab user's public SSH keys via the API - `gitlab_ssh_add_public_keys.sh` - uploads SSH keys from local files or standard input to the currently authenticated GitLab account. Specify pubkey files (default: `~/.ssh/id_rsa.pub`) or read from standard input for piping from adjacent tools - `gitlab_ssh_delete_public_keys.sh` - deletes given SSH keys from the currently authenticated GitLab account by key id or title regex match - `gitlab_validate_ci_yaml.sh` - validates a `.gitlab-ci.yml` file via the GitLab API - `bitbucket/*.sh` - [BitBucket](https://bitbucket.org/) API scripts: - `bitbucket_api.sh` - queries the BitBucket [API](https://developer.atlassian.com/bitbucket/api/2/reference/resource/). Can infer BitBucket user, repo and authentication token from local checkout or environment (`$BITBUCKET_USER`, `$BITBUCKET_TOKEN`) - `bitbucket_foreach_repo.sh` - executes a templated command for each BitBucket repo, replacing the `{user}` and `{repo}` in each iteration - `bitbucket_workspace_set_env_vars.sh` - adds / updates Bitbucket workspace-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `bitbucket_repo_set_env_vars.sh` - adds / updates Bitbucket repo-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `bitbucket_repo_set_description.sh` - sets the description for one or more repos using the BitBucket API - `bitbucket_enable_pipelines.sh` - enables the CI/CD pipelines for all repos - `bitbucket_disable_pipelines.sh` - disables the CI/CD pipelines for all repos - `bitbucket_repo_enable_pipeline.sh` - enables the CI/CD pipeline for a given repo - `bitbucket_repo_disable_pipeline.sh` - disables the CI/CD pipeline for a given repo - `bitbucket_ssh_get_public_keys.sh` - fetches the currently authenticated BitBucket user's public SSH keys via the API for piping to `~/.ssh/authorized_keys` or adjacent tools - `bitbucket_ssh_add_public_keys.sh` - uploads SSH keys from local files or standard input to the currently authenticated BitBucket account. Specify pubkey files (default: `~/.ssh/id_rsa.pub`) or read from standard input for piping from adjacent tools - `bitbucket_ssh_delete_public_keys.sh` - uploads SSH keys from local files or standard input to the currently authenticated BitBucket account. Specify pubkey files (default: `~/.ssh/id_rsa.pub`) or read from standard input for piping from adjacent tools See also [Knowledge Base notes for Git](https://github.com/HariSekhon/Knowledge-Base/blob/main/git.md). ### Markdown Maintain your Git `README.md` and similar Markdown documentation well. - `markdown/*`: - `markdown_generate_index.sh` - generates a markdown index list from the headings in a given markdown file such as `README.md` - `markdown_replace_index.sh` - replaces a markdown index section in a given markdown file using `markdown_generate_index.sh` - `markdown_replace_repos.sh` - replaces the repos block of a given markdown file. Used to keep my GitHub repos Other Repos sections updated - `mdl_list_indentations.sh`- runs Markdownlint `mdl` command and prefixes the spaces count to each offending line of MD005 (inconsistent list indentations). Workaround for [Markdownlint issue #520](https://github.com/markdownlint/markdownlint/issues/520) - `markdown_list_indentations.sh` - prefixes number of spaces before each list item for comparison to MarkdownLint MD005 inconsistent list indentation error - `markdown_columns_to_table.sh` - converts text columns separated by whitespace to a Markdown table with vertically aligned column pipe chars. Combine with scripts like `domains_subdomains_environments.sh` to generate the markdown documentation of your domains and subdomains per project and environment - `markdown_octocat_github_links.sh` - converts GitHub links like `` to shorthand links with an OctoCat emoji and without the redundant `https://github.com/` prefix such as [:octocat: HariSekhon/Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) - `markdown_replace_links_with_jsdelivr.sh` - replaces local GitHub repo file links in the given markdown file(s) with JSDelivr CDN links ### CI/CD - Continuous Integration / Continuous Deployment `jenkins/`, `terraform/`, `teamcity/`, `buildkite/`, `circlci/`, `travis/`, `azure_devops/`, ..., `cicd/` directories: - `appveyor_api.sh` - queries [AppVeyor](https://www.appveyor.com/)'s API with authentication - `azure_devops/*.sh` - [Azure DevOps](https://dev.azure.com/) scripts: - `azure_devops_api.sh` - queries Azure DevOps's API with authentication - `azure_devops_foreach_repo.sh` - executes a templated command for each Azure DevOps repo, replacing `{user}`, `{org}`, `{project}` and `{repo}` in each iteration - `azure_devops_to_github_migration.sh` - migrates one or all Azure DevOps git repos to GitHub, including all branches and sets the default branch to match via the APIs to maintain the same checkout behaviour - `azure_devops_disable_repos.sh` - disables one or more given Azure DevOps repos (to prevent further pushes to them after migration to GitHub) - `circleci/*.sh` - [CircleCI](https://circleci.com/) scripts: - `circleci_api.sh` - queries CircleCI's API with authentication - `circleci_project_set_env_vars.sh` - adds / updates CircleCI project-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `circleci_context_set_env_vars.sh` - adds / updates CircleCI context-level environment variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `circleci_project_delete_env_vars.sh` - deletes CircleCI project-level environment variable(s) via the API - `circleci_context_delete_env_vars.sh` - deletes CircleCI context-level environment variable(s) via the API - `circleci_local_execute.sh` - installs CircleCI CLI and executes `.circleci/config.yml` locally - `circleci_public_ips.sh` - lists [CircleCI](https://circleci.com) public IP addresses via dnsjson.com - `codeship_api.sh` - queries [CodeShip](https://codeship.com/)'s API with authentication - `drone_api.sh` - queries [Drone.io](https://drone.io/)'s API with authentication - `shippable_api.sh` - queries [Shippable](https://www.shippable.com/)'s API with authentication - `wercker_app_api.sh` - queries [Wercker](https://app.wercker.com/)'s Applications API with authentication - `gocd_api.sh` - queries [GoCD](https://www.gocd.org/)'s API - `gocd.sh` - one-touch [GoCD CI](https://www.gocd.org/): - launches in Docker - (re)creates config repo (`$PWD/setup/gocd_config_repo.json`) from which to source pipeline(s) (`.gocd.yml`) - detects and enables agent(s) to start building - call from any repo top level directory with a `.gocd.yml` config (all mine have it), mimicking structure of fully managed CI systems - `concourse.sh` - one-touch [Concourse CI](https://concourse-ci.org/): - launches in Docker - configures pipeline from `$PWD/.concourse.yml` - triggers build - tails results in terminal - prints recent build statuses at end - call from any repo top level directory with a `.concourse.yml` config (all mine have it), mimicking structure of fully managed CI systems - `fly.sh` - shortens [Concourse](https://concourse-ci.org/) `fly` command to not have to specify target all the time - `jenkins/*.sh` - [Jenkins CI](https://jenkins.io/) scripts: - `jenkins.sh` - one-touch [Jenkins CI](https://jenkins.io/): - launches Docker container - installs plugins - validates `Jenkinsfile` - configures job from `$PWD/setup/jenkins-job.xml` - sets Pipeline to git remote origin's `Jenkinsfile` - triggers build - tails results in terminal - call from any repo top level directory with a `Jenkinsfile` pipeline and `setup/jenkins-job.xml` (all mine have it) - `jenkins_api.sh` - queries the Jenkins Rest API, handles authentication, pre-fetches CSFR protection token crumb, supports many environment variables such as `$JENKINS_URL` for ease of use - `jenkins_jobs.sh` - lists Jenkins jobs (pipelines) - `jenkins_foreach_job.sh` - runs a templated command for each Jenkins job - `jenkins_jobs_download_configs.sh` - downloads all Jenkins job configs to xml files of the same name - `jenkins_job_config.sh` - gets or sets a Jenkins job's config - `jenkins_job_description.sh` - gets or sets a Jenkins job's description - `jenkins_job_enable.sh` - enables a Jenkins job by name - `jenkins_job_disable.sh` - disables a Jenkins job by name - `jenkins_job_trigger.sh` - triggers a Jenkins job by name - `jenkins_job_trigger_with_params.sh` - triggers a Jenkins job with parameters which can be passed as `--data KEY=VALUE` - `jenkins_jobs_enable.sh` - enables all Jenkins jobs/pipelines with names matching a given regex - `jenkins_jobs_disable.sh` - disables all Jenkins jobs/pipelines with names matching a given regex - `jenkins_builds.sh` - lists Jenkins latest builds for every job - `jenkins_cred_add_cert.sh` - creates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_add_kubernetes_sa.sh` - creates a Jenkins Kubernetes service account credential - `jenkins_cred_add_secret_file.sh` - creates a Jenkins secret file credential from a file - `jenkins_cred_add_secret_text.sh` - creates a Jenkins secret string credential from a string or a file - `jenkins_cred_add_ssh_key.sh` - creates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_add_user_pass.sh` - creates a Jenkins username/password credential - `jenkins_cred_delete.sh` - deletes a given Jenkins credential by id - `jenkins_cred_list.sh` - lists Jenkins credentials IDs and Names - `jenkins_cred_update_cert.sh` - updates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_update_kubernetes_sa.sh` - updates a Jenkins Kubernetes service account credential - `jenkins_cred_update_secret_file.sh` - updates a Jenkins secret file credential from a file - `jenkins_cred_update_secret_text.sh` - updates a Jenkins secret string credential from a string or a file - `jenkins_cred_update_ssh_key.sh` - updates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_update_user_pass.sh` - updates a Jenkins username/password credential - `jenkins_cred_set_cert.sh` - creates or updates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_set_kubernetes_sa.sh` - creates or updates a Jenkins Kubernetes service account credential - `jenkins_cred_set_secret_file.sh` - creates or updates a Jenkins secret file credential from a file - `jenkins_cred_set_secret_text.sh` - creates or updates a Jenkins secret string credential from a string or a file - `jenkins_cred_set_ssh_key.sh` - creates or updates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_set_user_pass.sh` - creates or updates a Jenkins username/password credential - `jenkins_cli.sh` - shortens `jenkins-cli.jar` command by auto-inferring basic configuations, auto-downloading the CLI if absent, inferrings a bunch of Jenkins related variables like `$JENKINS_URL`, `$JENKINS_CLI_ARGS` and authentication using `$JENKINS_USER`/`$JENKINS_PASSWORD`, or finds admin password from inside local docker container. Used heavily by `jenkins.sh` one-shot setup and the following scripts: - `jenkins_foreach_job_cli.sh` - runs a templated command for each Jenkins job - `jenkins_create_job_parallel_test_runs.sh` - creates a freestyle parameterized test sleep job and launches N parallel runs of it to test scaling and parallelization of [Jenkins on Kubernetes](https://github.com/HariSekhon/Kubernetes-configs#jenkins-on-kubernetes) agents - `jenkins_create_job_check_gcp_serviceaccount.sh` - creates a freestyle test job which runs a GCP Metadata query to determine the GCP serviceaccount the agent pod is operating under to check GKE Workload Identity integration - `jenkins_jobs_download_configs_cli.sh` - downloads all Jenkins job configs to xml files of the same name - `jenkins_cred_cli_add_cert.sh` - creates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_cli_add_kubernetes_sa.sh` - creates a Jenkins Kubernetes service account credential - `jenkins_cred_cli_add_secret_file.sh` - creates a Jenkins secret file credential from a file - `jenkins_cred_cli_add_secret_text.sh` - creates a Jenkins secret string credential from a string or a file - `jenkins_cred_cli_add_ssh_key.sh` - creates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_cli_add_user_pass.sh` - creates a Jenkins username/password credential - `jenkins_cred_cli_delete.sh` - deletes a given Jenkins credential by id - `jenkins_cred_cli_list.sh` - lists Jenkins credentials IDs and Names - `jenkins_cred_cli_update_cert.sh` - updates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_cli_update_kubernetes_sa.sh` - updates a Jenkins Kubernetes service account credential - `jenkins_cred_cli_update_secret_file.sh` - updates a Jenkins secret file credential from a file - `jenkins_cred_cli_update_secret_text.sh` - updates a Jenkins secret string credential from a string or a file - `jenkins_cred_cli_update_ssh_key.sh` - updates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_cli_update_user_pass.sh` - updates a Jenkins username/password credential - `jenkins_cred_cli_set_cert.sh` - creates or updates a Jenkins certificate credential from a PKCS#12 keystore - `jenkins_cred_cli_set_kubernetes_sa.sh` - creates or updates a Jenkins Kubernetes service account credential - `jenkins_cred_cli_set_secret_file.sh` - creates or updates a Jenkins secret file credential from a file - `jenkins_cred_cli_set_secret_text.sh` - creates or updates a Jenkins secret string credential from a string or a file - `jenkins_cred_cli_set_ssh_key.sh` - creates or updates a Jenkins SSH key credential from a string or an SSH private key file - `jenkins_cred_cli_set_user_pass.sh` - creates or updates a Jenkins username/password credential - `jenkins_password.sh` - gets Jenkins admin password from local docker container. Used by `jenkins_cli.sh` - `jenkins_plugins_latest_versions.sh` - finds the latest versions of given Jenkins plugins. Useful to programmatically upgrade your Jenkins on Kubernetes plugins defined in [values.yaml](https://github.com/HariSekhon/Kubernetes-configs/blob/6d9e34b74d3fa8f353b0fe56e74cea3af439e01a/jenkins/base/values.yaml#L145) - `check_jenkinsfiles.sh` - validates all `*Jenkinsfile*` files in the given directory trees using the online Jenkins validator - See also [Knowledge Base notes for Jenkins](https://github.com/HariSekhon/Knowledge-Base/blob/main/jenkins.md). - `teamcity/*.sh` - [TeamCity CI](https://www.jetbrains.com/teamcity/) scripts: - `teamcity.sh` - one-touch [TeamCity CI](https://www.jetbrains.com/teamcity/) cluster: - launches Docker containers with server and 1 agent - click proceed and accept the EULA - waits for server to initialize - waits for agent to register - authorizes agent - creates a VCS Root if `$PWD` has a `.teamcity.vcs.json` / `.teamcity.vcs.ssh.json` / `.teamcity.vcs.oauth.json` and corresponding `$TEAMCITY_SSH_KEY` or `$TEAMCITY_GITHUB_CLIENT_ID`+`$TEAMCITY_GITHUB_CLIENT_SECRET` environment variables - creates a Project and imports all settings and builds from the VCS Root - creates an admin user and an API token for you - see also: [TeamCity CI](https://github.com/HariSekhon/TeamCity-CI) config repo for importing pipelines - `teamcity_api.sh` - queries TeamCity's API, auto-handling authentication and other quirks of the API - `teamcity_create_project.sh` - creates a TeamCity project using the API - `teamcity_create_github_oauth_connection.sh` - creates a TeamCity GitHub OAuth VCS connection in the Root project, useful for bootstrapping projects from VCS configs - `teamcity_create_vcs_root.sh` - creates a TeamCity VCS root from a save configuration (XML or JSON), as downloaded by `teamcity_export_vcs_roots.sh` - `teamcity_upload_ssh_key.sh` - uploads an SSH private key to a TeamCity project (for use in VCS root connections) - `teamcity_agents.sh` - lists TeamCity agents, their connected state, authorized state, whether enabled and up to date - `teamcity_builds.sh` - lists the last 100 TeamCity builds along with the their state (eg. `finished`) and status (eg. `SUCCESS`/`FAILURE`) - `teamcity_buildtypes.sh` - lists TeamCity buildTypes (pipelines) along with the their project and IDs - `teamcity_buildtype_create.sh` - creates a TeamCity buildType from a local JSON configuration (see `teamcity_buildtypes_download.sh`) - `teamcity_buildtype_set_description_from_github.sh` - sync's a TeamCity buildType's description from its Github repo description - `teamcity_buildtypes_set_description_from_github.sh` - sync's all TeamCity buildType descriptions from their GitHub repos where available - `teamcity_export.sh` - downloads TeamCity configs to local JSON files in per-project directories mimicking native TeamCity directory structure and file naming - `teamcity_export_project_config.sh` - downloads TeamCity project config to local JSON files - `teamcity_export_buildtypes.sh` - downloads TeamCity buildType config to local JSON files - `teamcity_export_vcs_roots.sh` - downloads TeamCity VCS root config to local JSON files - `teamcity_projects.sh` - lists TeamCity project IDs and Names - `teamcity_project_set_versioned_settings.sh` - configures a project to track all changes to a VCS (eg. GitHub) - `teamcity_project_vcs_versioning.sh` - quickly toggle VCS versioning on/off for a given TeamCity project (useful for testing without auto-committing) - `teamcity_vcs_roots.sh` - lists TeamCity VCS root IDs and Names - `travis/*.sh` - [Travis CI](https://travis-ci.org/) API scripts (one of my all-time favourite CI systems): - `travis_api.sh` - queries the Travis CI API with authentication using `$TRAVIS_TOKEN` - `travis_repos.sh` - lists Travis CI repos - `travis_foreach_repo.sh` - executes a templated command against all Travis CI repos - `travis_repo_build.sh` - triggers a build for the given repo - `travis_repo_caches.sh` - lists caches for a given repo - `travis_repo_crons.sh` - lists crons for a given repo - `travis_repo_env_vars.sh` - lists environment variables for a given repo - `travis_repo_settings.sh` - lists settings for a given repo - `travis_repo_create_cron.sh` - creates a cron for a given repo and branch - `travis_repo_delete_crons.sh` - deletes all crons for a given repo - `travis_repo_delete_caches.sh` - deletes all caches for a given repo (sometimes clears build problems) - `travis_delete_cron.sh` - deletes a Travis CI cron by ID - `travis_repos_settings.sh` - lists settings for all repos - `travis_repos_caches.sh` - lists caches for all repos - `travis_repos_crons.sh` - lists crons for all repos - `travis_repos_create_cron.sh` - creates a cron for all repos - `travis_repos_delete_crons.sh` - deletes all crons for all repos - `travis_repos_delete_caches.sh` - deletes all caches for all repos - `travis_lint.sh` - lints a given `.travis.yml` using the API - `buildkite/*.sh` - [BuildKite](https://buildkite.com/) API scripts: - `buildkite_api.sh` - queries the BuildKite API, handling authentication using `$BUILDKITE_TOKEN` - `buildkite_pipelines.sh` - list buildkite pipelines for your `$BUILDKITE_ORGANIZATION` / `$BUILDKITE_USER` - `buildkite_foreach_pipeline.sh` - executes a templated command for each Buildkite pipeline, replacing the `{user}` and `{pipeline}` in each iteration - `buildkite_agent.sh` - runs a buildkite agent locally on Linux or Mac, or in Docker with choice of Linux distros - `buildkite_agents.sh` - lists the Buildkite agents connected along with their hostname, IP, started dated and agent details - `buildkite_pipelines.sh` - lists Buildkite pipelines - `buildkite_create_pipeline.sh` - create a Buildkite pipeline from a JSON configuration (like from `buildkite_get_pipeline.sh` or `buildkite_save_pipelines.sh`) - `buildkite_get_pipeline.sh` - gets details for a specific Buildkite pipeline in JSON format - `buildkite_update_pipeline.sh` - updates a BuildKite pipeline from a configuration provided via stdin or from a file saved via `buildkite_get_pipeline.sh` - `buildkite_patch_pipeline.sh` - updates a BuildKite pipeline from a partial configuration provided as an arg, via stdin, or from a file saved via `buildkite_get_pipeline.sh` - `buildkite_pipeline_skip_settings.sh` - lists the skip intermediate build settings for one or more given BuildKite pipelines - `buildkite_pipeline_set_skip_settings.sh` - configures given or all BuildKite pipelines to skip intermediate builds and cancel running builds in favour of latest build - `buildkite_cancel_scheduled_builds.sh` - cancels BuildKite scheduled builds (to clear a backlog due to offline agents and just focus on new builds) - `buildkite_cancel_running_builds.sh` - cancels BuildKite running builds (to clear them and restart new later eg. after agent / environment change / fix) - `buildkite_pipeline_disable_forked_pull_requests.sh` - disables forked pull request builds on a BuildKite pipeline to protect your build environment from arbitrary code execution security vulnerabilities - `buildkite_pipelines_vulnerable_forked_pull_requests.sh` - prints the status of each pipeline, should all return false, otherwise run the above script to close the vulnerability - `buildkite_rebuild_cancelled_builds.sh` - triggers rebuilds of last N cancelled builds in current pipeline - `buildkite_rebuild_failed_builds.sh` - triggers rebuilds of last N failed builds in current pipeline (eg. after agent restart / environment change / fix) - `buildkite_rebuild_all_pipelines_last_cancelled.sh` - triggers rebuilds of the last cancelled build in each pipeline in the organization - `buildkite_rebuild_all_pipelines_last_failed.sh` - triggers rebuilds of the last failed build in each pipeline in the organization - `buildkite_retry_jobs_dead_agents.sh` - triggers job retries where jobs failed due to killed agents, continuing builds from that point and replacing their false negative failed status with the real final status, slightly better than rebuilding entire jobs which happen under a new build - `buildkite_recreate_pipeline.sh` - recreates a pipeline to wipe out all stats (see url and badge caveats in `--help`) - `buildkite_running_builds.sh` - lists running builds and the agent they're running on - `buildkite_save_pipelines.sh` - saves all BuildKite pipelines in your `$BUILDKITE_ORGANIZATION` to local JSON files in `$PWD/.buildkite-pipelines/` - `buildkite_set_pipeline_description.sh` - sets the description of one or more pipelines using the BuildKite API - `buildkite_set_pipeline_description_from_github.sh` - sets a Buildkite pipeline description to match its source GitHub repo - `buildkite_sync_pipeline_descriptions_from_github.sh` - for all BuildKite pipelines sets each description to match its source GitHub repo - `buildkite_trigger.sh` - triggers BuildKite build job for a given pipeline - `buildkite_trigger_all.sh` - same as above but for all pipelines - `terraform_cloud_*.sh` - [Terraform Cloud](https://www.terraform.io/cloud) API scripts: - `terraform_cloud_api.sh` - queries the Cloudflare API, handling authentication using `$TERRAFORM_TOKEN` - `terraform_cloud_ip_ranges.sh` - returns the list of IP ranges for Terraform Cloud - `terraform_cloud_organizations.sh` - lists Terraform Cloud organizations - `terraform_cloud_workspaces.sh` - lists Terraform Cloud workspaces - `terraform_cloud_workspace_vars.sh` - lists Terraform Cloud workspace variables - `terraform_cloud_workspace_set_vars.sh` - adds / updates Terraform workspace-level sensitive environment/terraform variable(s) via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `terraform_cloud_workspace_delete_vars.sh` - deletes one or more Terraform workspace-level variables - `terraform_cloud_varsets.sh` - lists Terraform Cloud variable sets - `terraform_cloud_varset_vars.sh` - lists Terraform Cloud variables in on or all variables sets for the given organization - `terraform_cloud_varset_set_vars.sh` - adds / updates Terraform sensitive environment/terraform variable(s) in a given variable set via the API from `key=value` or shell export format, as args or via stdin (eg. piped from `aws_csv_creds.sh`) - `terraform_cloud_varset_delete_vars.sh` - deletes one or more Terraform variables in a given variable set - `terraform_*.sh` - [Terraform](https://www.terraform.io/) scripts: - `terraform_gcs_backend_version.sh` - determines the Terraform state version from the tfstate file in a GCS bucket found in a local given `backend.tf` - `terraform_gitlab_download_backend_variable.sh` - downloads backend.tf from a GitLab CI/CD variable to be able to quickly iterate plans locally - `terraform_import.sh` - finds given resource type in `./*.tf` code or Terraform plan output that are not in Terraform state and imports them - `terraform_import_aws_iam_users.sh` - parses Terraform plan output to import new `aws_iam_user` additions into Terraform state - `terraform_import_aws_iam_groups.sh` - parses Terraform plan output to import new `aws_iam_group` additions into Terraform state - `terraform_import_aws_iam_policies.sh` - parses Terraform plan output to import new `aws_iam_policies` additions, resolves their ARNs and imports them into Terraform state - `terraform_import_aws_sso_permission_sets.sh` - finds all `aws_ssoadmin_permission_set` in `./*.tf` code, resolves the ARNs and imports them to Terraform state - `terraform_import_aws_sso_account_assignments.sh` - parses Terraform plan output to import new `aws_ssoadmin_account_assignment` additions into Terraform state - `terraform_import_aws_sso_managed_policy_attachments.sh` - parses Terraform plan output to import new `aws_ssoadmin_account_assignment` additions into Terraform state - `terraform_import_aws_sso_permission_set_inline_policies.sh` - parses Terraform plan output to import new `aws_ssoadmin_permission_set_inline_policy` additions into Terraform state - `terraform_import_github_repos.sh` - finds all `github_repository` in `./*.tf` code or Terraform plan output that are not in Terraform state and imports them. See also `github_repos_not_in_terraform.sh` - `terraform_import_github_team.sh` - imports a given GitHub team into a given Terraform state resource, by first querying the GitHub API for the team ID needed to import into Terraform - `terraform_import_github_teams.sh` - finds all `github_team` in `./*.tf` code or Terraform plan output that are not in Terraform state, then queries the GitHub API for their IDs and imports them. See also `github_teams_not_in_terraform.sh` - `terraform_import_github_team_repos.sh` - finds all `github_team_repository` in Terraform plan that would be added, then queries the GitHub API for the repos and team IDs and if they both exist then imports them to Terraform state - `terraform_provider_count_sizes.sh` - finds duplicate Terraform providers and their sizes. Useful to find space wastage caused by using Terragrunt without configuring a unified Terraform Plugin Cache - `terraform_resources.sh` - external program to get all resource ids and attribute for a given resource type to work around Terraform splat expression limitation ([#19931](https://github.com/hashicorp/terraform/issues/19931)) - `terraform_managed_resource_types.sh` - quick parse of what Terraform resource types are found in `*.tf` files under the current or given directory tree. Useful to give you a quick glance of what services you are managing - `terraform_registry_url_extract.sh` - extracts the Terraform Registry URL in either `tfr://` or `https://registry.terraform.io/` format from a given string, file or standard input. Useful to fast load Terraform Module documentation via editor/IDE hotkeys (see [.vimrc](configs/.vimrc)). Based on `urlextract.sh` above - `terraform_registry_url_to_https.sh` - converts one or more Terraform Registry URLs from `tfr://` to `https://registry.terraform.io/` format - `terraform_registry_url_open.sh` - opens the Terraform Registry URL given as a string arg, file or standard input in either `tfr://` or `https://registry.terraform.io/` format - See also [Knowledge Base notes for Terraform](https://github.com/HariSekhon/Knowledge-Base/blob/main/terraform.md). - `checkov_resource_*.sh` - [Checkov](https://www.checkov.io/) resource counts - useful to estimate [Bridgecrew Cloud](https://www.bridgecrew.cloud/) costs which are charged per resource: - `checkov_resource_count.sh` - counts the number of resources Checkov is scanning in the current or given directory - `checkov_resource_count_all.sh` - counts the total number of resources Checkov is scanning across all given repo checkouts - `octopus_api.sh` - queries the [Octopus Deploy](https://octopus.com/) API - `sonarlint_generate_config.sh` - generates the `.sonarlint/connectedMode.json` config at the root of the Git repo from the `sonar-project.properties` file See also [Knowledge Base notes for CI/CD](https://github.com/HariSekhon/Knowledge-Base/blob/main/ci-cd.md). ### AI & IPaaS `ai/` and `ipaas/` directories: - `openai_api.sh` - queries the [OpenAI](https://openai.com/) (ChatGPT) API with authentication - `make_api.sh` - queries the [Make.com](https://www.make.com) API with authentication ### Internet Services `internet/`, `cloudflare/`, `pingdom/`, `terraform/` directories: - Pastebins - uploads files and copies the resulting URL to your clipboard: - code / text only - prompts to approve text / code before upload for safety: - `pastebin.sh` - uploads a file to , script auto-determines which syntax highlighting to add since API doesn't auto infer - `dpaste.sh` - uploads a file to , script auto-determines which syntax highlighting to add since API doesn't auto infer - `termbin.sh` - uploads a file to (site has no syntax highlighting) - all files, multimedia or text / code - prompts to approve text / code before upload for safety: - `0x0.sh` - uploads a file to (fast) - `imgur.sh` - uploads an image file to - `file.io.sh` - uploads a file to with 2 weeks, single download retention - `catbox.sh` - uploads a file to with permanent retention (slow) - `litterbox.sh` - uploads a file to with temporary retention (slow) - `digital_ocean_api.sh` / `doapi.sh` - queries the [Digital Ocean](https://www.digitalocean.com/) API with authentication - see also the Digital Ocean CLI `doctl` (`install/install_doctl.sh`) - `atlassian_ip_ranges.sh` - lists [Atlassian](https://www.atlassian.com/)'s IPv4 and/or IPv6 cidr ranges via its API - `circleci_public_ips.sh` - lists [CircleCI](https://circleci.com) public IP addresses via dnsjson.com - `cloudflare_*.sh` - [Cloudflare](https://www.cloudflare.com/) API queries and reports: - `cloudflare_api.sh` - queries the Cloudflare API with authentication - `cloudflare_ip_ranges.sh` - lists Cloudflare's IPv4 and/or IPv6 cidr ranges via its API - `cloudflare_custom_certificates.sh` - lists any custom SSL certificates in a given Cloudflare zone along with their status and expiry date - `cloudflare_dns_records.sh` - lists any Cloudflare DNS records for a zone, including the type and ttl - `cloudflare_dns_records_all_zones.sh` - same as above but for all zones - `cloudflare_dns_record_create.sh` - creates a DNS record in the given domain - `cloudflare_dns_record_update.sh` - updates a DNS record in the given domain - `cloudflare_dns_record_delete.sh` - deletes a DNS record in the given domain - `cloudflare_dns_record_details.sh` - lists the details for a DNS record in the given domain in JSON format for further pipe processing - `cloudflare_dnssec.sh` - lists the Cloudflare DNSSec status for all zones - `cloudflare_firewall_rules.sh` - lists Cloudflare Firewall rules, optionally with filter expression - `cloudflare_firewall_access_rules.sh` - lists Cloudflare Firewall Access rules, optionally with filter expression - `cloudflare_foreach_account.sh` - executes a templated command for each Cloudflare account, replacing the `{account_id}` and `{account_name}` in each iteration (useful for chaining with `cloudflare_api.sh`) - `cloudflare_foreach_zone.sh` - executes a templated command for each Cloudflare zone, replacing the `{zone_id}` and `{zone_name}` in each iteration (useful for chaining with `cloudflare_api.sh`, used by adjacent `cloudflare_*_all_zones.sh` scripts) - `cloudflare_purge_cache.sh` - purges the entire Cloudflare cache - `cloudflare_ssl_verified.sh` - gets the Cloudflare zone SSL verification status for a given zone - `cloudflare_ssl_verified_all_zones.sh` - same as above for all zones - `cloudflare_zones.sh` - lists Cloudflare zone names and IDs (needed for writing Terraform Cloudflare code) - `datadog_api.sh` - queries the [DataDog](https://www.datadoghq.com/) API with authentication - `dnsjson.sh` - queries dnsjson.com for DNS records - `domains_subdomains_environments.sh` - for a given list of domains, deduplicate and print dev / staging subdomains as well as root domain for prod. Used to generate a whole bunch of Ad Tech domains and pixel tracker subdomains for a project. Combine with `markdown_columns_to_table.sh` to generate the markdown documentation for your domains and subomains per project and environment - `gitguardian_api.sh` - queries the [GitGuardian](https://www.gitguardian.com/) API with authentication - `google_maps_link.sh` - queries for a search string, returns the first hit and then generates a stable fixed place ID url to the result. Useful for sharing in documentation links to places like [HariSekhon/Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) Travel pages - `jira_api.sh` - queries [Jira](https://www.atlassian.com/software/jira) API with authentication - `kong_api.sh` - queries the [Kong API Gateway](https://docs.konghq.com/gateway/latest/)'s Admin API, handling authentication if enabled - `traefik_api.sh` - queries the [Traefik](https://traefik.io/) API, handling authentication if enabled - `ngrok_api.sh` - queries the [NGrok](https://ngrok.com/) API with authentication - `pingdom_*.sh` - [Pingdom](https://www.pingdom.com/) API queries and reports for status, latency, average response times, latency averages by hour, SMS credits, outages periods and durations over the last year etc. - `pingdom_api.sh` - queries the Solarwinds [Pingdom](https://www.pingdom.com/) API with authentication - `pingdom_foreach_check.sh` - executes a templated command against each Pingdom check, replacing the `{check_id}` and `{check_name}` in each iteration - `pingdom_checks.sh` - show all Pingdom checks, status and latencies - `pingdom_checks_outages.sh` / `pingdom_checks_outages.sh` - show one or all Pingdom checks outage histories for the last year - `pingdom_checks_average_response_times.sh` - shows the average response times for all Pingdom checks for the last week - `pingdom_check_latency_by_hour.sh` / `pingdom_checks_latency_by_hour.sh` - shows the average latency for one or all Pingdom checks broken down by hour of the day, over the last week - `pingdom_sms_credits.sh` - gets the remaining number of Pingdom SMS credits - `terraform_cloud_api.sh` - queries [Terraform Cloud](https://www.terraform.io/cloud) API with authentication - `terraform_cloud_ip_ranges.sh` - returns the list of IP ranges for [Terraform Cloud](https://www.terraform.io/cloud) via the API, or optionally one or more of the ranges used by different functions - `wordpress.sh` - boots Wordpress in docker with a MySQL backend, and increases the upload_max_filesize to be able to restore a real world sized export backup - `wordpress_api.sh` - queries the Wordpress API with authentication - `wordpress_posts_without_category_tags.sh` - checks posts (articles) for categories without corresponding tags and prints the posts and their missing tags ### Java `java/` directory: - `java_show_classpath.sh` - shows Java classpaths, one per line, of currently running Java programs - `jvm_heaps*.sh` - show all your Java heap sizes for all running Java processes, and their total MB (for performance tuning and sizing) - Java Decompilers: - `java_decompile_jar.sh` - decompiles a Java JAR in /tmp, finds the main class and runs a Java decompiler on its main .class file using `jd_gui.sh` - `jd_gui.sh` - runs Java Decompiler JD GUI, downloading its jar the first time if it's not already present - `bytecode_viwer.sh` - runs Bytecode-Viewer GUI Java decompiler, downloading its jar the first time if it's not already present - `cfr.sh` - runs CFR command line Java decompiler, downloading its jar the first time if it's not already present - `procyon.sh` - runs Procyon command line Java decompiler, downloading its jar the first time if it's not already present See also [Knowledge Base notes for Java](https://github.com/HariSekhon/Knowledge-Base/blob/main/java.md) and [JVM Performance Tuning](https://github.com/HariSekhon/Knowledge-Base/blob/main/java-jvm-performance-tuning.md). ### Python `python/` directory: - `python_compile.sh` - byte-compiles Python scripts and libraries into `.pyo` optimized files - `python_pip_install.sh` - bulk installs PyPI modules from mix of arguments / file lists / stdin, accounting for User vs System installs, root vs user sudo, VirtualEnvs / Anaconda / GitHub Workflows/ Google Cloud Shell, Mac vs Linux library paths, and ignore failure option - `python_pip_install_if_absent.sh` - installs PyPI modules not already in Python libary path (OS or pip installed) for faster installations only where OS packages are already providing some of the modules, reducing time and failure rates in CI builds - `python_pip_install_for_script.sh` - installs PyPI modules for given script(s) if not already installed. Used for dynamic individual script dependency installation in the [DevOps Python tools](https://github.com/HariSekhon/DevOps-Python-tools) repo - `python_pip_reinstall_all_modules.sh` - reinstalls all PyPI modules which can fix some issues - `pythonpath.sh` - prints all Python libary search paths, one per line - `python_find_library_path.sh` - finds directory where a PyPI module is installed - without args finds the Python library base - `python_find_library_executable.sh` - finds directory where a PyPI module's CLI program is installed (system vs user, useful when it gets installed to a place that isn't in your `$PATH`, where `which` won't help) - `python_find_unused_pip_modules.sh` - finds PyPI modules that aren't used by any programs in the current directory tree - `python_find_duplicate_pip_requirements.sh` - finds duplicate PyPI modules listed for install under the directory tree (useful for deduping module installs in a project and across submodules) - `python_translate_import_module.sh` - converts Python import modules to PyPI module names, used by `python_pip_install_for_script.sh` - `python_translate_module_to_import.sh` - converts PyPI module names to Python import names, used by `python_pip_install_if_absent.sh` and `python_find_unused_pip_modules.sh` - `python_pyinstaller.sh` - creates [PyInstaller](https://pypi.org/project/pyinstaller/) self-contained Python programs with Python interpreter and all PyPI modules included - `python_pypi_versions.sh` - prints all available versions of a given PyPi module using the API See also [Knowledge Base notes for Python](https://github.com/HariSekhon/Knowledge-Base/blob/main/python.md). ### Perl `perl/` directory: - `perl_cpanm_install.sh` - bulk installs CPAN modules from mix of arguments / file lists / stdin, accounting for User vs System installs, root vs user sudo, [Perlbrew](https://perlbrew.pl/) / Google Cloud Shell environments, Mac vs Linux library paths, ignore failure option, auto finds and reads build failure log for quicker debugging showing root cause error in CI builds logs etc - `perl_cpanm_install_if_absent.sh` - installs CPAN modules not already in Perl libary path (OS or CPAN installed) for faster installations only where OS packages are already providing some of the modules, reducing time and failure rates in CI builds - `perl_cpanm_reinstall_all.sh` - re-installs all CPAN modules. Useful for trying to recompile XS modules on Macs after migration assistant from an Intel Mac to an ARM Silicon Mac leaves your home XS libraries broken as they're built for the wrong architecture - `perlpath.sh` - prints all Perl libary search paths, one per line - `perl_find_library_path.sh` - finds directory where a CPAN module is installed - without args finds the Perl library base - `perl_find_library_executable.sh` - finds directory where a CPAN module's CLI program is installed (system vs user, useful when it gets installed to a place that isn't in your `$PATH`, where `which` won't help) - `perl_find_unused_cpan_modules.sh` - finds CPAN modules that aren't used by any programs in the current directory tree - `perl_find_duplicate_cpan_requirements.sh` - finds duplicate CPAN modules listed for install more than once under the directory tree (useful for deduping module installs in a project and across submodules) - `perl_generate_fatpacks.sh` - creates [Fatpacks](https://metacpan.org/pod/App::FatPacker) - self-contained Perl programs with all CPAN modules built-in See also [Knowledge Base notes for Perl](https://github.com/HariSekhon/Knowledge-Base/blob/main/perl.md). ### Golang `packages/` directory: - `golang_install.sh` - bulk installs Golang modules from mix of arguments / file lists / stdin - `golang_install_if_absent.sh` - same as above but only if the package binary isn't already available in `$PATH` - `golang_rm_binaries.sh` - deletes binaries of the same name adjacent to `.go` files. Doesn't delete your `bin/` etc as these are often real deployed applications rather than development binaries ### Diagrams Used in [HariSekhon/Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) `diagrams/` directory: - `d2.sh` - generates a D2lang diagram, using its shebang if present for themes etc, and then opens the resulting image - `d2_generate_diagrams.sh` - generates all D2lang `.d2` diagrams found under the current or given directory, git reverting or deleting those that didn't generate properly (to work around [d2lang issue #2367](https://github.com/terrastruct/d2/issues/2367)) - `mermaidjs_generate_diagrams.sh` - generates all MermaidJS `.mmd` diagrams found under the current or given directory, git reverting or deleting those that didn't generate properly - `python_mingrammer_generate_diagrams.sh` - generates all Python Mingrammer `.py` diagrams found under the current or given directory ### Media `media/` directory: #### Images - `image_shrink.sh` - shrinks an image by resizing it (default 50%) to be able to upload it against limits on some websites. Shows the before and after stats and automatically opens the image to check it - `image_reduce_quality.sh` - shrinks an image size by reducing its quality (default to 80%) to be able to upload it against limits on some websites. Shows the before and after stats and automatically opens the image to check it - `image_trim_pixels.sh` - trims N pixels off one of the sides of an image. Useful to tweak screenshots before sharing them - `image_join_vertical.sh` - joins two images top and bottom after matching their widths so they align correctly - `image_join_horizontal.sh` - joins two images left and right after matching their heights so they align correctly - `imageopen.sh` - opens the given image file using whatever available tool is found on Linux or Mac - `d2.sh` - generates [D2lang diagram](https://github.com/HariSekhon/Diagrams-as-Code) and then opens the resulting image - `avif_to_png.sh` - converts an Avif image to PNG to be usable on websites that don't support Webp images like LinkedIn - `svg_to_png.sh` - converts an SVG image to PNG to be usable on websites that don't support SVG images like LinkedIn, Medium or Reddit - `webp_to_png.sh` - converts a Webp image to PNG to be usable on websites that don't support Webp images like Medium. Tries 2 different tools for webp unlike the next generic converter - `image_to_png.sh` - try to convert any image to PNG to be usable on fussy websites such as LinkedIn, Medium or Reddit #### Terminal Gif Capture Each of these three scripts creates an animated Git from running terminal commands and then opens the resulting gif. - `ttygif.sh` - uses `ttyrec` and `ttygif` - `asciinema.sh` - uses `asciinema` and `agg` - `terminalizer.sh` - uses Terminalizer #### Audio - `mp3_set_artist.sh` / `mp3_set_album.sh` - set the artist / album tag for all mp3 files under given directories. Useful for grouping artists/albums and audiobook author/books (eg. for correct importing into Mac's Books.app) - `mp3_set_track_name.sh` - set the track name metadata for mp3 files under given directories to follow their filenames. Useful for correctly displaying audiobook progress / chapters etc. - `mp3_set_track_order.sh` - set the track order metadata for mp3 files under given directories to follow the lexical file naming order. Useful for correctly ordering album songs and audiobook chapters (eg. for Mac's Books.app). Especially useful for enforcing global ordering on multi-CD audiobooks after grouping into a single audiobook using `mp3_set_album.sh` (otherwise default track numbers in each CD interleave in Mac's Books.app) #### Video - `avi_to_mp4.sh` - converts avi files to mp4 using ffmpeg. Useful to be able to play videos on devices like smart TVs that may not recognize newer codecs otherwise - `mkv_to_mp4.sh` - converts mkv files to mp4 using ffmpeg. Same use case as above - `youtube_download_video.sh` - downloads a YouTube video to mp4 with maximum quality and compatibility usng yt-dlp - `facebook_download_video.sh` - same as above for Facebook - `twitter_download_video.sh` - same as above for Twitter / X - `x_download_video.sh` - same as above for X / Twitter - `youtube_download_channel.sh` - downloads all videos from a given YouTube channel using yt-dlp - `video_to_720p_mp4` - converts one or more video files to 720p mp4 format using ffmpeg. Useful to make good trade-off of quality vs size for social media sharing - `vidopen.sh` - opens the given video file using whatever available tool is found on Linux or Mac See also [Knowledge Base notes for MultiMedia](https://github.com/HariSekhon/Knowledge-Base/blob/main/multimedia.md). ### Spotify 40+ [Spotify](https://www.spotify.com/) API scripts (used extensively to manage my [Spotify-Playlists](https://github.com/HariSekhon/Spotify-Playlists) repo). `spotify/` directory: - `spotify_playlists*.sh` - list playlists in either ` ` or JSON format - `spotify_playlist_tracks*.sh` - gets playlist contents as track URIs / `Artists - Track` / CSV format - useful for backups or exports between music systems - `spotify_backup.sh` - backs up all Spotify playlists as well as the ordered list of playlists - `spotify_backup_playlist*.sh` - backs up Spotify playlists to local files in both human readable `Artist - Track` format and Spotify URI format for easy restores or adding to new playlists - `spotify_backup_artists_followed.sh` - backs up the list of artists followed to Spotify URI and Names - `spotify_search*.sh` - searches Spotify's library for tracks / albums / artists getting results in human readable format, JSON, or URI formats for easy loading to Spotify playlists - `spotify_release_year.sh` - searches for a given track or album and finds the original release year - `spotify_uri_json.sh` - takes a Spotify URI and dumps its JSON for inspection. You can pass one of the following formats: `spotify::`, `http://open.spotify.com//`, `` - `spotify_uri_to_name.sh` - converts Spotify track / album / artist URIs to human readable `Artist - Track` or CSV format. Takes Spotify URIs, URL links or just IDs similar to `spotify_uri_json.sh` above. Reads URIs from files or standard input - `spotify_create_playlist.sh` - creates a Spotify playlist, either public or private - `spotify_rename_playlist.sh` - renames a Spotify playlist - `spotify_set_playlists_public.sh` / `spotify_set_playlists_private.sh` - sets one or more given Spotify playlists to public / private - `spotify_add_to_playlist.sh` - adds tracks to a given playlist. Takes a playlist name or ID and Spotify URIs in any form from files or standard input. Can be combined with many other tools listed here which output Spotify URIs, or appended from other playlists. Can also be used to restore a spotify playlist from backups - `spotify_delete_from_playlist.sh` - deletes tracks from a given playlist. Takes a playlist name or ID and Spotify URIs in any form from files or standard input, optionally prefixed with a track position to remove only specific occurrences (useful for removing duplicates from playlists) - `spotify_delete_from_playlist_if_in_other_playlists.sh` - deletes tracks from a given playlist if their URIs are found in the subsequently given playlists - `spotify_delete_from_playlist_if_track_in_other_playlists.sh` - deletes tracks from a given playlist if their 'Artist - Track' name match are found in the subsequently given playlists (less accurate than exact URI deletion above) - `spotify_duplicate_uri_in_playlist.sh` - finds duplicate Spotify URIs in a given playlist (these are guaranteed exact duplicate matches), returns all but the first occurrence and optionally their track positions (zero-indexed to align with the Spotify API for easy chaining with other tools) - `spotify_duplicate_tracks_in_playlist.sh` - finds duplicate Spotify tracks in a given playlist (these are idential `Artist - Track` name matches, which may be from different albums / singles) - `spotify_delete_duplicate_tracks_in_playlist.sh` - deletes duplicate Spotify tracks (name matched) in a given playlist using `spotify_duplicate_tracks_in_playlist.sh` and `spotify_delete_from_playlist.sh` - `spotify_delete_duplicate_track_uris_in_playlist.sh` - deletes duplicate Spotify URI tracks (identical) in a given playlist using `spotify_duplicate_uri_in_playlist.sh` and `spotify_delete_from_playlist.sh` - `spotify_delete_any_duplicates_in_playlist.sh` - calls both of the above scripts to first get rid of duplicate URIs and then remove any other duplicates by track name matches - `spotify_playlist_tracks_uri_in_year.sh` - finds track URIs in a playlist where their original release date is in a given year or decade (by regex match). This has to do a secondary Spotify track search lookup by name and relies on `normalize_tracknames.pl` from [HariSekhon/Spotify-tools](https://github.com/HariSekhon/Spotify-tools) being built and in the `$PATH` - `spotify_playlist_tracks_uri_by_year.sh` - returns track URIs from the given Spotify playlist for a specific year or range of years. Useful for filtering tracks to add to my best of each decade playlists. More efficient than `spotify_playlist_tracks_uri_in_year.sh`, without dependency on [HariSekhon/Spotify-tools](https://github.com/HariSekhon/Spotify-tools), but it only uses the year of the track version, which if an album track may not be the same year if it was first released as a single earlier - `spotify_playlist_tracks_uri_batch_by_year.sh` - Returns all track URIs from the given Spotify playlist(s) grouped by year or decade. Copies each batch to the clipboard, prints to stdout, and prompts to continue before printing the next batch. Useful for filtering tracks to add to my best of each year or decade playlists - `spotify_playlist_uri_offset.sh` - finds the offset of a given track URI in a given playlist, useful to find positions to resume processing a large playlist - `spotify_top_artists*.sh` - lists your top artists in URI or human readable format - `spotify_top_tracks*.sh` - lists top tracks in URI or human readable format - `spotify_playlist_top_artists.sh` - returns the top artists for a given Spotify playlist by counting unique track names for each artist. If HariSekhon/Spotify-tools is in the $PATH it uses normalize_tracknames.pl for greater accuracy to collapse multiple versions such as Radio Edit and Album Version to only count that same song once - `spotify_liked_tracks*.sh` - lists your `Liked Songs` in URI or human readable formats - `spotify_liked_artists*.sh` - list artists from `Liked Songs` in URI or human readable formats - `spotify_artists_followed*.sh` - lists all followed artists in URI or human readable formats - `spotify_artist_tracks.sh` - gets all track URIs for a given artist, from both albums and single for chain loading to playlists - `spotify_follow_artists.sh` - follows artists for the given URIs from files or standard input - `spotify_follow_top_artists.sh` - follows all artists in your current Spotify top artists list - `spotify_follow_liked_artists.sh` - follows artists with N or more tracks in your `Liked Songs` - `spotify_set_tracks_uri_to_liked.sh` - sets a list of spotify track URIs to 'Liked' so they appear in the `Liked Songs` playlist. Useful for marking all the tracks in your best playlists as favourite tracks, or for porting historical `Starred` tracks to the newer `Liked Songs` - `spotify_foreach_playlist.sh` - executes a templated command against all playlists, replacing `{playlist}` and `{playlist_id}` in each iteration - `spotify_playlist_name_to_id.sh` / `spotify_playlist_id_to_name.sh` - convert playlist names <=> IDs - `spotify_playlist_snapshot_id.sh` - returns the Snapshot ID of a given Spotify playlist, useful for detecting whether a playlist has changed or not to skip re-downloading it - `spotify_api_token.sh` - gets a Spotify authentication token using either [Client Credentials](https://developer.spotify.com/documentation/general/guides/authorization-guide/#client-credentials-flow) or [Authorization Code](https://developer.spotify.com/documentation/general/guides/authorization-guide/#authorization-code-flow) authentication flows, the latter being able to read/modify private user data, automatically used by `spotify_api.sh` - `spotify_api.sh` - query any Spotify [API](https://developer.spotify.com/documentation/web-api/reference/) endpoint with authentication, used by adjacent spotify scripts ### More Linux & Mac `bin/`, `install/`, `packages/`, `setup/` directories: - [Linux](https://en.wikipedia.org/wiki/Linux) / [Mac](https://en.wikipedia.org/wiki/MacOS) systems administration scripts: - `install/` - installation scripts for various OS packages (RPM, Deb, Apk) for various Linux distros ([Redhat RHEL](https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux) / [CentOS](https://www.centos.org/) / [Fedora](https://getfedora.org/), [Debian](https://www.debian.org/) / [Ubuntu](https://ubuntu.com/), [Alpine](https://alpinelinux.org/)) - install if absent scripts for Python, Perl, Ruby, NodeJS and Golang packages - good for minimizing the number of source code installs by first running the OS install scripts and then only building modules which aren't already detected as installed (provided by system packages), speeding up builds and reducing the likelihood of compile failures - install scripts for tarballs, Golang binaries, random 3rd party installers, [Jython](https://www.jython.org/) and build tools like [Gradle](https://gradle.org/) and [SBT](https://www.scala-sbt.org/) for when Linux distros don't provide packaged versions or where the packaged versions are too old - `packages/` - OS / Distro Package Management: - `install_packages.sh` - installs package lists from arguments, files or stdin on major linux distros and Mac, detecting the package manager and invoking the right install commands, with `sudo` if not root. Works on [RHEL](https://www.redhat.com/en) / [CentOS](https://www.centos.org/) / [Fedora](https://getfedora.org/), [Debian](https://www.debian.org/) / [Ubuntu](https://ubuntu.com/), [Alpine](https://alpinelinux.org/), and [Mac Homebrew](https://brew.sh/). Leverages and supports all features of the distro / OS specific install scripts listed below - `install_packages_if_absent.sh` - installs package lists if they're not already installed, saving time and minimizing install logs / CI logs, same support list as above - Redhat RHEL / CentOS: - `yum_install_packages.sh` / `yum_remove_packages.sh` - installs RPM lists from arguments, files or stdin. Handles Yum + Dnf behavioural differences, calls `sudo` if not root, auto-attempts variations of python/python2/python3 package names. Avoids yum slowness by checking if rpm is installed before attempting to install it, accepts `NO_FAIL=1` env var to ignore unavailable / changed package names (useful for optional packages or attempts for different package names across RHEL/CentOS/Fedora versions) - `yum_install_packages_if_absent.sh` - installs RPMs only if not already installed and not a metapackage provided by other packages (eg. `vim` metapackage provided by `vim-enhanced`), saving time and minimizing install logs / CI logs, plus all the features of `yum_install_packages.sh` above - `yum_upgrade_packages_if_outdated.sh` - upgrades RPMs only if they're outdated - `rpms_filter_installed.sh` / `rpms_filter_not_installed.sh` - pipe filter packages that are / are not installed for easy script piping - Debian / Ubuntu: - `apt_install_packages.sh` / `apt_remove_packages.sh` - installs Deb package lists from arguments, files or stdin. Auto calls `sudo` if not root, accepts `NO_FAIL=1` env var to ignore unavailable / changed package names (useful for optional packages or attempts for different package names across Debian/Ubuntu distros/versions) - `apt_install_packages_if_absent.sh` - installs Deb packages only if not already installed, saving time and minimizing install logs / CI logs, plus all the features of `apt_install_packages.sh` above - `apt_upgrade_packages_if_outdated.sh` - upgrades Deb packages only if they're outdated - `apt_wait.sh` - blocking wait on concurrent apt locks to avoid failures and continue when available, mimicking yum's waiting behaviour rather than error'ing out - `debs_filter_installed.sh` / `debs_filter_not_installed.sh` - pipe filter packages that are / are not installed for easy script piping - Alpine: - `apk_install_packages.sh` / `apk_remove_packages.sh` - installs Alpine apk package lists from arguments, files or stdin. Auto calls `sudo` if not root, accepts `NO_FAIL=1` env var to ignore unavailable / changed package names (useful for optional packages or attempts for different package names across Alpine versions) - `apk_install_packages_if_absent.sh` - installs Alpine apk packages only if not already installed, saving time and minimizing install logs / CI logs, plus all the features of `apk_install_packages.sh` above - `apk_upgrade_packages_if_outdated.sh` - upgrades Alpine apk packages only if they're outdated - `apk_filter_installed.sh` / `apk_filter_not_installed.sh` - pipe filter packages that are / are not installed for easy script piping - Mac: - `brew_install_packages.sh` / `brew_remove_packages.sh` - installs Mac Hombrew package lists from arguments, files or stdin. Accepts `NO_FAIL=1` env var to ignore unavailable / changed package names (useful for optional packages or attempts for different package names across versions) - `brew_install_packages_if_absent.sh` - installs Mac Homebrew packages only if not already installed, saving time and minimizing install logs / CI logs, plus all the features of `brew_install_packages.sh` above - `brew_upgrade_packages_if_outdated.sh` - upgrades Mac Homebrew packages only if they're outdated - `brew_filter_installed.sh` / `brew_filter_not_installed.sh` - pipe filter packages that are / are not installed for easy script piping - `brew_package_owns.sh` - finds which brew package owns a given filename argument - all builds across all my GitHub repos now `make system-packages` before `make pip` / `make cpan` to shorten how many packages need installing, reducing chances of build failures ### Builds, Languages & Linting `bin/`, `checks/`, `cicd/` or language specific directories: - `lint.sh` - lints one or more files, auto-determines the file types, parses lint headers and calls appropriate scripts and tools. Integrated with my custom `.vimrc` - `run.sh` - runs one or more files, auto-determines the file types, any run or arg headers and executes each file using the appropriate script or CLI tool. Integrated with my custom `.vimrc` - `check_*.sh` - extensive collection of generalized tests - these run against all my GitHub repos via [CI](https://harisekhon.github.io/CI-CD/). Some examples: - Programming language linting: - [Python](https://www.python.org/) (syntax, pep8, byte-compiling, reliance on asserts which can be disabled at runtime, except/pass etc.) - [Perl](https://www.perl.org/) - [Java](https://www.java.com/en/) - [Scala](https://www.scala-lang.org/) - [Ruby](https://www.ruby-lang.org/en/) - [Bash](https://www.gnu.org/software/bash/) / Shell - Misc (whitespace, custom code checks etc.) - Build System, Docker & CI linting: - [Make](https://www.gnu.org/software/make/) - [Maven](https://maven.apache.org/) - [SBT](https://www.scala-sbt.org/) - [Gradle](https://gradle.org/) - [Travis CI](https://travis-ci.org/) - [Circle CI](https://circleci.com/) - [GitLab CI](https://docs.gitlab.com/ee/ci/) - [Concourse CI](https://concourse-ci.org/) - [Codefresh CI](https://codefresh.io/) - [Dockerfiles](https://docs.docker.com/engine/reference/builder/) - [Docker Compose](https://docs.docker.com/compose/) - [Vagrantfiles](https://www.vagrantup.com/docs/vagrantfile) ## Individual Setup Parts Optional, only if you don't do the full `make install`. Install only OS system package dependencies and [AWS CLI](https://aws.amazon.com/cli/) via Python Pip (doesn't symlink anything to `$HOME`): ```shell make ``` Adds sourcing to `.bashrc` and `.bash_profile` and symlinks dot config files to `$HOME` (doesn't install OS system package dependencies): ```shell make link ``` undo via ```shell make unlink ``` Install only OS system package dependencies (doesn't include [AWS CLI](https://aws.amazon.com/cli/) or Python packages): ```shell make system-packages ``` Install [AWS CLI](https://aws.amazon.com/cli/): ```shell make aws ``` Install [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/): ```shell make azure ``` Install [GCP GCloud SDK](https://cloud.google.com/sdk) (includes CLI): ```shell make gcp ``` Install [GCP GCloud Shell](https://cloud.google.com/shell) environment (sets up persistent OS packages and all home directory configs): ```shell make gcp-shell ``` Install generically useful Python CLI tools and modules (includes [AWS CLI](https://aws.amazon.com/cli/), autopep8 etc): ```shell make python ``` ### Full Help ```shell > make help Usage: Common Options: make help show this message make build installs all dependencies - OS packages and any language libraries via native tools eg. pip, cpanm, gem, go etc that are not available via OS packages make build-retry retries 'make build' x 3 until success to try to mitigate temporary upstream repo failures triggering false alerts in CI systems make ci prints env, then runs 'build-retry' for more resilient CI builds with debugging make printenv prints environment variables, CPU cores, OS release, $PWD, Git branch, hashref etc. Useful for CI debugging make system-packages installs OS packages only (detects OS via whichever package manager is available) make test run tests make clean removes compiled / generated files, downloaded tarballs, temporary files etc. make submodules initialize and update submodules to the right release (done automatically by build / system-packages) make init same as above, often useful to do in CI systems to get access to additional submodule provided targets such as 'make ci' make cpan install any modules listed in any cpan-requirements.txt files if not already installed make pip install any modules listed in any requirements.txt files if not already installed make python-compile compile any python files found in the current directory and 1 level of subdirectory make pycompile make github open browser at github project make readme open browser at github's README make github-url print github url and copy to clipboard make status open browser at Github CI Builds overview Status page for all projects make ls print list of code files in project make wc show counts of files and lines Repo specific options: make install builds all script dependencies, installs AWS CLI, symlinks all config files to $HOME and adds sourcing of bash profile make link symlinks all config files to $HOME and adds sourcing of bash profile make unlink removes all symlinks pointing to this repo's config files and removes the sourcing lines from .bashrc and .bash_profile make python-desktop installs all Python Pip packages for desktop workstation listed in setup/pip-packages-desktop.txt make perl-desktop installs all Perl CPAN packages for desktop workstation listed in setup/cpan-packages-desktop.txt make ruby-desktop installs all Ruby Gem packages for desktop workstation listed in setup/gem-packages-desktop.txt make golang-desktop installs all Golang packages for desktop workstation listed in setup/go-packages-desktop.txt make nodejs-desktop installs all NodeJS packages for desktop workstation listed in setup/npm-packages-desktop.txt make desktop installs all of the above + many desktop OS packages listed in setup/ make mac-desktop all of the above + installs a bunch of major common workstation software packages like Ansible, Terraform, MiniKube, MiniShift, SDKman, Travis CI, CCMenu, Parquet tools etc. make linux-desktop make ls-scripts print list of scripts in this project, ignoring code libraries in lib/ and .bash.d/ make github-cli installs GitHub CLI make kubernetes installs Kubernetes kubectl and kustomize to ~/bin/ make terraform installs Terraform to ~/bin/ make vim installs Vundle and plugins make tmux installs TMUX TPM and plugin for kubernetes context make ccmenu installs and (re)configures CCMenu to watch this and all other major HariSekhon GitHub repos make status open the Github Status page of all my repos build statuses across all CI platforms make aws installs AWS CLI tools make azure installs Azure CLI make gcp installs Google Cloud SDK make digital-ocean installs Digital Ocean CLI make aws-shell sets up AWS Cloud Shell: installs core packages and links configs (maintains itself across future Cloud Shells via .aws_customize_environment hook) make gcp-shell sets up GCP Cloud Shell: installs core packages and links configs (maintains itself across future Cloud Shells via .customize_environment hook) make azure-shell sets up Azure Cloud Shell (limited compared to gcp-shell, doesn't install OS packages since there is no sudo) Now exiting usage help with status code 3 to explicitly prevent silent build failures from stray 'help' arguments make: *** [help] Error 3 ``` (`make help` exits with error code 3 like most of my programs to differentiate from build success to make sure a stray `help` argument doesn't cause silent build failure with exit code 0) ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Bash-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Bash-tools&Date) [git.io/bash-tools](https://git.io/bash-tools) ## More Core Repos ### Knowledge [![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) [![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) ### DevOps Code [![DevOps-Bash-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) [![DevOps-Python-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) [![DevOps-Perl-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) [![DevOps-Golang-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) ### Containerization [![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) [![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) ### CI/CD [![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) [![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) ### Databases - DBA - SQL [![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded [![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) [![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) [![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) [![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) [![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) ### Monitoring [![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) [![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates [![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) ### Desktop [![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) [![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) [![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) ### Spotify [![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) The rest of my original source repos are [here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. ================================================ FILE: STARCHARTS.md ================================================ # GitHub StarCharts ![Original Repos](https://img.shields.io/badge/Repos-20-blue?logo=github) ![Stars](https://img.shields.io/badge/Stars-7542-blue?logo=github) ![Forks](https://img.shields.io/badge/Forks-2603-blue?logo=github) ![Followers](https://img.shields.io/badge/Followers-1568-blue?logo=github) [![Azure DevOps Profile](https://img.shields.io/badge/Azure%20DevOps-HariSekhon-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub) [![GitHub Profile](https://img.shields.io/badge/GitHub-HariSekhon-2088FF?logo=github)](https://github.com/HariSekhon) [![GitLab Profile](https://img.shields.io/badge/GitLab-HariSekhon-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon) [![BitBucket Profile](https://img.shields.io/badge/BitBucket-HariSekhon-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon) [![GitStar Ranking Profile](https://img.shields.io/badge/GitStar%20Ranking-HariSekhon-blue?logo=github)](https://gitstar-ranking.com/HariSekhon) [git.io/hari-starcharts](https://git.io/hari-starcharts) generated by `github_generate_starcharts.md.sh` in [HariSekhon/DevOps-Bash-tools](https://github.com/HariSekhon/DevOps-Bash-tools) --- ## Hari Sekhon - DevOps Bash Tools [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/DevOps-Bash-tools?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/DevOps-Bash-tools?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/network) 1000+ DevOps Bash Scripts - AWS, GCP, Kubernetes, Docker, CI/CD, APIs, SQL, PostgreSQL, MySQL, Hive, Impala, Kafka, Hadoop, Jenkins, GitHub, GitLab, BitBucket, Azure DevOps, TeamCity, Spotify, MP3, LDAP, Code/Build Linting, pkg mgmt for Linux, Mac, Python, Perl, Ruby, NodeJS, Golang, Advanced dotfiles: .bashrc, .vimrc, .gitconfig, .screenrc, tmux.. [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Bash-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Bash-tools) --- ## Dockerfiles for DevOps, CI/CD, Big Data & NoSQL [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Dockerfiles) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Dockerfiles?logo=github)](https://github.com/HariSekhon/Dockerfiles/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Dockerfiles?logo=github)](https://github.com/HariSekhon/Dockerfiles/network) 50+ DockerHub public images for Docker & Kubernetes - DevOps, CI/CD, GitHub Actions, CircleCI, Jenkins, TeamCity, Alpine, CentOS, Debian, Fedora, Ubuntu, Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Consul, Riak [![Stargazers over time](https://starchart.cc/HariSekhon/Dockerfiles.svg)](https://starchart.cc/HariSekhon/Dockerfiles) --- ## Advanced Nagios Plugins Collection [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Nagios-Plugins) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Nagios-Plugins?logo=github)](https://github.com/HariSekhon/Nagios-Plugins/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Nagios-Plugins?logo=github)](https://github.com/HariSekhon/Nagios-Plugins/network) 450+ AWS, Hadoop, Cloud, Kafka, Docker, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Cloudera etc... [![Stargazers over time](https://starchart.cc/HariSekhon/Nagios-Plugins.svg)](https://starchart.cc/HariSekhon/Nagios-Plugins) --- ## Hari Sekhon - DevOps Python Tools [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) 80+ DevOps & Data CLI Tools - AWS, GCP, GCF Python Cloud Functions, Log Anonymizer, Spark, Hadoop, HBase, Hive, Impala, Linux, Docker, Spark Data Converters & Validators (Avro/Parquet/JSON/CSV/INI/XML/YAML), Travis CI, AWS CloudFormation, Elasticsearch, Solr etc. [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) --- ## Kubernetes configs [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Kubernetes-configs) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Kubernetes-configs?logo=github)](https://github.com/HariSekhon/Kubernetes-configs/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Kubernetes-configs?logo=github)](https://github.com/HariSekhon/Kubernetes-configs/network) Advanced Kubernetes YAML configs - Best Practices, Tips & Tricks, Production-Ready Checklist - experience from several production environments. AWS, GCP, Azure, ArgoCD, GKE, EKS, AKS, Nginx, Traefik, Kong, Cert Manager, CI/CD, Jenkins, Artifactory, TeamCity, GitHub Actions, Cloud SQL, FluxCD, Spinnaker, Selenium Grid, Moon, Helm + Kustomize [![Stargazers over time](https://starchart.cc/HariSekhon/Kubernetes-configs.svg)](https://starchart.cc/HariSekhon/Kubernetes-configs) --- ## SQL Scripts [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/SQL-scripts) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/SQL-scripts?logo=github)](https://github.com/HariSekhon/SQL-scripts/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/SQL-scripts?logo=github)](https://github.com/HariSekhon/SQL-scripts/network) 100+ SQL Scripts - PostgreSQL, MySQL, Google BigQuery, MariaDB, AWS Athena. DBA, Analytics, DevOps, performance engineering. Google BigQuery ML machine learning classification. [![Stargazers over time](https://starchart.cc/HariSekhon/SQL-scripts.svg)](https://starchart.cc/HariSekhon/SQL-scripts) --- ## Advanced HAProxy Configs for Big Data, NoSQL, Web and Infrastructure technologies [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/HAProxy-configs) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/HAProxy-configs?logo=github)](https://github.com/HariSekhon/HAProxy-configs/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/HAProxy-configs?logo=github)](https://github.com/HariSekhon/HAProxy-configs/network) 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Kubernetes, Elasticsearch, SolrCloud, HBase, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, Hue, ZooKeeper, SSH, RabbitMQ, Redis, Riak, Cloudera, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, Rancher etc. [![Stargazers over time](https://starchart.cc/HariSekhon/HAProxy-configs.svg)](https://starchart.cc/HariSekhon/HAProxy-configs) --- ## Hari Sekhon - Diagrams-as-Code [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Diagrams-as-Code) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Diagrams-as-Code?logo=github)](https://github.com/HariSekhon/Diagrams-as-Code/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Diagrams-as-Code?logo=github)](https://github.com/HariSekhon/Diagrams-as-Code/network) Cloud & DevOps Architecture Diagrams-as-Code in Python and D2 languages [![Stargazers over time](https://starchart.cc/HariSekhon/Diagrams-as-Code.svg)](https://starchart.cc/HariSekhon/Diagrams-as-Code) --- ## Code & Config Templates [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Templates) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Templates?logo=github)](https://github.com/HariSekhon/Templates/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Templates?logo=github)](https://github.com/HariSekhon/Templates/network) 100+ DevOps Code & Config templates for Kubernetes, AWS, GCP, Terraform, Docker, Packer, Jenkins, CircleCI, GitHub Actions, Lambda, AWS CodeBuild, GCP Cloud Build, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, Vagrantfile, M4 etc... [![Stargazers over time](https://starchart.cc/HariSekhon/Templates.svg)](https://starchart.cc/HariSekhon/Templates) --- ## Hari Sekhon - DevOps Perl Tools [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/DevOps-Perl-tools) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/DevOps-Perl-tools?logo=github)](https://github.com/HariSekhon/DevOps-Perl-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/DevOps-Perl-tools?logo=github)](https://github.com/HariSekhon/DevOps-Perl-tools/network) 25+ DevOps CLI Tools - Anonymizer, SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles), Hadoop HDFS & Hive tools, Solr/SolrCloud CLI, Nginx stats & HTTP(S) URL watchers for load-balanced web farms, Linux tools etc. [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Perl-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Perl-tools) --- ## Spotify Tools [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Spotify-tools) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Spotify-tools?logo=github)](https://github.com/HariSekhon/Spotify-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Spotify-tools?logo=github)](https://github.com/HariSekhon/Spotify-tools/network) Spotify Tools - Playlists Backups, Spotify CLI, URI translator, duplication detection / removal, API search queries, API automation etc. [![Stargazers over time](https://starchart.cc/HariSekhon/Spotify-tools.svg)](https://starchart.cc/HariSekhon/Spotify-tools) --- ## # Jenkins - Advanced Jenkinsfile & Groovy Shared Library [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Jenkins) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Jenkins?logo=github)](https://github.com/HariSekhon/Jenkins/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Jenkins?logo=github)](https://github.com/HariSekhon/Jenkins/network) Jenkins - Advanced Jenkinsfile & Groovy Shared Library of reusable functions and pipelines - including for AWS, GCP, Docker, Kubernetes, ArgoCD, Slack notifications, Git Merge, Terraform, Cloudflare, Jenkins Job Backups, most major Docker registries, DockerHub, GHCR, ECR, GCR, GAR, ACR, GitLab, Quay [![Stargazers over time](https://starchart.cc/HariSekhon/Jenkins.svg)](https://starchart.cc/HariSekhon/Jenkins) --- ## Hari Sekhon - Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, Security, AWS, GCP etc. [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Knowledge-Base) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Knowledge-Base?logo=github)](https://github.com/HariSekhon/Knowledge-Base/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Knowledge-Base?logo=github)](https://github.com/HariSekhon/Knowledge-Base/network) IT Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, AWS, GCP etc - gradually porting my large private knowledge base to public [![Stargazers over time](https://starchart.cc/HariSekhon/Knowledge-Base.svg)](https://starchart.cc/HariSekhon/Knowledge-Base) --- ## Hari Sekhon - DevOps Golang Tools [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/DevOps-Golang-tools) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/DevOps-Golang-tools?logo=github)](https://github.com/HariSekhon/DevOps-Golang-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/DevOps-Golang-tools?logo=github)](https://github.com/HariSekhon/DevOps-Golang-tools/network) DevOps Golang tools [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Golang-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Golang-tools) --- ## Terraform Templates [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Terraform) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Terraform?logo=github)](https://github.com/HariSekhon/Terraform/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Terraform?logo=github)](https://github.com/HariSekhon/Terraform/network) Terraform HCL code for AWS / GCP / Azure / GitHub management [![Stargazers over time](https://starchart.cc/HariSekhon/Terraform.svg)](https://starchart.cc/HariSekhon/Terraform) --- ## GitHub Actions [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/GitHub-Actions) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/GitHub-Actions?logo=github)](https://github.com/HariSekhon/GitHub-Actions/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/GitHub-Actions?logo=github)](https://github.com/HariSekhon/GitHub-Actions/network) GitHub Actions Reusable Workflows and Master Template [![Stargazers over time](https://starchart.cc/HariSekhon/GitHub-Actions.svg)](https://starchart.cc/HariSekhon/GitHub-Actions) --- ## Hari Sekhon - HashiCorp Packer templates [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Packer-templates) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Packer-templates?logo=github)](https://github.com/HariSekhon/Packer-templates/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Packer-templates?logo=github)](https://github.com/HariSekhon/Packer-templates/network) HashiCorp Packer templates to build portable virtual machines in OVA format for Ubuntu, Debian and Redhat based systems with automated installers Kickstart, Preseed and AutoInstaller / Cloud-Init. Useful for IoT edge sites, Kubernetes base systems and VM appliances to ship to customers [![Stargazers over time](https://starchart.cc/HariSekhon/Packer-templates.svg)](https://starchart.cc/HariSekhon/Packer-templates) --- ## Hari Sekhon - Spotify Playlists [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/Spotify-Playlists) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/Spotify-Playlists?logo=github)](https://github.com/HariSekhon/Spotify-Playlists/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/Spotify-Playlists?logo=github)](https://github.com/HariSekhon/Spotify-Playlists/network) 240+ playlists, 36,000+ tracks - in both Spotify URI and human-readable formats. Spotify Profile: . Spotify API tools are submodules of this repo. [![Stargazers over time](https://starchart.cc/HariSekhon/Spotify-Playlists.svg)](https://starchart.cc/HariSekhon/Spotify-Playlists) --- ## Hari Sekhon - Perl Library [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/lib) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/lib?logo=github)](https://github.com/HariSekhon/lib/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/lib?logo=github)](https://github.com/HariSekhon/lib/network) Perl Utility Library for my other repos [![Stargazers over time](https://starchart.cc/HariSekhon/lib.svg)](https://starchart.cc/HariSekhon/lib) --- ## CI/CD Status Page [![Repo on GitHub](https://img.shields.io/badge/GitHub-repo-blue?logo=github)](https://github.com/HariSekhon/CI-CD) [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/CI-CD?logo=github)](https://github.com/HariSekhon/CI-CD/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/CI-CD?logo=github)](https://github.com/HariSekhon/CI-CD/network) CI/CD Status page for Hari Sekhon's GitHub repos [![Stargazers over time](https://starchart.cc/HariSekhon/CI-CD.svg)](https://starchart.cc/HariSekhon/CI-CD) ================================================ FILE: STATUS.md ================================================ # CI/CD Status Page Moved to ================================================ FILE: ai/openai_api.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: /models # # Author: Hari Sekhon # Date: 2023-06-10 21:45:48 +0100 (Sat, 10 Jun 2023) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Queries the OpenAI API Automatically handles authentication via environment variable \$OPENAI_API_KEY If a member of multiple organizations then you must also set \$OPENAI_ORGANIZATION_ID Can specify \$CURL_OPTS for options to pass to curl or provide them as arguments Set up your API key here: https://platform.openai.com/account/api-keys If you are a member of multiple organizations, get the Organization ID here: https://platform.openai.com/account/org-settings API Reference: https://platform.openai.com/docs/api-reference/introduction Examples: List Models: ${0##*/} /models Retrieve Model: ${0##*/} /models/{model_id} ${0##*/} /models/gpt-3.5-turbo List Files in our org: ${0##*/} /files Get File metadata: ${0##*/} /files/{file_id} Get File content: ${0##*/} /files/{file_id}/content List fine-tuning jobs: ${0##*/} /fine-tunes Retrieve fine-tune: ${0##*/} /fine-tunes/{fine_tune_id} " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="/path []" url_base="https://api.openai.com/v1" help_usage "$@" min_args 1 "$@" check_env_defined OPENAI_API_KEY curl_api_opts "$@" url_path="$1" shift || : url_path="${url_path//https:\\/\\/api.openai.com\/v1}" url_path="${url_path##/}" export TOKEN="$OPENAI_API_KEY" if [ -n "${OPENAI_ORGANIZATION_ID:-}" ]; then CURL_OPTS+=(-H "OpenAI-Organization: $OPENAI_ORGANIZATION_ID") fi "$srcdir/../bin/curl_auth.sh" "$url_base/$url_path" ${CURL_OPTS:+"${CURL_OPTS[@]}"} "$@" | jq_debug_pipe_dump ================================================ FILE: applescript/app_names.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-11-18 17:03:28 +0400 (Mon, 18 Nov 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ $# -gt 0 ]; then cat < Label com.harisekhon.wakeup_script Program /Users/hari/github/bash-tools/applescript/wakeup_script.sh WatchPaths /var/db/.AppleSetupDone RunAtLoad ================================================ FILE: applescript/get_application_names.scpt ================================================ #!/usr/bin/env osascript -- vim:ts=4:sts=4:sw=4:et -- -- Author: Hari Sekhon -- Date: 2024-10-13 20:26:31 +0300 (Sun, 13 Oct 2024) -- -- https///github.com/HariSekhon/DevOps-Bash-tools -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- -- https://www.linkedin.com/in/HariSekhon -- -- ============================================================================ # -- A p p l e S c r i p t -- ============================================================================ # -- Gets the list of Applications running in the name format that can be passed to -- the adjacent script set_frontmost_process.scpt tell application "System Events" set appList to (name of every application process) end tell set output to "" repeat with appName in appList set output to output & appName & "\n" end repeat -- strip trailing newline set output to text 1 thru -2 of output -- doesn't come out right due to carriage returns --do shell script "echo " & quoted form of output -- even this outputs carriage returns --do shell script "echo " & quoted form of output & " | tr '\r' '\n'" -- annoying pop-up --display dialog output as text -- outputs to stderr instead of stdout, use implicit print of last value instead --log output output -- output is unsorted and sorting in Applescript requires crude in-code sorting like Bubblescript to passing array out -- to shell sort and back which results in a string formatting output one character per line BS, just wrap this in quick -- shell it's simpler ================================================ FILE: applescript/get_frontmost_process.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-02-28 13:05:26 +0000 (Mon, 28 Feb 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # tell application "System Events" set frontmostProcess to first process where it is frontmost name of frontmostProcess end tell ================================================ FILE: applescript/get_frontmost_process_title.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-12-05 16:30:05 +0000 (Mon, 05 Dec 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Returns the name and title of the current foreground app # # Borrowed from: # # https://stackoverflow.com/questions/5292204/macosx-get-foremost-window-title # # Tested on macOS 10.7 and 11.7 global frontApp, frontAppName, windowTitle set windowTitle to "" tell application "System Events" set frontApp to first application process whose frontmost is true set frontAppName to name of frontApp tell process frontAppName tell (1st window whose value of attribute "AXMain" is true) set windowTitle to value of attribute "AXTitle" end tell end tell end tell return {frontAppName, windowTitle} ================================================ FILE: applescript/get_mouse_coordinates.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-06-13 20:43:57 +0100 (Sat, 13 Jun 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Gets the coordinates of the mouse cursor # # other options: # # Cmd-Shift-4 - to take a select area screenshot displays the coordates, then press Esc to cancel # # MouseTools -location tell application "System Events" # doesn't work due to mouse not being defined set mousePosition to position of the mouse end tell ================================================ FILE: applescript/get_mouse_coordinates.sh ================================================ #!/usr/bin/env bash set -euo pipefail [ -n "${DEBUG:-}" ] && set -x MouseTools -location ================================================ FILE: applescript/is_screen_locked.py ================================================ #!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-12-05 22:03:56 +0000 (Mon, 05 Dec 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # """ Detects whether the macOS screen is locked If locked, prints 'true' and returns exit code 0 If unlocked, prints 'false' and returns exit code 1 Useful to avoid automated keystrokes or mouse_clicks while on locked screen which can make it hard to login back in """ from __future__ import print_function import sys import Quartz if __name__ == '__main__': # false positive # pylint: disable=no-member d = Quartz.CGSessionCopyCurrentDictionary() if 'CGSSessionScreenIsLocked' in d and d['CGSSessionScreenIsLocked'] == 1: print('true') sys.exit(0) else: print('false') sys.exit(1) ================================================ FILE: applescript/is_screensaver_running.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-12-05 17:44:03 +0000 (Mon, 05 Dec 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Returns 'true' or 'false' as to whether screensaver is running # # Useful to avoid sending clicks or keystrokes during screensaver, as this makes it hard to login # # XXX: Caveat: doesn't work on locked screen when screensaver isn't running so not comprehensive tell application "System Events" get running of screen saver preferences end tell ================================================ FILE: applescript/keystrokes.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-06-14 17:16:31 +0100 (Sun, 14 Jun 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/../lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Automates Keyboard strokes to automate tedious UI actions Performs N keyboard key code presses https://eastmanreference.com/complete-list-of-applescript-key-codes Sleeps for \$SLEEP_SECS (default: 1) between clicks to allow UIs to update and perform the next keystroke Starts each keystroke after \$START_DELAY seconds (default: 5) to give time to alt-tab back to your UI application and position the cursor If given num is negative, will run indefinitely until Control-C'd " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ...]" help_usage "$@" min_args 1 "$@" mac_only num="$1" start_delay="${START_DELAY:-5}" sleep_secs="${SLEEP_SECS:-1}" if ! [[ "$num" =~ ^-?[[:digit:]]+$ ]]; then usage "invalid non-integer '$num' given for first argument" fi if ! is_float "$start_delay"; then usage "invalid non-float '$START_DELAY' found in environment for \$START_DELAY" fi if ! is_float "$sleep_secs"; then usage "invalid non-float '$SLEEP_SECS' found in environment for \$SLEEP_SECS" fi shift || : read -r -a keys <<< "$@" timestamp "waiting for $start_delay secs before starting" sleep "$start_delay" timestamp "starting $num keystrokes" echo for ((i=1; ; i++)); do # if given num is negative, will run for infinity until Control-C'd if [ "$num" -ge 0 ] && [ "$i" -gt "$num" ]; then break fi for key in "${keys[@]}"; do if [[ "$key" =~ ^[[:digit:]][[:digit:]]+$ ]]; then timestamp "keystroke $i/$num keycode $key" osascript -e "tell application \"System Events\" to key code $key" || : else timestamp "keystroke $i/$num key $key" osascript -e "tell application \"System Events\" to keystroke \"$key\"" fi sleep "$sleep_secs.$RANDOM" # add $RANDOM up to 1 second jitter to make it harder to spot that this is perfectly automated clicking done done ================================================ FILE: applescript/mouse_clicks.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-06-13 20:47:12 +0100 (Sat, 13 Jun 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Click N times at coordinates x,y # # waits 5 seconds before starting # # easier to just use 'MouseTools -leftClick' command line, see adjacent mouse_clicks.sh # Incomplete, doesn't seem to work, use adjacent mouse_clicks.sh instead which works nicely set N to 10 # see adjacent get_mouse_coordinates.scpt for how to get the mouse coordinates set x to 2455 set y to 1273 do shell script "echo waiting 5 secs before starting clicking" delay 5 # repeat N times # want loop iterator variable to print the click we're on set i to 0 repeat while i < N tell application "System Events" click at {x,y} end tell #do shell script "echo click " & i #copy "click " & i to stdout set i to i + 1 delay 1 end repeat # looks like last thing printed overwrites all previous output :-/ copy "DONE" to stdout ================================================ FILE: applescript/mouse_clicks.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-06-14 17:16:31 +0100 (Sun, 14 Jun 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/../lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Automates Mouse Clicks to automate tedious UI actions Performs N mouse clicks at the sequence of X,Y coordinates given or the current mouse location if no coordinates Sleeps for \$SLEEP_SECS (default: 1) between clicks to allow UIs to update and perform the next click Starts clicking after \$START_DELAYS seconds (default: 5) to give time to alt-tab back to your UI application and position the cursor If given num is negative, will run indefinitely until Control-C'd " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ...]" help_usage "$@" min_args 1 "$@" mac_only num="$1" start_delay="${START_DELAY:-5}" sleep_secs="${SLEEP_SECS:-1}" if ! [[ "$num" =~ ^-?[[:digit:]]+$ ]]; then usage "invalid non-integer '$num' given for first argument" fi if ! is_float "$start_delay"; then usage "invalid non-float '$START_DELAY' found in environment for \$START_DELAY" fi if ! is_float "$sleep_secs"; then usage "invalid non-float '$SLEEP_SECS' found in environment for \$SLEEP_SECS" fi shift || : read -r -a coordinates <<< "$@" if ! type -P cliclick &>/dev/null; then brew install cliclick fi if [ -n "${coordinates:-}" ]; then for coordinate in "${coordinates[@]}"; do if ! [[ "$coordinate" =~ ^[[:digit:]]+,[[:digit:]]+$ ]]; then usage "invalid coordinate '$coordinate' given - must be in form x,y" fi done fi timestamp "Waiting for $start_delay secs before starting" sleep "$start_delay" timestamp "Starting $num mouse clicks every $sleep_secs" echo for ((i=1; ; i++)); do # if given num is negative, will run for infinity until Control-C'd if [ "$num" -ge 0 ] && [ "$i" -gt "$num" ]; then break fi if [ -n "${coordinates:-}" ]; then for coordinate in "${coordinates[@]}"; do x="${coordinate%,*}" y="${coordinate#*,}" timestamp "Mouse click $i/$num at $x , $y" # tool no longer available online #MouseTools -leftClick -x "$x" -y "$y" cliclick "c:$x,$y" sleep "$sleep_secs.$RANDOM" done else timestamp "Mouse click $i/$num at current mouse location" #MouseTools -leftClick cliclick "c:." sleep "$sleep_secs.$RANDOM" # add $RANDOM up to 1 second jitter to make it harder to spot that this is perfectly automated clicking fi done ================================================ FILE: applescript/mouse_clicks_remote_desktop.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-12-06 10:55:58 +0700 (Fri, 06 Dec 2024) # # https://github.com/HariSekhon # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/../lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Switches to Microsoft Remote Desktop, waits 10 seconds and then clicks the mouse once a minute to prevent the screensaver from coming on Workaround to Active Directory Group Policies that don't let you disable the screensaver Point the mouse to a safe location with no mouse click effect Then Cmd-Tab to Terminal, run this and let it switch back to Remote Desktop to keep the session open minute to prevent the screensaver from coming on " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 0 "$@" mac_only export START_DELAY="${START_DELAY:-10}" export SLEEP_SECS="${SLEEP_SECS:-60}" if ! is_float "$START_DELAY"; then usage "invalid non-float '$START_DELAY' found in environment for \$START_DELAY" fi if ! is_float "$SLEEP_SECS"; then usage "invalid non-float '$SLEEP_SECS' found in environment for \$SLEEP_SECS" fi timestamp "Switching foreground window to Remote Desktop" "$srcdir/set_frontmost_process.scpt" "Microsoft Remote Desktop" exec "$srcdir/mouse_clicks.sh" -1 ================================================ FILE: applescript/mouse_random_movements.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-12-06 08:09:48 +0700 (Fri, 06 Dec 2024) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/../lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Randomly moves the mouse around the screen Useful to prevent a screensaver kicking in on a Remote Desktop connection which has Active Directory Group Policies applied that doesn't let you disable the screensaver Sleeps for 10 seconds between mouse movements UPDATE: it turns out this doesn't stop Windows Virtual Desktop from going to screensaver. WORKAROUND: find a safe area in your WVD session to click without any effect, then Cmd-Tab to Terminal and run this: mouse_clicks_remote_desktop.sh " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[ ]" help_usage "$@" max_args 2 "$@" mac_only sleep_secs="${1:-10}" num="${2:--1}" if ! is_float "$sleep_secs"; then usage "invalid non-float argument given for sleep seconds: $sleep_secs" fi if [ "$sleep_secs" -lt 1 ]; then usage "Sleep seconds cannot be less than 1" fi if ! [[ "$num" =~ ^-?[[:digit:]]+$ ]]; then usage "invalid non-integer num movements given for first argument: $num" fi if ! type -P cliclick &>/dev/null; then brew install cliclick fi timestamp "Starting random mouse movements" echo screen_width="$(system_profiler SPDisplaysDataType | grep Resolution | awk '{print $2}')" screen_height="$(system_profiler SPDisplaysDataType | grep Resolution | awk '{print $4}')" for ((i=1; ; i++)); do # if given num is negative, will run for infinity until Control-C'd if [ "$num" -ge 0 ] && [ "$i" -gt "$num" ]; then break fi x="$((RANDOM % screen_width))" y="$((RANDOM % screen_height))" timestamp "Mouse movement $i/$num at $x , $y" cliclick "m:$x,$y" sleep "$sleep_secs" done ================================================ FILE: applescript/reopen_app.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: Shazam # # Author: Hari Sekhon # Date: 2025-11-02 00:44:40 +0300 (Sun, 02 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Uses Applescript to quit and re-open a given application Written to relaunch Shazam after deleting tracks from its DB using adjacent script to reflect the changes " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 1 "$@" app="$1" mac_only timestamp "Quitting and re-opening app: $app" osascript </dev/null; then "$srcdir/../install/install_homebrew.sh" fi if ! type SwitchAudioSource &>/dev/null; then brew install switchaudio-osx fi internal_mic="$(SwitchAudioSource -a | grep Microphone | grep -v -i -e iPhone -e AirPods || :)" if [[ $(wc -l <<< "$internal_mic") -ge 2 ]]; then die "ERROR: more than one microphone returned: $internal_mic " fi if [ -z "$internal_mic" ]; then die "ERROR: failed to determine internal mic" fi #SwitchAudioSource -t input -s "MacBook Pro Microphone" #timestamp "Switching to microphone: $internal_mic" SwitchAudioSource -t input -s "$internal_mic" ================================================ FILE: applescript/shazam_app_delete_track.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-11-02 00:22:03 +0300 (Sun, 02 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Deletes a single track from the local Mac's Shazam app sqlite database The Shazam app caches this while running so you will need to quit and re-open the app to see this change " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args='"" ""' help_usage "$@" num_args 2 "$@" mac_only artist="$1" track="$2" dbpath="$( find ~/Library/Group\ Containers \ -type f \ -path '*/*group.com.shazam/com.shazam.mac.Shazam/ShazamDataModel.sqlite' 2>/dev/null | head -n 1 )" if [ -z "$dbpath" ]; then die "Error: Could not locate ShazamDataModel.sqlite" fi timestamp "Found Shazam App DB: $dbpath" timestamp "Backing up DB before deleting" backup="${dbpath}.bak.$(date +%Y%m%d%H%M%S)" cp -v "$dbpath" "$backup" timestamp "Backup created at $backup" echo >&2 timestamp "Deleting track from DB: '$artist - $track'" echo >&2 # Delete from ZSHTAGRESULTMO using JOIN with ZSHARTISTMO # # sqlite3 `.parameter` is fragile and unsafe for arbitrary text, # and we end up with all kinds of shell injection and # quoting and newline issues with arbitrary data, so pre-generate # the variables with escaping using SQLite's own quoting engine # also fragile #artist_sql=$(sqlite3 ':memory:' "SELECT quote($( # printf "'%s'" "$(printf '%s' "$artist" | sed "s/'/''/g")" #));") # #track_sql=$(sqlite3 ':memory:' "SELECT quote($( # printf "'%s'" "$(printf '%s' "$track" | sed "s/'/''/g")" #));") artist_sql="$( sqlite3 -batch \ -noheader \ -list \ -cmd ".parameter clear" \ -cmd ".parameter set @v '$(sed "s/'/''/g" <<< "$artist")'" \ :memory: \ "SELECT quote(@v);" )" track_sql="$( sqlite3 -batch \ -noheader \ -list \ -cmd ".parameter clear" \ -cmd ".parameter set @v '$(sed "s/'/''/g" <<< "$track")'" \ :memory: \ "SELECT quote(@v);" )" sqlite3 -batch \ -bail \ "$dbpath" </dev/null; then uniq_order_preserved.pl else cat fi ================================================ FILE: applescript/shazam_search_spotify.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-11-02 00:30:16 +0300 (Sun, 02 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Dumps the local Mac Shazam app's tracks one at a time, searches the Spotify app for each one Shazam to Spotify apps workaround to Apple removing Spotify integration from Shazam Prompts between each track to proceed to search for the next one Optionally prompts to delete the track after searh from the Shazam local sqlite DB if this environment variable is set: export SHAZAM_APP_DELETE_TRACK_AFTER_SEARCH=1 Can optionally specify a number of tracks to stop after as an arg, or a timeframe today/yesterday/week/YYYY-MM-DD, You can set this args as an environment variable \$SHAZAM_APP_DUMP_NUM_TRACKS - the arg takes precedence though " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[ ]" help_usage "$@" max_args 2 "$@" mac_only num_tracks="${1:-${SHAZAM_APP_DUMP_NUM_TRACKS:-1}}" sqlite_db_path="${2:-}" case "$num_tracks" in # allow only these args to be passed to shazam_app_dump_tracks.sh today|yesterday|week|????-??-??) : ;; *) if ! [[ "$num_tracks" =~ ^-?[[:digit:]]+$ ]]; then die "Invalid argument given, must be an integer or one of today/yesterday/week/YYYY-MM-DD: $num_tracks" fi ;; esac relaunch_shazam(){ timestamp "Relaunching Shazam app to reflect removed tracks" "$srcdir/reopen_app.sh" Shazam sleep 2 # hit Escape key to to minimize Shazam back to the menu bar osascript -e "tell application \"System Events\" to key code 53" untrap exit } export -f relaunch_shazam trap_cmd 'relaunch_shazam' while IFS=$'\t' read -r artist _ track; do "$srcdir/spotify_app_search.sh" "$artist $track" if [ "${SHAZAM_APP_DELETE_TRACK_AFTER_SEARCH:-}" = 1 ]; then timestamp "Press enter to delete this track from the Shazam DB: $artist - $track" read -r < /dev/tty QUIET=1 "$srcdir/shazam_app_delete_track.sh" "$artist" "$track" else timestamp "Press enter to search for next track" read -r < /dev/tty fi done < <( "$srcdir/shazam_app_dump_tracks.sh" "$num_tracks" ${sqlite_db_path:+"$sqlite_db_path"} ) ================================================ FILE: applescript/shazam_search_spotify_then_delete_track.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-11-02 00:30:16 +0300 (Sun, 02 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Dumps the local Mac Shazam app's tracks one at a time, searches the Spotify app for each one, and then deletes it from the Shazam local sqlite DB upon an Enter key press to proceed to the next one Shazam to Spotify apps workaround to Apple removing Spotify integration from Shazam Can optionally specify a number of tracks to stop after as an arg, or a timeframe today/yesterday/week/YYYY-MM-DD, You can set this args as an environment variable \$SHAZAM_APP_DUMP_NUM_TRACKS - the arg takes precedence though " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[ ]" help_usage "$@" max_args 1 "$@" mac_only export SHAZAM_APP_DELETE_TRACK_AFTER_SEARCH=1 "$srcdir/shazam_search_spotify.sh" "$@" ================================================ FILE: applescript/shorten_text_selection.scpt ================================================ #!/usr/bin/env osascript # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-05-19 01:44:50 +0300 (Mon, 19 May 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Shortens the selected text in the prior window # ============================================================================ # # - Copies the selected text to the clipboard # - Replaces "and" with "&" # - Removes multiple blank lines between paragraphs (which result from the pbcopy/pbpaste pipeline otherwise) # - Pastes the clipboard text back over the selected text # # I use this a lot for LinkedIn comments in browser due to the short 1250 character limit # Tested on macOS 14 # switch to previous window tell application "System Events" key down command keystroke tab key up command end tell delay 0.3 # copy the selected text tell application "System Events" keystroke "c" using command down end tell delay 0.1 # - replace occurrences of the word "and" with "&" using sed with word boundaries # - crush out multiple blank lines to a single blank line between paragraphs # - this is correct the pbpaste | pbcopy copying back multiplying the blank lines do shell script "pbpaste | gsed -E 's/\\band\\b/\\&/g' | cat -s | pbcopy" delay 0.1 # paste the modified text over the original selection tell application "System Events" keystroke "v" using command down end tell ================================================ FILE: applescript/spotify_app_search.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-11-01 23:34:33 +0300 (Sat, 01 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Runs a search in the Spotify App on Mac using Applescript " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" min_args 1 "$@" mac_only query="$*" query="${query//\"/\\\"}" timestamp "Telling Spotify app to search for: $query" osascript \ -e 'tell application "Spotify" to activate' \ -e 'tell application "System Events" to keystroke "l" using {command down}' \ -e 'delay 0.2' \ -e "tell application \"System Events\" to keystroke \"$query\" & return" ================================================ FILE: applescript/start_app_at_login.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-11-06 22:17:22 +0200 (Thu, 06 Nov 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Adds an App to auto-start at Login using Applescript Checks the /Applications and \$HOME/Applications for the given app name (auto-tries both with and without .app extension so you can provide it either way) " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 1 "$@" arg="$1" HOME="${HOME:-$(cd && pwd)}" app="" for path in \ "/Applications/$arg.app" \ "/Applications/$arg" \ "/$HOME/Applications/$arg.app" \ "/$HOME/Applications/$arg"; do if [ -e "$path" ]; then timestamp "Found '$arg' at '$path'" app="$path" fi done if [ -z "$app" ]; then die "App '$arg' not found in /Applications or $HOME/Applications" fi timestamp "Setting '$app' to start at login" osascript -e "tell application \"System Events\" to make login item at end with properties {path:\"$app\", hidden:false}" ================================================ FILE: applescript/wakeup_script.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-04-17 22:26:11 +0400 (Wed, 17 Apr 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/../lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Script of actions to run whenever a Mac wakes up from sleep Currently this script just flushes the DNS cache to fix Chrome hitting ERR_NOT_FOUND errors when waking up on a VPN To set this up, edit the path to this script in the plist xml file and then load it: cp -fv $srcdir/com.harisekhon.wakeup_script.plist ~/Library/LaunchAgents/ launchctl load $srcdir/com.harisekhon.wakeup_script.plist " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 0 "$@" mac_only { timestamp "Running Mac wake up script: $0" timestamp "Flushing DNS cache" dscacheutil -flushcache timestamp "Reloading mDNSResponder" sudo killall -HUP mDNSResponder timestamp "Wake up script completed" echo } 2>&1 | tee -a "$srcdir/wakeup_script.log" ================================================ FILE: applescript/world_clock_cities.txt ================================================ # # Author: Hari Sekhon # Date: 2026-02-05 14:53:51 -0300 (Thu, 05 Feb 2026) # # vim:ts=4:sts=4:sw=4:et # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback # # https://www.linkedin.com/in/HariSekhon # # Cities to load into the World Clock on macOS # Was going to load this into Clock via Applescript but this app is extraordinarily bad for scripting London Prague Madrid Amsterdam Dubai Bangkok Cairo Chisinau Istanbul Tbilisi Cancun Bogota Santiago Buenos Aires New York Los Angeles #Palo Alto San Franciso #Marrakesh #Kyiv #Miami #Hanoi #Adelaide #Athens #Pontianak #Melbourne #Tunis ================================================ FILE: appveyor/appveyor_api.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-31 13:47:21 +0100 (Tue, 31 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # Queries AppVeyor API, auto-populating $APPVEYOR_TOKEN from environment and API url base for convenience # # https://kevinoid.github.io/appveyor-swagger/bootprint/ # # eg. # # appveyor_api.sh projects | jq set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_description="Queries the AppVeyor API, auto-populating the base and API tokens from the environment" # shellcheck disable=SC2034 usage_args="/path []" if [ -z "${APPVEYOR_TOKEN:-}" ]; then usage "APPVEYOR_TOKEN environment variable is not set (generate this from your Web UI Dashboard -> profile -> API AUTH TOKENS" fi if [ $# -lt 1 ]; then usage "no /path given to query in the API" fi for arg; do case "$arg" in -h|--help) usage ;; esac done url_path="${1##/}" shift || : export TOKEN="$APPVEYOR_TOKEN" "$srcdir/../bin/curl_auth.sh" -sS --fail "https://ci.appveyor.com/api/$url_path" "$@" ================================================ FILE: appveyor/appveyor_byoc.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-03-09 23:16:47 +0000 (Mon, 09 Mar 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" if [ -z "${APPVEYOR_TOKEN:-}" ]; then echo "\$APPVEYOR_TOKEN not found in environment" exit 1 fi export PATH="$PATH:/opt/appveyor/host-agent" if ! type -P appveyor-host-agent &>/dev/null; then "$srcdir/../install/install_appveyor_byoc.sh" clear fi # leading whitespace break PowerShell commands pwsh <&2 )" if [ -n "$account_info" ]; then account_name="$(jq -r '.Account.Name' <<< "$account_info")" echo "AccountName = $account_name" account_email="$(jq -r '.Account.Email' <<< "$account_info")" echo "AccountEmail = $account_email" fi aws iam get-account-summary | jq -r '.SummaryMap | to_entries | map(.key + " = " + (.value | tostring)) | .[]' | sort ================================================ FILE: aws/aws_accounts_missing_from_config.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-02-04 22:01:06 +0700 (Tue, 04 Feb 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" For a list of AWS Account IDs in stdin or files (containing one account id per line), finds those missing from AWS config You can override the config file location by setting environment variable AWS_CONFIG_FILE $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" #min_args 0 "$@" HOME="${HOME:-$(cd && pwd)}" aws_config="${AWS_CONFIG_FILE:-$HOME/.aws/config}" # force functions to log with timestamps export VERBOSE=1 account_ids="$( sed ' s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//; /^[[:space:]]*$/d ' "$@" )" timestamp "AWS Account IDs not found in: $aws_config" echo >&2 while read -r aws_account_id; do if ! is_aws_account_id "$aws_account_id"; then warn "Invalid AWS Account ID: $aws_account_id" fi if ! grep -Fq "$aws_account_id" "$aws_config"; then echo "$aws_account_id" fi done <<< "$account_ids" ================================================ FILE: aws/aws_batch_kill_stale_jobs.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-11-02 17:00:24 +0000 (Tue, 02 Nov 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Terminates AWS Batch jobs in a given queue older than N hours (default: 24) Useful to find and kill jobs that have become too long running, eg. more than 24 hours, or jobs far exceeding their expected time, including jobs that can get stuck with memory allocation errors on shared VMs May take a few seconds before the job(s) are actually terminated Requires AWS CLI to be configured and authenticated " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" queue="$1" hours="${2:-24}" script_basename="${0##*/}" "$srcdir/aws_batch_stale_jobs.sh" "$queue" "$hours" | jq -r '.[] | [.jobId,.jobName] | @tsv' | while read -r job_id job_name; do timestamp "Terminating job id: '$job_id', name: '$job_name'" aws batch terminate-job --job-id "$job_id" --reason "Job terminated by script $script_basename after running for longer than $hours hours" done ================================================ FILE: aws/aws_batch_stale_jobs.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-11-02 17:00:24 +0000 (Tue, 02 Nov 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists AWS Batch jobs in a given queue older than N hours (default: 24) Includes jobs stuck in pending and runnable states as these are usually stuck due to an environment/configuration issue if they've been pending for a long time Useful to find jobs that have become too long running, eg. more than 24 hours, or jobs far exceeding their expected time, including jobs that can get stuck with memory allocation errors on shared VMs Returns JSON list of jobs for further processing Requires AWS CLI to be configured and authenticated " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" queue="$1" hours="${2:-24}" # doesn't support floats #millis="$((hours * 3600 * 1000))" millis="$(bc -l <<< "$hours * 3600 * 1000" | sed 's/\..*$//')" epoch_millis="$(date +%s)000" before_epoch_millis="$((epoch_millis - millis))" # --filters only works on newer versions of CLIv2 so jq processing is more reliable across environments: # # https://github.com/aws/aws-cli/issues/6526 # #aws batch list-jobs --job-queue "$queue" --filters "name=BEFORE_CREATED_AT,values=$before_epoch" for state in SUBMITTED PENDING RUNNABLE STARTING RUNNING; do aws batch list-jobs --job-queue "$queue" --job-status "$state" | jq ".jobSummaryList[] | select(.createdAt <= $before_epoch_millis)" done | # slurp items back into an array jq -s . ================================================ FILE: aws/aws_billing_alarm.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-12-11 16:30:33 +0000 (Fri, 11 Dec 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Sets an AWS CloudWatch billing alarm to trigger as soon as you begin incurring any charges Creates an SNS topic and subscription for the given email address and links it to the above CloudWatch Alarm to email you as soon as your billing charges go over The alarm is set in the us-east-1 region (N. Virginia in the web console) because that is where the metric billing data accumulates, regardless of which region you actually use The first argument sets the alert threshold in USD - an alarm is raised once it goes above that amount The default threshold is 0.00 USD to alert on any charges for safety The second argument sets the email address to use in an SNS topic to notify you. If no email is given specified attempts to use the email from your local Git configuration. If neither is available, shows this usage mesage. XXX: You must also enable Receive Billing Alerts in the Billing Preferences page for the CloudWatch metrics to be populated by AWS Billing: https://console.aws.amazon.com/billing/home?#/preferences See the created alarm here: https://console.aws.amazon.com/cloudwatch/home?region=us-east-1 (notice the region must be us-east-1 as per description above) See Also: aws_budget_alarm.sh - newer method of doing this using AWS Budgets $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" threshold="${1:-0.00}" email="${2:-$(git config user.email || :)}" # XXX: region has to be us-east-1 because this is where the billing metric data accumulates regardless of which region you actually use region="us-east-1" sns_topic="AWS_Charges" if ! [[ "$threshold" =~ ^[[:digit:]]{1,4}(\.[[:digit:]]{1,2})?$ ]]; then usage "invalid threshold argument given - must be 0.01 - 9999.99 USD" fi if is_blank "$email"; then usage "email address not specified and could not determine email from git config" fi timestamp "Creating SNS topic to email '$email' in region '$region'" output="$(aws sns create-topic --name "$sns_topic" --region "$region" --output json)" # "arn:aws:sns:us-east-1:123456789012:AWS_Charges" sns_topic_arn="$(jq -r '.TopicArn' <<< "$output")" echo timestamp "Subscribing email address '$email' to topic '$sns_topic' in region '$region'" aws sns subscribe --topic-arn "$sns_topic_arn" --protocol email --notification-endpoint "$email" --region "$region" echo timestamp "Creating CloudWatch Alarm for AWS charges > $threshold USD in region '$region'" # --period 21600 = 6 hours (default) aws cloudwatch put-metric-alarm --alarm-name "AWS Charges" \ --alarm-description "Alerts on AWS charges greater than $threshold USD" \ --actions-enabled \ --alarm-actions "$sns_topic_arn" \ --region "$region" \ --namespace "AWS/Billing" \ --metric-name "EstimatedCharges" \ --dimensions "Name=Currency,Value=USD" \ --threshold "$threshold" \ --comparison-operator "GreaterThanThreshold" \ --statistic Maximum \ --period 21600 \ --evaluation-periods 1 ================================================ FILE: aws/aws_budget.json ================================================ { "BudgetName": "AWS_Charges", "BudgetType": "COST", "BudgetLimit": { "Amount": "", "Unit": "USD" }, "CostFilters": {}, "CostTypes": { "IncludeTax": true, "IncludeSubscription": true, "UseBlended": false, "IncludeRefund": false, "IncludeCredit": false, "IncludeUpfront": true, "IncludeRecurring": true, "IncludeOtherSubscription": true, "IncludeSupport": true, "IncludeDiscount": true, "UseAmortized": false }, "TimeUnit": "MONTHLY", "TimePeriod": { "Start": "2021-08-01T01:00:00+01:00", "End": "2087-06-15T01:00:00+01:00" } } ================================================ FILE: aws/aws_budget_alarm.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-08-02 15:57:07 +0100 (Mon, 02 Aug 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Creates an AWS Budget with an alarm if forecasted to go over 80% of total monthly budget, and another alarm if over 90% of monthly budget Creates an SNS topic and subscription for the given email address and links it to the above AWS Budgets Alarm to email you as soon as your billing charges are anticipated to go over the threshold. It also modifies the SNS topic's access policy to be accessible from the AWS Budgets service. The first argument sets the total monthly budget in USD - the 80% and 90% threshold alarms are based on that The default budget is 0.01 USD (will trigger a notification on any expenditure) The second argument sets the email address to use in an SNS topic to notify you. If no email is given specified attempts to use the email from your local Git configuration. If neither is available, shows this usage mesage. See the created AWS Budget here (Global): https://console.aws.amazon.com/billing/home#/budgets/overview $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" budget="${1:-0.01}" email="${2:-$(git config user.email || :)}" region="us-east-1" sns_topic="AWS_Charges" if ! [[ "$budget" =~ ^[[:digit:]]{1,4}(\.[[:digit:]]{1,2})?$ ]]; then usage "invalid budget argument given - must be 0.01 - 9999.99 USD" fi if is_blank "$email"; then usage "email address not specified and could not determine email from git config" fi timestamp "Creating SNS topic to email '$email' in region '$region'" output="$(aws sns create-topic --name "$sns_topic" --region "$region" --output json)" # "arn:aws:sns:us-east-1:123456789012:AWS_Charges" sns_topic_arn="$(jq -r '.TopicArn' <<< "$output")" echo timestamp "Subscribing email address '$email' to topic '$sns_topic' in region '$region'" aws sns subscribe --topic-arn "$sns_topic_arn" --protocol email --notification-endpoint "$email" --region "$region" echo timestamp "Getting account id" account_id="$(aws sts get-caller-identity --query Account --output text)" echo # https://docs.aws.amazon.com/cli/latest/reference/sns/set-topic-attributes.html timestamp "Updating access policy on SNS topic '$sns_topic' to allow AWS Budgets to use it" aws sns set-topic-attributes --topic-arn "$sns_topic_arn" --attribute-name Policy --attribute-value "$(sed "s//$sns_topic_arn/; s//$account_id/" "$srcdir/aws_budget_sns_access_policy.json")" --region "$region" echo # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/budgets/create-budget.html timestamp "Checking for existing AWS Budgets" budgets="$(aws budgets describe-budgets --account-id "$account_id" --query 'Budgets[*].BudgetName' --output text)" echo budget_name="$(jq -r .BudgetName < "$srcdir/aws_budget.json")" if grep -Fxq "$budget_name" <<< "$budgets"; then if [ -n "${REPLACE_BUDGET:-}" ]; then timestamp "deleting budget '$budget' to replace it" aws budgets delete-budget --account-id "$account_id" --budget-name "$budget_name" echo else echo "AWS Budget '$budget' already exists - you must delete it before running this" exit 0 fi fi timestamp "Creating AWS Budget with $budget USD budget and 80% forecasted threshold alarm" aws budgets create-budget --account-id "$account_id" --budget "$(sed "s//$budget/" "$srcdir/aws_budget.json")" --notifications-with-subscribers "$(sed "s//$sns_topic_arn/" "$srcdir/aws_budget_notification.json")" ================================================ FILE: aws/aws_budget_notification.json ================================================ [ { "Notification": { "NotificationType": "FORECASTED", "ComparisonOperator": "GREATER_THAN", "Threshold": 80.0, "ThresholdType": "PERCENTAGE", "NotificationState": "ALARM" }, "Subscribers": [ { "SubscriptionType": "SNS", "Address": "" } ] }, { "Notification": { "NotificationType": "ACTUAL", "ComparisonOperator": "GREATER_THAN", "Threshold": 80.0, "ThresholdType": "PERCENTAGE", "NotificationState": "ALARM" }, "Subscribers": [ { "SubscriptionType": "SNS", "Address": "" } ] }, { "Notification": { "NotificationType": "FORECASTED", "ComparisonOperator": "GREATER_THAN", "Threshold": 100.0, "ThresholdType": "PERCENTAGE", "NotificationState": "ALARM" }, "Subscribers": [ { "SubscriptionType": "SNS", "Address": "" } ] }, { "Notification": { "NotificationType": "ACTUAL", "ComparisonOperator": "GREATER_THAN", "Threshold": 100.0, "ThresholdType": "PERCENTAGE", "NotificationState": "ALARM" }, "Subscribers": [ { "SubscriptionType": "SNS", "Address": "" } ] } ] ================================================ FILE: aws/aws_budget_sns_access_policy.json ================================================ { "Version": "2008-10-17", "Id": "__default_policy_ID", "Statement": [ { "Sid": "AWSBudgets-notification", "Effect": "Allow", "Principal": { "Service": "budgets.amazonaws.com" }, "Action": "SNS:Publish", "Resource": "" }, { "Sid": "__default_statement_ID", "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": [ "SNS:GetTopicAttributes", "SNS:SetTopicAttributes", "SNS:AddPermission", "SNS:RemovePermission", "SNS:DeleteTopic", "SNS:Subscribe", "SNS:ListSubscriptionsByTopic", "SNS:Publish", "SNS:Receive" ], "Resource": "", "Condition": { "StringEquals": { "AWS:SourceOwner": "" } } } ] } ================================================ FILE: aws/aws_cli_create_credential.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-02-20 17:26:21 +0000 (Sat, 20 Feb 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Creates an AWS service account for CI/CD automation or AWS CLI to avoid having to re-login every day via SSO with 'aws sso login' Grants this service account Administator privileges in the current AWS account unless an alternative group or policy is specified Creates an IAM access key (deleting an older unused key if necessary), writes a CSV just as the UI download would, and outputs both shell export commands and configuration in the format for copying to your AWS profile in ~/.aws/credentials The following optional arguments can be given: - user name (default: \$USER-cli) - keyfile (default: ~/.aws/keys/\${user}_\${aws_account_id}_accessKeys.csv) - be careful if specifying this, a non-existent keyfile will create a new key, deleting the older of 2 existing keys if necessary to be able to create this - group/policy (default: Admins group or falls through to AdministratorAccess policy - checks for this group name first, or else policy by this name) This can also be used as a backup credential - this way if something accidentally happens to your AWS SSO you can still get into your account Idempotent - safe to re-run, will skip creating a user that already exists or CSV export that already exists $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ]" help_usage "$@" #min_args 1 "$@" user="${1:-$USER-cli}" #group="${2:-Admins}" #policy="${2:-AdministratorAccess}" groups_or_policies="${2:-}" default_group="Admins" default_policy="AdministratorAccess" aws_account_id="$(aws_account_id)" access_keys_csv="${3:-$HOME/.aws/keys/${user}_${aws_account_id}_accessKeys.csv}" export AWS_DEFAULT_OUTPUT=json aws_create_user_if_not_exists "$user" exports="$(aws_create_access_key_if_not_exists "$user" "$access_keys_csv")" group_exists(){ # causes a failure in the if policy test condition, probably due to early exit on one of the pipe commands set +o pipefail aws iam list-groups | jq -r '.Groups[].GroupName' | grep -Fixq "$1" || return 1 set -o pipefail } policy_exists(){ # causes a failure in the if policy test condition, probably due to early exit on one of the pipe commands set +o pipefail aws iam list-policies | jq -r '.Policies[].PolicyName' | grep -Fixq "$1" || return 1 set -o pipefail } grant_group_or_policy(){ local group_or_policy="$1" if group_exists "$group_or_policy"; then group="$group_or_policy" timestamp "Adding user '$user' to group '$group' on account '$aws_account_id'" aws iam add-user-to-group --user-name "$user" --group-name "$group" elif policy_exists "$group_or_policy"; then policy="$group_or_policy" timestamp "Determining ARN for policy '$policy'" policy_arn="$(aws iam list-policies | jq -r ".Policies[] | select(.PolicyName == \"$policy\") | .Arn")" timestamp "Determined policy ARN: $policy_arn" timestamp "Granting policy '$policy' permissions directly to user '$user' in account '$aws_account_id'" aws iam attach-user-policy --user-name "$user" --policy-arn "$policy_arn" else die "Group/Policy '$group_or_policy' not found in account '$aws_account_id'" fi echo } if [ -n "$groups_or_policies" ]; then for group_or_policy in ${groups_or_policies//,/ }; do grant_group_or_policy "$group_or_policy" done else if group_exists "$default_group"; then grant_group_or_policy "$default_group" elif policy_exists "$default_policy"; then grant_group_or_policy "$policy" else die "Neither default group '$default_group', nor default policy '$default_policy' in account '$aws_account_id'" fi fi echo echo "Set the following export commands in your environment to begin using this access key in your CLI immediately:" echo echo "$exports" echo echo "or add the following to your ~/.aws/credentials file:" echo aws_access_keys_exports_to_credentials <<< "$exports" echo echo ================================================ FILE: aws/aws_cloudformation_stacks_pending.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-12-11 12:22:59 +0000 (Fri, 11 Dec 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists CloudFormation stacks not marked as completed Useful with the 'watch' command or in a loop as a latch (hint: use grep) to check until there are no pending CloudFormation stacks before continuing Arguments are fed to AWS CLI eg. to set --region Output Format: $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" aws cloudformation list-stacks --output json "$@" | jq -r '.StackSummaries[] | [.StackStatus, .StackName, .TemplateDescription] | @tsv' | { grep -Ev '^([[:alnum:]_]+)?COMPLETE' || : ; } ================================================ FILE: aws/aws_cloudfront_distribution_for_origin.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-12-19 13:54:16 +0700 (Thu, 19 Dec 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Returns the AWS CloudFront ARN of the distribution which serves origins containing a given substring Useful for quickly finding the CloudFront ARN needed to give permissions to a private S3 bucket exposed via CloudFront $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" min_args 1 "$@" domain_substring="$1" json="$( aws cloudfront list-distributions \ --query "DistributionList.Items[*].{ARN:ARN, DomainNames:Origins.Items[*].DomainName}" \ --output json )" if [ "$json" = null ]; then echo "No CloudFront distributions found. Have you set the right \$AWS_PROFILE environment variable to the correct account?" >&2 exit 1 fi jq -r ".[] | select(.DomainNames | map(ascii_downcase | contains(\"$domain_substring\")) | any) | .ARN" <<< "$json" ================================================ FILE: aws/aws_cloudtrails_cloudwatch.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-17 16:24:52 +0000 (Fri, 17 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists Cloud Trails and their last delivery to CloudWatch Logs (should be recent) Output Format: CloudTrail_Name LastDeliveryTimestampToCloudWatchLogs (may be null) $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" export AWS_DEFAULT_OUTPUT=json #echo "Getting Cloud Trails" >&2 aws cloudtrail describe-trails | jq -r '.trailList[].Name' | while read -r name; do printf '%s\t' "$name" output="$(aws cloudtrail get-trail-status --name "$name" | jq -r '.LatestcloudwatchLogdDeliveryTime')" if [ -n "$output" ]; then echo "$output" echo "$output" else echo "NOT_LOGGING" fi done | sort | column -t ================================================ FILE: aws/aws_cloudtrails_event_selectors.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-17 16:24:52 +0000 (Fri, 17 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists Cloud Trails and their event selectors To check there is at least one event selector for each trail with IncludeManagementEvents set to true and ReadWriteType set to All Output Format: Name IncludeManagementEvents (boolean) ReadWriteType (All) DataResources (optional) $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" export AWS_DEFAULT_OUTPUT=json #echo "Getting Cloud Trails" >&2 aws cloudtrail describe-trails | jq -r '.trailList[].Name' | while read -r name; do echo -n "$name " aws cloudtrail get-event-selectors --trail-name "$name" | jq -r '.EventSelectors[] | [.IncludeManagementEvents, .ReadWriteType, .DataResources[]] | @tsv' done | sort | column -t ================================================ FILE: aws/aws_cloudtrails_s3_accesslogging.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-17 16:24:52 +0000 (Fri, 17 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists Cloud Trails buckets and their Access Logging prefix and target bucket Output Format: CloudTrail_S3_Bucket TargetPrefix TargetBucket If access logging isn't configured on the bucket, outputs: CloudTrail_S3_Bucket S3_ACCESS_LOGGING_NOT_CONFIGURED $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" export AWS_DEFAULT_OUTPUT=json aws cloudtrail describe-trails --query 'trailList[*].S3BucketName' | jq -r '.[]' | while read -r name; do printf '%s\t' "$name" output="$(aws s3api get-bucket-logging --bucket "$name" | jq -r '.LoggingEnabled | [.TargetPrefix, .TargetBucket] | @tsv')" if [ -z "$output" ]; then echo "S3_ACCESS_LOGGING_NOT_CONFIGURED" else echo "$output" fi done | sort | column -t ================================================ FILE: aws/aws_cloudtrails_s3_kms.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-21 18:25:39 +0000 (Tue, 21 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists Cloud Trails and whether their S3 buckets are KMS secured Output Format: CloudTrail_Name S3_KMS_secured (boolean) KMS_Key_Id $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" export AWS_DEFAULT_OUTPUT=json aws cloudtrail describe-trails | # more efficient jq -r '.trailList[] | [.Name, has("KmsKeyId"), .KmsKeyId // "N/A"] | @tsv' | #jq -r '.trailList[] | [.Name, .KmsKeyId] | @tsv' | #while read -r name keyid; do # kms_secured=false # if [ -n "$keyid" ]; then # kms_secured=true # else # keyid="N/A" # fi # printf "%s\t%s\t%s" "$name" "$kms_secured" "$keyid" #done | sort | column -t ================================================ FILE: aws/aws_cloudtrails_status.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-17 16:24:52 +0000 (Fri, 17 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists Cloud Trails and their status - if they're logging, multi-region and log file validation enabled Output Format: Name Logging (boolean) Multi-Region (boolean) Logfile Validation (boolean) $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" export AWS_DEFAULT_OUTPUT=json #echo "Getting Cloud Trails" >&2 aws cloudtrail describe-trails | jq -r '.trailList[] | [.Name, .IsMultiRegionTrail, .LogFileValidationEnabled] | @tsv' | while read -r name is_multi_region is_validation_enabled; do is_logging="$( aws cloudtrail get-trail-status --name "$name" | jq -r '.IsLogging' )" echo "$name $is_logging $is_multi_region $is_validation_enabled" done | sort | column -t ================================================ FILE: aws/aws_codecommit_csv_creds.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2022-03-30 11:13:53 +0100 (Wed, 30 Mar 2022) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Prints AWS CodeCommit Git credentials from a standard AWS HTTPS Git credentials CSV export file as shell export statements export GIT_USER=... export GIT_PASSWORD=... Useful to quickly switch your shell to some exported credentials from a service account for testing permissions or pipe to upload to a CI/CD system via an API, eg. the adjacent scripts: github_actions_repo*_set_secret.sh gitlab_*_set_env_vars.sh circleci_*_set_env_vars.sh bitbucket_*_set_env_vars.sh terraform_cloud_*_set_vars.sh kubectl_kv_to_secret.sh Examples: # format downloaded from the user's IAM -> Security Credentials -> section HTTPS Git credentials for AWS CodeCommit eval \$(${0##*/} hari_codecommit_credentials.csv) You can then use these credentials in commands, but note that if the \$GIT_PASSWORD contains slashes you will need to urlencode it: GIT_PASSWORD_URLENCODED=\"\$(urlencode.sh <<< \"\$GIT_PASSWORD\")\" git clone \"https://\$GIT_USER:\$GIT_PASSWORD_URLENCODED@git-codecommit.eu-west-2.amazonaws.com/v1/repos/myrepo\" " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="_codecommit_credentials.csv" help_usage "$@" min_args 1 "$@" csv="$1" if ! [ -f "$csv" ]; then die "ERROR: File not found: $csv" fi # people may rename their credentials file #if ! [[ "$csv" =~ ^.+_codecommit_credentials.csv ]]; then # die "ERROR: Wrong filename, should be in format *_codecommit_credentials.csv" #fi # XXX: this CSV credentials files come in DOS format unlike other CSV credential downloads for AWS CLI etc. if ! tr -d '\r' < "$csv" | grep -Fxq 'User Name,Password'; then die "ERROR: Expected 'User Name,Password' header not found in file '$csv'" fi lines="$(wc -l "$csv" | awk ' {print $1}')" if ! [ "$lines" -eq 2 ]; then die "ERROR: wrong number of lines found in CSV credentials file, expected 2, got $lines" fi tr -d '\r' < "$csv" | tail -n 1 | awk -F, '{ print "export GIT_USER="$1 print "export GIT_PASSWORD="$2 }' ================================================ FILE: aws/aws_config_all_types.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-21 17:20:46 +0000 (Tue, 21 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists AWS Config recorders, checking all resource types are supported (should be true) and includes global resources (should be true) eg. awsconfig true true $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" aws configservice describe-configuration-recorders --output json | jq -r '.ConfigurationRecorders[] | [.name, .recordingGroup.allSupported, .recordingGroup.includeGlobalResourceTypes] | @tsv' | column -t ================================================ FILE: aws/aws_config_recording.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-21 17:20:46 +0000 (Tue, 21 Jan 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists AWS Config recorders, their recording status (should be true) and their last status (should be success) eg. awsconfig true SUCCESS $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" aws configservice describe-configuration-recorder-status --output json | jq -r '.ConfigurationRecordersStatus[] | [.name, .recording, .lastStatus] | @tsv' | column -t ================================================ FILE: aws/aws_csv_creds.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-12-03 16:59:48 +0000 (Fri, 03 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Prints AWS credentials from a standard AWS CSV export file or access key export CSV as shell export statements export AWS_ACCESS_KEY_ID=AKIA... export AWS_SECRET_ACCESS_KEY=... Supports new user and new access key csv file formats eg. 'Download .csv file' when you create an AWS access key in the console Useful to quickly switch your shell to some exported credentials from a service account for testing permissions or pipe to upload to a CI/CD system via an API, eg. the adjacent scripts: jenkins_cred_add*.sh github_actions_repo*_set_secret.sh gitlab_*_set_env_vars.sh circleci_*_set_env_vars.sh bitbucket_*_set_env_vars.sh terraform_cloud_*_set_vars.sh kubectl_kv_to_secret.sh Examples: eval \$(${0##*/} new_user_credentials.csv) # format downloaded when creating a user eval \$(${0##*/} hari_accessKeys.csv) # format downloaded when creating an access key " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="credential.csv" help_usage "$@" min_args 1 "$@" csv="$1" if ! [ -f "$csv" ]; then die "ERROR: File not found: $csv" fi if ! grep -Fq 'AKIA' "$csv"; then die "ERROR: Access Key not found in file '$csv'" fi # for CSV created at access key creation time if tr -d '\r' < "$csv" | grep -Fq 'Access key ID,Secret access key'; then # access keys are prefixed with AKIA, skips header row by selecting the row with the AKIA key awk -F, '/AKIA/{ print "export AWS_ACCESS_KEY_ID="$1 print "export AWS_SECRET_ACCESS_KEY="$2 }' "$csv" | tr -d '\r' # for CSV created at user creation time elif tr -d '\r' < "$csv" | grep -Fq 'User name,Password,Access key ID,Secret access key,Console login link'; then awk -F, '/AKIA/{ print "export AWS_ACCESS_KEY_ID="$3 print "export AWS_SECRET_ACCESS_KEY="$4 }' "$csv" | tr -d '\r' else die "ERROR: unrecognized CSV header line, may have changed so code may need an update" fi ================================================ FILE: aws/aws_ec2_ami_boot.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-31 16:05:24 +0700 (Fri, 31 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Boots an EC2 instance from a given AMI for manual debugging Useful for interactive debugging when creating AMIs - checks if there is already an EC2 running instance tagged with your name and AMI ID - if yes, reuses it - if not, boots an EC2 instance from the AMI with the given security group, subnet id and SSH key-name given so you can SSH to its ec2-user - waits for the EC2 instance to boot - waits for the EC2 instance to pass its instance and system checks - determines the public or private IP address and outputs it to stdout for use in other scripts It's up to you to Terminate the instance as you may want to leave it running and then create an AMI from it when you've finished testing using this script: aws_ec2_ami_create_from_instance.sh You may want to run this adjacent wrapper script to drop you straight into an SSH prompt: aws_ec2_ami_boot_ssh.sh See Also: https://github.com/HariSekhon/Packer $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 5 "$@" max_args 6 "$@" ami_id="$1" instance_type="$2" security_group="$3" subnet_id="$4" ssh_key_name="$5" instance_profile="${6:-}" aws_validate_security_group_id "$security_group" aws_validate_subnet_id "$subnet_id" if ! is_blank "$instance_profile"; then if ! [[ "$instance_profile" =~ ^[A-Za-z0-9+=,.@_-]+$ ]]; then die "Invalid Instance Profile name: $instance_profile" fi fi instance_launched=0 user="${USER:-$(whoami)}" if is_blank "$user"; then die "Failed to determine username to tag the EC2 instance with" fi for((i=1; i <= 100 ; i++)); do instance_name="$user-$ami_id" timestamp "Checking if EC2 instance of AMI already exists: $instance_name" instance_id="$( aws ec2 describe-instances \ --filters "Name=tag:Name,Values=$instance_name" \ --query "Reservations[0].Instances[0].InstanceId" \ --output text )" if [ "$instance_id" != "None" ]; then timestamp "Checking the instance state isn't terminated or shutting down" instance_state="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query "Reservations[0].Instances[0].State.Name" \ --output text )" if grep -qi -e terminated -e shutting-down <<< "$instance_state"; then timestamp "This instance is already terminated / shutting down, will try a new instance name" echo >&2 continue fi fi if is_blank "$instance_id" || [ "$instance_id" = "None" ]; then timestamp "Launching EC2 instance: $instance_name" instance_id="$( aws ec2 run-instances \ --image-id "$ami_id" \ --count 1 \ --instance-type "$instance_type" \ --key-name "$ssh_key_name" \ --security-group-ids "$security_group" \ --subnet-id "$subnet_id" \ --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$instance_name}]" \ --query "Instances[0].InstanceId" \ --output text )" timestamp "Launched instance: $instance_id" fi instance_launched=1 break done if [ "$instance_launched" != 1 ]; then die "ERROR: Failed to launch instance" fi echo >&2 timestamp "Waiting for instance to be running..." aws ec2 wait instance-running --instance-ids "$instance_id" timestamp "Instance is running" echo >&2 get_instance_profile(){ local instance_id="$1" aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query "Reservations[0].Instances[0].IamInstanceProfile.Arn" \ --output text | sed 's|.*/||' } if ! is_blank "$instance_profile"; then instance_profile_attached=0 if [ "$(get_instance_profile "$instance_id")" = "$instance_profile" ]; then instance_profile_attached=1 else timestamp "Attaching instance profile: $instance_profile" aws ec2 associate-iam-instance-profile \ --instance-id "$instance_id" \ --iam-instance-profile Name="$instance_profile" echo >&2 timestamp "Waiting for profile to fully attach..." instance_profile_attached=0 for((i=1; i <= 100 ; i++)); do current_instance_profile="$(get_instance_profile "$instance_id")" if [ "$current_instance_profile" = "None" ]; then timestamp "No instance profile associated yet..." elif [ "$current_instance_profile" = "$instance_profile" ]; then timestamp "Instance profile attached" instance_profile_attached=1 break else timestamp "Waiting for instance profile to attach..." fi sleep 3 done fi if [ "$instance_profile_attached" != 1 ]; then die "Instance profile failed to attach, gave up waiting" fi fi "$srcdir/aws_ec2_wait_for_instance_ready.sh" "$instance_id" timestamp "EC2 instance running: $instance_name" echo "$instance_id" ================================================ FILE: aws/aws_ec2_ami_boot_ssh.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-31 16:05:24 +0700 (Fri, 31 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Boots an EC2 instance from a given AMI, determines the public or private IP, and drops you into an SSH shell Useful for interactive debugging when creating AMIs - checks if there is already an EC2 running instance tagged with your name and AMI ID - if yes, reuses it - if not, boots an EC2 instance from the AMI with the given security group, subnet id and SSH key-name given so you can SSH to its ec2-user - waits for the EC2 instance to boot - waits for the EC2 instance to pass its instance and system checks - determines the public or private IP address - SSH's to the EC2 instance - Assumes ~/.ssh/,pem is present locally to be able to log in to it It's up to you to Terminate the instance as you may want to leave it running and then create an AMI from it when you've finished testing using this script: aws_ec2_ami_create_from_instance.sh See Also: https://github.com/HariSekhon/Packer $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 5 "$@" max_args 6 "$@" ssh_key_name="$5" instance_id="$("$srcdir/aws_ec2_ami_boot.sh" "$@")" ip="$("$srcdir/aws_ec2_instance_ip.sh" "$instance_id")" timestamp "SSH'ing to EC2 instance" echo >&2 # this is a brand new instance so the SSH host key won't be trusted exec ssh \ -i ~/.ssh/"$ssh_key_name.pem" \ -o StrictHostKeyChecking=no \ ec2-user@"$ip" ================================================ FILE: aws/aws_ec2_ami_create_from_instance.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-11-12 16:42:30 +0400 (Tue, 12 Nov 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Creates an AWS EC2 AMI from an EC2 instance and waits for it to become available for use Outputs the AMI ID to stdout after it becomes available Useful to testing risky things on another EC2 vm cloned from that AMI Does not reboot the running EC2 instance for safety by default, which means a non-clean filesystem copy unless you shut it down first. To enforce a reboot of the EC2 instance (be careful in production!) you must set the environment variable: export AWS_EC2_REBOOT_INSTANCE=true See it in the list of AMIs afterwards using this command: aws ec2 describe-images --owners self --query 'Images[*].{ID:ImageId,Name:Name}' --output table Check the state of the AMI is finished: aws ec2 describe-images --image-ids \"\$AMI_ID\" --output table Investigate instance names quickly using adjacent script aws_ec2_instances.sh $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" " help_usage "$@" num_args 2 "$@" instance_name="$1" ami_name="$2" instance_id="$(VERBOSE=1 "$srcdir/aws_ec2_instance_name_to_id.sh" "$instance_name")" if ! is_instance_id "$instance_id"; then die "Invalid Instance ID returned, failed regex validation: $instance_id" fi echo >&2 no_reboot="--no-reboot" if [ "${AWS_EC2_REBOOT_INSTANCE:-}" = true ]; then timestamp "WARNING: AWS_EC2_REBOOT_INSTANCE environment variable is set to true" echo >&2 read -r -p "Are you sure you want to take down this EC2 instance '$instance_name'? (y/N) " answer check_yes "$answer" no_reboot="" echo >&2 fi timestamp "Creating AMI '$ami_name' from EC2 instance '$instance_name'" ami_id="$( aws ec2 create-image --instance-id "$instance_id" --name "$ami_name" "$no_reboot" | jq -r '.ImageId' )" echo >&2 if is_blank "$ami_id" || [ "$ami_id" = null ]; then die "Failed to get AMI ID" elif ! is_ami_id "$ami_id"; then die "Invalid AMI ID returned, failed regex validation: $ami_id" fi # special variable that increments - use as a built-in timer SECONDS=0 timestamp "Checking for AMI '$ami_name' to become ready..." echo >&2 while true; do state="$(aws ec2 describe-images --image-ids "$ami_id" | jq -r '.Images[0].State')" if [ "$state" = "available" ]; then timestamp "AMI '$ami_name' is now available after $SECONDS seconds" break elif [ "$SECONDS" -gt 1200 ]; then die "Waited for 20 minutes without AMI becoming available, something is wrong, aborting..." fi timestamp "Waiting for AMI '$ami_name' to become ready. State: $state" sleep 10 done echo "$ami_id" ================================================ FILE: aws/aws_ec2_ami_ids.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-11-13 14:42:01 +0400 (Wed, 13 Nov 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" List AWS EC2 AMI IDs in use in the current or given AWS account, one per line for processing in other scripts Used by: aws_info_ec2*.sh See also: aws_ec2_amis.sh $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" num_args 0 "$@" if [ $# -gt 0 ]; then aws_profile="$1" shift || : export AWS_PROFILE="$aws_profile" fi # false positive - want single quotes for * to be evaluated within AWS query not shell # shellcheck disable=SC2016 aws ec2 describe-instances \ --query 'Reservations[*].Instances[*].ImageId' \ --output text | tr '[:space:]' '\n' | sort -u | sed '/^[[:space:]]*$/d' ================================================ FILE: aws/aws_ec2_ami_name_to_id.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-11-13 14:42:01 +0400 (Wed, 13 Nov 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Returns an EC2 AMI ID from name Adds additional safety checks: - verifies no more than one AMI ID is returned - does a reverse lookup on the AMI ID to verify the name - if an AMI ID is passed, returns it as is for convenience Investigate AMI names and IDs quickly using adjacent script: aws_ec2_amis.sh $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" num_args 1 "$@" ami_name="$1" if is_ami_id "$ami_name"; then log "Given AMI name is already an AWS AMI ID, outputs as is: $ami_name" echo "$ami_name" exit 0 fi log "Determining EC2 AMI ID for name '$ami_name'" ami_id="$( aws ec2 describe-images \ --owners self \ --filters "Name=name,Values=$ami_name" \ --query 'Images[*].ImageId' \ --output text )" if is_blank "$ami_id"; then die "No EC2 AMI found with name '$ami_name'" fi if [ "$(awk '{print NF}' <<< "$ami_id")" -gt 1 ]; then cat >&2 <&2 # Volumes[*] should not be shell interpreted # shellcheck disable=SC2016 aws ec2 describe-volumes \ --filters 'Name=status,Values=available' \ --query 'Volumes[*].{ " VolumeID": VolumeId, " VolumeType": VolumeType, " Size": Size, " State": State, AvailabilityZone: AvailabilityZone, Name: Tags[?Key=="Name"].Value | [0], Environment: Tags[?Key=="Environment"].Value | [0], CreateTime: CreateTime }' \ --output table ================================================ FILE: aws/aws_ec2_instance_clone.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-11-12 16:42:30 +0400 (Tue, 12 Nov 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Clones an AWS EC2 instance by creating an AMI from the original and then booting a new instance from the AMI with the same settings as the original instance Useful to testing risky things on a separate EC2 instance, such as Server Administrator recovery of Tableau Does not reboot the running EC2 instance for safety by default, which means a non-clean filesystem copy unless you shut it down first. To enforce a reboot of the EC2 instance (be careful in production!) you must set the environment variable: export AWS_EC2_REBOOT_INSTANCE=true Uses adjacent scripts: aws_ec2_ami_create_from_instance.sh aws_ec2_instance_name_to_id.sh Investigate instance names quickly using adjacent script aws_ec2_instances.sh $usage_aws_cli_jq_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" " help_usage "$@" num_args 2 "$@" instance_name="$1" new_instance_name="$2" ami_name="instance-$instance_name-$(date '+%F_%H%M%S')" # this script has been updated to wait for the AMI state to become available ami_id="$("$srcdir/aws_ec2_ami_create_from_instance.sh" "$instance_name" "$ami_name")" echo >&2 if ! is_ami_id "$ami_id"; then die "Invalid AMI ID returned, failed regex validation: $ami_id" fi timestamp "Determining instance ID of original EC2 instance '$instance_name'" instance_id="$("$srcdir/aws_ec2_instance_name_to_id.sh" "$instance_name")" if ! is_instance_id "$instance_id"; then die "Invalid Instance ID returned, failed regex validation: $instance_id" fi timestamp "Determined instance ID to be: $instance_id" echo >&2 timestamp "Determining instance type of original instance" instance_type="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[*].Instances[*].InstanceType' \ --output text )" if is_blank "$instance_type"; then die "Failed to determine instance type" fi timestamp "Determined instance type to be: $instance_type" echo >&2 timestamp "Determining subnet ID of original instance" subnet_id="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[*].Instances[*].SubnetId' \ --output text )" if is_blank "$subnet_id"; then die "Failed to determine subnet ID" fi timestamp "Determined subnet ID to be: $subnet_id" echo >&2 timestamp "Determining key pair name of original instance" key_name="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[*].Instances[*].KeyName' \ --output text )" if is_blank "$key_name"; then die "Failed to determine key name" fi timestamp "Determined key pair name to be: $key_name" echo >&2 timestamp "Determining security group IDs of original instance" security_group_ids="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[*].Instances[*].SecurityGroups[*].GroupId' \ --output text | tr '\n' ',' | sed 's/,$//' )" if is_blank "$security_group_ids"; then die "Failed to determine security group IDs" fi timestamp "Determined security group ID to be: $security_group_ids" echo >&2 timestamp "Launching new EC2 instance from AMI '$ami_name'" new_instance_id="$( aws ec2 run-instances \ --image-id "$ami_id" \ --instance-type "$instance_type" \ --subnet-id "$subnet_id" \ --key-name "$key_name" \ --security-group-ids "$security_group_ids" \ --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$new_instance_name}]" | jq -r '.Instances[0].InstanceId' )" echo >&2 timestamp "Waiting for new EC2 instance '$new_instance_name' ($new_instance_id) to enter running state" echo >&2 # special variable that increments - use as a built-in timer SECONDS=0 while true; do state="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query 'Reservations[*].Instances[*].State.Name' \ --output text )" if [ "$state" = "running" ]; then timestamp "New instance '$new_instance_name' is now running after $SECONDS seconds" break elif [ "$SECONDS" -gt 1200 ]; then die "Waited for 20 minutes but instance did not enter running state, something is wrong, aborting..." fi timestamp "Waiting for instance '$new_instance_name' to enter running state. State: $state" sleep 10 done echo "$instance_id" ================================================ FILE: aws/aws_ec2_instance_ip.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-31 16:05:24 +0700 (Fri, 31 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Returns the IP address, public or private, or a given EC2 instance by name or instance id Used by: aws_ec2_ami_boot_ssh.sh $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 1 "$@" instance="$1" timestamp "Getting instance public IP for instance: $instance" instance_id="$("$srcdir/aws_ec2_instance_name_to_id.sh" "$instance")" timestamp "Instance ID: $instance_id" public_ip="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query "Reservations[0].Instances[0].PublicIpAddress" \ --output text )" if ! is_blank "$public_ip" && [ "$public_ip" != "None" ]; then ip="$public_ip" timestamp "Using instance public IP: $ip" else timestamp "No public IP found, getting instance private IP" private_ip="$( aws ec2 describe-instances \ --instance-ids "$instance_id" \ --query "Reservations[0].Instances[0].PrivateIpAddress" \ --output text )" ip="$private_ip" timestamp "Using instance private IP: $ip" fi timestamp "IP address is: $ip" echo "$ip" ================================================ FILE: aws/aws_ec2_instance_name_to_id.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-08-27 11:28:25 +0200 (Tue, 27 Aug 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Returns an EC2 instance ID from name Adds additional safety checks: - verifies no more than one instance ID is returned - does a reverse lookup on the instance ID to verify the name - if an instance ID is passed, returns it as is for convenience Called by adjacent scripts like: aws_ec2_ami_create_from_instance.sh aws_ec2_instance_terminate_by_name.sh Investigate instance names and IDs quickly using adjacent script: aws_ec2_instances.sh $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 1 "$@" instance_name="$1" if [[ "$instance_name" =~ ^i-[0-9a-f]{8,17}$ ]]; then log "Given instance name is already an AWS instance ID, outputs as is: $instance_name" echo "$instance_name" exit 0 fi log "Determining EC2 instance ID for name '$instance_name'" instance_id="$( aws ec2 describe-instances \ --filters "Name=tag:Name,Values=$instance_name" \ --query "Reservations[*].Instances[*].InstanceId" \ --output text )" if is_blank "$instance_id"; then die "No EC2 instance found with name '$instance_name'" fi if [ "$(awk '{print NF}' <<< "$instance_id")" -gt 1 ]; then cat >&2 <&2 read -r -p "Do you want to terminate instance '$instance_name' with id '$instance_id'? (y/N) " answer check_yes "$answer" aws ec2 terminate-instances --instance-ids "$instance_id" ================================================ FILE: aws/aws_ec2_instance_wait_for_ready.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-31 16:26:34 +0700 (Fri, 31 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Polls an AWS EC2 instance and waits for it to finish initializing to a ready state Used by adjacent script aws_eks_ami_create.sh Get the AWS EC2 Instance ID from the output of the 'aws ec2 run-instances' command - see above script Timeout secs defaults to 300 if not specified Check interval defaults to 5 seconds if not specified $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ]" help_usage "$@" min_args 1 "$@" instance_id="$1" timeout_secs="${2:-300}" check_interval_secs="${3:-5}" aws_validate_instance_id "$instance_id" if ! is_int "$timeout_secs"; then die "Invalid Timeout Secs, must be an integer: $timeout_secs" fi if ! is_int "$check_interval_secs"; then die "Invalid Check Interval Secs, must be an integer: $check_interval_secs" fi if [ "$timeout_secs" -lt 1 ]; then die "Invalid Timeout Secs cannot be less than 1: $timeout_secs" fi if [ "$check_interval_secs" -lt 1 ]; then die "Invalid Check Interval Secs cannot be less than 1: $check_interval_secs" fi timestamp "Waiting for AWS EC2 instance to become ready: $instance_id" timestamp "Check interval: $check_interval_secs secs" timestamp "Timeout max: $timeout_secs secs" SECONDS=0 timestamp "Checking EC2 instance status..." while true; do if [ "$SECONDS" -gt "$timeout_secs" ]; then die "ERROR: Timed out waiting $timeout_secs for instance to become ready" fi instance_checks="$( aws ec2 describe-instance-status \ --instance-ids "$instance_id" \ --query "InstanceStatuses[0].InstanceStatus.Status" \ --output text )" if [ "$instance_checks" = "ok" ]; then system_checks="$( aws ec2 describe-instance-status \ --instance-ids "$instance_id" \ --query "InstanceStatuses[0].SystemStatus.Status" \ --output text )" if [ "$instance_checks" = "ok" ] && [ "$system_checks" = "ok" ]; then timestamp "EC2 instance $instance_id is ready" break fi fi timestamp "AWS EC2 instance still initializing..." sleep "$check_interval_secs" done ================================================ FILE: aws/aws_ec2_instances.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-08-27 11:28:20 +0200 (Tue, 27 Aug 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" List AWS EC2 instances, their DNS names and States in an easy to read table output Useful for quickly investigating running instances and comparing to configured FQDN addresses in referencing software See also: aws_ec2_info.sh - gives similar info but also resolves AMI names and adds an architecture column aws_ec2_info_csv.sh - same as above but in quoted CSV format $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 0 "$@" # false positive - want single quotes for * to be evaluated within AWS query not shell # shellcheck disable=SC2016 # # prefixing the column headings with spaces forces them to come first so that we can get the State field # in the middle instead of end since AWS CLI seems to sort the columns lexically aws ec2 describe-instances \ --query 'Reservations[*].Instances[*].{ " Name": Tags[?Key==`Name`].Value | [0], " ID": InstanceId, " State": State.Name, " InstanceType": InstanceType, "Public DNS": publicDnsName, "Private DNS": PrivateDnsName }' \ --output table ================================================ FILE: aws/aws_ec2_launch_templates_ami_id.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-09 20:02:53 +0700 (Thu, 09 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" For each Launch Template lists the AMI ID of the latest version Useful to check EKS upgrades of node groups via Terragrunt have taken effect " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 0 "$@" # returns in format: # # ID NAME # launch_templates="$(aws ec2 describe-launch-templates --query "LaunchTemplates[].{Name:LaunchTemplateName,ID:LaunchTemplateId}" --output text)" while read -r _id name; do echo -n "$name " aws ec2 describe-launch-template-versions --launch-template-name "$name" | jq -r '.LaunchTemplateVersions[0].LaunchTemplateData.ImageId' done <<< "$launch_templates" # | #column -t ================================================ FILE: aws/aws_ecr_alternate_tags.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:30:51 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists all alternative tags for the given specific ECR docker image:tag If a container has multiple tags (eg. latest, v1, hashref), you can supply ':latest' to see which version has been tagged to 'latest' Each tag for the given : is output on a separate line for easy further piping and filtering, including the originally supplied tag If no tag is given, assumes 'latest' If the image isn't found in GCR, will return nothing and no error code since this is the default GCloud SDK behaviour $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" min_args 1 "$@" image_tag="$1" shift || : image="${image_tag%%:*}" tag="${image_tag##*:}" if [ -z "$tag" ] || [ "$tag" = "$image" ]; then tag="latest" fi aws ecr describe-images --repository-name "$image" --image-ids "imageTag=$tag" "$@" | jq -r '.imageDetails[].imageTags[]' | sort ================================================ FILE: aws/aws_ecr_delete_old_tags.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Deletes tags old than N days for a given AWS ECR image Useful to clean out old CI image builds to save S3 storage costs on old CI images you no longer use Prompts with the list of image:tags that it will delete before proceeding for safety. $usage_aws_cli_required See Also: aws_ecr_tags_old.sh - used by this script, lists all image:tag older than N days aws_ecr_tags_timestamps.sh - lists tags and timestamps - useful for comparing with the output from aws_ecr_tags_old.sh aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ]" help_usage "$@" min_args 2 "$@" image="$1" days="$2" shift || : shift || : image_tags="$("$srcdir/aws_ecr_tags_old.sh" "$image" "$days" "$@")" if [ -z "$image_tags" ]; then echo "No image:tags older than $days old" exit 0 fi echo echo "List of image:tags that will be deleted:" echo echo "$image_tags" echo read -r -p 'Are you sure you want to delete these image:tags listed above? (y/N) ' answer echo if [ "$answer" != "y" ]; then echo "Aborting..." exit 1 fi for image_tag in $image_tags; do "$srcdir/aws_ecr_delete_tag.sh" "$image_tag" "$@" done ================================================ FILE: aws/aws_ecr_delete_tag.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Deletes a tag for the given AWS ECR docker image $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" min_args 1 "$@" image_tag="$1" shift || : image="${image_tag%%:*}" tag="${image_tag##*:}" if [ -z "$tag" ] || [ "$tag" = "$image" ]; then usage "tag not given" fi timestamp "deleting tag '$tag' for ECR image '$image'" # negate the result of the pipe aws ecr batch-delete-image --repository-name "$image" --image-ids "imageTag=$tag" "$@" | if jq -e '.failures[0]'; then exit 1 fi ================================================ FILE: aws/aws_ecr_docker_build_push.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2021-10-22 15:11:27 +0100 (Fri, 22 Oct 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/utils.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Builds the local docker image using the Dockerfile in the current directory and pushes it to the AWS ECR registry Tags the docker image with the following and pushes all tags to AWS ECR: - latest - Git full hashref - Git branch - any Git tags, if found, for easy versioning support - date (YYYY-MM-DD) - datetimestamp (YYYYMMDDThhmmssZ] in UTC Requires AWS CLI to be installed and configured, as well as Docker to be running locally " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" " help_usage "$@" num_args 2 "$@" ECR="$1" REPO="$2" if ! is_aws_ecr_registry "$ECR"; then usage "Invalid ECR address given: $ECR" fi if ! [[ "$REPO" =~ ^[[:alnum:]/-]+$ ]]; then usage "Invalid Repo name given: $REPO" fi if is_CI; then docker version echo fi echo "* AWS ECR -> Docker login" # $AWS_DEFAULT_REGION should be set in env or profile "$srcdir/aws_ecr_docker_login.sh" "$ECR" echo echo "* Determining tags" hashref="$(git rev-parse HEAD)" git_branch="$(git rev-parse --abbrev-ref HEAD)" git_tags="$(git tag --points-at HEAD)" # can return multiple tags # must use date -u switch since --utc only works on Linux and not Mac date="$(date -u '+%F')" timestamp="$(date -u '+%FT%H%M%SZ')" # adding tags: # tags=" $git_branch $git_tags $date $timestamp " echo export DOCKER_BUILDKIT=1 # shellcheck disable=SC2046 docker build -t "$ECR/$REPO:$hashref" . \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from "$ECR/$REPO:latest" \ --cache-from "$ECR/$REPO:$hashref" \ $(for tag in $tags; do echo -n " --cache-from $ECR/$REPO:$tag"; done) echo for tag in latest $tags; do echo "* Tagging as '$tag'" docker tag "$ECR/$REPO:$hashref" "$ECR/$REPO:$tag" echo done # pushing latest last intentionally for a more atomic update for tag in "$hashref" $tags latest; do echo "* Pushing tag '$tag'" docker push "$ECR/$REPO:$tag" echo done ================================================ FILE: aws/aws_ecr_docker_login.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-12-17 02:39:54 +0700 (Tue, 17 Dec 2024) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Authenticates Docker to AWS ECR, inferring the ECR registry from the current AWS Account ID and Region If \$AWS_ACCOUNT_ID and \$AWS_DEFAULT_REGION are not set in the environment, tries to infer them from the current AWS config $usage_aws_cli_required, and also Docker must be installed " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" max_args 1 "$@" export AWS_ACCOUNT_ID="${AWS_ACCOUNT_ID:-$(aws_account_id)}" export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-$(aws_region)}" aws_ecr_registry="${1:-$(aws_ecr_registry)}" if ! is_aws_ecr_registry "$aws_ecr_registry"; then die "Invalid AWS ECR registry: $aws_ecr_registry" fi timestamp "Getting AWS ECR Login password and piping it into Docker for registry: $aws_ecr_registry" aws ecr get-login-password | docker login --username AWS --password-stdin "$aws_ecr_registry" ================================================ FILE: aws/aws_ecr_list_repos.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2024-07-02 19:54:54 +0200 (Tue, 02 Jul 2024) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists ECR repos, their docker image mutability and whether image scanning is enabled Output: $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="" help_usage "$@" num_args 0 "$@" aws ecr describe-repositories | jq -r ' .repositories[] | [ .repositoryName, .imageTagMutability, .imageScanningConfiguration.scanOnPush ] | @tsv ' | sort | column -t ================================================ FILE: aws/aws_ecr_list_tags.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists all tags for the given AWS ECR docker image Each tag for the given image is output on a separate line for easy further piping and filtering $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" image="$1" shift || : aws ecr list-images --repository "$image" "$@" | jq -r '.imageIds[].imageTag | select(.)' | sort ================================================ FILE: aws/aws_ecr_newest_image_tags.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists the tags for the given AWS ECR docker image with the newest creation date (eg. for tagging it as 'latest', see adjacent scripts aws_ecr_tag_latest.sh and aws_ecr_tag_newest_image_as_latest.sh) When a docker image has multiple tags (eg. v1, latest) then outputs each tag on a separate line for easy further piping and filtering $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" image="$1" shift || : newest_image_timestamp="$(aws ecr describe-images --repository-name "$image" "$@" | jq -r '.imageDetails[].imagePushedAt' | sort -r | head -n1)" aws ecr describe-images --repository-name "$image" "$@" | jq -r ".imageDetails[]? | select(.imagePushedAt == \"$newest_image_timestamp\") | .imageTags[]?" | sort ================================================ FILE: aws/aws_ecr_tag_branch.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Tags a given AWS ECR docker image:tag with the current branch name without pulling and pushing the docker image $usage_aws_cli_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" min_args 1 "$@" image_tag="$1" shift || : if ! [[ "$image_tag" =~ : ]]; then image_tag+=":latest" fi docker_image="${image_tag%%:*}" tag="${image_tag##*:}" # Jenkins provides GIT_BRANCH, TeamCity doesn't so normalize and determine it if not automatically set if [ -z "${BRANCH_NAME:-}" ]; then BRANCH_NAME="${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" fi BRANCH_NAME="${BRANCH_NAME##*/}" FORCE=1 "$srcdir/aws_ecr_tag_image.sh" "$docker_image:$tag" "$BRANCH_NAME" "$@" ================================================ FILE: aws/aws_ecr_tag_datetime.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Tags a given ECR docker image with it's creation Date and Timestamp without pulling and pushing the docker image The timestamp is the created time (either uploaded or created by Google Cloud Build) Tags are in the format: YYYY-MM-DD YYYY-MM-DDTHHMMSSZ (standard ISO UTC time without semi-colons which are invalid in docker tags) The timestamp will be normalized to UTC $usage_aws_cli_jq_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" num_args 1 "$@" image_tag="$1" shift || : if ! [[ "$image_tag" =~ : ]]; then image_tag+=":latest" fi image="${image_tag%%:*}" tag="${image_tag##*:}" if [ -z "$tag" ] || [ "$tag" = "$image" ] || [ "$tag" = "$image_tag" ]; then tag="latest" fi timestamp="$(aws ecr describe-images --repository-name "$image" --image-ids "imageTag=$tag" "$@" | jq -r '.imageDetails[].imagePushedAt' | sort -r | head -n1)" if [ -z "$timestamp" ]; then echo "Failed to determine timestamp from ECR for image '$image' with tag '$tag'" exit 1 fi if ! [[ "$timestamp" =~ ^[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}T[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}[+-][[:digit:]]{2}:[[:digit:]]{2}$ ]]; then echo "ECR timestamp not in expect YYYY-MM-DDTHH:MM:SS[+-]HH:MM format, API may have changed" exit 1 fi # normalize to UTC timestamp="$(date --utc --date="$timestamp" '+%FT%H%M%SZ')" date="${timestamp%T*}" FORCE=1 "$srcdir/aws_ecr_tag_image.sh" "$image:$tag" "$date" "$@" FORCE=1 "$srcdir/aws_ecr_tag_image.sh" "$image:$tag" "$timestamp" "$@" ================================================ FILE: aws/aws_ecr_tag_image.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2020-09-28 11:45:38 +0100 (Mon, 28 Sep 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-retag.html set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Tags an AWS ECR image with another tag without pulling + pushing the image If : isn't given, assumes 'latest' If the environment variable FORCE is set, will remove the new tag reference to ensure the new tagging takes effect $usage_aws_cli_jq_required Similar scripts: aws_ecr_tag_image_by_digest.sh - same as this script but locates the image to tag using a digest docker_registry_tag_image.sh - for private Docker Registries gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" min_args 2 "$@" image_tag="$1" new_tag="$2" shift || : shift || : image="${image_tag%%:*}" tag="${image_tag##*:}" if ! [[ "$image_tag" =~ : ]] && [ "$tag" = "$image" ]; then tag="latest" fi tstamp "getting manifest for image '$image:$tag'" manifest="$(aws ecr batch-get-image --repository-name "$image" --image-ids "imageTag=$tag" --query 'images[].imageManifest' --output text "$@")" if is_blank "$manifest"; then die "ERROR: no manifest returned, did you specify a valid image tag?" fi if [ -n "${FORCE:-}" ]; then "$srcdir/aws_ecr_delete_tag.sh" "$image:$new_tag" "$@" >/dev/null || : fi tstamp "tagging image '$image:$tag' with new tag '$new_tag'" aws ecr put-image --repository-name "$image" --image-tag "$new_tag" --image-manifest "$manifest" "$@" >/dev/null tstamp "tags for image '$image:$tag' are now:" aws ecr describe-images --repository-name "$image" --output json "$@" | jq -r ".imageDetails[] | select(.imageTags) | select(.imageTags[] == \"$tag\") | .imageTags[]" ================================================ FILE: aws/aws_ecr_tag_image_by_digest.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2020-09-28 11:45:38 +0100 (Mon, 28 Sep 2020) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # # https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-retag.html set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Tags an AWS ECR image digest with another tag without pulling + pushing the image Useful to recover an image and apply a new tag to it or if you want to more precisely tag an exact image than following another existing tag (which is usually easier but can be a moving target) If the environment variable FORCE is set, will remove the new tag reference to ensure the new tagging takes effect $usage_aws_cli_jq_required Similar scripts: aws_ecr_tag_image.sh - same as this script but locates the image using an existing tag docker_registry_tag_image.sh - for private Docker Registries gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 3 "$@" image="$1" digest="$2" new_tag="$3" if ! [[ "$digest" =~ : ]]; then digest="sha256:$digest" fi tstamp "getting manifest for image '$image' with digest '$digest'" manifest="$(aws ecr batch-get-image --repository-name "$image" --image-ids "imageDigest=$digest" --query 'images[].imageManifest' --output text "$@")" if is_blank "$manifest"; then die "ERROR: no manifest returned, did you specify a valid digest?" fi if [ -n "${FORCE:-}" ]; then "$srcdir/aws_ecr_delete_tag.sh" "$image:$new_tag" "$@" >/dev/null || : fi tstamp "tagging image '$image' with digest '$digest' with new tag '$new_tag'" aws ecr put-image --repository-name "$image" --image-tag "$new_tag" --image-manifest "$manifest" "$@" >/dev/null tstamp "tags for image '$image' with digest '$digest' are now:" aws ecr describe-images --repository-name "$image" --output json "$@" | jq -r ".imageDetails[] | select(.imageDigest) | select(.imageDigest == \"$digest\") | .imageTags[]" ================================================ FILE: aws/aws_ecr_tag_latest.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Tags a given AWS ECR docker image:tag as 'latest' without pulling and pushing the docker image $usage_aws_cli_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=": []" help_usage "$@" min_args 1 "$@" image_tag="$1" shift || : if ! [[ "$image_tag" =~ : ]]; then usage "tag suffix missing from docker image" fi docker_image="${image_tag%%:*}" tag="${image_tag##*:}" if is_blank "$tag"; then usage "tag suffix is blank" fi FORCE=1 "$srcdir/aws_ecr_tag_image.sh" "$docker_image:$tag" "latest" "$@" ================================================ FILE: aws/aws_ecr_tag_newest_as_latest.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Finds the newest build of a given AWS ECR docker image by creation date and tags it as 'latest' Does this via metadata API calls to avoid network transfer from any docker pull / docker push If an AWS ECR image has multiple tags, will take the longest tag which is assumed to be the most specific and therefore most likely to avoid collisions and race conditions of other tag updates happening concurrently $usage_aws_cli_required Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" image="$1" shift || : tags="$("$srcdir/aws_ecr_newest_image_tags.sh" "$image" "$@")" if [ -z "$tags" ]; then die "No tags were found for image '$image'... does it exist in ECR?" fi longest_tag="$(awk '{print length, $0}' <<< "$tags" | sort -nr | head -n 1 | awk '{print $2}')" "$srcdir/aws_ecr_tag_latest.sh" "$image:$longest_tag" "$@" ================================================ FILE: aws/aws_ecr_tags_old.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists old tags for a given AWS ECR image > \$days old The \$days threshold defaults to (365 * 2) ie. 2 years old You can grep and pipe this output to | xargs -L1 aws_ecr_delete_tag.sh to clean out old CI image builds to save S3 storage costs on old CI images you no longer use $usage_aws_cli_jq_required See Also: aws_ecr_tags_timestamps.sh - lists tags and timestamps - useful for comparing with the output from this script aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" [ ]" help_usage "$@" min_args 1 "$@" image="$1" # 2 years old images by default days_threshold="${2:-$((365 * 2))}" shift || : shift || : date_threshold="$(date '+%FT%T+00:00' --utc --date="$days_threshold days ago")" aws ecr describe-images --repository-name "$image" "$@" | jq -r ".imageDetails[] | select(.imagePushedAt < \"$date_threshold\") | [\"$image\" + \":\" + .imageTags[]] | .[]" ================================================ FILE: aws/aws_ecr_tags_timestamps.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # args: haritest:1.0 stable # # Author: Hari Sekhon # Date: 2021-12-10 11:53:32 +0000 (Fri, 10 Dec 2021) # # https://github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists all tags for the given AWS ECR docker image as well as their image upload timestamp, newest first Output Format: Each timestamp and tag for the given image is output tab separated on a separate line for easy further piping and filtering $usage_aws_cli_jq_required If you want to remove an extra tag from an existing image: aws ecr batch-delete-image --repository-name --image-ids \"imageTag=\" Similar scripts: aws_ecr_*.sh - scripts for AWS Elastic Container Registry gcr_*.sh - scripts for Google Container Registry " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args=" []" help_usage "$@" min_args 1 "$@" image="$1" shift || : aws ecr describe-images --repository "$image" "$@" | jq -r '.imageDetails | map(.imagePushedAt + "\t" + .imageTags[]?) | .[]' | sort -r ================================================ FILE: aws/aws_eks_addon_versions.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-01 01:52:09 +0700 (Wed, 01 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Lists the EKS addon versions available for the given cluster by checking its version before checking addons Requires either first arg of the EKS cluster name, or the environment variable \$EKS_CLUSTER If neither are given, checks clusters and if only one is found in account, uses that $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="[]" help_usage "$@" max_args 1 "$@" cluster="${1:-${EKS_CLUSTER:-}}" if is_blank "$cluster"; then cluster="$(aws_eks_cluster_if_only_one)" if ! is_blank "$cluster"; then timestamp "No cluster specified but only one found in this account, using that: $cluster" else usage "Need to define cluster name" fi fi timestamp "Getting cluster version for: $cluster" cluster_version="$(aws eks describe-cluster --name "$cluster" --query "cluster.version" --output text)" timestamp "Cluster version: $cluster_version" aws eks describe-addon-versions \ --kubernetes-version "$cluster_version" \ --addon-name vpc-cni \ --query 'addons[].addonVersions[].{Version: addonVersion, Defaultversion: compatibilities[0].defaultVersion}' \ --output table ================================================ FILE: aws/aws_eks_ami_create.sh ================================================ #!/usr/bin/env bash # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2025-01-31 16:05:24 +0700 (Fri, 31 Jan 2025) # # https///github.com/HariSekhon/DevOps-Bash-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # # https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090,SC1091 . "$srcdir/lib/aws.sh" # shellcheck disable=SC2034,SC2154 usage_description=" Create a custom EKS AMI quickly off the base EKS template and then running a shell script in it before saving it to a new AMI - finds the standard EKS AMI for the given version - checks if there is already an EC2 running instance tagged for this - boots an EC2 instance from the above AMI with the given security group, subnet id and SSH key-name given so you can SSH to its ec2-user - waits for the EC2 instance to boot - waits for the EC2 instance to pass its instance and system checks - determines the public or private IP address - scp's the local script to the instance /tmp - SSH's to execute the script (eg. to install the needed things) - Assumes ~/.ssh/,pem is present locally to be able to log in to it - Creates the AMI - Terminates the EC2 instance Uses adjacent script: aws_ec2_ami_boot.sh You should really use Packer instead, see https://github.com/HariSekhon/Packer But this script is an alternative and allowed me to debug something in a pinch $usage_aws_cli_required " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 usage_args="