Repository: open-telemetry/opentelemetry-network Branch: main Commit: 0a078f520732 Files: 972 Total size: 4.4 MB Directory structure: gitextract_vvvufxro/ ├── .clang-format ├── .devcontainer/ │ └── devcontainer.json ├── .git/ │ ├── HEAD │ ├── config │ ├── description │ ├── hooks/ │ │ ├── applypatch-msg.sample │ │ ├── commit-msg.sample │ │ ├── fsmonitor-watchman.sample │ │ ├── post-update.sample │ │ ├── pre-applypatch.sample │ │ ├── pre-commit.sample │ │ ├── pre-merge-commit.sample │ │ ├── pre-push.sample │ │ ├── pre-rebase.sample │ │ ├── pre-receive.sample │ │ ├── prepare-commit-msg.sample │ │ ├── push-to-checkout.sample │ │ ├── sendemail-validate.sample │ │ └── update.sample │ ├── index │ ├── info/ │ │ └── exclude │ ├── logs/ │ │ ├── HEAD │ │ └── refs/ │ │ ├── heads/ │ │ │ └── main │ │ └── remotes/ │ │ └── origin/ │ │ └── HEAD │ ├── objects/ │ │ └── pack/ │ │ ├── pack-71fb82fb82ed2e298be25eaa90192ef76cf2859f.idx │ │ ├── pack-71fb82fb82ed2e298be25eaa90192ef76cf2859f.pack │ │ ├── pack-71fb82fb82ed2e298be25eaa90192ef76cf2859f.promisor │ │ └── pack-71fb82fb82ed2e298be25eaa90192ef76cf2859f.rev │ ├── packed-refs │ ├── refs/ │ │ ├── heads/ │ │ │ └── main │ │ └── remotes/ │ │ └── origin/ │ │ └── HEAD │ └── shallow ├── .git-blame-ignore-revs ├── .github/ │ ├── .actionlint.yaml │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yaml │ │ ├── config.yml │ │ ├── feature_request.yaml │ │ └── other.yaml │ ├── actions/ │ │ └── build-tools-single-stage/ │ │ └── action.yml │ ├── pull_request_template.md │ ├── renovate.json5 │ └── workflows/ │ ├── build-and-release.yaml │ ├── build-and-test.yaml │ ├── build-benv-multiarch.yaml │ ├── build-benv-single-arch.yaml │ ├── fossa.yml │ ├── k8s-collector-integration.yaml │ ├── ossf-scorecard.yml │ ├── release-please.yml │ ├── scripts/ │ │ └── check-clang-format.sh │ └── trivy-scans.yml ├── .gitignore ├── .gitmodules ├── .release-please-manifest.json ├── CHANGELOG.md ├── CMakeLists.txt ├── Cargo.toml ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── RELEASING.md ├── VERSION ├── build-tools/ │ ├── .gitignore │ ├── .templates/ │ │ └── dependency/ │ │ └── Dockerfile │ ├── CMakeLists.txt │ ├── add_dependency.sh │ ├── aws_sdk/ │ │ └── Dockerfile │ ├── base/ │ │ └── Dockerfile │ ├── benv/ │ │ └── docker.d/ │ │ ├── cgroups │ │ ├── gdb │ │ ├── kernel-headers │ │ ├── pid-host │ │ ├── privileged │ │ └── vimrc │ ├── build.sh │ ├── build_directory.sh │ ├── check_missing.sh │ ├── cpp_misc/ │ │ ├── Dockerfile │ │ └── ccan/ │ │ ├── build_assert/ │ │ │ ├── _info │ │ │ ├── build_assert.h │ │ │ └── test/ │ │ │ ├── compile_fail-expr.c │ │ │ ├── compile_fail.c │ │ │ ├── compile_ok.c │ │ │ └── run-BUILD_ASSERT_OR_ZERO.c │ │ ├── check_type/ │ │ │ ├── _info │ │ │ ├── check_type.h │ │ │ └── test/ │ │ │ ├── compile_fail-check_type.c │ │ │ ├── compile_fail-check_type_unsigned.c │ │ │ ├── compile_fail-check_types_match.c │ │ │ └── run.c │ │ ├── compiler/ │ │ │ ├── _info │ │ │ ├── compiler.h │ │ │ └── test/ │ │ │ ├── compile_fail-printf.c │ │ │ └── run-is_compile_constant.c │ │ ├── container_of/ │ │ │ ├── _info │ │ │ ├── container_of.h │ │ │ └── test/ │ │ │ ├── compile_fail-bad-type.c │ │ │ ├── compile_fail-types.c │ │ │ ├── compile_fail-var-types.c │ │ │ └── run.c │ │ ├── hash/ │ │ │ ├── _info │ │ │ ├── hash.c │ │ │ ├── hash.h │ │ │ └── test/ │ │ │ ├── api-hash_stable.c │ │ │ └── run.c │ │ ├── licenses/ │ │ │ ├── BSD-MIT │ │ │ └── CC0 │ │ └── list/ │ │ ├── _info │ │ ├── list.c │ │ ├── list.h │ │ └── test/ │ │ ├── compile_ok-constant.c │ │ ├── helper.c │ │ ├── helper.h │ │ ├── run-check-corrupt.c │ │ ├── run-list_del_from-assert.c │ │ ├── run-list_prev-list_next.c │ │ ├── run-prepend_list.c │ │ ├── run-single-eval.c │ │ ├── run-with-debug.c │ │ └── run.c │ ├── final/ │ │ ├── Dockerfile │ │ ├── LICENSE.txt │ │ └── NOTICE.txt │ ├── get_tag.sh │ ├── libbpf/ │ │ └── Dockerfile │ ├── libmaxminddb/ │ │ └── Dockerfile │ ├── libuv/ │ │ └── Dockerfile │ └── nproc.sh ├── channel/ │ ├── CMakeLists.txt │ ├── buffered_writer.cc │ ├── buffered_writer.h │ ├── buffered_writer_test.cc │ ├── callbacks.h │ ├── channel.h │ ├── component.h │ ├── connection_caretaker.cc │ ├── connection_caretaker.h │ ├── double_write_channel.cc │ ├── double_write_channel.h │ ├── file_channel.cc │ ├── file_channel.h │ ├── ibuffered_writer.h │ ├── lz4_channel.cc │ ├── lz4_channel.h │ ├── mock_channel.h │ ├── network_channel.h │ ├── reconnecting_channel.cc │ ├── reconnecting_channel.h │ ├── tcp_channel.cc │ ├── tcp_channel.h │ ├── test_channel.cc │ ├── test_channel.h │ ├── tls_handler.cc │ ├── tls_handler.h │ ├── upstream_connection.cc │ └── upstream_connection.h ├── clang-format.sh ├── cmake/ │ ├── abseil.cmake │ ├── aws-sdk.cmake │ ├── cargo-test.cmake │ ├── cargo_build_rust.cmake │ ├── ccache.cmake │ ├── civetweb.cmake │ ├── clang.cmake │ ├── cpp-compiler.cmake │ ├── curl.cmake │ ├── curlpp.cmake │ ├── debug.cmake │ ├── defaults.cmake │ ├── docker-utils.cmake │ ├── executable.cmake │ ├── geoip.cmake │ ├── libbpf.cmake │ ├── libelf.cmake │ ├── llvm.cmake │ ├── lz4.cmake │ ├── openssl.cmake │ ├── protobuf.cmake │ ├── render.cmake │ ├── rust_cxxbridge.cmake │ ├── rust_main.cmake │ ├── sanitizer.cmake │ ├── shell.cmake │ ├── spdlog.cmake │ ├── test.cmake │ ├── tool.cmake │ ├── uv.cmake │ ├── xxd.cmake │ └── yamlcpp.cmake ├── collector/ │ ├── CMakeLists.txt │ ├── Dockerfile.k8s-collector │ ├── agent_log.h │ ├── cloud/ │ │ ├── CMakeLists.txt │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── collector.cc │ │ ├── collector.h │ │ ├── entrypoint.cc │ │ ├── entrypoint.sh │ │ ├── enumerator.cc │ │ ├── enumerator.h │ │ ├── ingest_connection.cc │ │ └── ingest_connection.h │ ├── constants.h │ ├── kernel/ │ │ ├── CMakeLists.txt │ │ ├── Dockerfile │ │ ├── bpf_handler.cc │ │ ├── bpf_handler.h │ │ ├── bpf_src/ │ │ │ ├── render_bpf.c │ │ │ ├── render_bpf.h │ │ │ ├── tcp-processor/ │ │ │ │ ├── bpf_data_channel.h │ │ │ │ ├── bpf_debug.h │ │ │ │ ├── bpf_http_protocol.h │ │ │ │ ├── bpf_inet_csk_accept.h │ │ │ │ ├── bpf_memory.h │ │ │ │ ├── bpf_tcp_events.h │ │ │ │ ├── bpf_tcp_processor.c │ │ │ │ ├── bpf_tcp_send_recv.h │ │ │ │ ├── bpf_tcp_socket.h │ │ │ │ ├── bpf_types.h │ │ │ │ ├── tcp-processor.py │ │ │ │ └── tcp_processor.h │ │ │ ├── vmlinux_compat.h │ │ │ └── vmlinux_extensions.h │ │ ├── buffered_poller.cc │ │ ├── buffered_poller.h │ │ ├── cgroup_handler.cc │ │ ├── cgroup_handler.h │ │ ├── cgroup_handler_test.cc │ │ ├── cgroup_prober.cc │ │ ├── cgroup_prober.h │ │ ├── dns/ │ │ │ ├── ares.h │ │ │ ├── ares_expand_name.c │ │ │ ├── ares_parse_a_aaaa_reply.c │ │ │ ├── ares_parse_query.c │ │ │ ├── c_ares_nameser.h │ │ │ └── dns.h │ │ ├── dns_requests.cc │ │ ├── dns_requests.h │ │ ├── entrypoint-kct.sh │ │ ├── entrypoint.cc │ │ ├── entrypoint.sh │ │ ├── fd_reader.cc │ │ ├── fd_reader.h │ │ ├── hostport_tuple.h │ │ ├── kernel_blacklist.h │ │ ├── kernel_collector.cc │ │ ├── kernel_collector.h │ │ ├── kernel_collector_restarter.cc │ │ ├── kernel_collector_restarter.h │ │ ├── kernel_collector_test.cc │ │ ├── kernel_collector_test_docker/ │ │ │ └── Dockerfile │ │ ├── kernel_symbols.cc │ │ ├── kernel_symbols.h │ │ ├── kernel_symbols_test.cc │ │ ├── main.cc │ │ ├── nat_handler.cc │ │ ├── nat_handler.h │ │ ├── nat_prober.cc │ │ ├── nat_prober.h │ │ ├── perf_poller.cc │ │ ├── perf_poller.h │ │ ├── perf_reader.cc │ │ ├── perf_reader.h │ │ ├── probe_handler.cc │ │ ├── probe_handler.h │ │ ├── proc_cmdline.cc │ │ ├── proc_cmdline.h │ │ ├── proc_net_reader.cc │ │ ├── proc_net_reader.h │ │ ├── proc_reader.cc │ │ ├── proc_reader.h │ │ ├── process_handler.cc │ │ ├── process_handler.h │ │ ├── process_prober.cc │ │ ├── process_prober.h │ │ ├── protocols/ │ │ │ ├── protocol_handler_base.cc │ │ │ ├── protocol_handler_base.h │ │ │ ├── protocol_handler_http.cc │ │ │ ├── protocol_handler_http.h │ │ │ ├── protocol_handler_unknown.cc │ │ │ ├── protocol_handler_unknown.h │ │ │ └── protocol_tools.h │ │ ├── socket_prober.cc │ │ ├── socket_prober.h │ │ ├── socket_table.h │ │ ├── tcp_data_handler.cc │ │ ├── tcp_data_handler.h │ │ ├── troubleshoot_item.h │ │ ├── troubleshooting.cc │ │ └── troubleshooting.h │ └── server_command.h ├── common/ │ ├── client_server_type.h │ ├── client_type.h │ ├── cloud_platform.h │ ├── collected_blob_type.h │ ├── collector_status.h │ ├── component.h │ ├── constants.h │ ├── host_info.h │ ├── http_status_code.h │ ├── intake_encoder.h │ ├── kernel_headers_source.h │ ├── linux_distro.h │ ├── operating_system.h │ └── port_protocol.h ├── config/ │ ├── CMakeLists.txt │ ├── config_file.cc │ ├── config_file.h │ ├── intake_config.cc │ └── intake_config.h ├── config.h.cmake_in ├── crates/ │ ├── build/ │ │ └── otn_link_build.rs │ ├── cloud-collector-bin/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── cloud-collector-sys/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src/ │ │ └── lib.rs │ ├── element-queue/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── layout.rs │ │ ├── lib.rs │ │ └── raw.rs │ ├── k8s-collector/ │ │ ├── Cargo.toml │ │ ├── src/ │ │ │ ├── collector.rs │ │ │ ├── config.rs │ │ │ ├── convert_to_meta.rs │ │ │ ├── encode.rs │ │ │ ├── lib.rs │ │ │ ├── matcher.rs │ │ │ ├── output.rs │ │ │ ├── tombstone_adapter.rs │ │ │ ├── types.rs │ │ │ └── writer.rs │ │ └── tests/ │ │ ├── collector_prop.rs │ │ └── integration_test.rs │ ├── k8s-collector-bin/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── k8s-relay-bin/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── k8s-relay-sys/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src/ │ │ └── lib.rs │ ├── kernel-collector-bin/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── kernel-collector-sys/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src/ │ │ └── lib.rs │ ├── otlp_export/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── lib.rs │ ├── perfect_hash_map/ │ │ ├── Cargo.toml │ │ ├── src/ │ │ │ └── lib.rs │ │ └── tests/ │ │ ├── basic.rs │ │ ├── drops.rs │ │ └── prop.rs │ ├── reducer/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── aggregation_core.rs │ │ ├── aggregation_framework.rs │ │ ├── aggregation_message_handler.rs │ │ ├── aggregator.rs │ │ ├── ffi.rs │ │ ├── internal_events.rs │ │ ├── lib.rs │ │ ├── metrics.rs │ │ ├── otlp_encoding.rs │ │ └── queue_handler.rs │ ├── reducer-bin/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── reducer-sys/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src/ │ │ └── lib.rs │ ├── render/ │ │ ├── ebpf_net/ │ │ │ ├── Cargo.toml │ │ │ ├── agent_internal/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── aggregation/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── cloud_collector/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── ingest/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── kernel_collector/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── logging/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ ├── matching/ │ │ │ │ ├── Cargo.toml │ │ │ │ └── src/ │ │ │ │ ├── encoder.rs │ │ │ │ ├── hash.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── parsed_message.rs │ │ │ │ └── wire_messages.rs │ │ │ └── src/ │ │ │ └── lib.rs │ │ └── test/ │ │ ├── Cargo.toml │ │ ├── app1/ │ │ │ ├── Cargo.toml │ │ │ └── src/ │ │ │ ├── encoder.rs │ │ │ ├── hash.rs │ │ │ ├── lib.rs │ │ │ ├── parsed_message.rs │ │ │ └── wire_messages.rs │ │ └── src/ │ │ └── lib.rs │ ├── render_parser/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── lib.rs │ │ └── message.rs │ └── timeslot/ │ ├── Cargo.toml │ └── src/ │ ├── fast_div.rs │ ├── lib.rs │ └── virtual_clock.rs ├── dev/ │ ├── CMakeLists.txt │ ├── benv-build.sh │ ├── benv-list.sh │ ├── benv-run.sh │ ├── commits_to_prs.sh │ ├── commits_to_stacked_prs.sh │ ├── devbox/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── boxes/ │ │ │ ├── .gitignore │ │ │ ├── centos-7/ │ │ │ │ ├── .gitignore │ │ │ │ └── build.sh │ │ │ ├── debian-bullseye/ │ │ │ │ ├── .gitignore │ │ │ │ └── build.sh │ │ │ ├── ubuntu-focal/ │ │ │ │ ├── .gitignore │ │ │ │ └── build.sh │ │ │ ├── ubuntu-jammy/ │ │ │ │ ├── .gitignore │ │ │ │ └── build.sh │ │ │ ├── ubuntu-lunar/ │ │ │ │ ├── .gitignore │ │ │ │ └── build.sh │ │ │ └── vagrant-base-boxes/ │ │ │ ├── base-bento-amazonlinux-2/ │ │ │ │ ├── .gitignore │ │ │ │ └── run.sh │ │ │ ├── base-centos-7/ │ │ │ │ ├── .gitignore │ │ │ │ └── run.sh │ │ │ ├── base-debian-bullseye/ │ │ │ │ ├── .gitignore │ │ │ │ └── run.sh │ │ │ ├── base-ubuntu-focal/ │ │ │ │ ├── .gitignore │ │ │ │ └── run.sh │ │ │ ├── base-ubuntu-jammy/ │ │ │ │ ├── .gitignore │ │ │ │ └── run.sh │ │ │ └── base-ubuntu-lunar/ │ │ │ ├── .gitignore │ │ │ └── run.sh │ │ ├── build.sh │ │ ├── run.sh │ │ └── source/ │ │ ├── .rgrc │ │ ├── Vagrantfile.packer.box.rb │ │ ├── cloud-collector.sh │ │ ├── collector-entrypoint.sh │ │ ├── devbox.packer.json │ │ ├── k8s/ │ │ │ ├── deploy.sh │ │ │ ├── ebpf-net-local-registry.yaml │ │ │ ├── ebpf-net-logging-exporter.yaml │ │ │ ├── ebpf-net-modify-otelcol-metricstransform-processor.yaml │ │ │ ├── ebpf-net-modify-otelcol-splunk-hec-exporter.yaml │ │ │ ├── ebpf-net-modify-reducer-enable-flow-logs.yaml │ │ │ ├── ebpf-net-modify-reducer.yaml │ │ │ ├── ebpf-net-use-otel-demo-otelcol.yaml │ │ │ ├── ebpf-net.yaml │ │ │ ├── init.sh │ │ │ ├── modify.sh │ │ │ └── otel-demo.yaml │ │ ├── k8s-collector.sh │ │ ├── kernel-collector.sh │ │ ├── otelcol-config.yaml │ │ ├── otelcol-gateway.sh │ │ ├── prometheus.yml │ │ ├── provision/ │ │ │ ├── core.sh │ │ │ ├── docker.sh │ │ │ ├── k8s.sh │ │ │ ├── kernel.sh │ │ │ ├── packages.sh │ │ │ ├── prometheus.sh │ │ │ ├── repo.sh │ │ │ ├── symlinks.sh │ │ │ └── upgrade.sh │ │ ├── reducer.sh │ │ └── test-kernel-collector.sh │ ├── docker-registry-login.sh │ ├── docker-registry-push.sh │ ├── docker-registry.sh │ ├── git-pull-request.sh │ ├── git-upstream-branch.sh │ ├── otel/ │ │ ├── otel-config.yaml │ │ ├── run-ncat.sh │ │ └── run-otel.sh │ ├── script/ │ │ ├── CMakeLists.txt │ │ ├── bash-error-lib.sh │ │ ├── benv-lib.sh │ │ └── docker-registry-lib.sh │ ├── selinux-bpf.sh │ └── strip-symbols.sh ├── dist/ │ ├── CMakeLists.txt │ ├── cloud-collector/ │ │ ├── cloud-collector.args │ │ ├── cloud-collector.service │ │ ├── cloud-collector.yaml │ │ ├── deb/ │ │ │ ├── conffiles │ │ │ ├── postinst │ │ │ ├── postrm │ │ │ └── prerm │ │ └── rpm/ │ │ ├── post.sh │ │ ├── postun.sh │ │ └── preun.sh │ ├── kernel-collector/ │ │ ├── deb/ │ │ │ ├── conffiles │ │ │ ├── postinst │ │ │ ├── postrm │ │ │ └── prerm │ │ ├── kernel-collector.args │ │ ├── kernel-collector.service │ │ ├── kernel-collector.yaml │ │ └── rpm/ │ │ ├── post.sh │ │ ├── postun.sh │ │ └── preun.sh │ └── reducer/ │ ├── deb/ │ │ ├── conffiles │ │ ├── postinst │ │ ├── postrm │ │ └── prerm │ ├── reducer.args │ ├── reducer.service │ ├── reducer.yaml │ └── rpm/ │ ├── post.sh │ ├── postun.sh │ └── preun.sh ├── docs/ │ ├── cloud-collector.md │ ├── data-model.md │ ├── developing.md │ ├── dns-and-http.md │ ├── k8s-collector.md │ ├── kernel-collector.md │ ├── metrics/ │ │ ├── dimensions.yaml │ │ ├── internal_metrics/ │ │ │ ├── dimensions.yaml │ │ │ └── metrics.yaml │ │ └── metrics.yaml │ ├── processes-and-cgroups.md │ ├── reducer/ │ │ └── architecture.md │ ├── reducer.md │ ├── render.md │ ├── roadmap.md │ ├── running-multiple-agents.md │ └── tcp-and-udp.md ├── geoip/ │ ├── CMakeLists.txt │ ├── geoip.cc │ ├── geoip.h │ └── geoip.inl ├── jitbuf/ │ ├── CMakeLists.txt │ ├── descriptor.h │ ├── descriptor_reader.cc │ ├── descriptor_reader.h │ ├── fixed_handler.c │ ├── fixed_handler.h │ ├── handler.cc │ ├── handler.h │ ├── jb.h │ ├── perfect_hash.h │ ├── service.h │ ├── transform_builder.cc │ ├── transform_builder.h │ ├── transformer.cc │ └── transformer.h ├── platform/ │ ├── CMakeLists.txt │ ├── bitops.h │ ├── debug.h │ ├── fastpass_linux.h │ ├── generic.h │ ├── linux-platform.h │ ├── memory.h │ ├── no-dpdk.h │ ├── platform.h │ ├── spin_lock.h │ ├── types.h │ ├── types_test.cc │ └── userspace-time.h ├── reducer/ │ ├── CMakeLists.txt │ ├── Dockerfile │ ├── aggregation/ │ │ ├── agg_core.cc │ │ ├── agg_core.h │ │ ├── agg_root_span.h │ │ ├── labels.h │ │ ├── labels.inl │ │ └── stat_counters.h │ ├── constants.h │ ├── constants.inl │ ├── copy_metrics.h │ ├── core.cc │ ├── core.h │ ├── core_base.h │ ├── core_base.inl │ ├── core_type.h │ ├── disabled_metrics.cc │ ├── disabled_metrics.h │ ├── disabled_metrics_test.cc │ ├── dns_cache.h │ ├── entrypoint.cc │ ├── entrypoint.h │ ├── entrypoint.sh │ ├── health_check.sh │ ├── ingest/ │ │ ├── agent_span.cc │ │ ├── agent_span.h │ │ ├── aws_network_interface_span.cc │ │ ├── aws_network_interface_span.h │ │ ├── cgroup_span.cc │ │ ├── cgroup_span.h │ │ ├── component.h │ │ ├── container_updater.h │ │ ├── flow_updater.cc │ │ ├── flow_updater.h │ │ ├── ingest_core.cc │ │ ├── ingest_core.h │ │ ├── ingest_worker.cc │ │ ├── ingest_worker.h │ │ ├── k8s_pod_span.cc │ │ ├── k8s_pod_span.h │ │ ├── npm_connection.cc │ │ ├── npm_connection.h │ │ ├── process_span.cc │ │ ├── process_span.h │ │ ├── shared_state.cc │ │ ├── shared_state.h │ │ ├── socket_span.cc │ │ ├── socket_span.h │ │ ├── tcp_server.cc │ │ ├── tcp_server.h │ │ ├── udp_socket_span.cc │ │ └── udp_socket_span.h │ ├── internal_metrics_encoder.h │ ├── internal_stats.h │ ├── json_formatter.cc │ ├── json_formatter.h │ ├── latency_accumulator.h │ ├── latency_accumulator.inl │ ├── load_balancer.h │ ├── logging/ │ │ ├── agg_core_stats_span.cc │ │ ├── agg_core_stats_span.h │ │ ├── component.h │ │ ├── connection_metrics.h │ │ ├── core_stats_span.cc │ │ ├── core_stats_span.h │ │ ├── ingest_core_stats_span.cc │ │ ├── ingest_core_stats_span.h │ │ ├── logger_span.cc │ │ ├── logger_span.h │ │ ├── logging_core.cc │ │ └── logging_core.h │ ├── matching/ │ │ ├── aws_enrichment_info.h │ │ ├── aws_enrichment_span.cc │ │ ├── aws_enrichment_span.h │ │ ├── component.h │ │ ├── flow_span.cc │ │ ├── flow_span.h │ │ ├── k8s_container_span.cc │ │ ├── k8s_container_span.h │ │ ├── k8s_pod_span.cc │ │ ├── k8s_pod_span.h │ │ ├── matching_core.cc │ │ └── matching_core.h │ ├── metric_info.cc │ ├── metric_info.h │ ├── null_publisher.cc │ ├── null_publisher.h │ ├── otlp_grpc_formatter.cc │ ├── otlp_grpc_formatter.h │ ├── otlp_grpc_publisher.cc │ ├── otlp_grpc_publisher.h │ ├── outbound_metrics.h │ ├── outbound_stats.h │ ├── prometheus_formatter.cc │ ├── prometheus_formatter.h │ ├── prometheus_handler.cc │ ├── prometheus_handler.h │ ├── prometheus_publisher.cc │ ├── prometheus_publisher.h │ ├── publisher.h │ ├── reducer.cc │ ├── reducer.h │ ├── reducer_config.cc │ ├── reducer_config.h │ ├── reducer_config.inl │ ├── rpc_queue_matrix.h │ ├── rpc_queue_matrix_test.cc │ ├── rpc_stats.cc │ ├── rpc_stats.h │ ├── rpc_stats.inl │ ├── stat_info.cc │ ├── stat_info.h │ ├── thread_safe_map.h │ ├── tsdb_format.h │ ├── tsdb_formatter.cc │ ├── tsdb_formatter.h │ ├── uid_key.cc │ ├── uid_key.h │ ├── util/ │ │ ├── CMakeLists.txt │ │ ├── blob_collector.cc │ │ ├── blob_collector.h │ │ ├── docker_image.h │ │ ├── docker_image.inl │ │ ├── index_dumper.cc │ │ ├── index_dumper.h │ │ ├── index_dumper.inl │ │ ├── signal_handler.cc │ │ ├── signal_handler.h │ │ ├── thread_ops.cc │ │ ├── thread_ops.h │ │ ├── time_tracker.cc │ │ ├── time_tracker.h │ │ ├── virtual_clock.cc │ │ ├── virtual_clock.h │ │ └── virtual_clock_test.cc │ ├── worker.cc │ ├── worker.h │ └── write_metrics.h ├── release-please-config.json ├── render/ │ ├── CMakeLists.txt │ └── ebpf_net.render ├── renderc/ │ ├── .gitignore │ ├── CMakeLists.txt │ ├── build.gradle │ ├── gradle/ │ │ ├── repositories.gradle │ │ ├── source-layout.gradle │ │ └── wrapper/ │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── io.opentelemetry.render/ │ │ ├── META-INF/ │ │ │ └── MANIFEST.MF │ │ ├── build.gradle │ │ └── src/ │ │ └── io/ │ │ └── opentelemetry/ │ │ └── render/ │ │ ├── GenerateRender.mwe2 │ │ ├── Render.xtext │ │ ├── RenderRuntimeModule.xtend │ │ ├── RenderStandaloneSetup.xtend │ │ ├── extensions/ │ │ │ ├── AppExtensions.xtend │ │ │ ├── FieldExtensions.xtend │ │ │ ├── FieldTypeExtensions.xtend │ │ │ ├── MessageExtensions.xtend │ │ │ ├── MetricFieldExtension.xtend │ │ │ ├── SpanExtensions.xtend │ │ │ ├── UtilityExtensions.xtend │ │ │ └── XPackedMessageExtensions.xtend │ │ ├── formatting/ │ │ │ └── RenderFormatter.xtend │ │ ├── generator/ │ │ │ ├── AppGenerator.xtend │ │ │ ├── AppPacker.xtend │ │ │ ├── BpfGenerator.xtend │ │ │ ├── ConnectionGenerator.xtend │ │ │ ├── HashGenerator.xtend │ │ │ ├── MessageGenerator.xtend │ │ │ ├── MetricsGenerator.xtend │ │ │ ├── PerfectHash.xtend │ │ │ ├── ProtocolGenerator.xtend │ │ │ ├── RenderGenerator.xtend │ │ │ ├── RustCargoGenerator.xtend │ │ │ ├── RustEncoderGenerator.xtend │ │ │ ├── RustMessageGenerator.xtend │ │ │ ├── SpanAutoDependencies.xtend │ │ │ ├── SpanGenerator.xtend │ │ │ ├── TransformBuilderGenerator.xtend │ │ │ └── WriterGenerator.xtend │ │ ├── scoping/ │ │ │ └── RenderScopeProvider.xtend │ │ └── validation/ │ │ └── RenderValidator.xtend │ ├── io.opentelemetry.render.standalone/ │ │ ├── build.gradle │ │ ├── plugin.properties │ │ └── src/ │ │ └── Main.xtend │ ├── settings.gradle │ └── test/ │ ├── CMakeLists.txt │ ├── render_test.cc │ └── test.render ├── scheduling/ │ ├── CMakeLists.txt │ ├── interval_scheduler.cc │ ├── interval_scheduler.h │ ├── job.h │ ├── timer.cc │ └── timer.h ├── scratchpad/ │ └── modules.md ├── test/ │ └── kernel/ │ ├── .gitignore │ ├── README.md │ ├── bootstrap.sh │ ├── distros-and-kernels.sh │ ├── gen-tests.sh │ ├── run-test.sh │ ├── run-tests.sh │ └── source/ │ ├── data/ │ │ ├── agent.sh │ │ ├── centos-provision.sh │ │ ├── debian-provision.sh │ │ ├── env-provision.sh │ │ ├── get-agent-shell.sh │ │ ├── reducer.sh │ │ ├── test-entrypoint.sh │ │ └── ubuntu-provision.sh │ └── runners/ │ ├── 0-setup.sh │ ├── 1-apply-selinux-policy.sh │ ├── 2-start-reducer.sh │ ├── 3-fetch.sh │ ├── 4-cached.sh │ ├── 5-pre-installed.sh │ ├── 6-cleanup.sh │ └── run-kernel-collector-test.sh ├── tools/ │ ├── CMakeLists.txt │ ├── aggregation_wire_to_json.cc │ ├── bpf_wire_to_json.cc │ ├── error_lookup.cc │ ├── intake_wire_to_json.cc │ └── matching_wire_to_json.cc ├── util/ │ ├── CMakeLists.txt │ ├── LRU.h │ ├── agent_id.cc │ ├── agent_id.h │ ├── args_parser.cc │ ├── args_parser.h │ ├── args_parser.inl │ ├── aws_instance_metadata.cc │ ├── aws_instance_metadata.h │ ├── base64.cc │ ├── base64.h │ ├── base64_test.cc │ ├── bits.h │ ├── bits_test.cc │ ├── boot_time.c │ ├── boot_time.h │ ├── buffer.h │ ├── cgroup_parser.cc │ ├── cgroup_parser.h │ ├── cgroup_parser_test.cc │ ├── circular_queue.h │ ├── circular_queue_cpp.h │ ├── code_timing.cc │ ├── code_timing.h │ ├── common_test.h │ ├── container_of.h │ ├── counter.h │ ├── counter.inl │ ├── counter_test.cc │ ├── curl_engine.cc │ ├── curl_engine.h │ ├── debug.h │ ├── defer.h │ ├── defer_test.cc │ ├── docker_host_config_metadata.cc │ ├── docker_host_config_metadata.h │ ├── docker_host_config_metadata.inl │ ├── element_queue.c │ ├── element_queue.h │ ├── element_queue_cpp.h │ ├── element_queue_writer.cc │ ├── element_queue_writer.h │ ├── enum.h │ ├── enum.inl │ ├── enum_operators.inl │ ├── enum_test.cc │ ├── environment_variables.cc │ ├── environment_variables.h │ ├── environment_variables.inl │ ├── error_handling.cc │ ├── error_handling.h │ ├── expected.h │ ├── expected_test.cc │ ├── fast_div.h │ ├── file_ops.cc │ ├── file_ops.h │ ├── fixed_hash.h │ ├── fixed_hash_test.cc │ ├── fmt_extensions.h │ ├── gauge.h │ ├── gauge.inl │ ├── gauge_test.cc │ ├── gcp_instance_metadata.cc │ ├── gcp_instance_metadata.h │ ├── histogram.h │ ├── ip_address.cc │ ├── ip_address.h │ ├── ip_address_test.cc │ ├── iterable_bitmap.h │ ├── jitter.h │ ├── jitter.inl │ ├── jitter_test.cc │ ├── json.h │ ├── json_converter.h │ ├── json_test.cc │ ├── k8s_metadata.cc │ ├── k8s_metadata.h │ ├── k8s_metadata.inl │ ├── lazy_array.h │ ├── log.cc │ ├── log.h │ ├── log_formatters.h │ ├── log_modifiers.h │ ├── log_modifiers_test.cc │ ├── log_whitelist.cc │ ├── log_whitelist.h │ ├── log_whitelist.inl │ ├── logger.h │ ├── lookup3.c │ ├── lookup3.h │ ├── lookup3_hasher.h │ ├── lookup3_hasher_test.cc │ ├── lz4_decompressor.cc │ ├── lz4_decompressor.h │ ├── meta.h │ ├── meta.inl │ ├── meta_test.cc │ ├── metric_store.h │ ├── nomad_metadata.cc │ ├── nomad_metadata.h │ ├── overloaded_visitor.h │ ├── parser_utils.h │ ├── perf_ring.c │ ├── perf_ring.h │ ├── perf_ring_cpp.h │ ├── pool.h │ ├── pool_allocator.c │ ├── pool_allocator.h │ ├── preprocessor.h │ ├── proc_io_view.inl │ ├── proc_ops.cc │ ├── proc_ops.h │ ├── proc_ops_test.cc │ ├── proc_stat_view.inl │ ├── proc_status_view.inl │ ├── process_state.h │ ├── protobuf_log.h │ ├── raii.h │ ├── random.cc │ ├── random.h │ ├── random.inl │ ├── random_test.cc │ ├── raw_json.h │ ├── render.h │ ├── resource_usage.h │ ├── resource_usage_reporter.cc │ ├── resource_usage_reporter.h │ ├── restful.cc │ ├── restful.h │ ├── restful.inl │ ├── short_string.h │ ├── signal_handler.cc │ ├── signal_handler.h │ ├── stop_watch.h │ ├── string.h │ ├── string.inl │ ├── string_view.h │ ├── string_view.inl │ ├── string_view_test.cc │ ├── system_ops.cc │ ├── system_ops.h │ ├── tdigest.cc │ ├── tdigest.h │ ├── tdigest_test.cc │ ├── time.cc │ ├── time.h │ ├── time_test.cc │ ├── utility.h │ ├── uv_helpers.cc │ ├── uv_helpers.h │ ├── version.cc │ ├── version.h │ └── version_config.h.cmake_in └── version.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .clang-format ================================================ BasedOnStyle: LLVM AlignAfterOpenBracket: AlwaysBreak AllowShortFunctionsOnASingleLine: Inline BinPackArguments: false BinPackParameters: false BraceWrapping: AfterFunction: true SplitEmptyFunction: false BreakBeforeBraces: Custom BreakStringLiterals: false ColumnLimit: 128 ConstructorInitializerAllOnOneLineOrOnePerLine: true ================================================ FILE: .devcontainer/devcontainer.json ================================================ { "name": "OpenTelemetry Network Dev", "image": "otel/opentelemetry-network-build-tools:latest", "features": { "ghcr.io/yonch/devcontainer-features/codex:1": {} }, "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", "mounts": [ "source=${localEnv:HOME}/.codex/auth.json,target=/home/user/.codex/auth.json,readonly,type=bind", "source=${localEnv:HOME}/.ccache,target=/home/user/.ccache,type=bind" ], "customizations": { "vscode": { "settings": {}, "extensions": [ "eamodio.gitlens" ] } }, "remoteUser": "root" } ================================================ FILE: .git/HEAD ================================================ ref: refs/heads/main ================================================ FILE: .git/config ================================================ [core] repositoryformatversion = 1 filemode = true bare = false logallrefupdates = true [remote "origin"] url = https://github.com/open-telemetry/opentelemetry-network tagOpt = --no-tags fetch = +refs/heads/main:refs/remotes/origin/main promisor = true partialclonefilter = blob:limit=1048576 [branch "main"] remote = origin merge = refs/heads/main ================================================ FILE: .git/description ================================================ Unnamed repository; edit this file 'description' to name the repository. ================================================ FILE: .git/hooks/applypatch-msg.sample ================================================ #!/bin/sh # # An example hook script to check the commit log message taken by # applypatch from an e-mail message. # # The hook should exit with non-zero status after issuing an # appropriate message if it wants to stop the commit. The hook is # allowed to edit the commit message file. # # To enable this hook, rename this file to "applypatch-msg". . git-sh-setup commitmsg="$(git rev-parse --git-path hooks/commit-msg)" test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} : ================================================ FILE: .git/hooks/commit-msg.sample ================================================ #!/bin/sh # # An example hook script to check the commit log message. # Called by "git commit" with one argument, the name of the file # that has the commit message. The hook should exit with non-zero # status after issuing an appropriate message if it wants to stop the # commit. The hook is allowed to edit the commit message file. # # To enable this hook, rename this file to "commit-msg". # Uncomment the below to add a Signed-off-by line to the message. # Doing this in a hook is a bad idea in general, but the prepare-commit-msg # hook is more suited to it. # # SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') # grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" # This example catches duplicate Signed-off-by lines. test "" = "$(grep '^Signed-off-by: ' "$1" | sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { echo >&2 Duplicate Signed-off-by lines. exit 1 } ================================================ FILE: .git/hooks/fsmonitor-watchman.sample ================================================ #!/usr/bin/perl use strict; use warnings; use IPC::Open2; # An example hook script to integrate Watchman # (https://facebook.github.io/watchman/) with git to speed up detecting # new and modified files. # # The hook is passed a version (currently 2) and last update token # formatted as a string and outputs to stdout a new update token and # all files that have been modified since the update token. Paths must # be relative to the root of the working tree and separated by a single NUL. # # To enable this hook, rename this file to "query-watchman" and set # 'git config core.fsmonitor .git/hooks/query-watchman' # my ($version, $last_update_token) = @ARGV; # Uncomment for debugging # print STDERR "$0 $version $last_update_token\n"; # Check the hook interface version if ($version ne 2) { die "Unsupported query-fsmonitor hook version '$version'.\n" . "Falling back to scanning...\n"; } my $git_work_tree = get_working_dir(); my $retry = 1; my $json_pkg; eval { require JSON::XS; $json_pkg = "JSON::XS"; 1; } or do { require JSON::PP; $json_pkg = "JSON::PP"; }; launch_watchman(); sub launch_watchman { my $o = watchman_query(); if (is_work_tree_watched($o)) { output_result($o->{clock}, @{$o->{files}}); } } sub output_result { my ($clockid, @files) = @_; # Uncomment for debugging watchman output # open (my $fh, ">", ".git/watchman-output.out"); # binmode $fh, ":utf8"; # print $fh "$clockid\n@files\n"; # close $fh; binmode STDOUT, ":utf8"; print $clockid; print "\0"; local $, = "\0"; print @files; } sub watchman_clock { my $response = qx/watchman clock "$git_work_tree"/; die "Failed to get clock id on '$git_work_tree'.\n" . "Falling back to scanning...\n" if $? != 0; return $json_pkg->new->utf8->decode($response); } sub watchman_query { my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') or die "open2() failed: $!\n" . "Falling back to scanning...\n"; # In the query expression below we're asking for names of files that # changed since $last_update_token but not from the .git folder. # # To accomplish this, we're using the "since" generator to use the # recency index to select candidate nodes and "fields" to limit the # output to file names only. Then we're using the "expression" term to # further constrain the results. my $last_update_line = ""; if (substr($last_update_token, 0, 1) eq "c") { $last_update_token = "\"$last_update_token\""; $last_update_line = qq[\n"since": $last_update_token,]; } my $query = <<" END"; ["query", "$git_work_tree", {$last_update_line "fields": ["name"], "expression": ["not", ["dirname", ".git"]] }] END # Uncomment for debugging the watchman query # open (my $fh, ">", ".git/watchman-query.json"); # print $fh $query; # close $fh; print CHLD_IN $query; close CHLD_IN; my $response = do {local $/; }; # Uncomment for debugging the watch response # open ($fh, ">", ".git/watchman-response.json"); # print $fh $response; # close $fh; die "Watchman: command returned no output.\n" . "Falling back to scanning...\n" if $response eq ""; die "Watchman: command returned invalid output: $response\n" . "Falling back to scanning...\n" unless $response =~ /^\{/; return $json_pkg->new->utf8->decode($response); } sub is_work_tree_watched { my ($output) = @_; my $error = $output->{error}; if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { $retry--; my $response = qx/watchman watch "$git_work_tree"/; die "Failed to make watchman watch '$git_work_tree'.\n" . "Falling back to scanning...\n" if $? != 0; $output = $json_pkg->new->utf8->decode($response); $error = $output->{error}; die "Watchman: $error.\n" . "Falling back to scanning...\n" if $error; # Uncomment for debugging watchman output # open (my $fh, ">", ".git/watchman-output.out"); # close $fh; # Watchman will always return all files on the first query so # return the fast "everything is dirty" flag to git and do the # Watchman query just to get it over with now so we won't pay # the cost in git to look up each individual file. my $o = watchman_clock(); $error = $output->{error}; die "Watchman: $error.\n" . "Falling back to scanning...\n" if $error; output_result($o->{clock}, ("/")); $last_update_token = $o->{clock}; eval { launch_watchman() }; return 0; } die "Watchman: $error.\n" . "Falling back to scanning...\n" if $error; return 1; } sub get_working_dir { my $working_dir; if ($^O =~ 'msys' || $^O =~ 'cygwin') { $working_dir = Win32::GetCwd(); $working_dir =~ tr/\\/\//; } else { require Cwd; $working_dir = Cwd::cwd(); } return $working_dir; } ================================================ FILE: .git/hooks/post-update.sample ================================================ #!/bin/sh # # An example hook script to prepare a packed repository for use over # dumb transports. # # To enable this hook, rename this file to "post-update". exec git update-server-info ================================================ FILE: .git/hooks/pre-applypatch.sample ================================================ #!/bin/sh # # An example hook script to verify what is about to be committed # by applypatch from an e-mail message. # # The hook should exit with non-zero status after issuing an # appropriate message if it wants to stop the commit. # # To enable this hook, rename this file to "pre-applypatch". . git-sh-setup precommit="$(git rev-parse --git-path hooks/pre-commit)" test -x "$precommit" && exec "$precommit" ${1+"$@"} : ================================================ FILE: .git/hooks/pre-commit.sample ================================================ #!/bin/sh # # An example hook script to verify what is about to be committed. # Called by "git commit" with no arguments. The hook should # exit with non-zero status after issuing an appropriate message if # it wants to stop the commit. # # To enable this hook, rename this file to "pre-commit". if git rev-parse --verify HEAD >/dev/null 2>&1 then against=HEAD else # Initial commit: diff against an empty tree object against=$(git hash-object -t tree /dev/null) fi # If you want to allow non-ASCII filenames set this variable to true. allownonascii=$(git config --type=bool hooks.allownonascii) # Redirect output to stderr. exec 1>&2 # Cross platform projects tend to avoid non-ASCII filenames; prevent # them from being added to the repository. We exploit the fact that the # printable range starts at the space character and ends with tilde. if [ "$allownonascii" != "true" ] && # Note that the use of brackets around a tr range is ok here, (it's # even required, for portability to Solaris 10's /usr/bin/tr), since # the square bracket bytes happen to fall in the designated range. test $(git diff-index --cached --name-only --diff-filter=A -z $against | LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 then cat <<\EOF Error: Attempt to add a non-ASCII file name. This can cause problems if you want to work with people on other platforms. To be portable it is advisable to rename the file. If you know what you are doing you can disable this check using: git config hooks.allownonascii true EOF exit 1 fi # If there are whitespace errors, print the offending file names and fail. exec git diff-index --check --cached $against -- ================================================ FILE: .git/hooks/pre-merge-commit.sample ================================================ #!/bin/sh # # An example hook script to verify what is about to be committed. # Called by "git merge" with no arguments. The hook should # exit with non-zero status after issuing an appropriate message to # stderr if it wants to stop the merge commit. # # To enable this hook, rename this file to "pre-merge-commit". . git-sh-setup test -x "$GIT_DIR/hooks/pre-commit" && exec "$GIT_DIR/hooks/pre-commit" : ================================================ FILE: .git/hooks/pre-push.sample ================================================ #!/bin/sh # An example hook script to verify what is about to be pushed. Called by "git # push" after it has checked the remote status, but before anything has been # pushed. If this script exits with a non-zero status nothing will be pushed. # # This hook is called with the following parameters: # # $1 -- Name of the remote to which the push is being done # $2 -- URL to which the push is being done # # If pushing without using a named remote those arguments will be equal. # # Information about the commits which are being pushed is supplied as lines to # the standard input in the form: # # # # This sample shows how to prevent push of commits where the log message starts # with "WIP" (work in progress). remote="$1" url="$2" zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" exit 1 fi fi done exit 0 ================================================ FILE: .git/hooks/pre-rebase.sample ================================================ #!/bin/sh # # Copyright (c) 2006, 2008 Junio C Hamano # # The "pre-rebase" hook is run just before "git rebase" starts doing # its job, and can prevent the command from running by exiting with # non-zero status. # # The hook is called with the following parameters: # # $1 -- the upstream the series was forked from. # $2 -- the branch being rebased (or empty when rebasing the current branch). # # This sample shows how to prevent topic branches that are already # merged to 'next' branch from getting rebased, because allowing it # would result in rebasing already published history. publish=next basebranch="$1" if test "$#" = 2 then topic="refs/heads/$2" else topic=`git symbolic-ref HEAD` || exit 0 ;# we do not interrupt rebasing detached HEAD fi case "$topic" in refs/heads/??/*) ;; *) exit 0 ;# we do not interrupt others. ;; esac # Now we are dealing with a topic branch being rebased # on top of master. Is it OK to rebase it? # Does the topic really exist? git show-ref -q "$topic" || { echo >&2 "No such branch $topic" exit 1 } # Is topic fully merged to master? not_in_master=`git rev-list --pretty=oneline ^master "$topic"` if test -z "$not_in_master" then echo >&2 "$topic is fully merged to master; better remove it." exit 1 ;# we could allow it, but there is no point. fi # Is topic ever merged to next? If so you should not be rebasing it. only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` only_next_2=`git rev-list ^master ${publish} | sort` if test "$only_next_1" = "$only_next_2" then not_in_topic=`git rev-list "^$topic" master` if test -z "$not_in_topic" then echo >&2 "$topic is already up to date with master" exit 1 ;# we could allow it, but there is no point. else exit 0 fi else not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` /usr/bin/perl -e ' my $topic = $ARGV[0]; my $msg = "* $topic has commits already merged to public branch:\n"; my (%not_in_next) = map { /^([0-9a-f]+) /; ($1 => 1); } split(/\n/, $ARGV[1]); for my $elem (map { /^([0-9a-f]+) (.*)$/; [$1 => $2]; } split(/\n/, $ARGV[2])) { if (!exists $not_in_next{$elem->[0]}) { if ($msg) { print STDERR $msg; undef $msg; } print STDERR " $elem->[1]\n"; } } ' "$topic" "$not_in_next" "$not_in_master" exit 1 fi <<\DOC_END This sample hook safeguards topic branches that have been published from being rewound. The workflow assumed here is: * Once a topic branch forks from "master", "master" is never merged into it again (either directly or indirectly). * Once a topic branch is fully cooked and merged into "master", it is deleted. If you need to build on top of it to correct earlier mistakes, a new topic branch is created by forking at the tip of the "master". This is not strictly necessary, but it makes it easier to keep your history simple. * Whenever you need to test or publish your changes to topic branches, merge them into "next" branch. The script, being an example, hardcodes the publish branch name to be "next", but it is trivial to make it configurable via $GIT_DIR/config mechanism. With this workflow, you would want to know: (1) ... if a topic branch has ever been merged to "next". Young topic branches can have stupid mistakes you would rather clean up before publishing, and things that have not been merged into other branches can be easily rebased without affecting other people. But once it is published, you would not want to rewind it. (2) ... if a topic branch has been fully merged to "master". Then you can delete it. More importantly, you should not build on top of it -- other people may already want to change things related to the topic as patches against your "master", so if you need further changes, it is better to fork the topic (perhaps with the same name) afresh from the tip of "master". Let's look at this example: o---o---o---o---o---o---o---o---o---o "next" / / / / / a---a---b A / / / / / / / / c---c---c---c B / / / / \ / / / / b---b C \ / / / / / \ / ---o---o---o---o---o---o---o---o---o---o---o "master" A, B and C are topic branches. * A has one fix since it was merged up to "next". * B has finished. It has been fully merged up to "master" and "next", and is ready to be deleted. * C has not merged to "next" at all. We would want to allow C to be rebased, refuse A, and encourage B to be deleted. To compute (1): git rev-list ^master ^topic next git rev-list ^master next if these match, topic has not merged in next at all. To compute (2): git rev-list master..topic if this is empty, it is fully merged to "master". DOC_END ================================================ FILE: .git/hooks/pre-receive.sample ================================================ #!/bin/sh # # An example hook script to make use of push options. # The example simply echoes all push options that start with 'echoback=' # and rejects all pushes when the "reject" push option is used. # # To enable this hook, rename this file to "pre-receive". if test -n "$GIT_PUSH_OPTION_COUNT" then i=0 while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" do eval "value=\$GIT_PUSH_OPTION_$i" case "$value" in echoback=*) echo "echo from the pre-receive-hook: ${value#*=}" >&2 ;; reject) exit 1 esac i=$((i + 1)) done fi ================================================ FILE: .git/hooks/prepare-commit-msg.sample ================================================ #!/bin/sh # # An example hook script to prepare the commit log message. # Called by "git commit" with the name of the file that has the # commit message, followed by the description of the commit # message's source. The hook's purpose is to edit the commit # message file. If the hook fails with a non-zero status, # the commit is aborted. # # To enable this hook, rename this file to "prepare-commit-msg". # This hook includes three examples. The first one removes the # "# Please enter the commit message..." help message. # # The second includes the output of "git diff --name-status -r" # into the message, just before the "git status" output. It is # commented because it doesn't cope with --amend or with squashed # commits. # # The third example adds a Signed-off-by line to the message, that can # still be edited. This is rarely a good idea. COMMIT_MSG_FILE=$1 COMMIT_SOURCE=$2 SHA1=$3 /usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" # case "$COMMIT_SOURCE,$SHA1" in # ,|template,) # /usr/bin/perl -i.bak -pe ' # print "\n" . `git diff --cached --name-status -r` # if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; # *) ;; # esac # SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') # git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" # if test -z "$COMMIT_SOURCE" # then # /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" # fi ================================================ FILE: .git/hooks/push-to-checkout.sample ================================================ #!/bin/sh # An example hook script to update a checked-out tree on a git push. # # This hook is invoked by git-receive-pack(1) when it reacts to git # push and updates reference(s) in its repository, and when the push # tries to update the branch that is currently checked out and the # receive.denyCurrentBranch configuration variable is set to # updateInstead. # # By default, such a push is refused if the working tree and the index # of the remote repository has any difference from the currently # checked out commit; when both the working tree and the index match # the current commit, they are updated to match the newly pushed tip # of the branch. This hook is to be used to override the default # behaviour; however the code below reimplements the default behaviour # as a starting point for convenient modification. # # The hook receives the commit with which the tip of the current # branch is going to be updated: commit=$1 # It can exit with a non-zero status to refuse the push (when it does # so, it must not modify the index or the working tree). die () { echo >&2 "$*" exit 1 } # Or it can make any necessary changes to the working tree and to the # index to bring them to the desired state when the tip of the current # branch is updated to the new commit, and exit with a zero status. # # For example, the hook can simply run git read-tree -u -m HEAD "$1" # in order to emulate git fetch that is run in the reverse direction # with git push, as the two-tree form of git read-tree -u -m is # essentially the same as git switch or git checkout that switches # branches while keeping the local changes in the working tree that do # not interfere with the difference between the branches. # The below is a more-or-less exact translation to shell of the C code # for the default behaviour for git's push-to-checkout hook defined in # the push_to_deploy() function in builtin/receive-pack.c. # # Note that the hook will be executed from the repository directory, # not from the working tree, so if you want to perform operations on # the working tree, you will have to adapt your code accordingly, e.g. # by adding "cd .." or using relative paths. if ! git update-index -q --ignore-submodules --refresh then die "Up-to-date check failed" fi if ! git diff-files --quiet --ignore-submodules -- then die "Working directory has unstaged changes" fi # This is a rough translation of: # # head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX if git cat-file -e HEAD 2>/dev/null then head=HEAD else head=$(git hash-object -t tree --stdin &2 exit 1 } unset GIT_DIR GIT_WORK_TREE cd "$worktree" && if grep -q "^diff --git " "$1" then validate_patch "$1" else validate_cover_letter "$1" fi && if test "$GIT_SENDEMAIL_FILE_COUNTER" = "$GIT_SENDEMAIL_FILE_TOTAL" then git config --unset-all sendemail.validateWorktree && trap 'git worktree remove -ff "$worktree"' EXIT && validate_series fi ================================================ FILE: .git/hooks/update.sample ================================================ #!/bin/sh # # An example hook script to block unannotated tags from entering. # Called by "git receive-pack" with arguments: refname sha1-old sha1-new # # To enable this hook, rename this file to "update". # # Config # ------ # hooks.allowunannotated # This boolean sets whether unannotated tags will be allowed into the # repository. By default they won't be. # hooks.allowdeletetag # This boolean sets whether deleting tags will be allowed in the # repository. By default they won't be. # hooks.allowmodifytag # This boolean sets whether a tag may be modified after creation. By default # it won't be. # hooks.allowdeletebranch # This boolean sets whether deleting branches will be allowed in the # repository. By default they won't be. # hooks.denycreatebranch # This boolean sets whether remotely creating branches will be denied # in the repository. By default this is allowed. # # --- Command line refname="$1" oldrev="$2" newrev="$3" # --- Safety check if [ -z "$GIT_DIR" ]; then echo "Don't run this script from the command line." >&2 echo " (if you want, you could supply GIT_DIR then run" >&2 echo " $0 )" >&2 exit 1 fi if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then echo "usage: $0 " >&2 exit 1 fi # --- Config allowunannotated=$(git config --type=bool hooks.allowunannotated) allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) denycreatebranch=$(git config --type=bool hooks.denycreatebranch) allowdeletetag=$(git config --type=bool hooks.allowdeletetag) allowmodifytag=$(git config --type=bool hooks.allowmodifytag) # check for no description projectdesc=$(sed -e '1q' "$GIT_DIR/description") case "$projectdesc" in "Unnamed repository"* | "") echo "*** Project description file hasn't been set" >&2 exit 1 ;; esac # --- Check types # if $newrev is 0000...0000, it's a commit to delete a ref. zero=$(git hash-object --stdin &2 echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 exit 1 fi ;; refs/tags/*,delete) # delete tag if [ "$allowdeletetag" != "true" ]; then echo "*** Deleting a tag is not allowed in this repository" >&2 exit 1 fi ;; refs/tags/*,tag) # annotated tag if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 then echo "*** Tag '$refname' already exists." >&2 echo "*** Modifying a tag is not allowed in this repository." >&2 exit 1 fi ;; refs/heads/*,commit) # branch if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then echo "*** Creating a branch is not allowed in this repository" >&2 exit 1 fi ;; refs/heads/*,delete) # delete branch if [ "$allowdeletebranch" != "true" ]; then echo "*** Deleting a branch is not allowed in this repository" >&2 exit 1 fi ;; refs/remotes/*,commit) # tracking branch ;; refs/remotes/*,delete) # delete tracking branch if [ "$allowdeletebranch" != "true" ]; then echo "*** Deleting a tracking branch is not allowed in this repository" >&2 exit 1 fi ;; *) # Anything else (is there anything else?) echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 exit 1 ;; esac # --- Finished exit 0 ================================================ FILE: .git/info/exclude ================================================ # git ls-files --others --exclude-from=.git/info/exclude # Lines that start with '#' are comments. # For a project mostly in C, the following would be a good set of # exclude patterns (uncomment them if you want to use them): # *.[oa] # *~ ================================================ FILE: .git/logs/HEAD ================================================ 0000000000000000000000000000000000000000 0a078f5207323d2be24cbdede1de3abb19bc55a4 appuser 1778556387 +0000 clone: from https://github.com/open-telemetry/opentelemetry-network ================================================ FILE: .git/logs/refs/heads/main ================================================ 0000000000000000000000000000000000000000 0a078f5207323d2be24cbdede1de3abb19bc55a4 appuser 1778556387 +0000 clone: from https://github.com/open-telemetry/opentelemetry-network ================================================ FILE: .git/logs/refs/remotes/origin/HEAD ================================================ 0000000000000000000000000000000000000000 0a078f5207323d2be24cbdede1de3abb19bc55a4 appuser 1778556387 +0000 clone: from https://github.com/open-telemetry/opentelemetry-network ================================================ FILE: .git/objects/pack/pack-71fb82fb82ed2e298be25eaa90192ef76cf2859f.promisor ================================================ 0a078f5207323d2be24cbdede1de3abb19bc55a4 refs/heads/main ================================================ FILE: .git/packed-refs ================================================ # pack-refs with: peeled fully-peeled sorted 0a078f5207323d2be24cbdede1de3abb19bc55a4 refs/remotes/origin/main ================================================ FILE: .git/refs/heads/main ================================================ 0a078f5207323d2be24cbdede1de3abb19bc55a4 ================================================ FILE: .git/refs/remotes/origin/HEAD ================================================ ref: refs/remotes/origin/main ================================================ FILE: .git/shallow ================================================ 0a078f5207323d2be24cbdede1de3abb19bc55a4 ================================================ FILE: .git-blame-ignore-revs ================================================ 5ac6cdf70d1b8f407e873a7a3fcf0bec07232c75 ================================================ FILE: .github/.actionlint.yaml ================================================ self-hosted-runner: labels: - ubuntu-24.04-arm ================================================ FILE: .github/CODEOWNERS ================================================ ##################################################### # # List of approvers for OpenTelemetry Collector Contrib # ##################################################### # # Learn about membership in OpenTelemetry community: # https://github.com/open-telemetry/community/blob/main/community-membership.md # # # Learn about CODEOWNERS file format: # https://help.github.com/en/articles/about-code-owners # # NOTE: Lines should be entered in the following format: # /.. # reducer/logging/ @open-telemetry/ebpf-approvers @bjandras @jimwsplk # Path separator and minimum of 1 space between component path and owners is # important for validation steps # * @open-telemetry/network-approvers ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yaml ================================================ name: Bug report description: Create a report to help us improve labels: ["bug", "needs triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this bug report! Please make sure to fill out the entire form below, providing as much context as you can in order to help us triage and track down your bug as quickly as possible. Before filing a bug, please be sure you have searched through [existing bugs](https://github.com/open-telemetry/opentelemetry-ebpf/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Abug) to see if an existing issue covers your bug. - type: textarea attributes: label: What happened? description: Please provide as much detail as you reasonably can. value: | ## Description ## Steps to Reproduce ## Expected Result ## Actual Result validations: required: true - type: input attributes: label: eBPF Collector version description: What version did you use? (e.g., `v0.4.0`, `1eb551b`, etc) validations: required: true - type: textarea attributes: label: Environment information description: Please provide any additional information about your installation. value: | ## Environment OS: (e.g., "Ubuntu 20.04") Compiler(if manually compiled): (e.g., "go 14.2") - type: textarea attributes: label: eBPF Collector configuration description: Please provide the configuration you are using (e.g. the YAML config file). placeholder: render: yaml - type: textarea attributes: label: Log output description: | Please copy and paste any relevant log output. render: shell - type: textarea attributes: label: Additional context description: Any additional information you think may be relevant to this issue. - type: dropdown attributes: label: Tip description: This element is static, used to render a helpful sub-heading for end-users and community members to help prioritize issues. Please leave as is. options: - [React](https://github.blog/news-insights/product-news/add-reactions-to-pull-requests-issues-and-comments/) with 👍 to help prioritize this issue. Please use comments to provide useful context, avoiding `+1` or `me too`, to help us triage it. Learn more [here](https://opentelemetry.io/community/end-user/issue-participation/). default: 0 ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: eBPF Collector Slack Channel url: https://cloud-native.slack.com/archives/C02AB15583A about: Please ask and answer questions here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yaml ================================================ name: Feature request description: Suggest an idea for this project labels: ["enhancement", "needs triage"] body: - type: textarea attributes: label: Is your feature request related to a problem? Please describe. description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] validations: required: true - type: textarea attributes: label: Describe the solution you'd like description: A clear and concise description of what you want to happen. validations: required: true - type: textarea attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. - type: textarea attributes: label: Additional context description: Add any other context or screenshots about the feature request here. - type: dropdown attributes: label: Tip description: This element is static, used to render a helpful sub-heading for end-users and community members to help prioritize issues. Please leave as is. options: - [React](https://github.blog/news-insights/product-news/add-reactions-to-pull-requests-issues-and-comments/) with 👍 to help prioritize this issue. Please use comments to provide useful context, avoiding `+1` or `me too`, to help us triage it. Learn more [here](https://opentelemetry.io/community/end-user/issue-participation/). default: 0 ================================================ FILE: .github/ISSUE_TEMPLATE/other.yaml ================================================ name: Other issue description: Create a new issue to help us improve the collector labels: ["needs triage"] body: - type: textarea attributes: label: Describe the issue you're reporting description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] validations: required: true - type: dropdown attributes: label: Tip description: This element is static, used to render a helpful sub-heading for end-users and community members to help prioritize issues. Please leave as is. options: - [React](https://github.blog/news-insights/product-news/add-reactions-to-pull-requests-issues-and-comments/) with 👍 to help prioritize this issue. Please use comments to provide useful context, avoiding `+1` or `me too`, to help us triage it. Learn more [here](https://opentelemetry.io/community/end-user/issue-participation/). default: 0 ================================================ FILE: .github/actions/build-tools-single-stage/action.yml ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 name: 'Build and Push Container' description: 'Build and push a Docker container with dependency management and registry caching' inputs: directory: description: 'Directory name to build' required: true registry: description: 'Container registry to use' required: true default: 'ghcr.io' registry_username: description: 'Registry username' required: true registry_password: description: 'Registry password/token' required: true image_prefix: description: 'Prefix for image names' required: false default: 'benv' ref: description: 'Git ref to checkout' required: false default: 'main' force_rebuild: description: 'Force rebuild and push even if image exists in registry' required: false default: 'false' arch: description: 'Target architecture (amd64 or arm64)' required: false default: 'amd64' outputs: image-tag: description: 'The computed image tag' value: ${{ steps.compute-recursive-tags.outputs.image-tag }} full-image-tag: description: 'The full image tag with registry' value: ${{ steps.compute-recursive-tags.outputs.full-image-tag }} image-exists: description: 'Whether the image already exists in registry' value: ${{ steps.check-exists.outputs.exists }} build-needed: description: 'Whether a build was needed' value: ${{ steps.check-exists.outputs.exists == 'false' }} runs: using: 'composite' steps: - name: Checkout sources uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ inputs.ref }} fetch-depth: 0 - name: Compute recursive tags for all directories id: compute-recursive-tags shell: bash env: DOCKER_TAG_PREFIX: ${{ github.repository_owner }}/ run: | DIRECTORY="build-tools/${{ inputs.directory }}" BASE_DIRECTORY="${{ inputs.directory }}" ARCH="${{ inputs.arch }}" # Define dependency mapping based on CMakeLists.txt declare -A DEPS DEPS["base"]="" DEPS["libuv"]="base" DEPS["cpp_misc"]="base" DEPS["libmaxminddb"]="base" DEPS["libbpf"]="base" DEPS["aws_sdk"]="base" DEPS["final"]="base libuv aws_sdk cpp_misc libmaxminddb libbpf" # Compute direct hashes for all directories upfront declare -A DIRECT_HASHES ALL_DIRS="base libuv cpp_misc libmaxminddb libbpf aws_sdk final" echo "Computing direct hashes..." >&2 for dir in $ALL_DIRS; do direct_hash=$(git log -1 --format=%h "build-tools/${dir}") DIRECT_HASHES[$dir]=$direct_hash echo "Direct hash for $dir: $direct_hash" >&2 done # Function to compute dependency closure (all transitive dependencies) compute_closure() { local target="$1" local visited_key="VISITED_$target" # Check for circular dependency if [[ -n "${!visited_key:-}" ]]; then echo "ERROR: Circular dependency detected for $target" >&2 exit 1 fi # Mark as visiting declare -g "$visited_key=1" # Start with direct dependencies local deps="${DEPS[$target]:-}" local closure_set="" # Add direct dependencies for dep in $deps; do closure_set="$closure_set $dep" # Recursively add their closures local dep_closure=$(compute_closure "$dep") closure_set="$closure_set $dep_closure" done # Remove duplicates by converting to array and back local unique_closure=($(echo $closure_set | tr ' ' '\n' | sort -u | tr '\n' ' ')) # Unmark visiting unset "$visited_key" echo "${unique_closure[@]}" } # Function to compute recursive hash using closure approach compute_recursive_hash() { local dir="$1" # Get the full dependency closure local closure=$(compute_closure "$dir") # Include the directory itself in the hash computation local all_dirs_for_hash="$dir $closure" # Sort all directories local sorted_dirs=($(echo $all_dirs_for_hash | tr ' ' '\n' | sort -u | tr '\n' ' ')) # Concatenate their direct hashes with dashes local hash_input="" for d in "${sorted_dirs[@]}"; do if [[ -n "$d" ]]; then if [[ -n "$hash_input" ]]; then hash_input="$hash_input-${DIRECT_HASHES[$d]}" else hash_input="${DIRECT_HASHES[$d]}" fi fi done # Use the dash-separated hashes directly as the tag local final_hash="$hash_input" echo "Closure for $dir: ${sorted_dirs[@]}" >&2 echo "Final hash for $dir: $final_hash" >&2 echo "$final_hash" } # Compute recursive hash for target directory RECURSIVE_HASH=$(compute_recursive_hash "$BASE_DIRECTORY") # Create image tag IMAGE_TAG="${{ github.repository_owner }}/opentelemetry-network-build-tools-cache:${BASE_DIRECTORY}-${ARCH}-${RECURSIVE_HASH}" FULL_IMAGE_TAG="${{ inputs.registry }}/${IMAGE_TAG}" echo "image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT echo "full-image-tag=${FULL_IMAGE_TAG}" >> $GITHUB_OUTPUT echo "recursive-hash=${RECURSIVE_HASH}" >> $GITHUB_OUTPUT echo "Computed recursive image tag: ${IMAGE_TAG}" >&2 echo "Full image tag: ${FULL_IMAGE_TAG}" >&2 echo "Recursive hash: ${RECURSIVE_HASH}" >&2 # Compute all dependency tags for build args echo "Computing all dependency tags..." >&2 for dir in $ALL_DIRS; do if [[ "$dir" != "$BASE_DIRECTORY" ]]; then dir_hash=$(compute_recursive_hash "$dir") dir_image_tag="${{ github.repository_owner }}/opentelemetry-network-build-tools-cache:${dir}-${ARCH}-${dir_hash}" dir_full_tag="${{ inputs.registry }}/${dir_image_tag}" # Export as environment variable for use in build args export "${dir}_IMAGE_TAG=${dir_full_tag}" echo "${dir}_IMAGE_TAG=${dir_full_tag}" >> $GITHUB_OUTPUT echo "Dependency: ${dir} -> ${dir_full_tag}" >&2 fi done - name: Check if image exists in registry id: check-exists shell: bash run: | FULL_IMAGE_TAG="${{ steps.compute-recursive-tags.outputs.full-image-tag }}" if [[ "${{ inputs.force_rebuild }}" == "true" ]]; then echo "exists=false" >> $GITHUB_OUTPUT echo "Force rebuild enabled - will rebuild ${FULL_IMAGE_TAG} regardless of registry state" elif docker manifest inspect "${FULL_IMAGE_TAG}" >/dev/null 2>&1; then echo "exists=true" >> $GITHUB_OUTPUT echo "Image ${FULL_IMAGE_TAG} already exists in registry" else echo "exists=false" >> $GITHUB_OUTPUT echo "Image ${FULL_IMAGE_TAG} does not exist in registry" fi - name: Initialize directory submodules if: steps.check-exists.outputs.exists == 'false' shell: bash run: | DIRECTORY="build-tools/${{ inputs.directory }}" echo "Initializing submodules for directory: ${DIRECTORY}" # Initialize submodules for the specific directory path git submodule update --init --recursive -- "${DIRECTORY}/" - name: Log in to Container Registry if: steps.check-exists.outputs.exists == 'false' uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ inputs.registry }} username: ${{ inputs.registry_username }} password: ${{ inputs.registry_password }} - name: Build and push image if: steps.check-exists.outputs.exists == 'false' shell: bash run: | DIRECTORY="build-tools/${{ inputs.directory }}" FULL_IMAGE_TAG="${{ steps.compute-recursive-tags.outputs.full-image-tag }}" # Start building the docker command BUILD_ARGS="--build-arg NPROC=$(nproc)" # Add all dependency image tags as build args using outputs from compute-recursive-tags step BUILD_ARGS="${BUILD_ARGS} --build-arg base_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.base_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg libuv_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libuv_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg cpp_misc_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.cpp_misc_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg libmaxminddb_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libmaxminddb_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg libbpf_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.libbpf_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg aws_sdk_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.aws_sdk_IMAGE_TAG }}" BUILD_ARGS="${BUILD_ARGS} --build-arg final_IMAGE_TAG=${{ steps.compute-recursive-tags.outputs.final_IMAGE_TAG }}" # Add environment-specific build args if they exist if [ -n "${BENV_BASE_IMAGE_DISTRO}" ]; then BUILD_ARGS="${BUILD_ARGS} --build-arg BENV_BASE_IMAGE_DISTRO=${BENV_BASE_IMAGE_DISTRO}" fi if [ -n "${BENV_BASE_IMAGE_VERSION}" ]; then BUILD_ARGS="${BUILD_ARGS} --build-arg BENV_BASE_IMAGE_VERSION=${BENV_BASE_IMAGE_VERSION}" fi # Add CMAKE_BUILD_TYPE (defaults to Release if not set) CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" BUILD_ARGS="${BUILD_ARGS} --build-arg CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" # Add BUILD_CFLAGS based on build type if [ "${CMAKE_BUILD_TYPE}" = "Debug" ]; then BUILD_ARGS="${BUILD_ARGS} --build-arg BUILD_CFLAGS='-O0 -g'" fi # Build the image echo "Building image: ${FULL_IMAGE_TAG}" echo "Build args: ${BUILD_ARGS}" docker build -t "${FULL_IMAGE_TAG}" ${BUILD_ARGS} "${DIRECTORY}/" # Always push intermediate builds to cache registry (dry_run only affects final Docker Hub push) echo "Pushing image to cache registry: ${FULL_IMAGE_TAG}" docker push "${FULL_IMAGE_TAG}" ================================================ FILE: .github/pull_request_template.md ================================================ **Description:** **Link to tracking Issue:** **Testing:** **Documentation:** ================================================ FILE: .github/renovate.json5 ================================================ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:best-practices", "helpers:pinGitHubActionDigestsToSemver" ], "packageRules": [ { "groupName": "all patch versions", "matchUpdateTypes": ["patch"], "schedule": ["before 8am every weekday"] }, { "matchUpdateTypes": ["minor", "major"], "schedule": ["before 8am on Monday"] } ], "labels": [ "dependencies" ] } ================================================ FILE: .github/workflows/build-and-release.yaml ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 name: build-and-release run-name: Publishing a release on: workflow_dispatch: inputs: release_type: description: "Release type" required: true type: choice options: - public - unofficial default: public ref: description: "Tag, branch or SHA to checkout" required: true type: string default: "main" image_prefix: description: "Prefix to use for destination image name" required: false type: string default: "opentelemetry-ebpf-" additional_tag: description: "Additional tag to use when pushing to docker repository" required: false type: string dry_run: description: "Build everything but don't actually push to repository" required: false type: boolean default: false workflow_call: inputs: release_type: description: "Release type" required: true type: string ref: description: "Tag, branch or SHA to checkout" required: true type: string image_prefix: description: "Prefix to use for destination image name" required: false type: string default: "opentelemetry-ebpf-" additional_tag: description: "Additional tag to use when pushing to docker repository" required: false type: string dry_run: description: "Build everything but don't actually push to repository" required: false type: boolean default: false permissions: contents: write packages: write env: BENV_IMAGE: ${{ vars.BENV_IMAGE || 'docker.io/otel/opentelemetry-network-build-tools' }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_REGISTRY: ${{ vars.DOCKER_REGISTRY }} DOCKER_NAMESPACE: ${{ vars.DOCKER_NAMESPACE }} IMAGE_PREFIX: ${{ inputs.image_prefix || 'opentelemetry-ebpf-' }} RELEASE_TYPE: ${{ inputs.release_type || 'public' }} DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }} jobs: compute-version: name: Compute version runs-on: ubuntu-24.04 outputs: git_short_hash: ${{ steps.version.outputs.git_short_hash }} short_version_number: ${{ steps.version.outputs.short_version_number }} full_version_number: ${{ steps.version.outputs.full_version_number }} github_tag: ${{ steps.version.outputs.github_tag }} steps: - name: Checkout sources uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ inputs.ref }} fetch-depth: 0 - name: Compute version numbers id: version run: | source ./version.sh git_short_hash=$(git rev-parse --short=8 HEAD) short_version_number="${EBPF_NET_MAJOR_VERSION}.${EBPF_NET_MINOR_VERSION}" full_version_number="${EBPF_NET_MAJOR_VERSION}.${EBPF_NET_MINOR_VERSION}.${EBPF_NET_PATCH_VERSION}" if [[ "${RELEASE_TYPE}" == "public" ]]; then github_tag=v${full_version_number} else github_tag=v${full_version_number}-${git_short_hash} fi echo "git_short_hash=${git_short_hash}" >> "$GITHUB_OUTPUT" echo "short_version_number=${short_version_number}" >> "$GITHUB_OUTPUT" echo "full_version_number=${full_version_number}" >> "$GITHUB_OUTPUT" echo "github_tag=${github_tag}" >> "$GITHUB_OUTPUT" build-per-arch: name: Build ${{ matrix.arch }} artifacts needs: compute-version strategy: matrix: include: - arch: amd64 runner: ubuntu-24.04 - arch: arm64 runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} steps: - name: Checkout sources uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ inputs.ref }} fetch-depth: 0 submodules: false path: src - name: Checkout ext/ submodules run: | cd $GITHUB_WORKSPACE/src git submodule update --init --recursive ext/ - name: Fetch build environment run: | docker pull $BENV_IMAGE - name: Create and fix permissions for output directory run: | mkdir -p $GITHUB_WORKSPACE/out sudo chown -R 1000:1000 $GITHUB_WORKSPACE/out - name: Build artifacts run: | docker run -d -p 5000:5000 --name registry docker.io/library/registry:2 docker run -t --rm \ --mount "type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock" \ --mount "type=bind,source=$GITHUB_WORKSPACE/src,destination=/home/user/src,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/out,destination=/home/user/out" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --network host \ --privileged \ $BENV_IMAGE \ ./build.sh pipeline-docker-registry docker pull localhost:5000/reducer docker pull localhost:5000/kernel-collector docker pull localhost:5000/cloud-collector docker pull localhost:5000/k8s-collector - name: Build packages run: | docker run -t --rm \ --mount "type=bind,source=$GITHUB_WORKSPACE/src,destination=/home/user/src,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/out,destination=/home/user/out" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --workdir /home/user/out \ $BENV_IMAGE \ cpack -G 'RPM;DEB' - name: Log-in to container registry run: | docker login --username="$DOCKER_USERNAME" --password-stdin $DOCKER_REGISTRY <<< "$DOCKER_PASSWORD" - name: Tag and push arch-specific images run: | docker_registry=$(sed -e 's,^https://,,' -e 's,/*$,,' <<< $DOCKER_REGISTRY) images=(reducer kernel-collector cloud-collector k8s-collector) for image in ${images[@]}; do image_name="${IMAGE_PREFIX}${image}" image_path="${docker_registry}/${DOCKER_NAMESPACE}/${image_name}" docker tag localhost:5000/$image ${image_path}:${{ matrix.arch }}-${{ needs.compute-version.outputs.git_short_hash }} if [[ "${DRY_RUN}" == "false" ]]; then docker push ${image_path}:${{ matrix.arch }}-${{ needs.compute-version.outputs.git_short_hash }} fi done docker stop registry || true docker rm registry || true - name: Upload packages uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: packages-${{ matrix.arch }} path: | out/opentelemetry-ebpf-*.rpm out/opentelemetry-ebpf-*.deb create-manifests: name: Create multi-arch manifests needs: [compute-version, build-per-arch] runs-on: ubuntu-24.04 if: ${{ inputs.dry_run != true }} steps: - name: Log-in to container registry run: | docker login --username="$DOCKER_USERNAME" --password-stdin $DOCKER_REGISTRY <<< "$DOCKER_PASSWORD" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Create and push multi-arch manifests run: | docker_registry=$(sed -e 's,^https://,,' -e 's,/*$,,' <<< $DOCKER_REGISTRY) git_short_hash="${{ needs.compute-version.outputs.git_short_hash }}" short_version="${{ needs.compute-version.outputs.short_version_number }}" full_version="${{ needs.compute-version.outputs.full_version_number }}" if [[ "${RELEASE_TYPE}" == "public" ]]; then tags=(latest latest-v${short_version} v${full_version}) else tags=(v${full_version}-${git_short_hash}) fi if [[ "${{ inputs.additional_tag }}" != "" ]]; then tags=(${tags[@]} "${{ inputs.additional_tag }}") fi images=(reducer kernel-collector cloud-collector k8s-collector) for image in ${images[@]}; do image_name="${IMAGE_PREFIX}${image}" image_path="${docker_registry}/${DOCKER_NAMESPACE}/${image_name}" amd64_tag="${image_path}:amd64-${git_short_hash}" arm64_tag="${image_path}:arm64-${git_short_hash}" for tag in ${tags[@]}; do docker buildx imagetools create --tag "${image_path}:${tag}" \ "${amd64_tag}" \ "${arm64_tag}" echo "Created multi-arch manifest: ${image_path}:${tag}" done done upload-release: name: Upload release artifacts needs: [compute-version, build-per-arch] runs-on: ubuntu-24.04 if: ${{ inputs.dry_run != true }} steps: - name: Download all packages uses: actions/download-artifact@v4 with: pattern: packages-* merge-multiple: true path: packages - name: Upload packages to Release uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe with: tag_name: ${{ needs.compute-version.outputs.github_tag }} prerelease: ${{ env.RELEASE_TYPE != 'public' }} files: packages/* ================================================ FILE: .github/workflows/build-and-test.yaml ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 name: build-and-test run-name: ${{ github.actor }} is running GitHub Actions on: push: branches: - main pull_request: paths: permissions: contents: read env: BENV_IMAGE: ${{ vars.BENV_IMAGE || 'docker.io/otel/opentelemetry-network-build-tools' }} # concurrency: # group: build-and-test-${{ github.event.pull_request_number || github.ref }} # cancel-in-progress: true jobs: clang-format-check: runs-on: ubuntu-24.04 name: clang-format-check steps: - name: Print github workspace run: | echo "github.workspace = ${{ github.workspace }}" echo "pr.ref = ${{github.event.pull_request.head.ref}}" echo "github.ref = ${{ github.ref }}" echo "$GITHUB_CONTEXT" - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - name: Runs format checker run: | # disable man page updates for faster apt install echo "set man-db/auto-update false" | sudo debconf-communicate || true sudo dpkg-reconfigure man-db sudo apt update sudo apt install -y --no-install-recommends clang-format-19 cd ${{ github.workspace }} ./.github/workflows/scripts/check-clang-format.sh outputs: date: ${{ steps.date.outputs.date }} cargo-test: name: cargo-test runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Run cargo test env: PASS: ${{ secrets.DOCKER_PASSWORD }} run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ docker run -t \ --rm \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ $BENV_IMAGE \ ./build.sh cargo-test build-reducer: name: build-reducer runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-reducer max-size: 1G verbose: 2 - name: build-reducer env: PASS: ${{ secrets.DOCKER_PASSWORD }} run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Start local registry for the build process docker run -d -p 5000:5000 --name registry docker.io/library/registry:2 # Build reducer with registry access # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --network host \ --privileged \ $BENV_IMAGE \ ./build.sh reducer-docker-registry # Export reducer container mkdir -p container-exports docker pull localhost:5000/reducer docker save localhost:5000/reducer > container-exports/reducer.tar # Clean up registry docker stop registry docker rm registry - name: Upload reducer container uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: reducer-container path: container-exports/reducer.tar if-no-files-found: error retention-days: 1 build-kernel-collector: name: build-kernel-collector runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-kernel-collector max-size: 1G verbose: 2 - name: build-kernel-collector env: PASS: ${{ secrets.DOCKER_PASSWORD }} run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Start local registry for the build process docker run -d -p 5000:5000 --name registry docker.io/library/registry:2 # Build kernel-collector with registry access # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --network host \ --privileged \ $BENV_IMAGE \ ./build.sh kernel-collector-docker-registry # Export kernel-collector container mkdir -p container-exports docker pull localhost:5000/kernel-collector docker save localhost:5000/kernel-collector > container-exports/kernel-collector.tar # Clean up registry docker stop registry docker rm registry - name: Upload kernel collector container uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: kernel-collector-container path: container-exports/kernel-collector.tar if-no-files-found: error retention-days: 1 build-kernel-collector-test: name: build-kernel-collector-test runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-kernel-collector-test max-size: 1G verbose: 2 - name: build kernel-collector-test container env: PASS: ${{ secrets.DOCKER_PASSWORD }} run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Start local registry for the build process docker run -d -p 5000:5000 --name registry docker.io/library/registry:2 # Build kernel-collector-test with registry access # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --network host \ --privileged \ $BENV_IMAGE \ ./build.sh kernel-collector-test-docker-registry # Export kernel-collector-test container mkdir -p container-exports docker pull localhost:5000/kernel-collector-test docker save localhost:5000/kernel-collector-test > container-exports/kernel-collector-test.tar # Clean up registry docker stop registry docker rm registry - name: Upload kernel-collector-test container uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: kernel-collector-test-container path: container-exports/kernel-collector-test.tar if-no-files-found: error retention-days: 1 build-k8s-collector: name: build-k8s-collector runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-k8s-collector max-size: 1G verbose: 2 - name: build k8s-collector run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Start local registry for the build process docker run -d -p 5000:5000 --name registry docker.io/library/registry:2 # Build k8s-collector with registry access # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --network host \ --privileged \ $BENV_IMAGE \ ./build.sh k8s-collector-docker-registry # Export k8s-collector container mkdir -p container-exports docker pull localhost:5000/k8s-collector docker save localhost:5000/k8s-collector > container-exports/k8s-collector.tar # Clean up registry docker stop registry docker rm registry - name: Upload k8s-collector container uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: k8s-collector-container path: container-exports/k8s-collector.tar if-no-files-found: error retention-days: 1 build-cloud-collector: name: build-cloud-collector runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-cloud-collector max-size: 1G verbose: 2 - name: build cloud-collector run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ $BENV_IMAGE \ ./build.sh cloud-collector build-run-unit-tests: name: build-run-unit-tests runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-run-unit-tests max-size: 1G verbose: 2 - name: run unit tests run: | echo "github.workspace = ${{ github.workspace }}" docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --env ARGS="--output-on-failure --repeat until-pass:3" \ --env SPDLOG_LEVEL="trace" \ $BENV_IMAGE \ ./build.sh unit_tests test build-run-unit-tests-with-asan-and-debug-flags: name: build-run-unit-tests-with-asan-and-debug-flags runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-run-unit-tests-asan-debug max-size: 1G verbose: 2 - name: build unit tests with asan and debug flags on then run all tests run: | docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Ensure ccache directory is writable inside container mkdir -p "${{ github.workspace }}/.ccache" chmod -R 777 "${{ github.workspace }}/.ccache" || true docker run -t \ --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock" \ --mount "type=bind,source=$(git rev-parse --show-toplevel),destination=/home/user/src,readonly" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ --env ARGS="--output-on-failure --repeat until-pass:3 -E render_test" \ --env SPDLOG_LEVEL="trace" \ $BENV_IMAGE \ ./build.sh --debug --asan unit_tests test build-deb-rpm: name: build-deb-rpm runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Cache and install ccache uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ccache version: 1.0 - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: key: ccache-build-packages max-size: 1G verbose: 2 - name: Build RPM/DEB packages run: | # Prepare output and ccache directories mkdir -p "${{ github.workspace }}/out" mkdir -p "${{ github.workspace }}/.ccache" # Ensure the build container user (uid 1000) can write here sudo chown -R 1000:1000 "${{ github.workspace }}/out" || true chmod -R 777 "${{ github.workspace }}/.ccache" || true docker pull $BENV_IMAGE git submodule update --init --recursive ext/ # Configure and build the project into /home/user/out docker run -t --rm \ --mount "type=bind,source=${{ github.workspace }}/.ccache,destination=/ccache" \ --mount "type=bind,source=${{ github.workspace }},destination=/home/user/src,readonly" \ --mount "type=bind,source=${{ github.workspace }}/out,destination=/home/user/out" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --env CCACHE_DIR=/ccache \ $BENV_IMAGE \ ./build.sh pipeline # Run CPack in the configured build directory docker run -t --rm \ --mount "type=bind,source=${{ github.workspace }},destination=/home/user/src,readonly" \ --mount "type=bind,source=${{ github.workspace }}/out,destination=/home/user/out" \ --env EBPF_NET_SRC_ROOT=/home/user/src \ --workdir /home/user/out \ $BENV_IMAGE \ cpack -G 'RPM;DEB' e2e-otel-jsonl: name: e2e-otel-jsonl needs: [build-reducer, build-kernel-collector, build-k8s-collector] runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Set up kind cluster uses: helm/kind-action@v1.10.0 with: version: v0.30.0 cluster_name: e2e-kind - name: Download reducer container uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: reducer-container path: ./container-exports - name: Download kernel-collector container uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: kernel-collector-container path: ./container-exports - name: Download k8s-collector container uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: k8s-collector-container path: ./container-exports - name: Load images and prepare config/output run: | set -euxo pipefail docker load < container-exports/reducer.tar docker load < container-exports/kernel-collector.tar docker load < container-exports/k8s-collector.tar mkdir -p e2e-out # ensure otelcol (which runs non-root) can write here regardless of uid mapping chmod 0777 e2e-out cat > otel-e2e-config.yaml <<'EOF' receivers: otlp: protocols: grpc: endpoint: 0.0.0.0:4317 processors: batch: {} exporters: file: path: /out/otel.jsonl # keep default json lines, rotate small just in case rotation: max_megabytes: 5 max_days: 1 format: json service: pipelines: metrics: receivers: [otlp] processors: [batch] exporters: [file] logs: receivers: [otlp] processors: [batch] exporters: [file] EOF - name: Start OpenTelemetry Collector run: | set -euxo pipefail # Use contrib distribution to ensure file exporter availability docker pull otel/opentelemetry-collector-contrib:0.102.0 docker run -d --rm \ --name otelcol-e2e \ --network host \ -v "$(pwd)/otel-e2e-config.yaml:/etc/otelcol/config.yaml:ro" \ -v "$(pwd)/e2e-out:/out" \ otel/opentelemetry-collector-contrib:0.102.0 \ --config=/etc/otelcol/config.yaml # Give the collector a moment to start sleep 3 docker logs --tail 50 otelcol-e2e || true - name: Start reducer run: | set -euxo pipefail REDUCER_ID=$(docker run -d --rm \ --name reducer-e2e \ --network host \ localhost:5000/reducer \ --port 8000 \ --prom 0.0.0.0:7000 \ --partitions-per-shard 1 \ --num-ingest-shards=1 \ --num-matching-shards=1 \ --num-aggregation-shards=1 \ --enable-aws-enrichment \ --enable-id-id \ --enable-flow-logs \ --enable-otlp-grpc-metrics \ --otlp-grpc-metrics-host 127.0.0.1 \ --otlp-grpc-metrics-port 4317 \ --log-console \ --debug \ --log-whitelist-all) echo "Reducer started: ${REDUCER_ID}" # brief wait to ensure listener is up sleep 5 docker logs --tail 100 reducer-e2e || true - name: Start kernel collector run: | set -euxo pipefail KERNEL_ID=$(docker run -d --rm \ --name kernel-collector-e2e \ --env EBPF_NET_INTAKE_HOST="127.0.0.1" \ --env EBPF_NET_INTAKE_PORT="8000" \ --env EBPF_NET_HOST_DIR="/hostfs" \ --privileged \ --network host \ --pid host \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume /sys/fs/cgroup:/hostfs/sys/fs/cgroup \ --volume /usr/src:/hostfs/usr/src \ --volume /lib/modules:/hostfs/lib/modules \ --volume /etc:/hostfs/etc \ --volume /var/cache:/hostfs/cache \ localhost:5000/kernel-collector \ --log-console) echo "Kernel collector started: ${KERNEL_ID}" # give it a moment to initialize BPF sleep 15 docker logs --tail 50 kernel-collector-e2e || true - name: Start k8s collector run: | set -euxo pipefail KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}" if [ ! -f "${KUBECONFIG_PATH}" ]; then echo "KUBECONFIG not found at ${KUBECONFIG_PATH}" ls -la "${HOME}" || true ls -la "${HOME}/.kube" || true exit 1 fi echo "Using kubeconfig at ${KUBECONFIG_PATH}" kubectl config current-context || true K8S_COLLECTOR_ID=$(docker run -d \ --name k8s-collector-e2e \ --network host \ --env EBPF_NET_INTAKE_HOST="127.0.0.1" \ --env EBPF_NET_INTAKE_PORT="8000" \ --env RUST_LOG="info" \ --env KUBECONFIG="/kubeconfig" \ --volume "${KUBECONFIG_PATH}:/kubeconfig:ro" \ localhost:5000/k8s-collector) echo "k8s-collector started: ${K8S_COLLECTOR_ID}" # allow it to connect and start watching sleep 10 docker logs --tail 100 k8s-collector-e2e || true - name: Generate traffic and wait for collection run: | set -euxo pipefail # Deploy Kubernetes traffic generator in the kind cluster KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}" if [ ! -f "${KUBECONFIG_PATH}" ]; then echo "KUBECONFIG not found at ${KUBECONFIG_PATH}" ls -la "${HOME}" || true ls -la "${HOME}/.kube" || true exit 1 fi kubectl create namespace e2e-kind || true cat > e2e-kind-traffic.yaml <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: e2e-wget namespace: e2e-kind labels: app: e2e-wget e2e-role: traffic spec: replicas: 10 selector: matchLabels: app: e2e-wget template: metadata: labels: app: e2e-wget e2e-role: traffic spec: containers: - name: wget image: busybox:1.36 command: - /bin/sh - -c - | echo "e2e-wget: starting in pod $(hostname)" echo "e2e-wget: uname: $(uname -a || echo 'uname failed')" echo "e2e-wget: /etc/resolv.conf:" cat /etc/resolv.conf || echo "e2e-wget: unable to read /etc/resolv.conf" echo "e2e-wget: testing DNS for example.com" nslookup example.com || echo "e2e-wget: nslookup example.com failed (command may be missing)" TARGET_URL="http://example.com" echo "e2e-wget: running wget to ${TARGET_URL}" if wget --timeout=15 --tries=3 -O /dev/null "${TARGET_URL}"; then echo "e2e-wget: wget succeeded" else ec=$? echo "e2e-wget: wget failed with exit code ${ec}" fi echo "e2e-wget: finished, sleeping" # keep the pod running long enough that it does not restart during the test sleep 600 EOF kubectl apply -f e2e-kind-traffic.yaml kubectl -n e2e-kind rollout status deploy/e2e-wget --timeout=180s kubectl -n e2e-kind get pods -o wide || true # Allow time for metrics to propagate and be exported through collectors sleep 60 - name: Stop services and collect logs if: always() run: | set -euxo pipefail mkdir -p e2e-logs || true docker ps || true # Capture full container logs to files for later inspection docker logs k8s-collector-e2e > e2e-logs/k8s-collector-e2e.log 2>&1 || true docker logs kernel-collector-e2e > e2e-logs/kernel-collector-e2e.log 2>&1 || true docker logs reducer-e2e > e2e-logs/reducer-e2e.log 2>&1 || true docker logs otelcol-e2e > e2e-logs/otelcol-e2e.log 2>&1 || true # Also emit a short tail to the job logs for quick debugging docker logs --tail 200 k8s-collector-e2e || true docker logs --tail 200 kernel-collector-e2e || true docker logs --tail 200 reducer-e2e || true docker logs --tail 200 otelcol-e2e || true docker stop k8s-collector-e2e || true docker rm k8s-collector-e2e || true docker stop kernel-collector-e2e || true docker stop reducer-e2e || true docker stop otelcol-e2e || true ls -la e2e-out || true sudo chmod -R a+r e2e-out || true # Show a small preview for debugging convenience if [ -f e2e-out/otel.jsonl ]; then head -n 50 e2e-out/otel.jsonl || true; fi - name: Upload OpenTelemetry JSONL if: always() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: e2e-otel-jsonl path: | e2e-out/* if-no-files-found: warn retention-days: 7 - name: Upload e2e container logs if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: e2e-container-logs path: | e2e-logs/* if-no-files-found: warn retention-days: 7 - name: Show tcp.bytes metrics from JSONL if: always() run: | set +e if [ ! -f e2e-out/otel.jsonl ]; then echo "e2e-out/otel.jsonl not found; skipping tcp.bytes display" exit 0 fi if ! command -v jq >/dev/null 2>&1; then echo "jq not found; attempting to install" sudo apt-get update && sudo apt-get install -y jq || true fi if ! command -v jq >/dev/null 2>&1; then echo "jq still not available; skipping tcp.bytes display" exit 0 fi echo "=== tcp.bytes metrics (pretty-printed) ===" jq ' .resourceMetrics[]? | .scopeMetrics[]? | .metrics[]? | select(.name == "tcp.bytes") ' e2e-out/otel.jsonl || true - name: Show wget pod logs if: always() run: | set +e KUBECONFIG_PATH="${KUBECONFIG:-$HOME/.kube/config}" if [ ! -f "${KUBECONFIG_PATH}" ]; then echo "KUBECONFIG not found; skipping wget pod logs" exit 0 fi echo "=== e2e-kind namespace pods ===" kubectl -n e2e-kind get pods -o wide || true echo "=== e2e-wget pod logs (tail) ===" kubectl -n e2e-kind logs -l app=e2e-wget --tail=100 || true run-kernel-collector-simple-tests: name: run-kernel-collector-simple-tests needs: [build-kernel-collector] runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: include: # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.4-20250721.013324' description: 'Kernel 5.4' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.10-20250507.063028' description: 'Kernel 5.10' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.15-20250507.063028' description: 'Kernel 5.15' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.1-20250507.063028' description: 'Kernel 6.1' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.6-20250507.063028' description: 'Kernel 6.6' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.12-20250507.063028' description: 'Kernel 6.12' timeout-minutes: 10 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Download kernel-collector container uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: kernel-collector-container path: ./container-exports - name: Cache and install LVH host dependencies uses: awalsh128/cache-apt-pkgs-action@latest with: # Match LVH action.yaml dependency_list exactly packages: cpu-checker qemu-system-x86 libvirt-daemon-system libvirt-clients bridge-utils virtinst virt-manager version: lvh-deps-1 - name: Run kernel collector simple tests on ${{ matrix.description }} uses: yonch/little-vm-helper@main with: test-name: kernel-collector-simple-test-${{ matrix.kernel }} image: 'complexity-test' image-version: ${{ matrix.kernel }} host-mount: ./ images-folder-parent: "/tmp" cpu: 2 mem: 2G cpu-kind: 'host,pmu=on' lvh-version: "v0.0.23" install-dependencies: 'true' verbose: 'true' cmd: | set -e # Exit on any error cd /host # Load container images docker load < container-exports/kernel-collector.tar # Start nc listener apt-get update && apt-get install -y netcat-openbsd echo "Starting netcat listener on port 8000..." nc -vl 8000 & nc_pid=$! echo "NC listener started with PID: $nc_pid" # Wait a moment for nc to start sleep 2 # Test: Verify kernel collector loads successfully with libbpf echo "=== Kernel Collector Simple Test with libbpf ===" # Run kernel collector and verify it starts successfully container_id=$(docker create \ --name "test-kernel-collector-libbpf" \ --env EBPF_NET_INTAKE_PORT="8000" \ --env EBPF_NET_INTAKE_HOST="127.0.0.1" \ --env EBPF_NET_HOST_DIR="/hostfs" \ --privileged --pid host --network host \ --volume /sys/fs/cgroup:/hostfs/sys/fs/cgroup \ --volume /etc:/hostfs/etc \ --volume /var/run/docker.sock:/var/run/docker.sock \ localhost:5000/kernel-collector --log-console --debug) echo "Starting kernel collector and running for 30 seconds..." docker start $container_id & collector_pid=$! # Wait for 30 seconds sleep 30 # Check if container is still running echo Checking if container is still running: if docker ps --filter "id=$container_id" --filter "status=running" --quiet > /dev/null; then echo "✓ Kernel collector loaded successfully and ran for 30 seconds" echo "---Kernel collector logs:" collector_logs=$(docker logs $container_id 2>&1 || true) echo "$collector_logs" # Fail if a crash was detected in the kernel collector logs if echo "$collector_logs" | grep -qi "CRASH DETECTED"; then echo "✗ Crash detected in kernel collector output - test failed" docker stop $container_id || true docker rm $container_id || true # Stop nc listener kill $nc_pid || true exit 1 fi # Check for error strings in the logs (exclude GCP metadata fetch errors which are expected) if echo "$collector_logs" | grep -i "error" | grep -v "Unable to fetch GCP metadata: error while fetching Google Cloud Platform instance metadata" > /dev/null 2>&1; then echo "✗ Found 'error' in kernel collector output - test failed" docker stop $container_id || true docker rm $container_id || true # Stop nc listener kill $nc_pid || true exit 1 fi docker stop $container_id || true docker rm $container_id || true # Stop nc listener kill $nc_pid || true exit 0 else echo "✗ Kernel collector failed to run properly" echo "---Kernel collector logs:" docker logs $container_id || true docker rm $container_id || true # Stop nc listener kill $nc_pid || true exit 1 fi - name: Stop qemu if: always() run: | sudo pkill -f qemu-system-x86_64 || true run-kernel-collector-tests: name: run-kernel-collector-tests needs: [build-reducer, build-kernel-collector-test] runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: include: # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.4-20250721.013324' description: 'Kernel 5.4' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.10-20250507.063028' description: 'Kernel 5.10' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '5.15-20250507.063028' description: 'Kernel 5.15' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.1-20250507.063028' description: 'Kernel 6.1' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.6-20250507.063028' description: 'Kernel 6.6' # renovate: datasource=docker depName=quay.io/lvh-images/complexity-test - kernel: '6.12-20250507.063028' description: 'Kernel 6.12' timeout-minutes: 10 steps: - name: Check out the codebase uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Download reducer container uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: reducer-container path: ./container-exports - name: Download kernel-collector-test container uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: kernel-collector-test-container path: ./container-exports - name: Cache and install LVH host dependencies uses: awalsh128/cache-apt-pkgs-action@latest with: # Match LVH action.yaml dependency_list exactly packages: cpu-checker qemu-system-x86 libvirt-daemon-system libvirt-clients bridge-utils virtinst virt-manager version: lvh-deps-1 - name: Run kernel collector tests on ${{ matrix.description }} uses: yonch/little-vm-helper@main with: test-name: kernel-collector-test-${{ matrix.kernel }} image: 'complexity-test' image-version: ${{ matrix.kernel }} host-mount: ./ images-folder-parent: "/tmp" cpu: 2 mem: 2G cpu-kind: 'host,pmu=on' lvh-version: "v0.0.23" install-dependencies: 'true' verbose: 'true' cmd: | set -e # Exit on any error cd /host # Load container images docker load < container-exports/reducer.tar docker load < container-exports/kernel-collector-test.tar # Create data directory mkdir -p data # Start reducer reducer_id=$(docker run --detach --rm \ --network=host \ localhost:5000/reducer \ --port 8000 \ --prom 0.0.0.0:7000 \ --partitions-per-shard 1 \ --num-ingest-shards=1 \ --num-matching-shards=1 \ --num-aggregation-shards=1 \ --enable-aws-enrichment \ --enable-otlp-grpc-metrics \ --log-console \ --debug) echo "Reducer started with ID: $reducer_id" # Wait a moment for reducer to start sleep 5 # Run kernel collector test and capture real exit code echo "Starting kernel collector test..." set +e # disable exit on error to capture exit status docker run --name kernel-collector-test-$$ \ --rm \ --env EBPF_NET_HOST_DIR="/hostfs" \ --privileged \ --network host \ --volume /sys/fs/cgroup:/hostfs/sys/fs/cgroup \ --volume /usr/src:/hostfs/usr/src \ --volume /lib/modules:/hostfs/lib/modules \ --volume /etc:/hostfs/etc \ --volume /var/cache:/hostfs/cache \ --volume /var/run/docker.sock:/var/run/docker.sock \ --env EBPF_NET_KERNEL_HEADERS_AUTO_FETCH="true" \ --env EBPF_NET_EXPORT_BPF_SRC_FILE="/hostfs/data/bpf.src.c" \ --volume "$(pwd)/data:/hostfs/data" \ localhost:5000/kernel-collector-test \ --log-console test_exit_code=$? set -e # re-enable exit on error # Stop reducer docker stop $reducer_id || true echo "Test completed with exit code: $test_exit_code" exit $test_exit_code - name: Stop qemu if: always() run: | sudo pkill -f qemu-system-x86_64 || true - name: Upload kernel-collector dumps if: always() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: kernel-collector-dumps-${{ matrix.kernel }} path: | data/*.json if-no-files-found: warn ================================================ FILE: .github/workflows/build-benv-multiarch.yaml ================================================ name: build-benv-multiarch on: workflow_dispatch: push: branches: [ main ] paths: - 'build-tools/**' - '.github/workflows/build-benv-multiarch.yaml' permissions: contents: read packages: write jobs: context: runs-on: ubuntu-latest outputs: short-sha: ${{ steps.s.outputs.s }} steps: - name: Checkout code uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Get short SHA id: s run: echo "s=$(git rev-parse --short=7 ${{ github.sha }})" >> "$GITHUB_OUTPUT" build-amd64: needs: context uses: ./.github/workflows/build-benv-single-arch.yaml permissions: contents: read packages: write with: runner: ubuntu-24.04 arch: amd64 ref: ${{ github.ref }} force_rebuild: false cache_registry: ghcr.io secrets: inherit build-arm64: needs: context uses: ./.github/workflows/build-benv-single-arch.yaml permissions: contents: read packages: write with: runner: ubuntu-24.04-arm arch: arm64 ref: ${{ github.ref }} force_rebuild: false cache_registry: ghcr.io secrets: inherit manifest: needs: [context, build-amd64, build-arm64] if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read packages: write env: GHCR_CACHE_REPO: ghcr.io/${{ github.repository_owner }}/opentelemetry-network-build-tools-cache DOCKER_REPO: docker.io/${{ vars.DOCKER_NAMESPACE || 'otel' }}/opentelemetry-network-build-tools SHORT_SHA: ${{ needs.context.outputs.short-sha }} AMD64_SRC: ${{ needs.build-amd64.outputs.final_image }} ARM64_SRC: ${{ needs.build-arm64.outputs.final_image }} steps: - name: Login to GHCR uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Create GHCR multi-arch cache manifest shell: bash run: | set -euo pipefail CACHE_TAG="$GHCR_CACHE_REPO:final-multiarch-$SHORT_SHA" docker buildx imagetools create --tag "$CACHE_TAG" "$AMD64_SRC" "$ARM64_SRC" docker buildx imagetools inspect "$CACHE_TAG" - name: Login to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: docker.io username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Create Docker Hub multi-arch manifest shell: bash run: | set -euo pipefail MAIN_TAG="$DOCKER_REPO:git-$SHORT_SHA" docker buildx imagetools create --tag "$MAIN_TAG" "$AMD64_SRC" "$ARM64_SRC" if [ "${GITHUB_REF##*/}" = "main" ]; then docker buildx imagetools create --tag "$DOCKER_REPO:latest" "$AMD64_SRC" "$ARM64_SRC" fi docker buildx imagetools inspect "$MAIN_TAG" ================================================ FILE: .github/workflows/build-benv-single-arch.yaml ================================================ name: build-benv-single-arch on: workflow_call: inputs: runner: description: Runner label (ubuntu-24.04 or ubuntu-24.04-arm) required: true type: string arch: description: Target architecture (amd64 or arm64) required: true type: string ref: description: Tag, branch or SHA to checkout required: false type: string default: main cache_registry: description: Cache registry to use (GHCR) required: false type: string default: ghcr.io force_rebuild: description: Force rebuild all containers (ignore cache) required: false type: boolean default: false outputs: final_image: description: Full image tag for the final stage in the cache registry value: ${{ jobs.build-final.outputs['full-image-tag'] }} permissions: contents: read packages: write env: CACHE_REGISTRY: ${{ inputs.cache_registry || 'ghcr.io' }} REF: ${{ inputs.ref || github.ref }} FORCE_REBUILD: ${{ inputs.force_rebuild || false }} jobs: build-base: name: Build base image runs-on: ${{ inputs.runner }} steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push base image uses: ./.github/actions/build-tools-single-stage/ with: directory: base arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-libuv: name: Build libuv image runs-on: ${{ inputs.runner }} needs: build-base steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push libuv image uses: ./.github/actions/build-tools-single-stage/ with: directory: libuv arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-cpp-misc: name: Build cpp_misc image runs-on: ${{ inputs.runner }} needs: build-base steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push cpp_misc image uses: ./.github/actions/build-tools-single-stage/ with: directory: cpp_misc arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-libmaxminddb: name: Build libmaxminddb image runs-on: ${{ inputs.runner }} needs: build-base steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push libmaxminddb image uses: ./.github/actions/build-tools-single-stage/ with: directory: libmaxminddb arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-libbpf: name: Build libbpf image runs-on: ${{ inputs.runner }} needs: build-base steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push libbpf image uses: ./.github/actions/build-tools-single-stage/ with: directory: libbpf arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-aws-sdk: name: Build aws_sdk image runs-on: ${{ inputs.runner }} needs: [build-base] steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push aws_sdk image uses: ./.github/actions/build-tools-single-stage/ with: directory: aws_sdk arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} build-final: name: Build final image runs-on: ${{ inputs.runner }} needs: [ build-base, build-libuv, build-aws-sdk, build-cpp-misc, build-libmaxminddb, build-libbpf ] outputs: image-tag: ${{ steps.build.outputs.image-tag }} full-image-tag: ${{ steps.build.outputs.full-image-tag }} steps: - name: Checkout repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: ref: ${{ env.REF }} - name: Build and push final image to cache registry id: build uses: ./.github/actions/build-tools-single-stage/ with: directory: final arch: ${{ inputs.arch }} registry: ${{ env.CACHE_REGISTRY }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} ref: ${{ env.REF }} force_rebuild: ${{ env.FORCE_REBUILD }} ================================================ FILE: .github/workflows/fossa.yml ================================================ name: FOSSA scanning on: push: branches: - main permissions: contents: read jobs: fossa: if: github.repository == 'open-telemetry/opentelemetry-network-build-tools' runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - uses: fossas/fossa-action@3ebcea1862c6ffbd5cf1b4d0bd6b3fe7bd6f2cac # v1.7.0 with: api-key: ${{secrets.FOSSA_API_KEY}} team: OpenTelemetry ================================================ FILE: .github/workflows/k8s-collector-integration.yaml ================================================ name: k8s-collector-integration on: workflow_call: push: branches: - main pull_request: paths: - "crates/k8s-collector/**" - ".github/workflows/k8s-collector-integration.yaml" jobs: build-k8s-collector-tests: runs-on: ubuntu-24.04 steps: - name: Check out the codebase uses: actions/checkout@v4 - name: Cache Rust build uses: Swatinem/rust-cache@v2 with: prefix-key: k8s-collector-integration-${{ runner.os }} - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable override: true - name: Build k8s-collector integration tests (no-run) run: | cargo test -p k8s-collector --tests --release --no-run - name: Collect k8s-collector integration test binary run: | set -euxo pipefail BIN=$(find target/release/deps -maxdepth 1 -type f -executable -name 'integration_test-*' | head -n 1) test -n "$BIN" || { echo "k8s-collector integration test binary not found"; exit 1; } cp "$BIN" k8s-collector-integration chmod +x k8s-collector-integration - name: Upload k8s-collector integration binary uses: actions/upload-artifact@v4 with: name: k8s-collector-integration path: k8s-collector-integration if-no-files-found: error k8s-collector-e2e: needs: build-k8s-collector-tests runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Check out the codebase uses: actions/checkout@v4 - name: Set up kind cluster uses: helm/kind-action@v1.10.0 with: version: v0.30.0 cluster_name: k8s-collector-e2e - name: Download k8s-collector integration binary uses: actions/download-artifact@v4 with: name: k8s-collector-integration path: ./ - name: Prepare test binary run: | chmod +x ./k8s-collector-integration || true - name: Run k8s-collector integration tests env: RUST_LOG: info run: | for i in {1..10}; do echo "k8s-collector integration test run ${i}/10" ./k8s-collector-integration --include-ignored --nocapture --test-threads=1 || exit 1 done ================================================ FILE: .github/workflows/ossf-scorecard.yml ================================================ name: OSSF Scorecard on: push: branches: - main schedule: - cron: "50 10 * * 3" # once a week workflow_dispatch: permissions: read-all jobs: analysis: if: github.repository == 'open-telemetry/opentelemetry-network-build-tools' runs-on: ubuntu-latest permissions: # Needed for Code scanning upload security-events: write # Needed for GitHub OIDC token if publish_results is true id-token: write steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: persist-credentials: false - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif publish_results: true # Upload the results as artifacts (optional). Commenting out will disable # uploads of run results in SARIF format to the repository Actions tab. # https://docs.github.com/en/actions/advanced-guides/storing-workflow-data-as-artifacts - name: "Upload artifact" uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@014f16e7ab1402f30e7c3329d33797e7948572db # v4.31.3 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/release-please.yml ================================================ name: release-please on: push: branches: [ main ] workflow_dispatch: {} permissions: contents: write pull-requests: write jobs: release-please: runs-on: ubuntu-24.04 outputs: release_created: ${{ steps.release.outputs.release_created }} tag_name: ${{ steps.release.outputs.tag_name }} steps: - name: Run Release Please id: release uses: googleapis/release-please-action@v4 with: token: ${{ secrets.GITHUB_TOKEN }} config-file: release-please-config.json manifest-file: .release-please-manifest.json build-and-release: needs: release-please if: ${{ needs.release-please.outputs.release_created == 'true' }} uses: ./.github/workflows/build-and-release.yaml with: release_type: public # Build from the tag that Release Please just created (e.g., v0.12.0). ref: ${{ needs.release-please.outputs.tag_name }} image_prefix: opentelemetry-ebpf- additional_tag: '' dry_run: false secrets: inherit ================================================ FILE: .github/workflows/scripts/check-clang-format.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 CLANG_FORMAT="clang-format-19" if ! command -v ${CLANG_FORMAT} then echo "ERROR: requires ${CLANG_FORMAT}" exit 1 fi RC=0 CMD="${CLANG_FORMAT} -Werror --dry-run -style=file" function check_file { if ! ${CMD} $1 then RC=1 fi } # Check that C and C++ source files are properly clang-formatted FILES=$(find ./geoip ./reducer ./test ./collector/kernel ./common ./tools \ -type f \ \( -name "*.c" \ -o -name "*.cc" \ -o -name "*.h" \ -o -name "*.inl" \) \ -print) for FILE in ${FILES} do check_file ${FILE} done exit ${RC} ================================================ FILE: .github/workflows/trivy-scans.yml ================================================ name: trivy scans on: push: branches: - main pull_request: paths: - '.github/workflows/trivy-scans.yml' - '.trivyignore' permissions: contents: read jobs: trivy-fs-scan: if: github.repository == 'open-telemetry/opentelemetry-network-build-tools' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Run trivy filesystem scan uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1 with: scan-type: 'fs' scan-ref: '.' skip-dirs: 'docs,cmake,ext' format: 'table' exit-code: '1' severity: 'CRITICAL,HIGH' ignore-unfixed: true vuln-type: 'os,library' timeout: 10m ================================================ FILE: .gitignore ================================================ *.o *.pyc *.bak *.so *.a *~ !.* *.backup *.aux *.log *.out *.profile *.bbl *.blg *.brf .autotools .cproject .*.cmd .*.d .deps .dirstamp *.la *.lo .libs .pydevproject .Rhistory .settings .settings/ .~lock*# *.xxd payload_trace.dat trace_stats.csv exec_at generated/ .DS_Store *.sw? *.orig target/ MANIFEST .sconsign.dblite CMakeFiles/ cmake_install.cmake CMakeCache.txt CTestTestfile.cmake Testing/ install_manifest.txt .idea/ aclocal.m4 configure autom4te.cache build-aux install-sh missing py-compile release/ out/ stamp-h1 src/Makefile config.h config.status libtool perf.data perf.data.old # Visual Studio Code .vscode/settings.json # Visual C++ cache files ipch/ # Gnu Global GPATH GRTAGS GTAGS #local Vagrantfile ================================================ FILE: .gitmodules ================================================ [submodule "ext/civetweb"] path = ext/civetweb url = https://github.com/civetweb/civetweb.git [submodule "build-tools/libmaxminddb/libmaxminddb"] path = build-tools/libmaxminddb/libmaxminddb url = https://github.com/maxmind/libmaxminddb [submodule "build-tools/libbpf/bpftool"] path = build-tools/libbpf/bpftool url = https://github.com/libbpf/bpftool.git [submodule "build-tools/cpp_misc/lz4"] path = build-tools/cpp_misc/lz4 url = https://github.com/lz4/lz4.git [submodule "build-tools/aws_sdk/aws-sdk-cpp"] path = build-tools/aws_sdk/aws-sdk-cpp url = https://github.com/aws/aws-sdk-cpp [submodule "build-tools/cpp_misc/json"] path = build-tools/cpp_misc/json url = https://github.com/nlohmann/json.git [submodule "build-tools/libuv/libuv"] path = build-tools/libuv/libuv url = https://github.com/libuv/libuv.git [submodule "build-tools/cpp_misc/spdlog"] path = build-tools/cpp_misc/spdlog url = https://github.com/gabime/spdlog.git [submodule "build-tools/cpp_misc/args"] path = build-tools/cpp_misc/args url = https://github.com/Taywee/args.git [submodule "build-tools/libbpf/libbpf"] path = build-tools/libbpf/libbpf url = https://github.com/libbpf/libbpf.git [submodule "build-tools/cpp_misc/yaml-cpp"] path = build-tools/cpp_misc/yaml-cpp url = https://github.com/jbeder/yaml-cpp.git [submodule "build-tools/cpp_misc/googletest"] path = build-tools/cpp_misc/googletest url = https://github.com/google/googletest.git [submodule "ext/vmlinux.h"] path = ext/vmlinux.h url = https://github.com/libbpf/vmlinux.h.git ================================================ FILE: .release-please-manifest.json ================================================ { ".": "0.11.0" } ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file by Release Please. ================================================ FILE: CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 cmake_minimum_required(VERSION 3.12) # Load container-friendly defaults (paths, build type, version vars) include(${CMAKE_CURRENT_LIST_DIR}/cmake/defaults.cmake) project( opentelemetry-ebpf VERSION ${EBPF_NET_MAJOR_VERSION}.${EBPF_NET_MINOR_VERSION}.${EBPF_NET_PATCH_VERSION} ) include(GNUInstallDirs) list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake ) include(FindPkgConfig) if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) # Building as a subproject (submodule). set(EBPF_NET_SUBPROJECT TRUE) endif() # Custom modules # include(cpp-compiler) include(ccache) include(docker-utils) include(sanitizer) include(executable) include(xxd) include(shell) include(debug) include(lz4) include(openssl) include(civetweb) include(curl) include(curlpp) include(spdlog) include(aws-sdk) include(geoip) include(protobuf) include(llvm) include(clang) include(libelf) include(test) include(uv) include(abseil) include(yamlcpp) include(libbpf) include(render) include(cargo-test) include(rust_cxxbridge) include_directories( ${PROJECT_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/include ) include_directories(${CMAKE_CURRENT_BINARY_DIR}) set(CONFIG_H_DIR ${CMAKE_CURRENT_BINARY_DIR}) add_custom_target(pipeline) add_custom_target(pipeline-docker) add_custom_target(pipeline-docker-registry) add_subdirectory(renderc) add_subdirectory(render) add_subdirectory(config) add_subdirectory(channel) add_subdirectory(platform) add_subdirectory(scheduling) add_subdirectory(util) add_subdirectory(geoip) add_subdirectory(jitbuf) add_subdirectory(collector) add_subdirectory(reducer) add_subdirectory(dev) add_subdirectory(tools) add_subdirectory(dist) configure_file(config.h.cmake_in config.h) configure_file(util/version_config.h.cmake_in util/version_config.h) add_dependencies(pipeline collectors reducer) add_dependencies(pipeline-docker collectors-docker) add_dependencies(pipeline-docker-registry collectors-docker-registry) ================================================ FILE: Cargo.toml ================================================ [workspace] resolver = "2" members = [ "crates/kernel-collector-sys", "crates/kernel-collector-bin", "crates/reducer-sys", "crates/reducer-bin", "crates/cloud-collector-sys", "crates/cloud-collector-bin", "crates/k8s-relay-sys", "crates/k8s-relay-bin", "crates/reducer", "crates/perfect_hash_map", "crates/render_parser", "crates/otlp_export", "crates/element-queue", "crates/timeslot", # Render encoders (ebpf_net) "crates/render/ebpf_net", "crates/render/ebpf_net/agent_internal", "crates/render/ebpf_net/aggregation", "crates/render/ebpf_net/cloud_collector", "crates/render/ebpf_net/ingest", "crates/render/ebpf_net/kernel_collector", "crates/render/ebpf_net/logging", "crates/render/ebpf_net/matching", "crates/k8s-collector", "crates/k8s-collector-bin", ] [workspace.dependencies] render_parser = { path = "crates/render_parser" } otlp_export = { path = "crates/otlp_export" } reducer = { path = "crates/reducer" } reducer-sys = { path = "crates/reducer-sys" } rc-hashmap = { version = "0.1" } ================================================ FILE: LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE.txt ================================================ OpenTelemetry eBPF ================== https://github.com/open-telemetry/opentelemetry-ebpf Copyright The OpenTelemetry Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Third party software ==================== ------------------------------------------------------------------------------- args https://github.com/Taywee/args Copyright (c) 2016-2017 Taylor C. Richberger and Pavel Belikov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- spdlog https://github.com/gabime/spdlog Copyright (c) 2016 Gabi Melman. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- ares https://github.com/c-ares/c-ares /* Copyright 1998 by the Massachusetts Institute of Technology. * Copyright (C) 2007-2013 by Daniel Stenberg * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ ------------------------------------------------------------------------------- bcc https://github.com/iovisor/bcc Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- abseil https://github.com/abseil/abseil-cpp Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- llvm https://github.com/llvm/llvm-project University of Illinois/NCSA Open Source License Copyright (c) 2003-2019 University of Illinois at Urbana-Champaign. All rights reserved. Developed by: LLVM Team University of Illinois at Urbana-Champaign http://llvm.org Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal with the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimers. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimers in the documentation and/or other materials provided with the distribution. * Neither the names of the LLVM Team, University of Illinois at Urbana-Champaign, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE. ============================================================================== Copyrights and Licenses for Third Party Software Distributed with LLVM: ============================================================================== The LLVM software contains code written by third parties. Such software will have its own individual LICENSE.TXT file in the directory in which it appears. This file will describe the copyrights, license, and restrictions which apply to that code. The disclaimer of warranty in the University of Illinois Open Source License applies to all code in the LLVM Distribution, and nothing in any of the other licenses gives permission to use the names of the LLVM Team or the University of Illinois to endorse or promote products derived from this Software. The following pieces of software have additional or alternate copyrights, licenses, and/or restrictions: Program Directory ------- --------- Google Test llvm/utils/unittest/googletest OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} ARM contributions llvm/lib/Target/ARM/LICENSE.TXT md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h ------------------------------------------------------------------------------- json https://github.com/nlohmann/json Copyright (c) 2013-2018 Niels Lohmann Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- lookup3 http://www.burtleburtle.net/bob/c/lookup3.c lookup3.c, by Bob Jenkins, May 2006, Public Domain. ------------------------------------------------------------------------------- libuv https://github.com/libuv/libuv libuv is licensed for use as follows: ==== Copyright (c) 2015-present libuv project contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ==== This license applies to parts of libuv originating from the https://github.com/joyent/libuv repository: ==== Copyright Joyent, Inc. and other Node contributors. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ==== This license applies to all parts of libuv that are not externally maintained libraries. The externally maintained libraries used by libuv are: - tree.h (from FreeBSD), copyright Niels Provos. Two clause BSD license. - inet_pton and inet_ntop implementations, contained in src/inet.c, are copyright the Internet Systems Consortium, Inc., and licensed under the ISC license. - stdint-msvc2008.h (from msinttypes), copyright Alexander Chemeris. Three clause BSD license. - pthread-fixes.c, copyright Google Inc. and Sony Mobile Communications AB. Three clause BSD license. - android-ifaddrs.h, android-ifaddrs.c, copyright Berkeley Software Design Inc, Kenneth MacKay and Emergya (Cloud4all, FP7/2007-2013, grant agreement n° 289016). Three clause BSD license. ------------------------------------------------------------------------------- openssl https://github.com/openssl/openssl /* ==================================================================== * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. All advertising materials mentioning features or use of this * software must display the following acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" * * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to * endorse or promote products derived from this software without * prior written permission. For written permission, please contact * openssl-core@openssl.org. * * 5. Products derived from this software may not be called "OpenSSL" * nor may "OpenSSL" appear in their names without prior written * permission of the OpenSSL Project. * * 6. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit (http://www.openssl.org/)" * * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * ==================================================================== * * This product includes cryptographic software written by Eric Young * (eay@cryptsoft.com). This product includes software written by Tim * Hudson (tjh@cryptsoft.com). * */ Original SSLeay License ----------------------- /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * * This package is an SSL implementation written * by Eric Young (eay@cryptsoft.com). * The implementation was written so as to conform with Netscapes SSL. * * This library is free for commercial and non-commercial use as long as * the following conditions are aheared to. The following conditions * apply to all code found in this distribution, be it the RC4, RSA, * lhash, DES, etc., code; not just the SSL code. The SSL documentation * included with this distribution is covered by the same copyright terms * except that the holder is Tim Hudson (tjh@cryptsoft.com). * * Copyright remains Eric Young's, and as such any Copyright notices in * the code are not to be removed. * If this package is used in a product, Eric Young should be given attribution * as the author of the parts of the library used. * This can be in the form of a textual message at program startup or * in documentation (online or textual) provided with the package. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * "This product includes cryptographic software written by * Eric Young (eay@cryptsoft.com)" * The word 'cryptographic' can be left out if the rouines from the library * being used are not cryptographic related :-). * 4. If you include any Windows specific code (or a derivative thereof) from * the apps directory (application code) you must include an acknowledgement: * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" * * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * The licence and distribution terms for any publically available version or * derivative of this code cannot be changed. i.e. this code cannot simply be * copied and put under another distribution licence * [including the GNU Public Licence.] */ ------------------------------------------------------------------------------- grpc https://github.com/grpc/grpc Copyright 2014 gRPC authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- curl https://github.com/curl/curl COPYRIGHT AND PERMISSION NOTICE Copyright (c) 1996 - 2017, Daniel Stenberg, , and many contributors, see the THANKS file. All rights reserved. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization of the copyright holder. ------------------------------------------------------------------------------- curlpp https://github.com/jpbarrette/curlpp Copyright (c) <2002-2004> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (cURLpp), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- aws-sdk-cpp https://github.com/aws/aws-sdk-cpp Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- google-cloud-cpp https://github.com/googleapis/google-cloud-cpp Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- ccan https://github.com/rustyrussell/ccan The list module is licensed under BSD-MIT: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. The build_assert, check_type, compiler, container_of and hash modules are licensed under CC0 (Public domain): Statement of Purpose The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. 1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; moral rights retained by the original author(s) and/or performer(s); publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; rights protecting the extraction, dissemination, use and reuse of data in a Work; database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. 2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. 3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. 4. Limitations and Disclaimers. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. ------------------------------------------------------------------------------- lz4 https://github.com/lz4/lz4 This repository uses 2 different licenses : - all files in the `lib` directory use a BSD 2-Clause license - all other files use a GPLv2 license, unless explicitly stated otherwise Relevant license is reminded at the top of each source file, and with presence of COPYING or LICENSE file in associated directories. This model is selected to emphasize that files in the `lib` directory are designed to be included into 3rd party applications, while all other files, in `programs`, `tests` or `examples`, receive more limited attention and support for such scenario. ------------------------------------------------------------------------------- yaml-cpp https://github.com/jbeder/yaml-cpp Copyright (c) 2008-2015 Jesse Beder. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- librdkafka https://github.com/edenhill/librdkafka librdkafka - Apache Kafka C driver library Copyright (c) 2012-2020, Magnus Edenhill All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- libmaxminddb https://github.com/maxmind/libmaxminddb Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- gradle https://github.com/gradle/gradle Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ------------------------------------------------------------------------------- demangle https://github.com/juchem/demangle BSD 3-Clause License Copyright (c) 2016, Marcelo Juchem All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: README.md ================================================ # OpenTelemetry eBPF # The OpenTelemetry eBPF project develops components that collect and analyze telemetry from the operating system, cloud, and container orchestrators. Its initial focus is on collecting network data to enable users to gain insight into their distributed applications. The _kernel collector_ gathers low level telemetry straight from the Linux kernel using [eBPF](https://ebpf.io/). It does so with negligible compute and network overheads. The _kubernetes collector_ and _cloud collector_ gather workload metadata. This telemetry is then sent to the _reducer_, which enriches and aggregates it. The reducer outputs metrics to the OpenTelemetry collector. ## Building ## For instructions on how to build the repository, see the [Developer Guide](docs/developing.md). ## Running ## For instructions on how to get OpenTelemetry-eBPF up-and-running, check the documentation for the individual components: - [reducer](docs/reducer.md) - [kernel-collector](docs/kernel-collector.md) - [cloud-collector](docs/cloud-collector.md) - [k8s-collector](docs/k8s-collector.md) ## Contributing ## Check out the [Developer Guide](docs/developing.md). See the [Roadmap](docs/roadmap.md) for an overwiew of the project's goals. ### Maintainers - [Borko Jandras](https://github.com/bjandras) - [Jim Wilson](https://github.com/jmw51798), DataDog - [Jonathan Perry](https://github.com/yonch) For more information about the maintainer role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#maintainer). ### Approvers - [Samiur Arif](https://github.com/samiura), Netbox Labs - Actively seeking approvers to review pull requests For more information about the approver role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#approver). ### Triagers - Actively seeking contributors to triage issues ### Emeritus Triagers - [Antoine Toulme](https://github.com/atoulme), Splunk For more information about the triager role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#triager). ## Questions ## You can connect with us in our [slack channel](https://cloud-native.slack.com/archives/C02AB15583A). The OpenTelemetry eBPF special interest group (SIG) meets regularly, and the meetting is held every week on Tuesday at 09:00 Pacific time. See the [eBPF Workgroup Meeting Notes](https://docs.google.com/document/d/13GK915hdDQ9sUYzUIWi4pOfJK68EE935ugutUgL3yOw) for a summary description of past meetings. ================================================ FILE: RELEASING.md ================================================ # Making a public release Public builds are those intended to be used by the general audience. ## Release procedure (Release Please) 1. Ensure that changes on `main` use Conventional Commit messages so Release Please can determine the correct next version. 1. Wait for the `release-please` workflow to open a Release PR, or trigger it manually from https://github.com/open-telemetry/opentelemetry-network/actions/workflows/release-please.yml. 1. Review the Release PR: - Confirm that `VERSION` and `CHANGELOG.md` are updated as expected. - Make any edits you need to the release notes in the PR description. 1. Merge the Release PR. 1. Once merged, Release Please will: - Create a Git tag `vMAJOR.MINOR.PATCH`. - Publish a GitHub Release with the generated release notes. 1. The `build-and-release` workflow will run automatically on the `release.published` event: - It checks out the release tag, builds RPM/DEB packages and container images, and uploads the RPM/DEB packages to the existing GitHub Release. - It pushes container images to the configured registry with tags: - `latest` - `latest-vMAJOR.MINOR` - `vMAJOR.MINOR.PATCH` 1. Verify the published GitHub Release at https://github.com/open-telemetry/opentelemetry-network/releases and confirm the assets. 1. On the Docker registry (for example https://hub.docker.com/r/otel/), confirm that the following images are tagged with the new tags: * opentelemetry-ebpf-reducer * opentelemetry-ebpf-kernel-collector * opentelemetry-ebpf-cloud-collector * opentelemetry-ebpf-k8s-watcher * opentelemetry-ebpf-k8s-relay If needed, you can still trigger the `build-and-release` workflow manually from https://github.com/open-telemetry/opentelemetry-network/actions/workflows/build-and-release.yaml with `release_type: public` to rebuild artifacts for an existing tag. ## Unofficial builds Unofficial builds, while public, are not intended for the use by the general audience. Because of that, no unofficial build should be tagged in such a way that it gets used automatically – no tagging with latest or with a version tag. Use a personal fork of the opentelemetry-network repository on GitHub. Make sure that the following repository secrets and variables are set up for the fork (Settings -> Secrets and variables -> Actions): variables: - DOCKER_REGISTRY: registry, such as quay.io - DOCKER_NAMESPACE: image name, such as o11ytest/network-explorer-debug secrets: - DOCKER_USERNAME: username to login to the registry - DOCKER_PASSWORD: password to loging to the registry Navigate to Actions -> build-and-release, click the "Run workflow" button. Select unofficial for the release type. Use whichever commit SHA, tag or branch name you wish to make a build of. The resulting docker images will be tagged with vMAJOR.MINOR.PATCH-GITHASH tag. ================================================ FILE: VERSION ================================================ 0.11.0 ================================================ FILE: build-tools/.gitignore ================================================ /Debug/ /Release/ ================================================ FILE: build-tools/.templates/dependency/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # DEPENDENCY_NAME ARG base_IMAGE_TAG FROM $base_IMAGE_TAG ARG NPROC WORKDIR $HOME COPY DEPENDENCY_NAME DEPENDENCY_NAME WORKDIR $HOME/DEPENDENCY_NAME # add build/install commands here, e.g.: #RUN ./bootstrap #RUN ./configure --prefix=$HOME/install \ # --enable-static #RUN nice make -j$NPROC && make install ================================================ FILE: build-tools/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 cmake_minimum_required (VERSION 3.5) project (opentelemetry-ebpf-build-tools VERSION 0.1.0) # Architecture: # # The build is comprised of multiple directories, each making one Docker image. # Each such directory can use other docker images as substrates or for artefacts. # The build system should rebuild a container when: # 1. There was a more recent commit into the directory, or # 2. One of the dependencies was rebuilt, or # 3. The build was cleaned, or # 4. docker doesn't have the images (e.g., because a dev explicitly erased it) # # We maintain two files for each directory inside the build-status directory: # A. `missing`: touch'd if docker doesn't already have an image for the most recent # commit into the directory # B. `built`: touch'd when we've successfully created a docker image # # Each directory's docker build depends on its `missing` and the `built` of its dependencies, # and outputs (touch's) its own `built`. # # On every run, the build system always checks if the most recent commit to the directory is in # docker, and if not touches `missing`. This handles (1), and (4). Furthermore, if `missing` # is itself not on the filesystem, then it is touched -- this handles (3). When one of the # dependencies is rebuilt, this causes a rebuild of the container, solving (2). set(STATUS_DIR "build-status") # a dummy target to force checks against docker add_custom_command( OUTPUT "dummy_target_to_force_rebuild" COMMAND true ) include(ProcessorCount) ProcessorCount(NPROCS) if(${NPROCS} GREATER 1) # don't use up all the cores, leave at least one for other processes and # scheduling to avoid trashing math(EXPR NPROCS "${NPROCS} - 1") endif() message(STATUS "using ${NPROCS} parallel jobs to build") option(BENV_UNMINIMIZE "whether or not to unminimize the benv image" OFF) function(build_directory NAME) cmake_parse_arguments(P "" "" "DEPENDS" ${ARGN}) # the missing filename set(MISSING_FILENAME ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${NAME}/missing) # update the `missing` file add_custom_target(check_missing_${NAME} # OUTPUT # ${MISSING_FILENAME} # our real output COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${NAME} COMMAND ${CMAKE_SOURCE_DIR}/check_missing.sh ${NAME} ${MISSING_FILENAME} # DEPENDS "dummy_target_to_force_rebuild" # fake, to force the check to always run ) # for each dependent directory, make command line parameters to pass docker the # directory's resulting docker tag set(DOCKER_PARAMS) # the image tags for the dependencies to pass to docker set(DEPENDS_FILES) # the `built` files of the dependencies list(APPEND DOCKER_PARAMS "--build-arg" "NPROC=${NPROCS}") foreach(DEP ${P_DEPENDS}) list(APPEND DOCKER_PARAMS "--build-arg" "${DEP}_IMAGE_TAG=$$(${CMAKE_SOURCE_DIR}/get_tag.sh" "${DEP})") list(APPEND DEPENDS_FILES "${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${DEP}/built") endforeach() if(DEFINED ENV{BENV_BASE_IMAGE_DISTRO}) message(STATUS "using $ENV{BENV_BASE_IMAGE_DISTRO} as base image distro for ${NAME}") list(APPEND DOCKER_PARAMS "--build-arg" "BENV_BASE_IMAGE_DISTRO=$ENV{BENV_BASE_IMAGE_DISTRO}") endif() if(DEFINED ENV{BENV_BASE_IMAGE_VERSION}) message(STATUS "using $ENV{BENV_BASE_IMAGE_VERSION} as base image version for ${NAME}") list(APPEND DOCKER_PARAMS "--build-arg" "BENV_BASE_IMAGE_VERSION=$ENV{BENV_BASE_IMAGE_VERSION}") endif() if(BENV_UNMINIMIZE) list(APPEND DOCKER_PARAMS "--build-arg" "BENV_UNMINIMIZE=true") endif() list(APPEND DOCKER_PARAMS "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") if(CMAKE_BUILD_TYPE STREQUAL "Debug") list(APPEND DOCKER_PARAMS "--build-arg" "RESTRICTED_NPROC='1'") list(APPEND DOCKER_PARAMS "--build-arg" "BUILD_CFLAGS='-O0 -g'") list(APPEND DOCKER_PARAMS "--build-arg" "CONFIGURE_ENABLE_DEBUG='--enable-debug'") list(APPEND DOCKER_PARAMS "--build-arg" "CONFIGURE_DEBUG='--debug'") list(APPEND DOCKER_PARAMS "--build-arg" "CONFIGURE_RELEASE_DEBUG='--debug'") else() list(APPEND DOCKER_PARAMS "--build-arg" "RESTRICTED_NPROC=${NPROCS}") list(APPEND DOCKER_PARAMS "--build-arg" "GRPC_BUILD_CFLAGS='-Wno-error=class-memaccess -Wno-error=ignored-qualifiers -Wno-error=stringop-truncation'") list(APPEND DOCKER_PARAMS "--build-arg" "CONFIGURE_RELEASE_DEBUG='--release'") endif() add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${NAME}/built COMMAND docker build -t "$$(${CMAKE_SOURCE_DIR}/get_tag.sh" "${NAME})" ${DOCKER_PARAMS} ${CMAKE_SOURCE_DIR}/${NAME} COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${NAME}/built DEPENDS ${P_DEPENDS} check_missing_${NAME} ${MISSING_FILENAME} ${DEPENDS_FILES} ) add_custom_target(${NAME} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${STATUS_DIR}/${NAME}/built) endfunction(build_directory) build_directory(base) build_directory(openssl DEPENDS base) build_directory(curl DEPENDS base openssl) build_directory(libuv DEPENDS base) build_directory(aws_sdk DEPENDS base openssl curl) build_directory(cpp_misc DEPENDS base) build_directory(grpc_cpp DEPENDS base abseil_cpp openssl) build_directory(gcp_cpp DEPENDS base openssl curl grpc_cpp) build_directory(abseil_cpp DEPENDS base) build_directory(libmaxminddb DEPENDS base) build_directory(libbpf DEPENDS base) #gen:dep-dir build_directory( final DEPENDS base openssl curl libuv aws_sdk cpp_misc grpc_cpp abseil_cpp libmaxminddb gcp_cpp libbpf ) if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_custom_target(benv ALL DEPENDS final COMMAND docker tag "$$(${CMAKE_SOURCE_DIR}/get_tag.sh" "final)" debug-build-env) else() add_custom_target(debug-benv ALL DEPENDS final COMMAND docker tag "$$(${CMAKE_SOURCE_DIR}/get_tag.sh" "final)" build-env) endif() ================================================ FILE: build-tools/add_dependency.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 set -e if [ -z "$2" ]; then echo "usage: $0 dependency_name dependency_git_url" exit 1 fi dep_name="$1" dep_repo="$2" echo "adding dependency $dep_name" echo " from repo $dep_repo" mkdir "$dep_name" sed "s/DEPENDENCY_NAME/$dep_name/g" \ ".templates/dependency/Dockerfile" \ > "$dep_name/Dockerfile" git add "$dep_name/Dockerfile" sed -i \ -e "s/^#gen:dep-dir\$/build_directory($dep_name DEPENDS base)\n&/g" \ -e "s/^\(build_directory(final DEPENDS base .*\))\$/\1 $dep_name)/g" \ CMakeLists.txt git add CMakeLists.txt sed -i \ -e "s/^#gen:dep-arg\$/ARG ${dep_name}_IMAGE_TAG\n&/g" \ -e "s/^#gen:dep-from\$/FROM \\\$${dep_name}_IMAGE_TAG as build-${dep_name}\n&/g" \ -e "s/^#gen:dep-copy\$/COPY --from=build-${dep_name} \\\$HOME\/install \\\$HOME\/install\n&/g" \ final/Dockerfile git add final/Dockerfile git submodule add "$dep_repo" "$dep_name/$dep_name" git commit -m "adding dependency $dep_name" echo echo "ACTION REQUIRED:" echo "update file \`$dep_name/Dockerfile\` with proper build instructions then update commit with:" echo echo " editor \"$dep_name/Dockerfile\" \\" echo " && git add \"$dep_name/Dockerfile\" \\" echo " && git commit --amend --no-edit" ================================================ FILE: build-tools/aws_sdk/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # AWS SDK ARG base_IMAGE_TAG ARG CMAKE_BUILD_TYPE FROM $base_IMAGE_TAG AS build ARG NPROC WORKDIR $HOME COPY --chown=${UID}:${GID} aws-sdk-cpp aws-sdk-cpp WORKDIR $HOME/build/aws-sdk-cpp RUN cmake \ -DCUSTOM_MEMORY_MANAGEMENT=0 \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_ONLY="ec2;s3" \ -DFORCE_CURL=ON \ -DUSE_OPENSSL=ON \ -DENABLE_TESTING=OFF \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_PREFIX:PATH=$HOME/install \ $HOME/aws-sdk-cpp RUN nice make -j${NPROC:-3} RUN nice make install # Runtime stage - copy only necessary artifacts FROM $base_IMAGE_TAG COPY --from=build $HOME/install $HOME/install ================================================ FILE: build-tools/base/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 ARG BENV_BASE_IMAGE_DISTRO=debian ARG BENV_BASE_IMAGE_VERSION=trixie@sha256:01a723bf5bfb21b9dda0c9a33e0538106e4d02cce8f557e118dd61259553d598 FROM ${BENV_BASE_IMAGE_DISTRO}:${BENV_BASE_IMAGE_VERSION} AS build-main ################ DEPENDENCIES ################ # fixes for some of the build bugs/warnings in docker RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections ENV BENV_BASE_IMAGE_DISTRO=${BENV_BASE_IMAGE_DISTRO} ENV BENV_BASE_IMAGE_VERSION=${BENV_BASE_IMAGE_VERSION} # Package definitions ARG PKG_CORE="wget curl git gnupg bc aptitude netcat-openbsd sudo" ARG PKG_TEXT="xxd sed ripgrep less jq" ARG PKG_COMPILERS="g++" ARG PKG_BUILD="ninja-build" ARG PKG_LINTERS="clang-format-19 clang-tidy-19 shellcheck" ARG PKG_MANAGERS="pkg-config rpm" ARG PKG_KERNEL="dkms build-essential" ARG PKG_DEV="gdb cgdb tmux strace" ARG PKG_LIBS="libc-ares-dev libelf-dev libssl-dev libzstd-dev libgrpc-dev libcurl4-openssl-dev libabsl-dev protobuf-compiler-grpc libcurlpp-dev libgrpc++-dev libprotobuf-dev" ARG PKG_PY_TEST="python3-pytest python3-dev python3-pip python3-setuptools python3-wheel pylint" ARG PKG_JAVA="default-jdk-headless" ARG PKG_LLVM="llvm-19-dev libclang-19-dev clang-19 libpolly-19-dev" ARG PKG_MAKE="cmake ccache autoconf autoconf-archive automake libtool make" ARG PKG_BCC="bison flex zip" ARG PKG_LIBBPF="zip pkg-config libelf-dev zlib1g-dev libbfd-dev libcap-dev" # setup apt (add non-free, contrib and backports) RUN [ "$BENV_BASE_IMAGE_DISTRO" != 'debian' ] || [ "$BENV_BASE_IMAGE_VERSION" != 'sid' ] \ || cat > /etc/apt/sources.list << EOF \ deb http://deb.debian.org/debian/ $BENV_BASE_IMAGE_VERSION main non-free contrib \ deb http://deb.debian.org/debian-security/ $BENV_BASE_IMAGE_VERSION/updates main non-free contrib \ deb http://deb.debian.org/debian/ $BENV_BASE_IMAGE_VERSION-updates main non-free contrib \ deb http://deb.debian.org/debian/ $BENV_BASE_IMAGE_VERSION-backports main non-free contrib \ EOF # Update, upgrade, and install all packages RUN apt-get -y update && \ apt-get -y install --no-install-recommends apt-utils && \ apt-get upgrade -y --no-install-recommends && \ apt-get -y install --no-install-recommends \ $PKG_CORE \ $PKG_TEXT \ $PKG_COMPILERS \ $PKG_BUILD \ $PKG_LINTERS \ $PKG_MANAGERS \ $PKG_KERNEL \ $PKG_DEV \ $PKG_LIBS \ $PKG_PY_TEST \ $PKG_JAVA \ $PKG_LLVM \ $PKG_LIBBPF && \ apt-get -y install \ $PKG_MAKE \ $PKG_BCC && \ apt-get upgrade -y --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* ################ ENVIRONMENT ################ ARG UNAME=user ARG UID=1000 ARG GNAME=user ARG GID=1000 RUN set -x; \ # These commands are allowed to fail (it happens for root, for example). # The result will be checked in the next RUN. userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \ groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \ groupadd -g ${GID} ${GNAME}; \ useradd -u $UID -g $GID -G sudo -ms /bin/bash ${UNAME}; \ echo "${UNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers USER ${UNAME}:${GNAME} ENV HOME=/home/${UNAME} WORKDIR $HOME RUN set -ex; \ id | grep "uid=${UID}(${UNAME}) gid=${GID}(${GNAME})" || (echo "ERROR: User ID verification failed" && exit 1); \ sudo ls || (echo "ERROR: sudo test failed" && exit 1); \ pwd | grep "^/home/${UNAME}" || (echo "ERROR: Working directory verification failed" && exit 1); \ echo $HOME | grep "^/home/${UNAME}" || (echo "ERROR: HOME directory verification failed" && exit 1); \ touch $HOME/test || (echo "ERROR: File creation test failed" && exit 1); \ rm $HOME/test || (echo "ERROR: File removal test failed" && exit 1) # setup path in both ENV and shell profile ENV PATH="$HOME/install/bin:/usr/local/go/bin:$PATH" RUN echo 'export PATH="$HOME/install/bin:/usr/local/go/bin:$PATH"' >> $HOME/.profile # set UID, GID ENV UID=${UID} ENV GID=${GID} # note: we do not export UNAME as it interferes with LZ4 builds ================================================ FILE: build-tools/benv/docker.d/cgroups ================================================ #!/bin/bash docker_args+=( --mount "type=bind,source=/sys/fs/cgroup,destination=/sys/fs/cgroup,readonly" ) ================================================ FILE: build-tools/benv/docker.d/gdb ================================================ #!/bin/bash docker_args+=( --cap-add=SYS_PTRACE --security-opt seccomp=unconfined ) ================================================ FILE: build-tools/benv/docker.d/kernel-headers ================================================ #!/bin/bash docker_args+=( --mount "type=bind,source=/lib/modules/`uname --kernel-release`,destination=/lib/modules/`uname --kernel-release`,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/build,destination=/lib/modules/`uname --kernel-release`/build,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/build/scripts,destination=/lib/modules/`uname --kernel-release`/build/scripts,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/build/tools,destination=/lib/modules/`uname --kernel-release`/build/tools,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/source,destination=/lib/modules/`uname --kernel-release`/source,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/source/scripts,destination=/lib/modules/`uname --kernel-release`/source/scripts,readonly" --mount "type=bind,source=/lib/modules/`uname --kernel-release`/source/tools,destination=/lib/modules/`uname --kernel-release`/source/tools,readonly" ) ================================================ FILE: build-tools/benv/docker.d/pid-host ================================================ #!/bin/bash docker_args+=( --pid=host ) ================================================ FILE: build-tools/benv/docker.d/privileged ================================================ #!/bin/bash docker_args+=( --privileged ) ================================================ FILE: build-tools/benv/docker.d/vimrc ================================================ #!/bin/bash docker_args+=( --mount "type=bind,source=$HOME/.vim,destination=/root/.vim,readonly" --mount "type=bind,source=$HOME/.vimrc,destination=/root/.vimrc,readonly" ) ================================================ FILE: build-tools/build.sh ================================================ #!/bin/bash -e # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # Builds the build environment -- a container image which is then used # to build the project in the main repo. # Call with `VERBOSE=1` for verbose output # e.g.: `./build.sh VERBOSE=1` # use env variables BENV_BASE_IMAGE_DISTRO and BENV_BASE_IMAGE_VERSION # to customize the base image used to build benv: # # BENV_BASE_IMAGE_DISTRO=debian BENV_BASE_IMAGE_VERSION=testing ./build.sh # use command line argument -DBENV_UNMINIMIZE=ON to unminimize the benv image # Note: the `--jobs` flag requires git 2.8.0 or later # Call with 'debug' to build a debug version of the build-env if [[ "$1" == "--help" ]]; then echo "usage: $0 [{--help | debug}]" echo echo " --help: shows this help message" echo " debug: builds benv with debug builds of 3rd party libraries" exit 0 fi set -x nproc="$(./nproc.sh)" git submodule update --init --recursive --jobs "${nproc}" # If this is a debug build set some extra flags to rename the benv image if [[ "$1" == "debug" ]]; then # Debug build echo Enabling debug build EXTRA_CMAKE_OPTIONS=-DCMAKE_BUILD_TYPE=Debug export DOCKER_TAG_PREFIX=debug- shift mkdir -p Debug cd Debug CMAKE_SOURCE_DIR=.. else # Release build echo Enabling release build EXTRA_CMAKE_OPTIONS=-DCMAKE_BUILD_TYPE=Release mkdir -p Release cd Release CMAKE_SOURCE_DIR=.. fi cmake $EXTRA_CMAKE_OPTIONS "$@" $CMAKE_SOURCE_DIR if [[ "$1" == "--cmake-only" ]]; then echo "==============================================" echo "cmake completed - skipping make and docker tag" echo "==============================================" exit 0 fi make if [[ -n "${BENV_BASE_IMAGE_DISTRO}" ]] && [[ -n "${BENV_BASE_IMAGE_VERSION}" ]]; then echo docker tag ${DOCKER_TAG_PREFIX}build-env:latest "${DOCKER_TAG_PREFIX}build-env:${BENV_BASE_IMAGE_DISTRO}-${BENV_BASE_IMAGE_VERSION}" docker tag ${DOCKER_TAG_PREFIX}build-env:latest "${DOCKER_TAG_PREFIX}build-env:${BENV_BASE_IMAGE_DISTRO}-${BENV_BASE_IMAGE_VERSION}" fi ================================================ FILE: build-tools/build_directory.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # This script gets the latest git modification hash for a specific directory (DIR=$1). # If docker doesn't have an image named ${BENV_PREFIX}-${DIR}:${VERSION_HASH}, # builds that image in DIR. SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$1" shift 1 IMAGE_TAG=$(${SCRIPTDIR}/get_tag.sh ${DIR}) EXISTING=$(docker images --filter "reference=${IMAGE_TAG}" -q) # if the docker image already exists, we're done if [ "${EXISTING}" != "" ] then echo ${IMAGE_TAG}: exists exit 0 fi echo ${IMAGE_TAG}: does not exist, building docker build ${DIR} -t ${IMAGE_TAG} $@ ================================================ FILE: build-tools/check_missing.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # For a docker image directory DIR ($1), checks if that version is not in docker. If so, # touches FILENAME ($2). If FILENAME itself is missing, creates it. SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$1" FILENAME="$2" IMAGE_TAG=$(${SCRIPTDIR}/get_tag.sh ${DIR}) EXISTING=$(docker images --filter "reference=${IMAGE_TAG}" -q) # if the docker image doesn't exists, touch the file if [ "${EXISTING}" == "" ] then echo "No existing image ${IMAGE_TAG}. Touching ${FILENAME}." touch "${FILENAME}" exit 0 fi # if `missing` is itself not on the filesystem, create it # does the version file exist if [ ! -f "${FILENAME}" ] then echo "File ${FILENAME} does not exist when checking for existing image ${IMAGE_TAG}. Touching. $(pwd)" # doesn't exist, just create it. touch "${FILENAME}" exit 0 fi #echo "${IMAGE_TAG} and ${FILENAME} exist. No change." ================================================ FILE: build-tools/cpp_misc/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # various c++ libraries: # - LZ4 # - yaml-cpp # - args # - nlohmann_json # - spdlog # - ccan # - googletest ARG base_IMAGE_TAG FROM $base_IMAGE_TAG AS build ARG CMAKE_BUILD_TYPE ARG BUILD_CFLAGS ARG NPROC # LZ4 WORKDIR $HOME COPY --chown=${UID}:${GID} lz4 lz4 WORKDIR $HOME/lz4 RUN make prefix=$HOME/install install # yaml-cpp WORKDIR $HOME COPY --chown=${UID}:${GID} yaml-cpp yaml-cpp WORKDIR $HOME/build/yaml-cpp RUN cmake \ -G Ninja \ -DCMAKE_INSTALL_PREFIX:PATH=$HOME/install \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ $HOME/yaml-cpp RUN nice ninja && ninja install # args WORKDIR $HOME COPY --chown=${UID}:${GID} args args WORKDIR $HOME/args RUN make DESTDIR=$HOME/install install # nlohmann_json WORKDIR $HOME COPY --chown=${UID}:${GID} json json WORKDIR $HOME/json RUN cmake \ "-DCMAKE_INSTALL_PREFIX=$HOME/install" \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DJSON_BuildTests=OFF \ . RUN nice cmake --build . --target install -j ${NPROC:-3} --config $CMAKE_BUILD_TYPE # spdlog WORKDIR $HOME COPY --chown=${UID}:${GID} spdlog spdlog WORKDIR $HOME/spdlog RUN cmake \ "-DCMAKE_INSTALL_PREFIX=$HOME/install" \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DSPDLOG_BUILD_BENCH=OFF \ -DSPDLOG_BUILD_EXAMPLES=OFF \ -DSPDLOG_BUILD_TESTING=OFF \ . RUN nice cmake --build . --target install -j ${NPROC:-3} --config $CMAKE_BUILD_TYPE # ccan WORKDIR $HOME COPY --chown=${UID}:${GID} ccan ccan RUN tar -cf- ccan/**/*.h | tar -xvf- -C $HOME/install/include # googletest WORKDIR $HOME COPY --chown=${UID}:${GID} googletest googletest WORKDIR $HOME/googletest RUN cmake \ "-DCMAKE_INSTALL_PREFIX=$HOME/install" \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DBUILD_GMOCK=ON \ -DINSTALL_GTEST=OFF \ . RUN nice cmake --build . -j ${NPROC:-3} --config $CMAKE_BUILD_TYPE RUN cp lib/*.a $HOME/install/lib RUN cp -R googletest/include/gtest $HOME/install/include RUN cp -R googlemock/include/gmock $HOME/install/include # Runtime stage - copy only necessary artifacts FROM $base_IMAGE_TAG COPY --from=build $HOME/install $HOME/install ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/_info ================================================ #include #include #include "config.h" /** * build_assert - routines for build-time assertions * * This code provides routines which will cause compilation to fail should some * assertion be untrue: such failures are preferable to run-time assertions, * but much more limited since they can only depends on compile-time constants. * * These assertions are most useful when two parts of the code must be kept in * sync: it is better to avoid such cases if possible, but seconds best is to * detect invalid changes at build time. * * For example, a tricky piece of code might rely on a certain element being at * the start of the structure. To ensure that future changes don't break it, * you would catch such changes in your code like so: * * Example: * #include * #include * * struct foo { * char string[5]; * int x; * }; * * static char *foo_string(struct foo *foo) * { * // This trick requires that the string be first in the structure * BUILD_ASSERT(offsetof(struct foo, string) == 0); * return (char *)foo; * } * * License: CC0 (Public domain) * Author: Rusty Russell */ int main(int argc, char *argv[]) { if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) /* Nothing. */ return 0; return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/build_assert.h ================================================ /* CC0 (Public domain) - see LICENSE file for details */ #ifndef CCAN_BUILD_ASSERT_H #define CCAN_BUILD_ASSERT_H /** * BUILD_ASSERT - assert a build-time dependency. * @cond: the compile-time condition which must be true. * * Your compile will fail if the condition isn't true, or can't be evaluated * by the compiler. This can only be used within a function. * * Example: * #include * ... * static char *foo_to_char(struct foo *foo) * { * // This code needs string to be at start of foo. * BUILD_ASSERT(offsetof(struct foo, string) == 0); * return (char *)foo; * } */ #define BUILD_ASSERT(cond) \ do { (void) sizeof(char [1 - 2*!(cond)]); } while(0) /** * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression. * @cond: the compile-time condition which must be true. * * Your compile will fail if the condition isn't true, or can't be evaluated * by the compiler. This can be used in an expression: its value is "0". * * Example: * #define foo_to_char(foo) \ * ((char *)(foo) \ * + BUILD_ASSERT_OR_ZERO(offsetof(struct foo, string) == 0)) */ #define BUILD_ASSERT_OR_ZERO(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) #endif /* CCAN_BUILD_ASSERT_H */ ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/test/compile_fail-expr.c ================================================ #include int main(int argc, char *argv[]) { #ifdef FAIL return BUILD_ASSERT_OR_ZERO(1 == 0); #else return 0; #endif } ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/test/compile_fail.c ================================================ #include int main(int argc, char *argv[]) { #ifdef FAIL BUILD_ASSERT(1 == 0); #endif return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/test/compile_ok.c ================================================ #include int main(int argc, char *argv[]) { BUILD_ASSERT(1 == 1); return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/build_assert/test/run-BUILD_ASSERT_OR_ZERO.c ================================================ #include #include int main(int argc, char *argv[]) { plan_tests(1); ok1(BUILD_ASSERT_OR_ZERO(1 == 1) == 0); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/check_type/_info ================================================ #include #include #include "config.h" /** * check_type - routines for compile time type checking * * C has fairly weak typing: ints get automatically converted to longs, signed * to unsigned, etc. There are some cases where this is best avoided, and * these macros provide methods for evoking warnings (or build errors) when * a precise type isn't used. * * On compilers which don't support typeof() these routines are less effective, * since they have to use sizeof() which can only distiguish between types of * different size. * * License: CC0 (Public domain) * Author: Rusty Russell */ int main(int argc, char *argv[]) { if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) { #if !HAVE_TYPEOF printf("ccan/build_assert\n"); #endif return 0; } return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/check_type/check_type.h ================================================ /* CC0 (Public domain) - see LICENSE file for details */ #ifndef CCAN_CHECK_TYPE_H #define CCAN_CHECK_TYPE_H #define HAVE_TYPEOF 1 /** * check_type - issue a warning or build failure if type is not correct. * @expr: the expression whose type we should check (not evaluated). * @type: the exact type we expect the expression to be. * * This macro is usually used within other macros to try to ensure that a macro * argument is of the expected type. No type promotion of the expression is * done: an unsigned int is not the same as an int! * * check_type() always evaluates to 0. * * If your compiler does not support typeof, then the best we can do is fail * to compile if the sizes of the types are unequal (a less complete check). * * Example: * // They should always pass a 64-bit value to _set_some_value! * #define set_some_value(expr) \ * _set_some_value((check_type((expr), uint64_t), (expr))) */ /** * check_types_match - issue a warning or build failure if types are not same. * @expr1: the first expression (not evaluated). * @expr2: the second expression (not evaluated). * * This macro is usually used within other macros to try to ensure that * arguments are of identical types. No type promotion of the expressions is * done: an unsigned int is not the same as an int! * * check_types_match() always evaluates to 0. * * If your compiler does not support typeof, then the best we can do is fail * to compile if the sizes of the types are unequal (a less complete check). * * Example: * // Do subtraction to get to enclosing type, but make sure that * // pointer is of correct type for that member. * #define container_of(mbr_ptr, encl_type, mbr) \ * (check_types_match((mbr_ptr), &((encl_type *)0)->mbr), \ * ((encl_type *) \ * ((char *)(mbr_ptr) - offsetof(enclosing_type, mbr)))) */ #if HAVE_TYPEOF #define check_type(expr, type) \ ((typeof(expr) *)0 != (type *)0) #define check_types_match(expr1, expr2) \ ((typeof(expr1) *)0 != (typeof(expr2) *)0) #else #include /* Without typeof, we can only test the sizes. */ #define check_type(expr, type) \ BUILD_ASSERT_OR_ZERO(sizeof(expr) == sizeof(type)) #define check_types_match(expr1, expr2) \ BUILD_ASSERT_OR_ZERO(sizeof(expr1) == sizeof(expr2)) #endif /* HAVE_TYPEOF */ #endif /* CCAN_CHECK_TYPE_H */ ================================================ FILE: build-tools/cpp_misc/ccan/check_type/test/compile_fail-check_type.c ================================================ #include int main(int argc, char *argv[]) { #ifdef FAIL check_type(argc, char); #endif return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/check_type/test/compile_fail-check_type_unsigned.c ================================================ #include int main(int argc, char *argv[]) { #ifdef FAIL #if HAVE_TYPEOF check_type(argc, unsigned int); #else /* This doesn't work without typeof, so just fail */ #error "Fail without typeof" #endif #endif return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/check_type/test/compile_fail-check_types_match.c ================================================ #include int main(int argc, char *argv[]) { unsigned char x = argc; #ifdef FAIL check_types_match(argc, x); #endif return x; } ================================================ FILE: build-tools/cpp_misc/ccan/check_type/test/run.c ================================================ #include #include int main(int argc, char *argv[]) { int x = 0, y = 0; plan_tests(9); ok1(check_type(argc, int) == 0); ok1(check_type(&argc, int *) == 0); ok1(check_types_match(argc, argc) == 0); ok1(check_types_match(argc, x) == 0); ok1(check_types_match(&argc, &x) == 0); ok1(check_type(x++, int) == 0); ok(x == 0, "check_type does not evaluate expression"); ok1(check_types_match(x++, y++) == 0); ok(x == 0 && y == 0, "check_types_match does not evaluate expressions"); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/compiler/_info ================================================ #include #include #include "config.h" /** * compiler - macros for common compiler extensions * * Abstracts away some compiler hints. Currently these include: * - COLD * For functions not called in fast paths (aka. cold functions) * - PRINTF_FMT * For functions which take printf-style parameters. * - CONST_FUNCTION * For functions which return the same value for same parameters. * - NEEDED * For functions and variables which must be emitted even if unused. * - UNNEEDED * For functions and variables which need not be emitted if unused. * - UNUSED * For parameters which are not used. * - IS_COMPILE_CONSTANT() * For using different tradeoffs for compiletime vs runtime evaluation. * * License: CC0 (Public domain) * Author: Rusty Russell * * Example: * #include * #include * #include * * // Example of a (slow-path) logging function. * static int log_threshold = 2; * static void COLD PRINTF_FMT(2,3) * logger(int level, const char *fmt, ...) * { * va_list ap; * va_start(ap, fmt); * if (level >= log_threshold) * vfprintf(stderr, fmt, ap); * va_end(ap); * } * * int main(int argc, char *argv[]) * { * if (argc != 1) { * logger(3, "Don't want %i arguments!\n", argc-1); * return 1; * } * return 0; * } */ int main(int argc, char *argv[]) { /* Expect exactly one argument */ if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) { return 0; } return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/compiler/compiler.h ================================================ /* CC0 (Public domain) - see LICENSE file for details */ #ifndef CCAN_COMPILER_H #define CCAN_COMPILER_H #ifndef COLD #if HAVE_ATTRIBUTE_COLD /** * COLD - a function is unlikely to be called. * * Used to mark an unlikely code path and optimize appropriately. * It is usually used on logging or error routines. * * Example: * static void COLD moan(const char *reason) * { * fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno)); * } */ #define COLD __attribute__((cold)) #else #define COLD #endif #endif #ifndef NORETURN #if HAVE_ATTRIBUTE_NORETURN /** * NORETURN - a function does not return * * Used to mark a function which exits; useful for suppressing warnings. * * Example: * static void NORETURN fail(const char *reason) * { * fprintf(stderr, "Error: %s (%s)\n", reason, strerror(errno)); * exit(1); * } */ #define NORETURN __attribute__((noreturn)) #else #define NORETURN #endif #endif #ifndef PRINTF_FMT #if HAVE_ATTRIBUTE_PRINTF /** * PRINTF_FMT - a function takes printf-style arguments * @nfmt: the 1-based number of the function's format argument. * @narg: the 1-based number of the function's first variable argument. * * This allows the compiler to check your parameters as it does for printf(). * * Example: * void PRINTF_FMT(2,3) my_printf(const char *prefix, const char *fmt, ...); */ #define PRINTF_FMT(nfmt, narg) \ __attribute__((format(__printf__, nfmt, narg))) #else #define PRINTF_FMT(nfmt, narg) #endif #endif #ifndef CONST_FUNCTION #if HAVE_ATTRIBUTE_CONST /** * CONST_FUNCTION - a function's return depends only on its argument * * This allows the compiler to assume that the function will return the exact * same value for the exact same arguments. This implies that the function * must not use global variables, or dereference pointer arguments. */ #define CONST_FUNCTION __attribute__((const)) #else #define CONST_FUNCTION #endif #endif #if HAVE_ATTRIBUTE_UNUSED #ifndef UNNEEDED /** * UNNEEDED - a variable/function may not be needed * * This suppresses warnings about unused variables or functions, but tells * the compiler that if it is unused it need not emit it into the source code. * * Example: * // With some preprocessor options, this is unnecessary. * static UNNEEDED int counter; * * // With some preprocessor options, this is unnecessary. * static UNNEEDED void add_to_counter(int add) * { * counter += add; * } */ #define UNNEEDED __attribute__((unused)) #endif #ifndef NEEDED #if HAVE_ATTRIBUTE_USED /** * NEEDED - a variable/function is needed * * This suppresses warnings about unused variables or functions, but tells * the compiler that it must exist even if it (seems) unused. * * Example: * // Even if this is unused, these are vital for debugging. * static NEEDED int counter; * static NEEDED void dump_counter(void) * { * printf("Counter is %i\n", counter); * } */ #define NEEDED __attribute__((used)) #else /* Before used, unused functions and vars were always emitted. */ #define NEEDED __attribute__((unused)) #endif #endif #ifndef UNUSED /** * UNUSED - a parameter is unused * * Some compilers (eg. gcc with -W or -Wunused) warn about unused * function parameters. This suppresses such warnings and indicates * to the reader that it's deliberate. * * Example: * // This is used as a callback, so needs to have this prototype. * static int some_callback(void *unused UNUSED) * { * return 0; * } */ #define UNUSED __attribute__((unused)) #endif #else #ifndef UNNEEDED #define UNNEEDED #endif #ifndef NEEDED #define NEEDED #endif #ifndef UNUSED #define UNUSED #endif #endif #ifndef IS_COMPILE_CONSTANT #if HAVE_BUILTIN_CONSTANT_P /** * IS_COMPILE_CONSTANT - does the compiler know the value of this expression? * @expr: the expression to evaluate * * When an expression manipulation is complicated, it is usually better to * implement it in a function. However, if the expression being manipulated is * known at compile time, it is better to have the compiler see the entire * expression so it can simply substitute the result. * * This can be done using the IS_COMPILE_CONSTANT() macro. * * Example: * enum greek { ALPHA, BETA, GAMMA, DELTA, EPSILON }; * * // Out-of-line version. * const char *greek_name(enum greek greek); * * // Inline version. * static inline const char *_greek_name(enum greek greek) * { * switch (greek) { * case ALPHA: return "alpha"; * case BETA: return "beta"; * case GAMMA: return "gamma"; * case DELTA: return "delta"; * case EPSILON: return "epsilon"; * default: return "**INVALID**"; * } * } * * // Use inline if compiler knows answer. Otherwise call function * // to avoid copies of the same code everywhere. * #define greek_name(g) \ * (IS_COMPILE_CONSTANT(greek) ? _greek_name(g) : greek_name(g)) */ #define IS_COMPILE_CONSTANT(expr) __builtin_constant_p(expr) #else /* If we don't know, assume it's not. */ #define IS_COMPILE_CONSTANT(expr) 0 #endif #endif #ifndef WARN_UNUSED_RESULT #if HAVE_WARN_UNUSED_RESULT /** * WARN_UNUSED_RESULT - warn if a function return value is unused. * * Used to mark a function where it is extremely unlikely that the caller * can ignore the result, eg realloc(). * * Example: * // buf param may be freed by this; need return value! * static char *WARN_UNUSED_RESULT enlarge(char *buf, unsigned *size) * { * return realloc(buf, (*size) *= 2); * } */ #define WARN_UNUSED_RESULT __attribute__((warn_unused_result)) #else #define WARN_UNUSED_RESULT #endif #endif #endif /* CCAN_COMPILER_H */ ================================================ FILE: build-tools/cpp_misc/ccan/compiler/test/compile_fail-printf.c ================================================ #include static void PRINTF_FMT(2,3) my_printf(int x, const char *fmt, ...) { } int main(int argc, char *argv[]) { unsigned int i = 0; my_printf(1, "Not a pointer " #ifdef FAIL "%p", #if !HAVE_ATTRIBUTE_PRINTF #error "Unfortunately we don't fail if !HAVE_ATTRIBUTE_PRINTF." #endif #else "%i", #endif i); return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/compiler/test/run-is_compile_constant.c ================================================ #include #include int main(int argc, char *argv[]) { plan_tests(2); ok1(!IS_COMPILE_CONSTANT(argc)); #if HAVE_BUILTIN_CONSTANT_P ok1(IS_COMPILE_CONSTANT(7)); #else pass("If !HAVE_BUILTIN_CONSTANT_P, IS_COMPILE_CONSTANT always false"); #endif return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/container_of/_info ================================================ #include #include #include "config.h" /** * container_of - routine for upcasting * * It is often convenient to create code where the caller registers a pointer * to a generic structure and a callback. The callback might know that the * pointer points to within a larger structure, and container_of gives a * convenient and fairly type-safe way of returning to the enclosing structure. * * This idiom is an alternative to providing a void * pointer for every * callback. * * Example: * #include * #include * * struct timer { * void *members; * }; * * struct info { * int my_stuff; * struct timer timer; * }; * * static void register_timer(struct timer *timer) * { * //... * } * * static void my_timer_callback(struct timer *timer) * { * struct info *info = container_of(timer, struct info, timer); * printf("my_stuff is %u\n", info->my_stuff); * } * * int main(void) * { * struct info info = { .my_stuff = 1 }; * * register_timer(&info.timer); * // ... * return 0; * } * * License: CC0 (Public domain) * Author: Rusty Russell */ int main(int argc, char *argv[]) { if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) { printf("ccan/check_type\n"); return 0; } return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/container_of/container_of.h ================================================ /* CC0 (Public domain) - see LICENSE file for details */ #ifndef CCAN_CONTAINER_OF_H #define CCAN_CONTAINER_OF_H #include #include /** * container_of - get pointer to enclosing structure * @member_ptr: pointer to the structure member * @containing_type: the type this member is within * @member: the name of this member within the structure. * * Given a pointer to a member of a structure, this macro does pointer * subtraction to return the pointer to the enclosing type. * * Example: * struct foo { * int fielda, fieldb; * // ... * }; * struct info { * int some_other_field; * struct foo my_foo; * }; * * static struct info *foo_to_info(struct foo *foo) * { * return container_of(foo, struct info, my_foo); * } */ #define container_of(member_ptr, containing_type, member) \ ((containing_type *) \ ((char *)(member_ptr) \ - container_off(containing_type, member)) \ + check_types_match(*(member_ptr), ((containing_type *)0)->member)) /** * container_off - get offset to enclosing structure * @containing_type: the type this member is within * @member: the name of this member within the structure. * * Given a pointer to a member of a structure, this macro does * typechecking and figures out the offset to the enclosing type. * * Example: * struct foo { * int fielda, fieldb; * // ... * }; * struct info { * int some_other_field; * struct foo my_foo; * }; * * static struct info *foo_to_info(struct foo *foo) * { * size_t off = container_off(struct info, my_foo); * return (void *)((char *)foo - off); * } */ #define container_off(containing_type, member) \ offsetof(containing_type, member) /** * container_of_var - get pointer to enclosing structure using a variable * @member_ptr: pointer to the structure member * @container_var: a pointer of same type as this member's container * @member: the name of this member within the structure. * * Given a pointer to a member of a structure, this macro does pointer * subtraction to return the pointer to the enclosing type. * * Example: * static struct info *foo_to_i(struct foo *foo) * { * struct info *i = container_of_var(foo, i, my_foo); * return i; * } */ #if HAVE_TYPEOF #define container_of_var(member_ptr, container_var, member) \ container_of(member_ptr, typeof(*container_var), member) #else #define container_of_var(member_ptr, container_var, member) \ ((void *)((char *)(member_ptr) - \ container_off_var(container_var, member))) #endif /** * container_off_var - get offset of a field in enclosing structure * @container_var: a pointer to a container structure * @member: the name of a member within the structure. * * Given (any) pointer to a structure and a its member name, this * macro does pointer subtraction to return offset of member in a * structure memory layout. * */ #if HAVE_TYPEOF #define container_off_var(var, member) \ container_off(typeof(*var), member) #else #define container_off_var(var, member) \ ((char *)&(var)->member - (char *)(var)) #endif #endif /* CCAN_CONTAINER_OF_H */ ================================================ FILE: build-tools/cpp_misc/ccan/container_of/test/compile_fail-bad-type.c ================================================ #include #include struct foo { int a; char b; }; int main(int argc, char *argv[]) { struct foo foo = { .a = 1, .b = 2 }; int *intp = &foo.a; char *p; #ifdef FAIL /* p is a char *, but this gives a struct foo * */ p = container_of(intp, struct foo, a); #else p = (char *)intp; #endif return p == NULL; } ================================================ FILE: build-tools/cpp_misc/ccan/container_of/test/compile_fail-types.c ================================================ #include #include struct foo { int a; char b; }; int main(int argc, char *argv[]) { struct foo foo = { .a = 1, .b = 2 }, *foop; int *intp = &foo.a; #ifdef FAIL /* b is a char, but intp is an int * */ foop = container_of(intp, struct foo, b); #else foop = NULL; #endif (void) foop; /* Suppress unused-but-set-variable warning. */ return intp == NULL; } ================================================ FILE: build-tools/cpp_misc/ccan/container_of/test/compile_fail-var-types.c ================================================ #include #include struct foo { int a; char b; }; int main(int argc, char *argv[]) { struct foo foo = { .a = 1, .b = 2 }, *foop; int *intp = &foo.a; #ifdef FAIL /* b is a char, but intp is an int * */ foop = container_of_var(intp, foop, b); #if !HAVE_TYPEOF #error "Unfortunately we don't fail if we don't have typeof." #endif #else foop = NULL; #endif (void) foop; /* Suppress unused-but-set-variable warning. */ return intp == NULL; } ================================================ FILE: build-tools/cpp_misc/ccan/container_of/test/run.c ================================================ #include #include struct foo { int a; char b; }; int main(int argc, char *argv[]) { struct foo foo = { .a = 1, .b = 2 }; int *intp = &foo.a; char *charp = &foo.b; plan_tests(8); ok1(container_of(intp, struct foo, a) == &foo); ok1(container_of(charp, struct foo, b) == &foo); ok1(container_of_var(intp, &foo, a) == &foo); ok1(container_of_var(charp, &foo, b) == &foo); ok1(container_off(struct foo, a) == 0); ok1(container_off(struct foo, b) == offsetof(struct foo, b)); ok1(container_off_var(&foo, a) == 0); ok1(container_off_var(&foo, b) == offsetof(struct foo, b)); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/hash/_info ================================================ #include #include /** * hash - routines for hashing bytes * * When creating a hash table it's important to have a hash function * which mixes well and is fast. This package supplies such functions. * * The hash functions come in two flavors: the normal ones and the * stable ones. The normal ones can vary from machine-to-machine and * may change if we find better or faster hash algorithms in future. * The stable ones will always give the same results on any computer, * and on any version of this package. * * License: CC0 (Public domain) * Maintainer: Rusty Russell * Author: Bob Jenkins */ int main(int argc, char *argv[]) { if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) { printf("ccan/build_assert\n"); return 0; } return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/hash/hash.c ================================================ /* CC0 (Public domain) - see LICENSE file for details */ /* ------------------------------------------------------------------------------- lookup3.c, by Bob Jenkins, May 2006, Public Domain. These are functions for producing 32-bit hashes for hash table lookup. hash_word(), hashlittle(), hashlittle2(), hashbig(), mix(), and final() are externally useful functions. Routines to test the hash are included if SELF_TEST is defined. You can use this free for any purpose. It's in the public domain. It has no warranty. You probably want to use hashlittle(). hashlittle() and hashbig() hash byte arrays. hashlittle() is is faster than hashbig() on little-endian machines. Intel and AMD are little-endian machines. On second thought, you probably want hashlittle2(), which is identical to hashlittle() except it returns two 32-bit hashes for the price of one. You could implement hashbig2() if you wanted but I haven't bothered here. If you want to find a hash of, say, exactly 7 integers, do a = i1; b = i2; c = i3; mix(a,b,c); a += i4; b += i5; c += i6; mix(a,b,c); a += i7; final(a,b,c); then use c as the hash value. If you have a variable length array of 4-byte integers to hash, use hash_word(). If you have a byte array (like a character string), use hashlittle(). If you have several byte arrays, or a mix of things, see the comments above hashlittle(). Why is this so big? I read 12 bytes at a time into 3 4-byte integers, then mix those integers. This is fast (you can do a lot more thorough mixing with 12*3 instructions on 3 integers than you can with 3 instructions on 1 byte), but shoehorning those bytes into integers efficiently is messy. ------------------------------------------------------------------------------- */ //#define SELF_TEST 1 #if 0 #include /* defines printf for tests */ #include /* defines time_t for timings in the test */ #include /* defines uint32_t etc */ #include /* attempt to define endianness */ #ifdef linux # include /* attempt to define endianness */ #endif /* * My best guess at if you are big-endian or little-endian. This may * need adjustment. */ #if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ __BYTE_ORDER == __LITTLE_ENDIAN) || \ (defined(i386) || defined(__i386__) || defined(__i486__) || \ defined(__i586__) || defined(__i686__) || defined(__x86_64) || \ defined(vax) || defined(MIPSEL)) # define HASH_LITTLE_ENDIAN 1 # define HASH_BIG_ENDIAN 0 #elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \ __BYTE_ORDER == __BIG_ENDIAN) || \ (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) # define HASH_LITTLE_ENDIAN 0 # define HASH_BIG_ENDIAN 1 #else #define HASH_LITTLE_ENDIAN 1 #define HASH_BIG_ENDIAN 0 #endif #endif /* old hash.c headers. */ #include "hash.h" #if HAVE_LITTLE_ENDIAN #define HASH_LITTLE_ENDIAN 1 #define HASH_BIG_ENDIAN 0 #elif HAVE_BIG_ENDIAN #define HASH_LITTLE_ENDIAN 0 #define HASH_BIG_ENDIAN 1 #else #define HASH_LITTLE_ENDIAN 1 #define HASH_BIG_ENDIAN 0 #endif #define hashsize(n) ((uint32_t)1<<(n)) #define hashmask(n) (hashsize(n)-1) #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) /* ------------------------------------------------------------------------------- mix -- mix 3 32-bit values reversibly. This is reversible, so any information in (a,b,c) before mix() is still in (a,b,c) after mix(). If four pairs of (a,b,c) inputs are run through mix(), or through mix() in reverse, there are at least 32 bits of the output that are sometimes the same for one pair and different for another pair. This was tested for: * pairs that differed by one bit, by two bits, in any combination of top bits of (a,b,c), or in any combination of bottom bits of (a,b,c). * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed the output delta to a Gray code (a^(a>>1)) so a string of 1's (as is commonly produced by subtraction) look like a single 1-bit difference. * the base values were pseudorandom, all zero but one bit set, or all zero plus a counter that starts at zero. Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that satisfy this are 4 6 8 16 19 4 9 15 3 18 27 15 14 9 3 7 17 3 Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for "differ" defined as + with a one-bit base and a two-bit delta. I used http://burtleburtle.net/bob/hash/avalanche.html to choose the operations, constants, and arrangements of the variables. This does not achieve avalanche. There are input bits of (a,b,c) that fail to affect some output bits of (a,b,c), especially of a. The most thoroughly mixed value is c, but it doesn't really even achieve avalanche in c. This allows some parallelism. Read-after-writes are good at doubling the number of bits affected, so the goal of mixing pulls in the opposite direction as the goal of parallelism. I did what I could. Rotates seem to cost as much as shifts on every machine I could lay my hands on, and rotates are much kinder to the top and bottom bits, so I used rotates. ------------------------------------------------------------------------------- */ #define mix(a,b,c) \ { \ a -= c; a ^= rot(c, 4); c += b; \ b -= a; b ^= rot(a, 6); a += c; \ c -= b; c ^= rot(b, 8); b += a; \ a -= c; a ^= rot(c,16); c += b; \ b -= a; b ^= rot(a,19); a += c; \ c -= b; c ^= rot(b, 4); b += a; \ } /* ------------------------------------------------------------------------------- final -- final mixing of 3 32-bit values (a,b,c) into c Pairs of (a,b,c) values differing in only a few bits will usually produce values of c that look totally different. This was tested for * pairs that differed by one bit, by two bits, in any combination of top bits of (a,b,c), or in any combination of bottom bits of (a,b,c). * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed the output delta to a Gray code (a^(a>>1)) so a string of 1's (as is commonly produced by subtraction) look like a single 1-bit difference. * the base values were pseudorandom, all zero but one bit set, or all zero plus a counter that starts at zero. These constants passed: 14 11 25 16 4 14 24 12 14 25 16 4 14 24 and these came close: 4 8 15 26 3 22 24 10 8 15 26 3 22 24 11 8 15 26 3 22 24 ------------------------------------------------------------------------------- */ #define final(a,b,c) \ { \ c ^= b; c -= rot(b,14); \ a ^= c; a -= rot(c,11); \ b ^= a; b -= rot(a,25); \ c ^= b; c -= rot(b,16); \ a ^= c; a -= rot(c,4); \ b ^= a; b -= rot(a,14); \ c ^= b; c -= rot(b,24); \ } /* -------------------------------------------------------------------- This works on all machines. To be useful, it requires -- that the key be an array of uint32_t's, and -- that the length be the number of uint32_t's in the key The function hash_word() is identical to hashlittle() on little-endian machines, and identical to hashbig() on big-endian machines, except that the length has to be measured in uint32_ts rather than in bytes. hashlittle() is more complicated than hash_word() only because hashlittle() has to dance around fitting the key bytes into registers. -------------------------------------------------------------------- */ uint32_t hash_u32( const uint32_t *k, /* the key, an array of uint32_t values */ size_t length, /* the length of the key, in uint32_ts */ uint32_t initval) /* the previous hash, or an arbitrary value */ { uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + (((uint32_t)length)<<2) + initval; /*------------------------------------------------- handle most of the key */ while (length > 3) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 3; k += 3; } /*------------------------------------------- handle the last 3 uint32_t's */ switch(length) /* all the case statements fall through */ { case 3 : c+=k[2]; case 2 : b+=k[1]; case 1 : a+=k[0]; final(a,b,c); case 0: /* case 0: nothing left to add */ break; } /*------------------------------------------------------ report the result */ return c; } /* ------------------------------------------------------------------------------- hashlittle() -- hash a variable-length key into a 32-bit value k : the key (the unaligned variable-length array of bytes) length : the length of the key, counting by bytes val2 : IN: can be any 4-byte value OUT: second 32 bit hash. Returns a 32-bit value. Every bit of the key affects every bit of the return value. Two keys differing by one or two bits will have totally different hash values. Note that the return value is better mixed than val2, so use that first. The best hash table sizes are powers of 2. There is no need to do mod a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask. For example, if you need only 10 bits, do h = (h & hashmask(10)); In which case, the hash table should have hashsize(10) elements. If you are hashing n strings (uint8_t **)k, do it like this: for (i=0, h=0; i 12) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 12; k += 3; } /*----------------------------- handle the last (probably partial) block */ /* * "k[2]&0xffffff" actually reads beyond the end of the string, but * then masks off the part it's not allowed to read. Because the * string is aligned, the masked-off tail is in the same word as the * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). * * Not on my testing with gcc 4.5 on an intel i5 CPU, at least --RR. */ #if 0 switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=k[1]&0xffffff; a+=k[0]; break; case 6 : b+=k[1]&0xffff; a+=k[0]; break; case 5 : b+=k[1]&0xff; a+=k[0]; break; case 4 : a+=k[0]; break; case 3 : a+=k[0]&0xffffff; break; case 2 : a+=k[0]&0xffff; break; case 1 : a+=k[0]&0xff; break; case 0 : return c; /* zero length strings require no mixing */ } #else /* make valgrind happy */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]; break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ case 1 : a+=k8[0]; break; case 0 : return c; } #endif /* !valgrind */ } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ const uint8_t *k8; /*--------------- all but last block: aligned reads and different mixing */ while (length > 12) { a += k[0] + (((uint32_t)k[1])<<16); b += k[2] + (((uint32_t)k[3])<<16); c += k[4] + (((uint32_t)k[5])<<16); mix(a,b,c); length -= 12; k += 6; } /*----------------------------- handle the last (probably partial) block */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[4]+(((uint32_t)k[5])<<16); b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=k[4]; b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=k[2]; a+=k[0]+(((uint32_t)k[1])<<16); break; case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]+(((uint32_t)k[1])<<16); break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=k[0]; break; case 1 : a+=k8[0]; break; case 0 : return c; /* zero length requires no mixing */ } } else { /* need to read the key one byte at a time */ const uint8_t *k = (const uint8_t *)key; /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; a += ((uint32_t)k[1])<<8; a += ((uint32_t)k[2])<<16; a += ((uint32_t)k[3])<<24; b += k[4]; b += ((uint32_t)k[5])<<8; b += ((uint32_t)k[6])<<16; b += ((uint32_t)k[7])<<24; c += k[8]; c += ((uint32_t)k[9])<<8; c += ((uint32_t)k[10])<<16; c += ((uint32_t)k[11])<<24; mix(a,b,c); length -= 12; k += 12; } /*-------------------------------- last block: affect all 32 bits of (c) */ switch(length) /* all the case statements fall through */ { case 12: c+=((uint32_t)k[11])<<24; case 11: c+=((uint32_t)k[10])<<16; case 10: c+=((uint32_t)k[9])<<8; case 9 : c+=k[8]; case 8 : b+=((uint32_t)k[7])<<24; case 7 : b+=((uint32_t)k[6])<<16; case 6 : b+=((uint32_t)k[5])<<8; case 5 : b+=k[4]; case 4 : a+=((uint32_t)k[3])<<24; case 3 : a+=((uint32_t)k[2])<<16; case 2 : a+=((uint32_t)k[1])<<8; case 1 : a+=k[0]; break; case 0 : return c; } } final(a,b,c); *val2 = b; return c; } /* * hashbig(): * This is the same as hash_word() on big-endian machines. It is different * from hashlittle() on all machines. hashbig() takes advantage of * big-endian byte ordering. */ static uint32_t hashbig( const void *key, size_t length, uint32_t *val2) { uint32_t a,b,c; union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */ /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)length) + *val2; u.ptr = key; if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) { const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ const uint8_t *k8; /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 12; k += 3; } /*----------------------------- handle the last (probably partial) block */ /* * "k[2]<<8" actually reads beyond the end of the string, but * then shifts out the part it's not allowed to read. Because the * string is aligned, the illegal read is in the same word as the * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). * * Not on my testing with gcc 4.5 on an intel i5 CPU, at least --RR. */ #if 0 switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break; case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break; case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break; case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=k[1]&0xffffff00; a+=k[0]; break; case 6 : b+=k[1]&0xffff0000; a+=k[0]; break; case 5 : b+=k[1]&0xff000000; a+=k[0]; break; case 4 : a+=k[0]; break; case 3 : a+=k[0]&0xffffff00; break; case 2 : a+=k[0]&0xffff0000; break; case 1 : a+=k[0]&0xff000000; break; case 0 : return c; /* zero length strings require no mixing */ } #else /* make valgrind happy */ k8 = (const uint8_t *)k; switch(length) /* all the case statements fall through */ { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=((uint32_t)k8[10])<<8; /* fall through */ case 10: c+=((uint32_t)k8[9])<<16; /* fall through */ case 9 : c+=((uint32_t)k8[8])<<24; /* fall through */ case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=((uint32_t)k8[6])<<8; /* fall through */ case 6 : b+=((uint32_t)k8[5])<<16; /* fall through */ case 5 : b+=((uint32_t)k8[4])<<24; /* fall through */ case 4 : a+=k[0]; break; case 3 : a+=((uint32_t)k8[2])<<8; /* fall through */ case 2 : a+=((uint32_t)k8[1])<<16; /* fall through */ case 1 : a+=((uint32_t)k8[0])<<24; break; case 0 : return c; } #endif /* !VALGRIND */ } else { /* need to read the key one byte at a time */ const uint8_t *k = (const uint8_t *)key; /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ while (length > 12) { a += ((uint32_t)k[0])<<24; a += ((uint32_t)k[1])<<16; a += ((uint32_t)k[2])<<8; a += ((uint32_t)k[3]); b += ((uint32_t)k[4])<<24; b += ((uint32_t)k[5])<<16; b += ((uint32_t)k[6])<<8; b += ((uint32_t)k[7]); c += ((uint32_t)k[8])<<24; c += ((uint32_t)k[9])<<16; c += ((uint32_t)k[10])<<8; c += ((uint32_t)k[11]); mix(a,b,c); length -= 12; k += 12; } /*-------------------------------- last block: affect all 32 bits of (c) */ switch(length) /* all the case statements fall through */ { case 12: c+=k[11]; case 11: c+=((uint32_t)k[10])<<8; case 10: c+=((uint32_t)k[9])<<16; case 9 : c+=((uint32_t)k[8])<<24; case 8 : b+=k[7]; case 7 : b+=((uint32_t)k[6])<<8; case 6 : b+=((uint32_t)k[5])<<16; case 5 : b+=((uint32_t)k[4])<<24; case 4 : a+=k[3]; case 3 : a+=((uint32_t)k[2])<<8; case 2 : a+=((uint32_t)k[1])<<16; case 1 : a+=((uint32_t)k[0])<<24; break; case 0 : return c; } } final(a,b,c); *val2 = b; return c; } /* I basically use hashlittle here, but use native endian within each * element. This delivers least-surprise: hash such as "int arr[] = { * 1, 2 }; hash_stable(arr, 2, 0);" will be the same on big and little * endian machines, even though a bytewise hash wouldn't be. */ uint64_t hash64_stable_64(const void *key, size_t n, uint64_t base) { const uint64_t *k = key; uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)n*8) + (base >> 32) + base; while (n > 3) { a += (uint32_t)k[0]; b += (uint32_t)(k[0] >> 32); c += (uint32_t)k[1]; mix(a,b,c); a += (uint32_t)(k[1] >> 32); b += (uint32_t)k[2]; c += (uint32_t)(k[2] >> 32); mix(a,b,c); n -= 3; k += 3; } switch (n) { case 2: a += (uint32_t)k[0]; b += (uint32_t)(k[0] >> 32); c += (uint32_t)k[1]; mix(a,b,c); a += (uint32_t)(k[1] >> 32); break; case 1: a += (uint32_t)k[0]; b += (uint32_t)(k[0] >> 32); break; case 0: return c; } final(a,b,c); return ((uint64_t)b << 32) | c; } uint64_t hash64_stable_32(const void *key, size_t n, uint64_t base) { const uint32_t *k = key; uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)n*4) + (base >> 32) + base; while (n > 3) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); n -= 3; k += 3; } switch (n) { case 2: b += (uint32_t)k[1]; case 1: a += (uint32_t)k[0]; break; case 0: return c; } final(a,b,c); return ((uint64_t)b << 32) | c; } uint64_t hash64_stable_16(const void *key, size_t n, uint64_t base) { const uint16_t *k = key; uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)n*2) + (base >> 32) + base; while (n > 6) { a += (uint32_t)k[0] + ((uint32_t)k[1] << 16); b += (uint32_t)k[2] + ((uint32_t)k[3] << 16); c += (uint32_t)k[4] + ((uint32_t)k[5] << 16); mix(a,b,c); n -= 6; k += 6; } switch (n) { case 5: c += (uint32_t)k[4]; case 4: b += ((uint32_t)k[3] << 16); case 3: b += (uint32_t)k[2]; case 2: a += ((uint32_t)k[1] << 16); case 1: a += (uint32_t)k[0]; break; case 0: return c; } final(a,b,c); return ((uint64_t)b << 32) | c; } uint64_t hash64_stable_8(const void *key, size_t n, uint64_t base) { uint32_t b32 = base + (base >> 32); uint32_t lower = hashlittle(key, n, &b32); return ((uint64_t)b32 << 32) | lower; } uint32_t hash_any(const void *key, size_t length, uint32_t base) { if (HASH_BIG_ENDIAN) return hashbig(key, length, &base); else return hashlittle(key, length, &base); } uint32_t hash_stable_64(const void *key, size_t n, uint32_t base) { return hash64_stable_64(key, n, base); } uint32_t hash_stable_32(const void *key, size_t n, uint32_t base) { return hash64_stable_32(key, n, base); } uint32_t hash_stable_16(const void *key, size_t n, uint32_t base) { return hash64_stable_16(key, n, base); } uint32_t hash_stable_8(const void *key, size_t n, uint32_t base) { return hashlittle(key, n, &base); } /* Jenkins' lookup8 is a 64 bit hash, but he says it's obsolete. Use * the plain one and recombine into 64 bits. */ uint64_t hash64_any(const void *key, size_t length, uint64_t base) { uint32_t b32 = base + (base >> 32); uint32_t lower; if (HASH_BIG_ENDIAN) lower = hashbig(key, length, &b32); else lower = hashlittle(key, length, &b32); return ((uint64_t)b32 << 32) | lower; } #ifdef SELF_TEST /* used for timings */ void driver1() { uint8_t buf[256]; uint32_t i; uint32_t h=0; time_t a,z; time(&a); for (i=0; i<256; ++i) buf[i] = 'x'; for (i=0; i<1; ++i) { h = hashlittle(&buf[0],1,h); } time(&z); if (z-a > 0) printf("time %d %.8x\n", z-a, h); } /* check that every input bit changes every output bit half the time */ #define HASHSTATE 1 #define HASHLEN 1 #define MAXPAIR 60 #define MAXLEN 70 void driver2() { uint8_t qa[MAXLEN+1], qb[MAXLEN+2], *a = &qa[0], *b = &qb[1]; uint32_t c[HASHSTATE], d[HASHSTATE], i=0, j=0, k, l, m=0, z; uint32_t e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE]; uint32_t x[HASHSTATE],y[HASHSTATE]; uint32_t hlen; printf("No more than %d trials should ever be needed \n",MAXPAIR/2); for (hlen=0; hlen < MAXLEN; ++hlen) { z=0; for (i=0; i>(8-j)); c[0] = hashlittle(a, hlen, m); b[i] ^= ((k+1)<>(8-j)); d[0] = hashlittle(b, hlen, m); /* check every bit is 1, 0, set, and not set at least once */ for (l=0; lz) z=k; if (k==MAXPAIR) { printf("Some bit didn't change: "); printf("%.8x %.8x %.8x %.8x %.8x %.8x ", e[0],f[0],g[0],h[0],x[0],y[0]); printf("i %d j %d m %d len %d\n", i, j, m, hlen); } if (z==MAXPAIR) goto done; } } } done: if (z < MAXPAIR) { printf("Mix success %2d bytes %2d initvals ",i,m); printf("required %d trials\n", z/2); } } printf("\n"); } /* Check for reading beyond the end of the buffer and alignment problems */ void driver3() { uint8_t buf[MAXLEN+20], *b; uint32_t len; uint8_t q[] = "This is the time for all good men to come to the aid of their country..."; uint32_t h; uint8_t qq[] = "xThis is the time for all good men to come to the aid of their country..."; uint32_t i; uint8_t qqq[] = "xxThis is the time for all good men to come to the aid of their country..."; uint32_t j; uint8_t qqqq[] = "xxxThis is the time for all good men to come to the aid of their country..."; uint32_t ref,x,y; uint8_t *p; printf("Endianness. These lines should all be the same (for values filled in):\n"); printf("%.8x %.8x %.8x\n", hash_word((const uint32_t *)q, (sizeof(q)-1)/4, 13), hash_word((const uint32_t *)q, (sizeof(q)-5)/4, 13), hash_word((const uint32_t *)q, (sizeof(q)-9)/4, 13)); p = q; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qq[1]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qqq[2]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qqqq[3]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); printf("\n"); /* check that hashlittle2 and hashlittle produce the same results */ i=47; j=0; hashlittle2(q, sizeof(q), &i, &j); if (hashlittle(q, sizeof(q), 47) != i) printf("hashlittle2 and hashlittle mismatch\n"); /* check that hash_word2 and hash_word produce the same results */ len = 0xdeadbeef; i=47, j=0; hash_word2(&len, 1, &i, &j); if (hash_word(&len, 1, 47) != i) printf("hash_word2 and hash_word mismatch %x %x\n", i, hash_word(&len, 1, 47)); /* check hashlittle doesn't read before or after the ends of the string */ for (h=0, b=buf+1; h<8; ++h, ++b) { for (i=0; i #include #include /* Stolen mostly from: lookup3.c, by Bob Jenkins, May 2006, Public Domain. * * http://burtleburtle.net/bob/c/lookup3.c */ /** * hash - fast hash of an array for internal use * @p: the array or pointer to first element * @num: the number of elements to hash * @base: the base number to roll into the hash (usually 0) * * The memory region pointed to by p is combined with the base to form * a 32-bit hash. * * This hash will have different results on different machines, so is * only useful for internal hashes (ie. not hashes sent across the * network or saved to disk). * * It may also change with future versions: it could even detect at runtime * what the fastest hash to use is. * * See also: hash64, hash_stable. * * Example: * #include * #include * #include * #include * * // Simple demonstration: idential strings will have the same hash, but * // two different strings will probably not. * int main(int argc, char *argv[]) * { * uint32_t hash1, hash2; * * if (argc != 3) * err(1, "Usage: %s ", argv[0]); * * hash1 = hash(argv[1], strlen(argv[1]), 0); * hash2 = hash(argv[2], strlen(argv[2]), 0); * printf("Hash is %s\n", hash1 == hash2 ? "same" : "different"); * return 0; * } */ #define hash(p, num, base) hash_any((p), (num)*sizeof(*(p)), (base)) /** * hash_stable - hash of an array for external use * @p: the array or pointer to first element * @num: the number of elements to hash * @base: the base number to roll into the hash (usually 0) * * The array of simple integer types pointed to by p is combined with * the base to form a 32-bit hash. * * This hash will have the same results on different machines, so can * be used for external hashes (ie. hashes sent across the network or * saved to disk). The results will not change in future versions of * this module. * * Note that it is only legal to hand an array of simple integer types * to this hash (ie. char, uint16_t, int64_t, etc). In these cases, * the same values will have the same hash result, even though the * memory representations of integers depend on the machine * endianness. * * See also: * hash64_stable * * Example: * #include * #include * #include * #include * * int main(int argc, char *argv[]) * { * if (argc != 2) * err(1, "Usage: %s ", argv[0]); * * printf("Hash stable result is %u\n", * hash_stable(argv[1], strlen(argv[1]), 0)); * return 0; * } */ #define hash_stable(p, num, base) \ (BUILD_ASSERT_OR_ZERO(sizeof(*(p)) == 8 || sizeof(*(p)) == 4 \ || sizeof(*(p)) == 2 || sizeof(*(p)) == 1) + \ sizeof(*(p)) == 8 ? hash_stable_64((p), (num), (base)) \ : sizeof(*(p)) == 4 ? hash_stable_32((p), (num), (base)) \ : sizeof(*(p)) == 2 ? hash_stable_16((p), (num), (base)) \ : hash_stable_8((p), (num), (base))) /** * hash_u32 - fast hash an array of 32-bit values for internal use * @key: the array of uint32_t * @num: the number of elements to hash * @base: the base number to roll into the hash (usually 0) * * The array of uint32_t pointed to by @key is combined with the base * to form a 32-bit hash. This is 2-3 times faster than hash() on small * arrays, but the advantage vanishes over large hashes. * * This hash will have different results on different machines, so is * only useful for internal hashes (ie. not hashes sent across the * network or saved to disk). */ uint32_t hash_u32(const uint32_t *key, size_t num, uint32_t base); /** * hash_string - very fast hash of an ascii string * @str: the nul-terminated string * * The string is hashed, using a hash function optimized for ASCII and * similar strings. It's weaker than the other hash functions. * * This hash may have different results on different machines, so is * only useful for internal hashes (ie. not hashes sent across the * network or saved to disk). The results will be different from the * other hash functions in this module, too. */ static inline uint32_t hash_string(const char *string) { /* This is Karl Nelson 's X31 hash. * It's a little faster than the (much better) lookup3 hash(): 56ns vs * 84ns on my 2GHz Intel Core Duo 2 laptop for a 10 char string. */ uint32_t ret; for (ret = 0; *string; string++) ret = (ret << 5) - ret + *string; return ret; } /** * hash64 - fast 64-bit hash of an array for internal use * @p: the array or pointer to first element * @num: the number of elements to hash * @base: the 64-bit base number to roll into the hash (usually 0) * * The memory region pointed to by p is combined with the base to form * a 64-bit hash. * * This hash will have different results on different machines, so is * only useful for internal hashes (ie. not hashes sent across the * network or saved to disk). * * It may also change with future versions: it could even detect at runtime * what the fastest hash to use is. * * See also: hash. * * Example: * #include * #include * #include * #include * * // Simple demonstration: idential strings will have the same hash, but * // two different strings will probably not. * int main(int argc, char *argv[]) * { * uint64_t hash1, hash2; * * if (argc != 3) * err(1, "Usage: %s ", argv[0]); * * hash1 = hash64(argv[1], strlen(argv[1]), 0); * hash2 = hash64(argv[2], strlen(argv[2]), 0); * printf("Hash is %s\n", hash1 == hash2 ? "same" : "different"); * return 0; * } */ #define hash64(p, num, base) hash64_any((p), (num)*sizeof(*(p)), (base)) /** * hash64_stable - 64 bit hash of an array for external use * @p: the array or pointer to first element * @num: the number of elements to hash * @base: the base number to roll into the hash (usually 0) * * The array of simple integer types pointed to by p is combined with * the base to form a 64-bit hash. * * This hash will have the same results on different machines, so can * be used for external hashes (ie. hashes sent across the network or * saved to disk). The results will not change in future versions of * this module. * * Note that it is only legal to hand an array of simple integer types * to this hash (ie. char, uint16_t, int64_t, etc). In these cases, * the same values will have the same hash result, even though the * memory representations of integers depend on the machine * endianness. * * See also: * hash_stable * * Example: * #include * #include * #include * #include * * int main(int argc, char *argv[]) * { * if (argc != 2) * err(1, "Usage: %s ", argv[0]); * * printf("Hash stable result is %llu\n", * (long long)hash64_stable(argv[1], strlen(argv[1]), 0)); * return 0; * } */ #define hash64_stable(p, num, base) \ (BUILD_ASSERT_OR_ZERO(sizeof(*(p)) == 8 || sizeof(*(p)) == 4 \ || sizeof(*(p)) == 2 || sizeof(*(p)) == 1) + \ sizeof(*(p)) == 8 ? hash64_stable_64((p), (num), (base)) \ : sizeof(*(p)) == 4 ? hash64_stable_32((p), (num), (base)) \ : sizeof(*(p)) == 2 ? hash64_stable_16((p), (num), (base)) \ : hash64_stable_8((p), (num), (base))) /** * hashl - fast 32/64-bit hash of an array for internal use * @p: the array or pointer to first element * @num: the number of elements to hash * @base: the base number to roll into the hash (usually 0) * * This is either hash() or hash64(), on 32/64 bit long machines. */ #define hashl(p, num, base) \ (BUILD_ASSERT_OR_ZERO(sizeof(long) == sizeof(uint32_t) \ || sizeof(long) == sizeof(uint64_t)) + \ (sizeof(long) == sizeof(uint64_t) \ ? hash64((p), (num), (base)) : hash((p), (num), (base)))) /* Our underlying operations. */ uint32_t hash_any(const void *key, size_t length, uint32_t base); uint32_t hash_stable_64(const void *key, size_t n, uint32_t base); uint32_t hash_stable_32(const void *key, size_t n, uint32_t base); uint32_t hash_stable_16(const void *key, size_t n, uint32_t base); uint32_t hash_stable_8(const void *key, size_t n, uint32_t base); uint64_t hash64_any(const void *key, size_t length, uint64_t base); uint64_t hash64_stable_64(const void *key, size_t n, uint64_t base); uint64_t hash64_stable_32(const void *key, size_t n, uint64_t base); uint64_t hash64_stable_16(const void *key, size_t n, uint64_t base); uint64_t hash64_stable_8(const void *key, size_t n, uint64_t base); /** * hash_pointer - hash a pointer for internal use * @p: the pointer value to hash * @base: the base number to roll into the hash (usually 0) * * The pointer p (not what p points to!) is combined with the base to form * a 32-bit hash. * * This hash will have different results on different machines, so is * only useful for internal hashes (ie. not hashes sent across the * network or saved to disk). * * Example: * #include * * // Code to keep track of memory regions. * struct region { * struct region *chain; * void *start; * unsigned int size; * }; * // We keep a simple hash table. * static struct region *region_hash[128]; * * static void add_region(struct region *r) * { * unsigned int h = hash_pointer(r->start, 0); * * r->chain = region_hash[h]; * region_hash[h] = r->chain; * } * * static struct region *find_region(const void *start) * { * struct region *r; * * for (r = region_hash[hash_pointer(start, 0)]; r; r = r->chain) * if (r->start == start) * return r; * return NULL; * } */ static inline uint32_t hash_pointer(const void *p, uint32_t base) { if (sizeof(p) % sizeof(uint32_t) == 0) { /* This convoluted union is the right way of aliasing. */ union { uint32_t a[sizeof(p) / sizeof(uint32_t)]; const void *p; } u; u.p = p; return hash_u32(u.a, sizeof(p) / sizeof(uint32_t), base); } else return hash(&p, 1, base); } #endif /* HASH_H */ ================================================ FILE: build-tools/cpp_misc/ccan/hash/test/api-hash_stable.c ================================================ #include #include #include #include #define ARRAY_WORDS 5 int main(int argc, char *argv[]) { unsigned int i; uint8_t u8array[ARRAY_WORDS]; uint16_t u16array[ARRAY_WORDS]; uint32_t u32array[ARRAY_WORDS]; uint64_t u64array[ARRAY_WORDS]; /* Initialize arrays. */ for (i = 0; i < ARRAY_WORDS; i++) { u8array[i] = i; u16array[i] = i; u32array[i] = i; u64array[i] = i; } plan_tests(264); /* hash_stable is API-guaranteed. */ ok1(hash_stable(u8array, ARRAY_WORDS, 0) == 0x1d4833cc); ok1(hash_stable(u8array, ARRAY_WORDS, 1) == 0x37125e2 ); ok1(hash_stable(u8array, ARRAY_WORDS, 2) == 0x330a007a); ok1(hash_stable(u8array, ARRAY_WORDS, 4) == 0x7b0df29b); ok1(hash_stable(u8array, ARRAY_WORDS, 8) == 0xe7e5d741); ok1(hash_stable(u8array, ARRAY_WORDS, 16) == 0xaae57471); ok1(hash_stable(u8array, ARRAY_WORDS, 32) == 0xc55399e5); ok1(hash_stable(u8array, ARRAY_WORDS, 64) == 0x67f21f7 ); ok1(hash_stable(u8array, ARRAY_WORDS, 128) == 0x1d795b71); ok1(hash_stable(u8array, ARRAY_WORDS, 256) == 0xeb961671); ok1(hash_stable(u8array, ARRAY_WORDS, 512) == 0xc2597247); ok1(hash_stable(u8array, ARRAY_WORDS, 1024) == 0x3f5c4d75); ok1(hash_stable(u8array, ARRAY_WORDS, 2048) == 0xe65cf4f9); ok1(hash_stable(u8array, ARRAY_WORDS, 4096) == 0xf2cd06cb); ok1(hash_stable(u8array, ARRAY_WORDS, 8192) == 0x443041e1); ok1(hash_stable(u8array, ARRAY_WORDS, 16384) == 0xdfc618f5); ok1(hash_stable(u8array, ARRAY_WORDS, 32768) == 0x5e3d5b97); ok1(hash_stable(u8array, ARRAY_WORDS, 65536) == 0xd5f64730); ok1(hash_stable(u8array, ARRAY_WORDS, 131072) == 0x372bbecc); ok1(hash_stable(u8array, ARRAY_WORDS, 262144) == 0x7c194c8d); ok1(hash_stable(u8array, ARRAY_WORDS, 524288) == 0x16cbb416); ok1(hash_stable(u8array, ARRAY_WORDS, 1048576) == 0x53e99222); ok1(hash_stable(u8array, ARRAY_WORDS, 2097152) == 0x6394554a); ok1(hash_stable(u8array, ARRAY_WORDS, 4194304) == 0xd83a506d); ok1(hash_stable(u8array, ARRAY_WORDS, 8388608) == 0x7619d9a4); ok1(hash_stable(u8array, ARRAY_WORDS, 16777216) == 0xfe98e5f6); ok1(hash_stable(u8array, ARRAY_WORDS, 33554432) == 0x6c262927); ok1(hash_stable(u8array, ARRAY_WORDS, 67108864) == 0x3f0106fd); ok1(hash_stable(u8array, ARRAY_WORDS, 134217728) == 0xc91e3a28); ok1(hash_stable(u8array, ARRAY_WORDS, 268435456) == 0x14229579); ok1(hash_stable(u8array, ARRAY_WORDS, 536870912) == 0x9dbefa76); ok1(hash_stable(u8array, ARRAY_WORDS, 1073741824) == 0xb05c0c78); ok1(hash_stable(u8array, ARRAY_WORDS, 2147483648U) == 0x88f24d81); ok1(hash_stable(u16array, ARRAY_WORDS, 0) == 0xecb5f507); ok1(hash_stable(u16array, ARRAY_WORDS, 1) == 0xadd666e6); ok1(hash_stable(u16array, ARRAY_WORDS, 2) == 0xea0f214c); ok1(hash_stable(u16array, ARRAY_WORDS, 4) == 0xae4051ba); ok1(hash_stable(u16array, ARRAY_WORDS, 8) == 0x6ed28026); ok1(hash_stable(u16array, ARRAY_WORDS, 16) == 0xa3917a19); ok1(hash_stable(u16array, ARRAY_WORDS, 32) == 0xf370f32b); ok1(hash_stable(u16array, ARRAY_WORDS, 64) == 0x807af460); ok1(hash_stable(u16array, ARRAY_WORDS, 128) == 0xb4c8cd83); ok1(hash_stable(u16array, ARRAY_WORDS, 256) == 0xa10cb5b0); ok1(hash_stable(u16array, ARRAY_WORDS, 512) == 0x8b7d7387); ok1(hash_stable(u16array, ARRAY_WORDS, 1024) == 0x9e49d1c ); ok1(hash_stable(u16array, ARRAY_WORDS, 2048) == 0x288830d1); ok1(hash_stable(u16array, ARRAY_WORDS, 4096) == 0xbe078a43); ok1(hash_stable(u16array, ARRAY_WORDS, 8192) == 0xa16d5d88); ok1(hash_stable(u16array, ARRAY_WORDS, 16384) == 0x46839fcd); ok1(hash_stable(u16array, ARRAY_WORDS, 32768) == 0x9db9bd4f); ok1(hash_stable(u16array, ARRAY_WORDS, 65536) == 0xedff58f8); ok1(hash_stable(u16array, ARRAY_WORDS, 131072) == 0x95ecef18); ok1(hash_stable(u16array, ARRAY_WORDS, 262144) == 0x23c31b7d); ok1(hash_stable(u16array, ARRAY_WORDS, 524288) == 0x1d85c7d0); ok1(hash_stable(u16array, ARRAY_WORDS, 1048576) == 0x25218842); ok1(hash_stable(u16array, ARRAY_WORDS, 2097152) == 0x711d985c); ok1(hash_stable(u16array, ARRAY_WORDS, 4194304) == 0x85470eca); ok1(hash_stable(u16array, ARRAY_WORDS, 8388608) == 0x99ed4ceb); ok1(hash_stable(u16array, ARRAY_WORDS, 16777216) == 0x67b3710c); ok1(hash_stable(u16array, ARRAY_WORDS, 33554432) == 0x77f1ab35); ok1(hash_stable(u16array, ARRAY_WORDS, 67108864) == 0x81f688aa); ok1(hash_stable(u16array, ARRAY_WORDS, 134217728) == 0x27b56ca5); ok1(hash_stable(u16array, ARRAY_WORDS, 268435456) == 0xf21ba203); ok1(hash_stable(u16array, ARRAY_WORDS, 536870912) == 0xd48d1d1 ); ok1(hash_stable(u16array, ARRAY_WORDS, 1073741824) == 0xa542b62d); ok1(hash_stable(u16array, ARRAY_WORDS, 2147483648U) == 0xa04c7058); ok1(hash_stable(u32array, ARRAY_WORDS, 0) == 0x13305f8c); ok1(hash_stable(u32array, ARRAY_WORDS, 1) == 0x171abf74); ok1(hash_stable(u32array, ARRAY_WORDS, 2) == 0x7646fcc7); ok1(hash_stable(u32array, ARRAY_WORDS, 4) == 0xa758ed5); ok1(hash_stable(u32array, ARRAY_WORDS, 8) == 0x2dedc2e4); ok1(hash_stable(u32array, ARRAY_WORDS, 16) == 0x28e2076b); ok1(hash_stable(u32array, ARRAY_WORDS, 32) == 0xb73091c5); ok1(hash_stable(u32array, ARRAY_WORDS, 64) == 0x87daf5db); ok1(hash_stable(u32array, ARRAY_WORDS, 128) == 0xa16dfe20); ok1(hash_stable(u32array, ARRAY_WORDS, 256) == 0x300c63c3); ok1(hash_stable(u32array, ARRAY_WORDS, 512) == 0x255c91fc); ok1(hash_stable(u32array, ARRAY_WORDS, 1024) == 0x6357b26); ok1(hash_stable(u32array, ARRAY_WORDS, 2048) == 0x4bc5f339); ok1(hash_stable(u32array, ARRAY_WORDS, 4096) == 0x1301617c); ok1(hash_stable(u32array, ARRAY_WORDS, 8192) == 0x506792c9); ok1(hash_stable(u32array, ARRAY_WORDS, 16384) == 0xcd596705); ok1(hash_stable(u32array, ARRAY_WORDS, 32768) == 0xa8713cac); ok1(hash_stable(u32array, ARRAY_WORDS, 65536) == 0x94d9794); ok1(hash_stable(u32array, ARRAY_WORDS, 131072) == 0xac753e8); ok1(hash_stable(u32array, ARRAY_WORDS, 262144) == 0xcd8bdd20); ok1(hash_stable(u32array, ARRAY_WORDS, 524288) == 0xd44faf80); ok1(hash_stable(u32array, ARRAY_WORDS, 1048576) == 0x2547ccbe); ok1(hash_stable(u32array, ARRAY_WORDS, 2097152) == 0xbab06dbc); ok1(hash_stable(u32array, ARRAY_WORDS, 4194304) == 0xaac0e882); ok1(hash_stable(u32array, ARRAY_WORDS, 8388608) == 0x443f48d0); ok1(hash_stable(u32array, ARRAY_WORDS, 16777216) == 0xdff49fcc); ok1(hash_stable(u32array, ARRAY_WORDS, 33554432) == 0x9ce0fd65); ok1(hash_stable(u32array, ARRAY_WORDS, 67108864) == 0x9ddb1def); ok1(hash_stable(u32array, ARRAY_WORDS, 134217728) == 0x86096f25); ok1(hash_stable(u32array, ARRAY_WORDS, 268435456) == 0xe713b7b5); ok1(hash_stable(u32array, ARRAY_WORDS, 536870912) == 0x5baeffc5); ok1(hash_stable(u32array, ARRAY_WORDS, 1073741824) == 0xde874f52); ok1(hash_stable(u32array, ARRAY_WORDS, 2147483648U) == 0xeca13b4e); ok1(hash_stable(u64array, ARRAY_WORDS, 0) == 0x12ef6302); ok1(hash_stable(u64array, ARRAY_WORDS, 1) == 0xe9aeb406); ok1(hash_stable(u64array, ARRAY_WORDS, 2) == 0xc4218ceb); ok1(hash_stable(u64array, ARRAY_WORDS, 4) == 0xb3d11412); ok1(hash_stable(u64array, ARRAY_WORDS, 8) == 0xdafbd654); ok1(hash_stable(u64array, ARRAY_WORDS, 16) == 0x9c336cba); ok1(hash_stable(u64array, ARRAY_WORDS, 32) == 0x65059721); ok1(hash_stable(u64array, ARRAY_WORDS, 64) == 0x95b5bbe6); ok1(hash_stable(u64array, ARRAY_WORDS, 128) == 0xe7596b84); ok1(hash_stable(u64array, ARRAY_WORDS, 256) == 0x503622a2); ok1(hash_stable(u64array, ARRAY_WORDS, 512) == 0xecdcc5ca); ok1(hash_stable(u64array, ARRAY_WORDS, 1024) == 0xc40d0513); ok1(hash_stable(u64array, ARRAY_WORDS, 2048) == 0xaab25e4d); ok1(hash_stable(u64array, ARRAY_WORDS, 4096) == 0xcc353fb9); ok1(hash_stable(u64array, ARRAY_WORDS, 8192) == 0x18e2319f); ok1(hash_stable(u64array, ARRAY_WORDS, 16384) == 0xfddaae8d); ok1(hash_stable(u64array, ARRAY_WORDS, 32768) == 0xef7976f2); ok1(hash_stable(u64array, ARRAY_WORDS, 65536) == 0x86359fc9); ok1(hash_stable(u64array, ARRAY_WORDS, 131072) == 0x8b5af385); ok1(hash_stable(u64array, ARRAY_WORDS, 262144) == 0x80d4ee31); ok1(hash_stable(u64array, ARRAY_WORDS, 524288) == 0x42f5f85b); ok1(hash_stable(u64array, ARRAY_WORDS, 1048576) == 0x9a6920e1); ok1(hash_stable(u64array, ARRAY_WORDS, 2097152) == 0x7b7c9850); ok1(hash_stable(u64array, ARRAY_WORDS, 4194304) == 0x69573e09); ok1(hash_stable(u64array, ARRAY_WORDS, 8388608) == 0xc942bc0e); ok1(hash_stable(u64array, ARRAY_WORDS, 16777216) == 0x7a89f0f1); ok1(hash_stable(u64array, ARRAY_WORDS, 33554432) == 0x2dd641ca); ok1(hash_stable(u64array, ARRAY_WORDS, 67108864) == 0x89bbd391); ok1(hash_stable(u64array, ARRAY_WORDS, 134217728) == 0xbcf88e31); ok1(hash_stable(u64array, ARRAY_WORDS, 268435456) == 0xfa7a3460); ok1(hash_stable(u64array, ARRAY_WORDS, 536870912) == 0x49a37be0); ok1(hash_stable(u64array, ARRAY_WORDS, 1073741824) == 0x1b346394); ok1(hash_stable(u64array, ARRAY_WORDS, 2147483648U) == 0x6c3a1592); ok1(hash64_stable(u8array, ARRAY_WORDS, 0) == 16887282882572727244ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 1) == 12032777473133454818ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 2) == 18183407363221487738ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 4) == 17860764172704150171ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 8) == 18076051600675559233ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 16) == 9909361918431556721ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 32) == 12937969888744675813ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 64) == 5245669057381736951ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 128) == 4376874646406519665ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 256) == 14219974419871569521ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 512) == 2263415354134458951ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 1024) == 4953859694526221685ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 2048) == 3432228642067641593ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 4096) == 1219647244417697483ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 8192) == 7629939424585859553ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 16384) == 10041660531376789749ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 32768) == 13859885793922603927ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 65536) == 15069060338344675120ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 131072) == 818163430835601100ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 262144) == 14914314323019517069ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 524288) == 17518437749769352214ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 1048576) == 14920048004901212706ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 2097152) == 8758567366332536138ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 4194304) == 6226655736088907885ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 8388608) == 13716650013685832100ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 16777216) == 305325651636315638ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 33554432) == 16784147606583781671ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 67108864) == 16509467555140798205ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 134217728) == 8717281234694060584ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 268435456) == 8098476701725660537ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 536870912) == 16345871539461094006ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 1073741824) == 3755557000429964408ULL); ok1(hash64_stable(u8array, ARRAY_WORDS, 2147483648U) == 15017348801959710081ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 0) == 1038028831307724039ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 1) == 10155473272642627302ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 2) == 5714751190106841420ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 4) == 3923885607767527866ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 8) == 3931017318293995558ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 16) == 1469696588339313177ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 32) == 11522218526952715051ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 64) == 6953517591561958496ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 128) == 7406689491740052867ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 256) == 10101844489704093104ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 512) == 12511348870707245959ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 1024) == 1614019938016861468ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 2048) == 5294796182374592721ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 4096) == 16089570706643716675ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 8192) == 1689302638424579464ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 16384) == 1446340172370386893ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 32768) == 16535503506744393039ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 65536) == 3496794142527150328ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 131072) == 6568245367474548504ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 262144) == 9487676460765485949ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 524288) == 4519762130966530000ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 1048576) == 15623412069215340610ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 2097152) == 544013388676438108ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 4194304) == 5594904760290840266ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 8388608) == 18098755780041592043ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 16777216) == 6389168672387330316ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 33554432) == 896986127732419381ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 67108864) == 13232626471143901354ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 134217728) == 53378562890493093ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 268435456) == 10072361400297824771ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 536870912) == 14511948118285144529ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 1073741824) == 6981033484844447277ULL); ok1(hash64_stable(u16array, ARRAY_WORDS, 2147483648U) == 5619339091684126808ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 0) == 3037571077312110476ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 1) == 14732398743825071988ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 2) == 14949132158206672071ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 4) == 1291370080511561429ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 8) == 10792665964172133092ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 16) == 14250138032054339435ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 32) == 17136741522078732741ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 64) == 3260193403318236635ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 128) == 10526616652205653536ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 256) == 9019690373358576579ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 512) == 6997491436599677436ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 1024) == 18302783371416533798ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 2048) == 10149320644446516025ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 4096) == 7073759949410623868ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 8192) == 17442399482223760073ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 16384) == 2983906194216281861ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 32768) == 4975845419129060524ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 65536) == 594019910205413268ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 131072) == 11903010186073691112ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 262144) == 7339636527154847008ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 524288) == 15243305400579108736ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 1048576) == 16737926245392043198ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 2097152) == 15725083267699862972ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 4194304) == 12527834265678833794ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 8388608) == 13908436455987824848ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 16777216) == 9672773345173872588ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 33554432) == 2305314279896710501ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 67108864) == 1866733780381408751ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 134217728) == 11906263969465724709ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 268435456) == 5501594918093830069ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 536870912) == 15823785789276225477ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 1073741824) == 17353000723889475410ULL); ok1(hash64_stable(u32array, ARRAY_WORDS, 2147483648U) == 7494736910655503182ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 0) == 9765419389786481410ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 1) == 11182806172127114246ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 2) == 2559155171395472619ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 4) == 3311692033324815378ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 8) == 1297175419505333844ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 16) == 617896928653569210ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 32) == 1517398559958603553ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 64) == 4504821917445110758ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 128) == 1971743331114904452ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 256) == 6177667912354374306ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 512) == 15570521289777792458ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 1024) == 9204559632415917331ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 2048) == 9008982669760028237ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 4096) == 14803537660281700281ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 8192) == 2873966517448487327ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 16384) == 5859277625928363661ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 32768) == 15520461285618185970ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 65536) == 16746489793331175369ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 131072) == 514952025484227461ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 262144) == 10867212269810675249ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 524288) == 9822204377278314587ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 1048576) == 3295088921987850465ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 2097152) == 7559197431498053712ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 4194304) == 1667267269116771849ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 8388608) == 2916804068951374862ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 16777216) == 14422558383125688561ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 33554432) == 10083112683694342602ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 67108864) == 7222777647078298513ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 134217728) == 18424513674048212529ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 268435456) == 14913668581101810784ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 536870912) == 14377721174297902048ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 1073741824) == 6031715005667500948ULL); ok1(hash64_stable(u64array, ARRAY_WORDS, 2147483648U) == 4827100319722378642ULL); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/hash/test/run.c ================================================ #include #include #include #include #include #define ARRAY_WORDS 5 int main(int argc, char *argv[]) { unsigned int i, j, k; uint32_t array[ARRAY_WORDS], val; char array2[sizeof(array) + sizeof(uint32_t)]; uint32_t results[256]; /* Initialize array. */ for (i = 0; i < ARRAY_WORDS; i++) array[i] = i; plan_tests(39); /* Hash should be the same, indep of memory alignment. */ val = hash(array, ARRAY_WORDS, 0); for (i = 0; i < sizeof(uint32_t); i++) { memcpy(array2 + i, array, sizeof(array)); ok(hash(array2 + i, ARRAY_WORDS, 0) != val, "hash matched at offset %i", i); } /* Hash of random values should have random distribution: * check one byte at a time. */ for (i = 0; i < sizeof(uint32_t); i++) { unsigned int lowest = -1U, highest = 0; memset(results, 0, sizeof(results)); for (j = 0; j < 256000; j++) { for (k = 0; k < ARRAY_WORDS; k++) array[k] = random(); results[(hash(array, ARRAY_WORDS, 0) >> i*8)&0xFF]++; } for (j = 0; j < 256; j++) { if (results[j] < lowest) lowest = results[j]; if (results[j] > highest) highest = results[j]; } /* Expect within 20% */ ok(lowest > 800, "Byte %i lowest %i", i, lowest); ok(highest < 1200, "Byte %i highest %i", i, highest); diag("Byte %i, range %u-%u", i, lowest, highest); } /* Hash of random values should have random distribution: * check one byte at a time. */ for (i = 0; i < sizeof(uint64_t); i++) { unsigned int lowest = -1U, highest = 0; memset(results, 0, sizeof(results)); for (j = 0; j < 256000; j++) { for (k = 0; k < ARRAY_WORDS; k++) array[k] = random(); results[(hash64(array, sizeof(array)/sizeof(uint64_t), 0) >> i*8)&0xFF]++; } for (j = 0; j < 256; j++) { if (results[j] < lowest) lowest = results[j]; if (results[j] > highest) highest = results[j]; } /* Expect within 20% */ ok(lowest > 800, "Byte %i lowest %i", i, lowest); ok(highest < 1200, "Byte %i highest %i", i, highest); diag("Byte %i, range %u-%u", i, lowest, highest); } /* Hash of pointer values should also have random distribution. */ for (i = 0; i < sizeof(uint32_t); i++) { unsigned int lowest = -1U, highest = 0; char *p = malloc(256000); memset(results, 0, sizeof(results)); for (j = 0; j < 256000; j++) results[(hash_pointer(p + j, 0) >> i*8)&0xFF]++; free(p); for (j = 0; j < 256; j++) { if (results[j] < lowest) lowest = results[j]; if (results[j] > highest) highest = results[j]; } /* Expect within 20% */ ok(lowest > 800, "hash_pointer byte %i lowest %i", i, lowest); ok(highest < 1200, "hash_pointer byte %i highest %i", i, highest); diag("hash_pointer byte %i, range %u-%u", i, lowest, highest); } if (sizeof(long) == sizeof(uint32_t)) ok1(hashl(array, ARRAY_WORDS, 0) == hash(array, ARRAY_WORDS, 0)); else ok1(hashl(array, ARRAY_WORDS, 0) == hash64(array, ARRAY_WORDS, 0)); /* String hash: weak, so only test bottom byte */ for (i = 0; i < 1; i++) { unsigned int num = 0, cursor, lowest = -1U, highest = 0; char p[5]; memset(results, 0, sizeof(results)); memset(p, 'A', sizeof(p)); p[sizeof(p)-1] = '\0'; for (;;) { for (cursor = 0; cursor < sizeof(p)-1; cursor++) { p[cursor]++; if (p[cursor] <= 'z') break; p[cursor] = 'A'; } if (cursor == sizeof(p)-1) break; results[(hash_string(p) >> i*8)&0xFF]++; num++; } for (j = 0; j < 256; j++) { if (results[j] < lowest) lowest = results[j]; if (results[j] > highest) highest = results[j]; } /* Expect within 20% */ ok(lowest > 35000, "hash_pointer byte %i lowest %i", i, lowest); ok(highest < 53000, "hash_pointer byte %i highest %i", i, highest); diag("hash_pointer byte %i, range %u-%u", i, lowest, highest); } return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/licenses/BSD-MIT ================================================ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: build-tools/cpp_misc/ccan/licenses/CC0 ================================================ Statement of Purpose The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. 1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; moral rights retained by the original author(s) and/or performer(s); publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; rights protecting the extraction, dissemination, use and reuse of data in a Work; database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. 2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. 3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. 4. Limitations and Disclaimers. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. ================================================ FILE: build-tools/cpp_misc/ccan/list/_info ================================================ #include #include #include "config.h" /** * list - double linked list routines * * The list header contains routines for manipulating double linked lists. * It defines two types: struct list_head used for anchoring lists, and * struct list_node which is usually embedded in the structure which is placed * in the list. * * Example: * #include * #include * #include * #include * * struct parent { * const char *name; * struct list_head children; * unsigned int num_children; * }; * * struct child { * const char *name; * struct list_node list; * }; * * int main(int argc, char *argv[]) * { * struct parent p; * struct child *c; * unsigned int i; * * if (argc < 2) * errx(1, "Usage: %s parent children...", argv[0]); * * p.name = argv[1]; * list_head_init(&p.children); * p.num_children = 0; * for (i = 2; i < argc; i++) { * c = malloc(sizeof(*c)); * c->name = argv[i]; * list_add(&p.children, &c->list); * p.num_children++; * } * * printf("%s has %u children:", p.name, p.num_children); * list_for_each(&p.children, c, list) * printf("%s ", c->name); * printf("\n"); * return 0; * } * * License: BSD-MIT * Author: Rusty Russell */ int main(int argc, char *argv[]) { if (argc != 2) return 1; if (strcmp(argv[1], "depends") == 0) { printf("ccan/container_of\n"); return 0; } return 1; } ================================================ FILE: build-tools/cpp_misc/ccan/list/list.c ================================================ /* Licensed under BSD-MIT - see LICENSE file for details */ #include #include #include "list.h" static void *corrupt(const char *abortstr, const struct list_node *head, const struct list_node *node, unsigned int count) { if (abortstr) { fprintf(stderr, "%s: prev corrupt in node %p (%u) of %p\n", abortstr, node, count, head); abort(); } return NULL; } struct list_node *list_check_node(const struct list_node *node, const char *abortstr) { const struct list_node *p, *n; int count = 0; for (p = node, n = node->next; n != node; p = n, n = n->next) { count++; if (n->prev != p) return corrupt(abortstr, node, n, count); } /* Check prev on head node. */ if (node->prev != p) return corrupt(abortstr, node, node, 0); return (struct list_node *)node; } struct list_head *list_check(const struct list_head *h, const char *abortstr) { if (!list_check_node(&h->n, abortstr)) return NULL; return (struct list_head *)h; } ================================================ FILE: build-tools/cpp_misc/ccan/list/list.h ================================================ /* Licensed under BSD-MIT - see LICENSE file for details */ #ifndef CCAN_LIST_H #define CCAN_LIST_H #include #include #include #include /** * struct list_node - an entry in a doubly-linked list * @next: next entry (self if empty) * @prev: previous entry (self if empty) * * This is used as an entry in a linked list. * Example: * struct child { * const char *name; * // Linked list of all us children. * struct list_node list; * }; */ struct list_node { struct list_node *next, *prev; }; /** * struct list_head - the head of a doubly-linked list * @h: the list_head (containing next and prev pointers) * * This is used as the head of a linked list. * Example: * struct parent { * const char *name; * struct list_head children; * unsigned int num_children; * }; */ struct list_head { struct list_node n; }; /** * list_check - check head of a list for consistency * @h: the list_head * @abortstr: the location to print on aborting, or NULL. * * Because list_nodes have redundant information, consistency checking between * the back and forward links can be done. This is useful as a debugging check. * If @abortstr is non-NULL, that will be printed in a diagnostic if the list * is inconsistent, and the function will abort. * * Returns the list head if the list is consistent, NULL if not (it * can never return NULL if @abortstr is set). * * See also: list_check_node() * * Example: * static void dump_parent(struct parent *p) * { * struct child *c; * * printf("%s (%u children):\n", p->name, p->num_children); * list_check(&p->children, "bad child list"); * list_for_each(&p->children, c, list) * printf(" -> %s\n", c->name); * } */ struct list_head *list_check(const struct list_head *h, const char *abortstr); /** * list_check_node - check node of a list for consistency * @n: the list_node * @abortstr: the location to print on aborting, or NULL. * * Check consistency of the list node is in (it must be in one). * * See also: list_check() * * Example: * static void dump_child(const struct child *c) * { * list_check_node(&c->list, "bad child list"); * printf("%s\n", c->name); * } */ struct list_node *list_check_node(const struct list_node *n, const char *abortstr); #ifdef CCAN_LIST_DEBUG #define list_debug(h) list_check((h), __func__) #define list_debug_node(n) list_check_node((n), __func__) #else #define list_debug(h) (h) #define list_debug_node(n) (n) #endif /** * LIST_HEAD_INIT - initializer for an empty list_head * @name: the name of the list. * * Explicit initializer for an empty list. * * See also: * LIST_HEAD, list_head_init() * * Example: * static struct list_head my_list = LIST_HEAD_INIT(my_list); */ #define LIST_HEAD_INIT(name) { { &name.n, &name.n } } /** * LIST_HEAD - define and initialize an empty list_head * @name: the name of the list. * * The LIST_HEAD macro defines a list_head and initializes it to an empty * list. It can be prepended by "static" to define a static list_head. * * See also: * LIST_HEAD_INIT, list_head_init() * * Example: * static LIST_HEAD(my_global_list); */ /*#define LIST_HEAD(name) \ struct list_head name = LIST_HEAD_INIT(name) */ /** * list_head_init - initialize a list_head * @h: the list_head to set to the empty list * * Example: * ... * struct parent *parent = malloc(sizeof(*parent)); * * list_head_init(&parent->children); * parent->num_children = 0; */ static inline void list_head_init(struct list_head *h) { h->n.next = h->n.prev = &h->n; } /** * list_add - add an entry at the start of a linked list. * @h: the list_head to add the node to * @n: the list_node to add to the list. * * The list_node does not need to be initialized; it will be overwritten. * Example: * struct child *child = malloc(sizeof(*child)); * * child->name = "marvin"; * list_add(&parent->children, &child->list); * parent->num_children++; */ static inline void list_add(struct list_head *h, struct list_node *n) { n->next = h->n.next; n->prev = &h->n; h->n.next->prev = n; h->n.next = n; (void)list_debug(h); } /** * list_add_tail - add an entry at the end of a linked list. * @h: the list_head to add the node to * @n: the list_node to add to the list. * * The list_node does not need to be initialized; it will be overwritten. * Example: * list_add_tail(&parent->children, &child->list); * parent->num_children++; */ static inline void list_add_tail(struct list_head *h, struct list_node *n) { n->next = &h->n; n->prev = h->n.prev; h->n.prev->next = n; h->n.prev = n; (void)list_debug(h); } /** * list_empty - is a list empty? * @h: the list_head * * If the list is empty, returns true. * * Example: * assert(list_empty(&parent->children) == (parent->num_children == 0)); */ static inline bool list_empty(const struct list_head *h) { (void)list_debug(h); return h->n.next == &h->n; } /** * list_del - delete an entry from an (unknown) linked list. * @n: the list_node to delete from the list. * * Note that this leaves @n in an undefined state; it can be added to * another list, but not deleted again. * * See also: * list_del_from() * * Example: * list_del(&child->list); * parent->num_children--; */ static inline void list_del(struct list_node *n) { (void)list_debug_node(n); n->next->prev = n->prev; n->prev->next = n->next; #ifdef CCAN_LIST_DEBUG /* Catch use-after-del. */ n->next = n->prev = NULL; #endif } /** * list_del_from - delete an entry from a known linked list. * @h: the list_head the node is in. * @n: the list_node to delete from the list. * * This explicitly indicates which list a node is expected to be in, * which is better documentation and can catch more bugs. * * See also: list_del() * * Example: * list_del_from(&parent->children, &child->list); * parent->num_children--; */ static inline void list_del_from(struct list_head *h, struct list_node *n) { #ifdef CCAN_LIST_DEBUG { /* Thorough check: make sure it was in list! */ struct list_node *i; for (i = h->n.next; i != n; i = i->next) assert(i != &h->n); } #endif /* CCAN_LIST_DEBUG */ /* Quick test that catches a surprising number of bugs. */ assert(!list_empty(h)); list_del(n); } /** * list_entry - convert a list_node back into the structure containing it. * @n: the list_node * @type: the type of the entry * @member: the list_node member of the type * * Example: * // First list entry is children.next; convert back to child. * child = list_entry(parent->children.n.next, struct child, list); * * See Also: * list_top(), list_for_each() */ #define list_entry(n, type, member) container_of(n, type, member) /** * list_top - get the first entry in a list * @h: the list_head * @type: the type of the entry * @member: the list_node member of the type * * If the list is empty, returns NULL. * * Example: * struct child *first; * first = list_top(&parent->children, struct child, list); * if (!first) * printf("Empty list!\n"); */ #define list_top(h, type, member) \ ((type *)list_top_((h), list_off_(type, member))) static inline const void *list_top_(const struct list_head *h, size_t off) { if (list_empty(h)) return NULL; return (const char *)h->n.next - off; } /** * list_pop - remove the first entry in a list * @h: the list_head * @type: the type of the entry * @member: the list_node member of the type * * If the list is empty, returns NULL. * * Example: * struct child *one; * one = list_pop(&parent->children, struct child, list); * if (!one) * printf("Empty list!\n"); */ #define list_pop(h, type, member) \ ((type *)list_pop_((h), list_off_(type, member))) static inline void *list_pop_(const struct list_head *h, size_t off) { struct list_node *n; if (list_empty(h)) return NULL; n = h->n.next; list_del(n); return (char *)n - off; } /** * list_tail - get the last entry in a list * @h: the list_head * @type: the type of the entry * @member: the list_node member of the type * * If the list is empty, returns NULL. * * Example: * struct child *last; * last = list_tail(&parent->children, struct child, list); * if (!last) * printf("Empty list!\n"); */ #define list_tail(h, type, member) \ ((type *)list_tail_((h), list_off_(type, member))) static inline const void *list_tail_(const struct list_head *h, size_t off) { if (list_empty(h)) return NULL; return (const char *)h->n.prev - off; } /** * list_for_each - iterate through a list. * @h: the list_head (warning: evaluated multiple times!) * @i: the structure containing the list_node * @member: the list_node member of the structure * * This is a convenient wrapper to iterate @i over the entire list. It's * a for loop, so you can break and continue as normal. * * Example: * list_for_each(&parent->children, child, list) * printf("Name: %s\n", child->name); */ #define list_for_each(h, i, member) \ list_for_each_off(h, i, list_off_var_(i, member)) /** * list_for_each_rev - iterate through a list backwards. * @h: the list_head * @i: the structure containing the list_node * @member: the list_node member of the structure * * This is a convenient wrapper to iterate @i over the entire list. It's * a for loop, so you can break and continue as normal. * * Example: * list_for_each_rev(&parent->children, child, list) * printf("Name: %s\n", child->name); */ #define list_for_each_rev(h, i, member) \ for (i = container_of_var(list_debug(h)->n.prev, i, member); \ &i->member != &(h)->n; \ i = container_of_var(i->member.prev, i, member)) /** * list_for_each_safe - iterate through a list, maybe during deletion * @h: the list_head * @i: the structure containing the list_node * @nxt: the structure containing the list_node * @member: the list_node member of the structure * * This is a convenient wrapper to iterate @i over the entire list. It's * a for loop, so you can break and continue as normal. The extra variable * @nxt is used to hold the next element, so you can delete @i from the list. * * Example: * struct child *next; * list_for_each_safe(&parent->children, child, next, list) { * list_del(&child->list); * parent->num_children--; * } */ #define list_for_each_safe(h, i, nxt, member) \ list_for_each_safe_off(h, i, nxt, list_off_var_(i, member)) /** * list_next - get the next entry in a list * @h: the list_head * @i: a pointer to an entry in the list. * @member: the list_node member of the structure * * If @i was the last entry in the list, returns NULL. * * Example: * struct child *second; * second = list_next(&parent->children, first, list); * if (!second) * printf("No second child!\n"); */ #define list_next(h, i, member) \ ((list_typeof(i))list_entry_or_null(list_debug(h), \ (i)->member.next, \ list_off_var_((i), member))) /** * list_prev - get the previous entry in a list * @h: the list_head * @i: a pointer to an entry in the list. * @member: the list_node member of the structure * * If @i was the first entry in the list, returns NULL. * * Example: * first = list_prev(&parent->children, second, list); * if (!first) * printf("Can't go back to first child?!\n"); */ #define list_prev(h, i, member) \ ((list_typeof(i))list_entry_or_null(list_debug(h), \ (i)->member.prev, \ list_off_var_((i), member))) /** * list_append_list - empty one list onto the end of another. * @to: the list to append into * @from: the list to empty. * * This takes the entire contents of @from and moves it to the end of * @to. After this @from will be empty. * * Example: * struct list_head adopter; * * list_append_list(&adopter, &parent->children); * assert(list_empty(&parent->children)); * parent->num_children = 0; */ static inline void list_append_list(struct list_head *to, struct list_head *from) { struct list_node *from_tail = list_debug(from)->n.prev; struct list_node *to_tail = list_debug(to)->n.prev; /* Sew in head and entire list. */ to->n.prev = from_tail; from_tail->next = &to->n; to_tail->next = &from->n; from->n.prev = to_tail; /* Now remove head. */ list_del(&from->n); list_head_init(from); } /** * list_prepend_list - empty one list into the start of another. * @to: the list to prepend into * @from: the list to empty. * * This takes the entire contents of @from and moves it to the start * of @to. After this @from will be empty. * * Example: * list_prepend_list(&adopter, &parent->children); * assert(list_empty(&parent->children)); * parent->num_children = 0; */ static inline void list_prepend_list(struct list_head *to, struct list_head *from) { struct list_node *from_tail = list_debug(from)->n.prev; struct list_node *to_head = list_debug(to)->n.next; /* Sew in head and entire list. */ to->n.next = &from->n; from->n.prev = &to->n; to_head->prev = from_tail; from_tail->next = to_head; /* Now remove head. */ list_del(&from->n); list_head_init(from); } /** * list_for_each_off - iterate through a list of memory regions. * @h: the list_head * @i: the pointer to a memory region wich contains list node data. * @off: offset(relative to @i) at which list node data resides. * * This is a low-level wrapper to iterate @i over the entire list, used to * implement all oher, more high-level, for-each constructs. It's a for loop, * so you can break and continue as normal. * * WARNING! Being the low-level macro that it is, this wrapper doesn't know * nor care about the type of @i. The only assumtion made is that @i points * to a chunk of memory that at some @offset, relative to @i, contains a * properly filled `struct node_list' which in turn contains pointers to * memory chunks and it's turtles all the way down. Whith all that in mind * remember that given the wrong pointer/offset couple this macro will * happilly churn all you memory untill SEGFAULT stops it, in other words * caveat emptor. * * It is worth mentioning that one of legitimate use-cases for that wrapper * is operation on opaque types with known offset for `struct list_node' * member(preferably 0), because it allows you not to disclose the type of * @i. * * Example: * list_for_each_off(&parent->children, child, * offsetof(struct child, list)) * printf("Name: %s\n", child->name); */ #define list_for_each_off(h, i, off) \ for (i = (typeof(i))list_node_to_off_(list_debug(h)->n.next, (off)); \ list_node_from_off_((void *)i, (off)) != &(h)->n; \ i = (typeof(i))list_node_to_off_(list_node_from_off_((void *)i, (off))->next, \ (off))) /** * list_for_each_safe_off - iterate through a list of memory regions, maybe * during deletion * @h: the list_head * @i: the pointer to a memory region wich contains list node data. * @nxt: the structure containing the list_node * @off: offset(relative to @i) at which list node data resides. * * For details see `list_for_each_off' and `list_for_each_safe' * descriptions. * * Example: * list_for_each_safe_off(&parent->children, child, * next, offsetof(struct child, list)) * printf("Name: %s\n", child->name); */ #define list_for_each_safe_off(h, i, nxt, off) \ for (i = (typeof(i))list_node_to_off_(list_debug(h)->n.next, (off)), \ nxt = (typeof(nxt))list_node_to_off_(list_node_from_off_(i, (off))->next, \ (off)); \ list_node_from_off_(i, (off)) != &(h)->n; \ i = nxt, \ nxt = (typeof(nxt))list_node_to_off_(list_node_from_off_(i, (off))->next, \ (off))) /* Other -off variants. */ #define list_entry_off(n, type, off) \ ((type *)list_node_from_off_((n), (off))) #define list_head_off(h, type, off) \ ((type *)list_head_off((h), (off))) #define list_tail_off(h, type, off) \ ((type *)list_tail_((h), (off))) #define list_add_off(h, n, off) \ list_add((h), list_node_from_off_((n), (off))) #define list_del_off(n, off) \ list_del(list_node_from_off_((n), (off))) #define list_del_from_off(h, n, off) \ list_del_from(h, list_node_from_off_((n), (off))) /* Offset helper functions so we only single-evaluate. */ static inline void *list_node_to_off_(struct list_node *node, size_t off) { return (void *)((char *)node - off); } static inline struct list_node *list_node_from_off_(void *ptr, size_t off) { return (struct list_node *)((char *)ptr + off); } /* Get the offset of the member, but make sure it's a list_node. */ #define list_off_(type, member) \ (container_off(type, member) + \ check_type(((type *)0)->member, struct list_node)) #define list_off_var_(var, member) \ (container_off_var(var, member) + \ check_type(var->member, struct list_node)) #if HAVE_TYPEOF #define list_typeof(var) typeof(var) #else #define list_typeof(var) void * #endif /* Returns member, or NULL if at end of list. */ static inline void *list_entry_or_null(const struct list_head *h, struct list_node *n, size_t off) { if (n == &h->n) return NULL; return (char *)n - off; } #endif /* CCAN_LIST_H */ ================================================ FILE: build-tools/cpp_misc/ccan/list/test/compile_ok-constant.c ================================================ #include #include #include #include #include struct child { const char *name; struct list_node list; }; static bool children(const struct list_head *list) { return !list_empty(list); } static const struct child *first_child(const struct list_head *list) { return list_top(list, struct child, list); } static const struct child *last_child(const struct list_head *list) { return list_tail(list, struct child, list); } static void check_children(const struct list_head *list) { list_check(list, "bad child list"); } static void print_children(const struct list_head *list) { const struct child *c; list_for_each(list, c, list) printf("%s\n", c->name); } int main(void) { LIST_HEAD(h); children(&h); first_child(&h); last_child(&h); check_children(&h); print_children(&h); return 0; } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/helper.c ================================================ #include #include #include #include #include "helper.h" #define ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING \ (42) struct opaque { struct list_node list; size_t secret_offset; char secret_drawer[42]; }; static bool not_randomized = true; struct opaque *create_opaque_blob(void) { struct opaque *blob = calloc(1, sizeof(struct opaque)); if (not_randomized) { srandom((int)time(NULL)); not_randomized = false; } blob->secret_offset = random() % (sizeof(blob->secret_drawer)); blob->secret_drawer[blob->secret_offset] = ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING; return blob; } bool if_blobs_know_the_secret(struct opaque *blob) { bool answer = true; int i; for (i = 0; i < sizeof(blob->secret_drawer) / sizeof(blob->secret_drawer[0]); i++) if (i != blob->secret_offset) answer = answer && (blob->secret_drawer[i] == 0); else answer = answer && (blob->secret_drawer[blob->secret_offset] == ANSWER_TO_THE_ULTIMATE_QUESTION_OF_LIFE_THE_UNIVERSE_AND_EVERYTHING); return answer; } void destroy_opaque_blob(struct opaque *blob) { free(blob); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/helper.h ================================================ /* These are in a separate C file so we can test undefined structures. */ struct opaque; typedef struct opaque opaque_t; opaque_t *create_opaque_blob(void); bool if_blobs_know_the_secret(opaque_t *blob); void destroy_opaque_blob(opaque_t *blob); ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-check-corrupt.c ================================================ #include #include #include #include #include #include /* We don't actually want it to exit... */ static jmp_buf aborted; #define abort() longjmp(aborted, 1) #define fprintf my_fprintf static char printf_buffer[1000]; static int my_fprintf(FILE *stream, const char *format, ...) { va_list ap; int ret; va_start(ap, format); ret = vsprintf(printf_buffer, format, ap); va_end(ap); return ret; } #include #include #include int main(int argc, char *argv[]) { struct list_head list; struct list_node n1; char expect[100]; plan_tests(9); /* Empty list. */ list.n.next = &list.n; list.n.prev = &list.n; ok1(list_check(&list, NULL) == &list); /* Bad back ptr */ list.n.prev = &n1; /* Non-aborting version. */ ok1(list_check(&list, NULL) == NULL); /* Aborting version. */ sprintf(expect, "test message: prev corrupt in node %p (0) of %p\n", &list, &list); if (setjmp(aborted) == 0) { list_check(&list, "test message"); fail("list_check on empty with bad back ptr didn't fail!"); } else { ok1(strcmp(printf_buffer, expect) == 0); } /* n1 in list. */ list.n.next = &n1; list.n.prev = &n1; n1.prev = &list.n; n1.next = &list.n; ok1(list_check(&list, NULL) == &list); ok1(list_check_node(&n1, NULL) == &n1); /* Bad back ptr */ n1.prev = &n1; ok1(list_check(&list, NULL) == NULL); ok1(list_check_node(&n1, NULL) == NULL); /* Aborting version. */ sprintf(expect, "test message: prev corrupt in node %p (1) of %p\n", &n1, &list); if (setjmp(aborted) == 0) { list_check(&list, "test message"); fail("list_check on n1 bad back ptr didn't fail!"); } else { ok1(strcmp(printf_buffer, expect) == 0); } sprintf(expect, "test message: prev corrupt in node %p (0) of %p\n", &n1, &n1); if (setjmp(aborted) == 0) { list_check_node(&n1, "test message"); fail("list_check_node on n1 bad back ptr didn't fail!"); } else { ok1(strcmp(printf_buffer, expect) == 0); } return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-list_del_from-assert.c ================================================ #define CCAN_LIST_DEBUG 1 #include #include #include #include #include #include #include int main(int argc, char *argv[]) { struct list_head list1, list2; struct list_node n1, n2, n3; pid_t child; int status; plan_tests(1); list_head_init(&list1); list_head_init(&list2); list_add(&list1, &n1); list_add(&list2, &n2); list_add_tail(&list2, &n3); child = fork(); if (child) { wait(&status); } else { /* This should abort. */ list_del_from(&list1, &n3); exit(0); } ok1(WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT); list_del_from(&list2, &n3); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-list_prev-list_next.c ================================================ #include #include #include #include "helper.h" struct parent { const char *name; unsigned int num_children; struct list_head children; }; struct child { const char *name; struct list_node list; }; int main(int argc, char *argv[]) { struct parent parent; struct child c1, c2, c3; const struct parent *p; const struct child *c; plan_tests(20); parent.num_children = 0; list_head_init(&parent.children); c1.name = "c1"; list_add(&parent.children, &c1.list); ok1(list_next(&parent.children, &c1, list) == NULL); ok1(list_prev(&parent.children, &c1, list) == NULL); c2.name = "c2"; list_add_tail(&parent.children, &c2.list); ok1(list_next(&parent.children, &c1, list) == &c2); ok1(list_prev(&parent.children, &c1, list) == NULL); ok1(list_next(&parent.children, &c2, list) == NULL); ok1(list_prev(&parent.children, &c2, list) == &c1); c3.name = "c3"; list_add_tail(&parent.children, &c3.list); ok1(list_next(&parent.children, &c1, list) == &c2); ok1(list_prev(&parent.children, &c1, list) == NULL); ok1(list_next(&parent.children, &c2, list) == &c3); ok1(list_prev(&parent.children, &c2, list) == &c1); ok1(list_next(&parent.children, &c3, list) == NULL); ok1(list_prev(&parent.children, &c3, list) == &c2); /* Const variants */ p = &parent; c = &c2; ok1(list_next(&p->children, &c1, list) == &c2); ok1(list_prev(&p->children, &c1, list) == NULL); ok1(list_next(&p->children, c, list) == &c3); ok1(list_prev(&p->children, c, list) == &c1); ok1(list_next(&parent.children, c, list) == &c3); ok1(list_prev(&parent.children, c, list) == &c1); ok1(list_next(&p->children, &c3, list) == NULL); ok1(list_prev(&p->children, &c3, list) == &c2); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-prepend_list.c ================================================ #include #include #include #include static bool list_expect(struct list_head *h, ...) { va_list ap; struct list_node *n = &h->n, *expected; va_start(ap, h); while ((expected = va_arg(ap, struct list_node *)) != NULL) { n = n->next; if (n != expected) return false; } return (n->next == &h->n); } int main(int argc, char *argv[]) { struct list_head h1, h2; struct list_node n[4]; plan_tests(40); list_head_init(&h1); list_head_init(&h2); /* Append an empty list to an empty list. */ list_append_list(&h1, &h2); ok1(list_empty(&h1)); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); /* Prepend an empty list to an empty list. */ list_prepend_list(&h1, &h2); ok1(list_empty(&h1)); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); /* Append an empty list to a non-empty list */ list_add(&h1, &n[0]); list_append_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[0], NULL)); /* Prepend an empty list to a non-empty list */ list_prepend_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[0], NULL)); /* Append a non-empty list to an empty list. */ list_append_list(&h2, &h1); ok1(list_empty(&h1)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h2, &n[0], NULL)); /* Prepend a non-empty list to an empty list. */ list_prepend_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[0], NULL)); /* Prepend a non-empty list to non-empty list. */ list_add(&h2, &n[1]); list_prepend_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[1], &n[0], NULL)); /* Append a non-empty list to non-empty list. */ list_add(&h2, &n[2]); list_append_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[1], &n[0], &n[2], NULL)); /* Prepend a 2-entry list to a 2-entry list. */ list_del_from(&h1, &n[2]); list_add(&h2, &n[2]); list_add_tail(&h2, &n[3]); list_prepend_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[2], &n[3], &n[1], &n[0], NULL)); /* Append a 2-entry list to a 2-entry list. */ list_del_from(&h1, &n[2]); list_del_from(&h1, &n[3]); list_add(&h2, &n[2]); list_add_tail(&h2, &n[3]); list_append_list(&h1, &h2); ok1(list_empty(&h2)); ok1(list_check(&h1, NULL)); ok1(list_check(&h2, NULL)); ok1(list_expect(&h1, &n[1], &n[0], &n[2], &n[3], NULL)); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-single-eval.c ================================================ /* Make sure macros only evaluate their args once. */ #include #include #include struct parent { const char *name; struct list_head children; unsigned int num_children; int eval_count; }; struct child { const char *name; struct list_node list; }; static LIST_HEAD(static_list); #define ref(obj, counter) ((counter)++, (obj)) int main(int argc, char *argv[]) { struct parent parent; struct child c1, c2, c3, *c, *n; unsigned int i; unsigned int static_count = 0, parent_count = 0, list_count = 0, node_count = 0; struct list_head list = LIST_HEAD_INIT(list); plan_tests(74); /* Test LIST_HEAD, LIST_HEAD_INIT, list_empty and check_list */ ok1(list_empty(ref(&static_list, static_count))); ok1(static_count == 1); ok1(list_check(ref(&static_list, static_count), NULL)); ok1(static_count == 2); ok1(list_empty(ref(&list, list_count))); ok1(list_count == 1); ok1(list_check(ref(&list, list_count), NULL)); ok1(list_count == 2); parent.num_children = 0; list_head_init(ref(&parent.children, parent_count)); ok1(parent_count == 1); /* Test list_head_init */ ok1(list_empty(ref(&parent.children, parent_count))); ok1(parent_count == 2); ok1(list_check(ref(&parent.children, parent_count), NULL)); ok1(parent_count == 3); c2.name = "c2"; list_add(ref(&parent.children, parent_count), &c2.list); ok1(parent_count == 4); /* Test list_add and !list_empty. */ ok1(!list_empty(ref(&parent.children, parent_count))); ok1(parent_count == 5); ok1(c2.list.next == &parent.children.n); ok1(c2.list.prev == &parent.children.n); ok1(parent.children.n.next == &c2.list); ok1(parent.children.n.prev == &c2.list); /* Test list_check */ ok1(list_check(ref(&parent.children, parent_count), NULL)); ok1(parent_count == 6); c1.name = "c1"; list_add(ref(&parent.children, parent_count), &c1.list); ok1(parent_count == 7); /* Test list_add and !list_empty. */ ok1(!list_empty(ref(&parent.children, parent_count))); ok1(parent_count == 8); ok1(c2.list.next == &parent.children.n); ok1(c2.list.prev == &c1.list); ok1(parent.children.n.next == &c1.list); ok1(parent.children.n.prev == &c2.list); ok1(c1.list.next == &c2.list); ok1(c1.list.prev == &parent.children.n); /* Test list_check */ ok1(list_check(ref(&parent.children, parent_count), NULL)); ok1(parent_count == 9); c3.name = "c3"; list_add_tail(ref(&parent.children, parent_count), &c3.list); ok1(parent_count == 10); /* Test list_add_tail and !list_empty. */ ok1(!list_empty(ref(&parent.children, parent_count))); ok1(parent_count == 11); ok1(parent.children.n.next == &c1.list); ok1(parent.children.n.prev == &c3.list); ok1(c1.list.next == &c2.list); ok1(c1.list.prev == &parent.children.n); ok1(c2.list.next == &c3.list); ok1(c2.list.prev == &c1.list); ok1(c3.list.next == &parent.children.n); ok1(c3.list.prev == &c2.list); /* Test list_check */ ok1(list_check(ref(&parent.children, parent_count), NULL)); ok1(parent_count == 12); /* Test list_check_node */ ok1(list_check_node(&c1.list, NULL)); ok1(list_check_node(&c2.list, NULL)); ok1(list_check_node(&c3.list, NULL)); /* Test list_top */ ok1(list_top(ref(&parent.children, parent_count), struct child, list) == &c1); ok1(parent_count == 13); /* Test list_tail */ ok1(list_tail(ref(&parent.children, parent_count), struct child, list) == &c3); ok1(parent_count == 14); /* Test list_for_each. */ i = 0; list_for_each(&parent.children, c, list) { switch (i++) { case 0: ok1(c == &c1); break; case 1: ok1(c == &c2); break; case 2: ok1(c == &c3); break; } if (i > 2) break; } ok1(i == 3); /* Test list_for_each_safe, list_del and list_del_from. */ i = 0; list_for_each_safe(&parent.children, c, n, list) { switch (i++) { case 0: ok1(c == &c1); list_del(ref(&c->list, node_count)); ok1(node_count == 1); break; case 1: ok1(c == &c2); list_del_from(ref(&parent.children, parent_count), ref(&c->list, node_count)); ok1(node_count == 2); break; case 2: ok1(c == &c3); list_del_from(ref(&parent.children, parent_count), ref(&c->list, node_count)); ok1(node_count == 3); break; } ok1(list_check(ref(&parent.children, parent_count), NULL)); if (i > 2) break; } ok1(i == 3); ok1(parent_count == 19); ok1(list_empty(ref(&parent.children, parent_count))); ok1(parent_count == 20); /* Test list_top/list_tail on empty list. */ ok1(list_top(ref(&parent.children, parent_count), struct child, list) == NULL); ok1(parent_count == 21); ok1(list_tail(ref(&parent.children, parent_count), struct child, list) == NULL); ok1(parent_count == 22); return exit_status(); } ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run-with-debug.c ================================================ /* Just like run.c, but with all debug checks enabled. */ #define CCAN_LIST_DEBUG 1 #include ================================================ FILE: build-tools/cpp_misc/ccan/list/test/run.c ================================================ #include #include #include #include "helper.h" struct parent { const char *name; struct list_head children; unsigned int num_children; }; struct child { const char *name; struct list_node list; }; static LIST_HEAD(static_list); int main(int argc, char *argv[]) { struct parent parent; struct child c1, c2, c3, *c, *n; unsigned int i; struct list_head list = LIST_HEAD_INIT(list); opaque_t *q, *nq; struct list_head opaque_list = LIST_HEAD_INIT(opaque_list); plan_tests(68); /* Test LIST_HEAD, LIST_HEAD_INIT, list_empty and check_list */ ok1(list_empty(&static_list)); ok1(list_check(&static_list, NULL)); ok1(list_empty(&list)); ok1(list_check(&list, NULL)); parent.num_children = 0; list_head_init(&parent.children); /* Test list_head_init */ ok1(list_empty(&parent.children)); ok1(list_check(&parent.children, NULL)); c2.name = "c2"; list_add(&parent.children, &c2.list); /* Test list_add and !list_empty. */ ok1(!list_empty(&parent.children)); ok1(c2.list.next == &parent.children.n); ok1(c2.list.prev == &parent.children.n); ok1(parent.children.n.next == &c2.list); ok1(parent.children.n.prev == &c2.list); /* Test list_check */ ok1(list_check(&parent.children, NULL)); c1.name = "c1"; list_add(&parent.children, &c1.list); /* Test list_add and !list_empty. */ ok1(!list_empty(&parent.children)); ok1(c2.list.next == &parent.children.n); ok1(c2.list.prev == &c1.list); ok1(parent.children.n.next == &c1.list); ok1(parent.children.n.prev == &c2.list); ok1(c1.list.next == &c2.list); ok1(c1.list.prev == &parent.children.n); /* Test list_check */ ok1(list_check(&parent.children, NULL)); c3.name = "c3"; list_add_tail(&parent.children, &c3.list); /* Test list_add_tail and !list_empty. */ ok1(!list_empty(&parent.children)); ok1(parent.children.n.next == &c1.list); ok1(parent.children.n.prev == &c3.list); ok1(c1.list.next == &c2.list); ok1(c1.list.prev == &parent.children.n); ok1(c2.list.next == &c3.list); ok1(c2.list.prev == &c1.list); ok1(c3.list.next == &parent.children.n); ok1(c3.list.prev == &c2.list); /* Test list_check */ ok1(list_check(&parent.children, NULL)); /* Test list_check_node */ ok1(list_check_node(&c1.list, NULL)); ok1(list_check_node(&c2.list, NULL)); ok1(list_check_node(&c3.list, NULL)); /* Test list_top */ ok1(list_top(&parent.children, struct child, list) == &c1); /* Test list_pop */ ok1(list_pop(&parent.children, struct child, list) == &c1); ok1(list_top(&parent.children, struct child, list) == &c2); list_add(&parent.children, &c1.list); /* Test list_tail */ ok1(list_tail(&parent.children, struct child, list) == &c3); /* Test list_for_each. */ i = 0; list_for_each(&parent.children, c, list) { switch (i++) { case 0: ok1(c == &c1); break; case 1: ok1(c == &c2); break; case 2: ok1(c == &c3); break; } if (i > 2) break; } ok1(i == 3); /* Test list_for_each_rev. */ i = 0; list_for_each_rev(&parent.children, c, list) { switch (i++) { case 0: ok1(c == &c3); break; case 1: ok1(c == &c2); break; case 2: ok1(c == &c1); break; } if (i > 2) break; } ok1(i == 3); /* Test list_for_each_safe, list_del and list_del_from. */ i = 0; list_for_each_safe(&parent.children, c, n, list) { switch (i++) { case 0: ok1(c == &c1); list_del(&c->list); break; case 1: ok1(c == &c2); list_del_from(&parent.children, &c->list); break; case 2: ok1(c == &c3); list_del_from(&parent.children, &c->list); break; } ok1(list_check(&parent.children, NULL)); if (i > 2) break; } ok1(i == 3); ok1(list_empty(&parent.children)); /* Test list_for_each_off. */ list_add_tail(&opaque_list, (struct list_node *)create_opaque_blob()); list_add_tail(&opaque_list, (struct list_node *)create_opaque_blob()); list_add_tail(&opaque_list, (struct list_node *)create_opaque_blob()); i = 0; list_for_each_off(&opaque_list, q, 0) { i++; ok1(if_blobs_know_the_secret(q)); } ok1(i == 3); /* Test list_for_each_safe_off, list_del_off and list_del_from_off. */ i = 0; list_for_each_safe_off(&opaque_list, q, nq, 0) { switch (i++) { case 0: ok1(if_blobs_know_the_secret(q)); list_del_off(q, 0); destroy_opaque_blob(q); break; case 1: ok1(if_blobs_know_the_secret(q)); list_del_from_off(&opaque_list, q, 0); destroy_opaque_blob(q); break; case 2: ok1(c == &c3); list_del_from_off(&opaque_list, q, 0); destroy_opaque_blob(q); break; } ok1(list_check(&opaque_list, NULL)); if (i > 2) break; } ok1(i == 3); ok1(list_empty(&opaque_list)); /* Test list_top/list_tail/list_pop on empty list. */ ok1(list_top(&parent.children, struct child, list) == NULL); ok1(list_tail(&parent.children, struct child, list) == NULL); ok1(list_pop(&parent.children, struct child, list) == NULL); return exit_status(); } ================================================ FILE: build-tools/final/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 ARG base_IMAGE_TAG ARG libuv_IMAGE_TAG ARG aws_sdk_IMAGE_TAG ARG cpp_misc_IMAGE_TAG ARG libmaxminddb_IMAGE_TAG ARG libbpf_IMAGE_TAG #gen:dep-arg FROM $base_IMAGE_TAG as build-main FROM $libuv_IMAGE_TAG as build-libuv FROM $aws_sdk_IMAGE_TAG as build-aws-sdk FROM $cpp_misc_IMAGE_TAG as build-cpp-misc FROM $libmaxminddb_IMAGE_TAG as build-libmaxminddb FROM $libbpf_IMAGE_TAG as build-libbpf #gen:dep-from # Bring everything together FROM build-main AS build-result ARG RUST_TOOLCHAIN=stable # Package definitions # Include fuse-overlayfs so rootless podman can run on overlayfs (e.g., in containers) ARG PKG_DOCKER="podman uidmap fuse-overlayfs" ARG PKG_KERNEL_TOOLS="kmod selinux-utils" ARG PKG_CORE_TOOLS="pass" ARG PKG_DEV_TOOLS="vim-nox lsof silversearcher-ag ssh" ARG PKG_AWS_TOOLS="awscli" ARG BENV_JAVA_VERSION=21 ARG PKG_EXTRA_PACKAGES="openjdk-${BENV_JAVA_VERSION}-jdk-headless google-cloud-sdk google-cloud-sdk-skaffold" ARG PKG_PYTHON_LIBS="python3-ijson python3-docker" RUN sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list' && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.gpg && \ sudo apt-get -y update && \ sudo apt-get install --no-install-recommends -y \ $PKG_DOCKER \ $PKG_KERNEL_TOOLS \ $PKG_CORE_TOOLS \ $PKG_DEV_TOOLS \ $PKG_AWS_TOOLS \ $PKG_EXTRA_PACKAGES \ $PKG_PYTHON_LIBS \ libcap2-bin && \ # START fix podman permissions -- see comment below \ sudo chmod 0755 /usr/bin/newuidmap /usr/bin/newgidmap && \ sudo setcap cap_setuid=ep /usr/bin/newuidmap && \ sudo setcap cap_setgid=ep /usr/bin/newgidmap && \ sudo apt-get autoremove --purge -y libcap2-bin && \ # END fix podman permissions \ sudo apt-get clean && \ sudo rm -rf /var/lib/apt/lists/* # Ensure podman uses fuse-overlayfs in rootless environments where kernel overlay is unavailable RUN mkdir -p $HOME/.config/containers && \ printf "[storage]\n" > $HOME/.config/containers/storage.conf && \ printf "driver = \"overlay\"\n" >> $HOME/.config/containers/storage.conf && \ printf "[storage.options]\n" >> $HOME/.config/containers/storage.conf && \ printf "mount_program = \"/usr/bin/fuse-overlayfs\"\n" >> $HOME/.config/containers/storage.conf # For info on the fix to podman in container, see https://samuel.forestier.app/blog/security/podman-rootless-in-podman-rootless-the-debian-way # Replace setuid bits by proper file capabilities for uidmap binaries. # See . ## java version required by render framework parser RUN case $(uname -m) in \ x86_64) sudo update-alternatives --set java /usr/lib/jvm/java-${BENV_JAVA_VERSION}-openjdk-amd64/bin/java && \ sudo update-alternatives --set javac /usr/lib/jvm/java-${BENV_JAVA_VERSION}-openjdk-amd64/bin/javac \ ;; \ aarch64) sudo update-alternatives --set java /usr/lib/jvm/java-${BENV_JAVA_VERSION}-openjdk-arm64/bin/java && \ sudo update-alternatives --set javac /usr/lib/jvm/java-${BENV_JAVA_VERSION}-openjdk-arm64/bin/javac \ ;; \ esac # gradle RUN sudo wget https://services.gradle.org/distributions/gradle-8.14.3-bin.zip -O /usr/local/lib/gradle.zip # Rust toolchain for building Rust components RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain ${RUST_TOOLCHAIN} && \ . "$HOME/.cargo/env" && \ rustup component add rustfmt clippy && \ cargo install cxxbridge-cmd ENV PATH="${HOME}/.cargo/bin:${PATH}" # Preprocessor for BPF used by cmake RUN pip3 install --break-system-packages pcpp # install GitHub's gh CLI RUN (type -p wget >/dev/null || (sudo apt update && sudo apt install wget -y)) \ && sudo mkdir -p -m 755 /etc/apt/keyrings \ && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ && cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ && sudo mkdir -p -m 755 /etc/apt/sources.list.d \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ && sudo apt update \ && sudo apt install gh -y \ && sudo apt-get clean \ && sudo rm -rf /var/lib/apt/lists/* # add a script to setup build inside of container # to be run after we build the image. RUN ln -s $HOME/src/dev/benv-build.sh build.sh # Licensing information # COPY LICENSE.txt $HOME/ COPY NOTICE.txt $HOME/ # Create a link in $HOME/install to /install so any values in compiled # projects that mention $HOME/install are still valid RUN ln -s /install $HOME/install # copy artifacts from individual builds COPY --from=build-libuv $HOME/install /install COPY --from=build-aws-sdk $HOME/install /install COPY --from=build-cpp-misc $HOME/install /install COPY --from=build-libmaxminddb $HOME/install /install COPY --from=build-libbpf $HOME/install /install #gen:dep-copy ARG BENV_UNMINIMIZE=false RUN (which unminimize && $BENV_UNMINIMIZE && (yes | sudo unminimize)) || true RUN echo 'if [ -e "$HOME/src/dev/build-env/profile" ]; then source "$HOME/src/dev/build-env/profile"; fi' >> $HOME/.profile ================================================ FILE: build-tools/final/LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: build-tools/final/NOTICE.txt ================================================ OpenTelemetry-eBPF Build Tools ============================== https://github.com/open-telemetry/opentelemetry-ebpf-build-tools Copyright The OpenTelemetry Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Third party software ==================== ------------------------------------------------------------------------------- args https://github.com/Taywee/args Copyright (c) 2016-2017 Taylor C. Richberger and Pavel Belikov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- spdlog https://github.com/gabime/spdlog Copyright (c) 2016 Gabi Melman. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- ares https://github.com/c-ares/c-ares /* Copyright 1998 by the Massachusetts Institute of Technology. * Copyright (C) 2007-2013 by Daniel Stenberg * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ ------------------------------------------------------------------------------- bcc https://github.com/iovisor/bcc Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- abseil https://github.com/abseil/abseil-cpp Apache License Version 2.0, January 2004 https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- json https://github.com/nlohmann/json Copyright (c) 2013-2018 Niels Lohmann Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- lookup3 http://www.burtleburtle.net/bob/c/lookup3.c lookup3.c, by Bob Jenkins, May 2006, Public Domain. ------------------------------------------------------------------------------- libuv https://github.com/libuv/libuv libuv is licensed for use as follows: ==== Copyright (c) 2015-present libuv project contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ==== This license applies to parts of libuv originating from the https://github.com/joyent/libuv repository: ==== Copyright Joyent, Inc. and other Node contributors. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ==== This license applies to all parts of libuv that are not externally maintained libraries. The externally maintained libraries used by libuv are: - tree.h (from FreeBSD), copyright Niels Provos. Two clause BSD license. - inet_pton and inet_ntop implementations, contained in src/inet.c, are copyright the Internet Systems Consortium, Inc., and licensed under the ISC license. - stdint-msvc2008.h (from msinttypes), copyright Alexander Chemeris. Three clause BSD license. - pthread-fixes.c, copyright Google Inc. and Sony Mobile Communications AB. Three clause BSD license. - android-ifaddrs.h, android-ifaddrs.c, copyright Berkeley Software Design Inc, Kenneth MacKay and Emergya (Cloud4all, FP7/2007-2013, grant agreement n° 289016). Three clause BSD license. ------------------------------------------------------------------------------- openssl https://github.com/openssl/openssl /* ==================================================================== * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. All advertising materials mentioning features or use of this * software must display the following acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" * * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to * endorse or promote products derived from this software without * prior written permission. For written permission, please contact * openssl-core@openssl.org. * * 5. Products derived from this software may not be called "OpenSSL" * nor may "OpenSSL" appear in their names without prior written * permission of the OpenSSL Project. * * 6. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by the OpenSSL Project * for use in the OpenSSL Toolkit (http://www.openssl.org/)" * * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * ==================================================================== * * This product includes cryptographic software written by Eric Young * (eay@cryptsoft.com). This product includes software written by Tim * Hudson (tjh@cryptsoft.com). * */ Original SSLeay License ----------------------- /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * * This package is an SSL implementation written * by Eric Young (eay@cryptsoft.com). * The implementation was written so as to conform with Netscapes SSL. * * This library is free for commercial and non-commercial use as long as * the following conditions are aheared to. The following conditions * apply to all code found in this distribution, be it the RC4, RSA, * lhash, DES, etc., code; not just the SSL code. The SSL documentation * included with this distribution is covered by the same copyright terms * except that the holder is Tim Hudson (tjh@cryptsoft.com). * * Copyright remains Eric Young's, and as such any Copyright notices in * the code are not to be removed. * If this package is used in a product, Eric Young should be given attribution * as the author of the parts of the library used. * This can be in the form of a textual message at program startup or * in documentation (online or textual) provided with the package. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * "This product includes cryptographic software written by * Eric Young (eay@cryptsoft.com)" * The word 'cryptographic' can be left out if the rouines from the library * being used are not cryptographic related :-). * 4. If you include any Windows specific code (or a derivative thereof) from * the apps directory (application code) you must include an acknowledgement: * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" * * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * The licence and distribution terms for any publically available version or * derivative of this code cannot be changed. i.e. this code cannot simply be * copied and put under another distribution licence * [including the GNU Public Licence.] */ ------------------------------------------------------------------------------- grpc https://github.com/grpc/grpc Copyright 2014 gRPC authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- curl https://github.com/curl/curl COPYRIGHT AND PERMISSION NOTICE Copyright (c) 1996 - 2017, Daniel Stenberg, , and many contributors, see the THANKS file. All rights reserved. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization of the copyright holder. ------------------------------------------------------------------------------- curlpp https://github.com/jpbarrette/curlpp Copyright (c) <2002-2004> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (cURLpp), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- aws-sdk-cpp https://github.com/aws/aws-sdk-cpp Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- google-cloud-cpp https://github.com/googleapis/google-cloud-cpp Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- ccan https://github.com/rustyrussell/ccan The list module is licensed under BSD-MIT: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. The build_assert, check_type, compiler, container_of and hash modules are licensed under CC0 (Public domain): Statement of Purpose The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. 1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; moral rights retained by the original author(s) and/or performer(s); publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; rights protecting the extraction, dissemination, use and reuse of data in a Work; database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. 2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. 3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. 4. Limitations and Disclaimers. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. ------------------------------------------------------------------------------- lz4 https://github.com/lz4/lz4 This repository uses 2 different licenses : - all files in the `lib` directory use a BSD 2-Clause license - all other files use a GPLv2 license, unless explicitly stated otherwise Relevant license is reminded at the top of each source file, and with presence of COPYING or LICENSE file in associated directories. This model is selected to emphasize that files in the `lib` directory are designed to be included into 3rd party applications, while all other files, in `programs`, `tests` or `examples`, receive more limited attention and support for such scenario. ------------------------------------------------------------------------------- yaml-cpp https://github.com/jbeder/yaml-cpp Copyright (c) 2008-2015 Jesse Beder. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- libmaxminddb https://github.com/maxmind/libmaxminddb Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- gradle https://github.com/gradle/gradle Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: build-tools/get_tag.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # This script gets a tag for the given directory (DIR=$1) from the latest git modification. # The image+tag will be ${BENV_PREFIX}-${DIR}:${VERSION_HASH} SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$1" BENV_PREFIX="${DOCKER_TAG_PREFIX}benv" VERSION_HASH=$(git log -1 --format=%h $SCRIPTDIR/${DIR}) IMAGE_TAG="${BENV_PREFIX}-${DIR}:${VERSION_HASH}" echo ${IMAGE_TAG} ================================================ FILE: build-tools/libbpf/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # compile our own libbpf ARG base_IMAGE_TAG FROM $base_IMAGE_TAG AS build ARG CMAKE_BUILD_TYPE ARG RESTRICTED_NPROC WORKDIR $HOME COPY --chown=${UID}:${GID} libbpf libbpf COPY --chown=${UID}:${GID} bpftool bpftool # Build libbpf first WORKDIR $HOME/libbpf/src RUN make -j ${RESTRICTED_NPROC:-1} DESTDIR=$HOME/install install # Build bpftool (it will use the libbpf we just built) WORKDIR $HOME/bpftool/src RUN make -j ${RESTRICTED_NPROC:-1} DESTDIR=$HOME/install install # Runtime stage - copy only necessary artifacts FROM $base_IMAGE_TAG COPY --from=build $HOME/install $HOME/install ================================================ FILE: build-tools/libmaxminddb/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # libmaxminddb ARG base_IMAGE_TAG FROM $base_IMAGE_TAG AS build ARG CONFIGURE_ENABLE_DEBUG ARG NPROC WORKDIR $HOME COPY --chown=${UID}:${GID} libmaxminddb libmaxminddb WORKDIR $HOME/libmaxminddb RUN ./bootstrap RUN ./configure ${CONFIGURE_ENABLE_DEBUG} --prefix=$HOME/install \ --enable-static RUN nice make -j${NPROC:-3} && make install # Runtime stage - copy only necessary artifacts FROM $base_IMAGE_TAG COPY --from=build $HOME/install $HOME/install ================================================ FILE: build-tools/libuv/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # libuv ARG base_IMAGE_TAG FROM $base_IMAGE_TAG AS build ARG NPROC ARG BUILD_CFLAGS WORKDIR $HOME RUN sudo apt remove -y libuv1 COPY --chown=${UID}:${GID} libuv libuv WORKDIR $HOME/libuv RUN ./autogen.sh WORKDIR $HOME/build/libuv RUN $HOME/libuv/configure --prefix=$HOME/install RUN CFLAGS=`echo ${BUILD_CFLAGS} | sed 's/\\\\ / /g'`; nice make -j${NPROC:-3} && make install # Runtime stage - copy only necessary artifacts FROM $base_IMAGE_TAG COPY --from=build $HOME/install $HOME/install ================================================ FILE: build-tools/nproc.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 if which nproc > /dev/null; then echo "$(((`nproc --all` + 1) / 2))" elif which sysctl > /dev/null; then sysctl -n hw.physicalcpu else # some default value - erring on the conservative side to avoid cache trashing echo 2 fi ================================================ FILE: channel/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 add_library( file_channel STATIC file_channel.cc ) target_link_libraries( file_channel file_ops logging ) add_library( double_write_channel STATIC double_write_channel.cc ) target_link_libraries( double_write_channel logging ) add_library( buffered_writer STATIC buffered_writer.cc ) target_link_libraries( buffered_writer logging ) add_library( tcp_channel STATIC tcp_channel.cc ) target_link_libraries( tcp_channel error_handling uv_helpers libuv-interface logging ) add_library( lz4_channel STATIC lz4_channel.cc ) target_link_libraries( lz4_channel logging lz4 ) add_library( upstream_connection STATIC upstream_connection.cc ) target_link_libraries( upstream_connection double_write_channel lz4_channel buffered_writer logging ) add_library( reconnecting_channel STATIC reconnecting_channel.cc ) target_link_libraries( reconnecting_channel upstream_connection spdlog libuv-interface render_ebpf_net_artifacts ) add_library( connection_caretaker STATIC connection_caretaker.cc ) target_link_libraries( connection_caretaker agent_id aws_instance_metadata gcp_instance_metadata intake_config element_queue_writer fastpass_util render_ebpf_net_artifacts logging tcp_channel absl::strings versions ) add_library( test_channel STATIC test_channel.cc ) target_link_libraries( test_channel render_ebpf_net_ingest json llvm logging ) add_unit_test(buffered_writer LIBS buffered_writer element_queue_writer llvm) ================================================ FILE: channel/buffered_writer.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include namespace channel { BufferedWriter::BufferedWriter(Channel &channel, u32 buf_size) : buf_size_(buf_size), write_start_loc_(0), write_finish_loc_(0), channel_(channel) { buf_ = (u8 *)malloc(buf_size * sizeof(u8)); if (buf_ == NULL) throw std::runtime_error("BufferedWriter: failed to allocate memory\n"); } BufferedWriter::~BufferedWriter() { /* if we're in a consistent state, try flushing the buffer */ if (write_start_loc_ == write_finish_loc_) { flush(); } free(buf_); } Expected BufferedWriter::start_write(u32 length) { /* if requesting more than buffer maximum size, bad request */ if (length > buf_size_) { LOG::error( "BufferedWriter::start_write: requesting more than buffer maximum size" " (requested={}, buf_size={})", length, buf_size_); return {unexpected, std::make_error_code(std::errc::no_buffer_space)}; } /* is there enough space in the current buffer? */ if (buf_size_ - write_start_loc_ < length) { if (auto error = flush()) { LOG::error( "BufferedWriter::start_write: failed to flush the channel and there's" " not enough space in the current buffer to return - check for channel" " errors prior to this one (requested={}, buf_size={} offset={})", length, buf_size_, write_start_loc_); return {unexpected, error}; } } assert(buf_size_ - write_start_loc_ >= length); /* mark the end of the write */ write_finish_loc_ = write_start_loc_ + length; /* return a pointer to the start of the write */ return &buf_[write_start_loc_]; } void BufferedWriter::finish_write() { write_start_loc_ = write_finish_loc_; } std::error_code BufferedWriter::flush() { /* we shouldn't be in the middle of a write */ assert(write_start_loc_ == write_finish_loc_); if (write_start_loc_ == 0) { return {}; } // TODO: it should never throw try { if (is_writable()) { if (auto error = channel_.send(buf_, write_start_loc_)) { return error; } } } catch (...) { return std::make_error_code(std::errc::invalid_argument); } write_start_loc_ = write_finish_loc_ = 0; return {}; } void BufferedWriter::reset() { write_start_loc_ = write_finish_loc_ = 0; } u32 BufferedWriter::buf_size() const { return buf_size_; } bool BufferedWriter::is_writable() const { return channel_.is_open(); } } // namespace channel ================================================ FILE: channel/buffered_writer.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include namespace channel { class Channel; /** * A class that enables writing through a buffer so send() calls don't have to * happen for every message. When buffers are exhausted, they are sent into * the given Channel. */ class BufferedWriter : public ::IBufferedWriter { public: /** * c'tor * throws if buff_ can't be malloc-ed * @param channel: the channel on which to send messages * @param buffsize: how many bytes used to batch the sent messages */ BufferedWriter(Channel &channel, u32 buf_size); /** * d'tor */ virtual ~BufferedWriter(); /** * batches as many entries into buffer as possible before calling * send_buffer(). always flushes the buffer at the end. * @see PerfPoller::poll * * @returns: on success, where caller should write the data. nullptr when * the requested length is larger than buf_size_, or if call to * flush() fails. */ Expected start_write(u32 length) override; /** * Finishes the current write */ void finish_write() override; /** * Flushes the buffer to the channel if it's non-empty. * * @return -EINVAL if a send() fails, 0 if successful */ std::error_code flush() override; /** * Abandons the current buffered data */ void reset(); /** * Returns the buffer size */ u32 buf_size() const override; bool is_writable() const override; private: u8 *buf_; const u32 buf_size_; /* where next or active write starts */ u32 write_start_loc_; /* where active write will finish */ u32 write_finish_loc_; Channel &channel_; }; } // namespace channel ================================================ FILE: channel/buffered_writer_test.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include namespace { using ::testing::_; using ::testing::IsFalse; using ::testing::IsTrue; using ::testing::NotNull; using ::testing::Return; using ::testing::Test; static constexpr u32 default_buffer_size = 32; class BufferedWriterTest : public Test { protected: void SetUp() override { writer_.reset(new channel::BufferedWriter(mock_channel_, default_buffer_size)); } ::channel::MockChannel mock_channel_; std::unique_ptr writer_; }; TEST_F(BufferedWriterTest, empty_writer) { EXPECT_CALL(mock_channel_, send(_, _)).Times(0); EXPECT_EQ(writer_->buf_size(), default_buffer_size); } TEST_F(BufferedWriterTest, one_flush) { ON_CALL(mock_channel_, is_open()).WillByDefault(Return(true)); EXPECT_CALL(mock_channel_, send(_, 24)).Times(1); EXPECT_THAT(writer_->start_write(24), IsTrue()); EXPECT_THAT(*writer_->start_write(24), NotNull()); writer_->finish_write(); EXPECT_THAT(writer_->flush(), IsFalse()); } } // namespace ================================================ FILE: channel/callbacks.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include namespace channel { class Callbacks { public: /** * virtual d'tor */ virtual ~Callbacks() {} /** * Callback with ready data. * * The default implementation ignores received data. * * @returns how many bytes were consumed */ virtual u32 received_data(u8 const *data, int length) { return length; } u32 received_data(std::basic_string_view data) { return received_data(data.data(), data.size()); } u32 received_data(std::string_view data) { return received_data(reinterpret_cast(data.data()), data.size()); } /** * An error occurred on the channel, or -ENOLINK on EOF */ virtual void on_error(int error) {} /** * The link finished closing */ virtual void on_closed() {} /** * Connected */ virtual void on_connect() {} }; } /* namespace channel */ ================================================ FILE: channel/channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include namespace channel { /** * An interface that allows reading and writing data to a pipe/socket/etc */ class Channel { public: /** * Virtual d'tor */ virtual ~Channel() {} /** * Sends data onto the channel. */ virtual std::error_code send(const u8 *data, int data_len) = 0; inline std::error_code send(std::basic_string_view data) { return send(data.data(), data.size()); } inline std::error_code send(std::string_view data) { return send(reinterpret_cast(data.data()), data.size()); } /** * Flushes any internal buffers. */ virtual std::error_code flush() { return {}; } /** * Closes the channel. */ virtual void close() {} virtual bool is_open() const = 0; }; } /* namespace channel */ ================================================ FILE: channel/component.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAMESPACE channel #define ENUM_NAME Component #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(none, 0, "") \ X(tls, 1, "") \ X(reconnecting_channel, 2, "") \ X(tcp, 3, "") \ X(upstream, 4, "") #define ENUM_DEFAULT none #include ================================================ FILE: channel/connection_caretaker.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include namespace channel { namespace { void heartbeat_timer_cb(uv_timer_t *timer) { auto *caretaker = (ConnectionCaretaker *)(timer->data); caretaker->send_heartbeat(); } } // namespace ConnectionCaretaker::ConnectionCaretaker( std::string_view hostname, ClientType client_type, config_labels_t const &config_labels, uv_loop_t *loop, ebpf_net::ingest::Writer &writer, std::chrono::milliseconds metadata_timeout, std::chrono::milliseconds heartbeat_interval, std::function flush_cb, std::function set_compression_cb, std::function on_connected_cb) : hostname_(hostname), client_type_(client_type), config_labels_(config_labels), loop_(loop), heartbeat_interval_(heartbeat_interval), flush_cb_(std::move(flush_cb)), set_compression_cb_(std::move(set_compression_cb)), on_connected_cb_(std::move(on_connected_cb)), writer_(writer) { assert(loop != nullptr); assert(heartbeat_interval_.count() > 0); LOG::trace_in(CloudPlatform::aws, "--- resolving AWS metadata ---"); if (auto aws_metadata = AwsMetadata::fetch(metadata_timeout)) { aws_metadata_.emplace(std::move(aws_metadata.value())); aws_metadata_->print_instance_metadata(); aws_metadata_->print_interfaces(); } else { LOG::warn("Unable to fetch AWS metadata: {}", aws_metadata.error().what()); } LOG::trace_in(CloudPlatform::gcp, "--- resolving GCP metadata ---"); if (auto gcp_metadata = GcpInstanceMetadata::fetch(metadata_timeout)) { gcp_metadata_.emplace(std::move(gcp_metadata.value())); gcp_metadata_->print(); } else { LOG::warn("Unable to fetch GCP metadata: {}", gcp_metadata.error().what()); } int res = uv_timer_init(loop_, &heartbeat_timer_); if (res != 0) { throw std::runtime_error("Cannot init heartbeat_timer"); } heartbeat_timer_.data = this; } ConnectionCaretaker::~ConnectionCaretaker() { stop_heartbeat(); uv_close((uv_handle_t *)&heartbeat_timer_, NULL); } void ConnectionCaretaker::send_metadata_header() { set_compression_cb_(false); LOG::info("initiating connection of {} collector version {}", client_type_, versions::release); writer_.version_info(versions::release.major(), versions::release.minor(), versions::release.patch()); flush(); set_compression_cb_(true); writer_.connect(static_cast(client_type_), jb_blob(hostname_)); writer_.report_cpu_cores(std::thread::hardware_concurrency()); flush(); #define make_buf_from_field(struct_name, field, buf_name) \ struct struct_name __##struct_name##__##buf_name; \ char buf_name[sizeof(__##struct_name##__##buf_name.field)] = {}; for (auto const &label : config_labels_) { writer_.set_config_label(jb_blob{label.first}, jb_blob{label.second}); } flush(); if (aws_metadata_) { writer_.cloud_platform(static_cast(CloudPlatform::aws)); if (auto const &account_id = aws_metadata_->account_id()) { LOG::trace_in(CloudPlatform::aws, "reporting aws account id: {}", account_id.value()); writer_.cloud_platform_account_info(jb_blob{account_id.value()}); } else { LOG::trace_in(CloudPlatform::aws, "no aws account id to report"); } auto id = aws_metadata_->id().value(); if (id.starts_with(std::string_view("i-"))) { id.remove_prefix(2); } writer_.set_node_info( jb_blob{aws_metadata_->az().value()}, jb_blob{aws_metadata_->iam_role().value()}, jb_blob{id}, jb_blob{aws_metadata_->type().value()}); flush(); for (auto const &interface : aws_metadata_->network_interfaces()) { for (auto const &ipv4 : interface.private_ipv4s()) { struct sockaddr_in private_sa; int res = inet_pton(AF_INET, ipv4.c_str(), &(private_sa.sin_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__private_ipv4_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.private_ipv4_addr(private_sa.sin_addr.s_addr, (u8 *)vpc_id_buf); } for (auto const &ipv6 : interface.ipv6s()) { struct sockaddr_in6 sa; int res = inet_pton(AF_INET6, ipv6.c_str(), &(sa.sin6_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__ipv6_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.ipv6_addr(sa.sin6_addr.s6_addr, (u8 *)vpc_id_buf); } for (auto const &mapped_ipv4 : interface.mapped_ipv4s()) { struct sockaddr_in public_sa; int res = inet_pton(AF_INET, mapped_ipv4.first.c_str(), &(public_sa.sin_addr)); if (res != 1) { continue; } struct sockaddr_in private_sa; res = inet_pton(AF_INET, mapped_ipv4.second.c_str(), &(private_sa.sin_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__public_to_private_ipv4, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.public_to_private_ipv4(public_sa.sin_addr.s_addr, private_sa.sin_addr.s_addr, (u8 *)vpc_id_buf); } } } else if (gcp_metadata_) { writer_.cloud_platform(static_cast(CloudPlatform::gcp)); // TODO: obtain account_id for GCP and uncomment below // writer_.cloud_platform_account_info(jb_blob{account_id}); writer_.set_node_info( jb_blob{gcp_metadata_->az()}, jb_blob{gcp_metadata_->role()}, jb_blob{gcp_metadata_->hostname()}, jb_blob{gcp_metadata_->type()}); flush(); for (auto const &interface : gcp_metadata_->network_interfaces()) { if (auto const ipv4 = interface.ipv4()) { make_buf_from_field(jb_ingest__private_ipv4_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.private_ipv4_addr(ipv4->as_int(), (u8 *)vpc_id_buf); for (auto const &public_ip : interface.public_ips()) { make_buf_from_field(jb_ingest__public_to_private_ipv4, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.public_to_private_ipv4(public_ip.as_int(), ipv4->as_int(), (u8 *)vpc_id_buf); } } else if (auto const ipv6 = interface.ipv6()) { uint8_t ipv6_buffer[16]; ipv6->write_to(ipv6_buffer); make_buf_from_field(jb_ingest__ipv6_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.ipv6_addr(ipv6_buffer, (u8 *)vpc_id_buf); } } } else { writer_.cloud_platform(static_cast(CloudPlatform::unknown)); writer_.set_node_info(jb_blob{/* az */}, jb_blob{/* role */}, jb_blob{hostname_}, jb_blob{/* instance_type */}); // no network interface data (public/private ip) to send } writer_.metadata_complete(0); flush(); #undef make_buf_from_field on_connected_cb_(); } void ConnectionCaretaker::flush() { flush_cb_(); } void ConnectionCaretaker::start_heartbeat() { int res = uv_timer_start(&heartbeat_timer_, heartbeat_timer_cb, heartbeat_interval_.count(), heartbeat_interval_.count()); if (res != 0) { LOG::error("Cannot start heartbeat_timer: {}", uv_err_name(res)); } } void ConnectionCaretaker::stop_heartbeat() { uv_timer_stop(&heartbeat_timer_); } void ConnectionCaretaker::send_heartbeat() { LOG::debug("sending heartbeat for {} collector", client_type_); if (writer_.is_writable()) { writer_.heartbeat(); flush(); } } void ConnectionCaretaker::set_connected() { LOG::info("collector {} connected to host", hostname_); send_metadata_header(); start_heartbeat(); } void ConnectionCaretaker::set_disconnected() { LOG::info("collector {} disconnected from host", hostname_); stop_heartbeat(); } } // namespace channel ================================================ FILE: channel/connection_caretaker.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include #include #include namespace channel { // ConnectionCaretaker handles common tasks of agent->server connection. // // Current implementation does followings: // 1. Sends back initial metadata, including agent version, and configuration labels. // 2. Sends back heartbeat signal to server periodically. // // This class is NOT thread-safe. class ConnectionCaretaker { public: using config_labels_t = std::map; // Constructor // // |config_data|: Configuration labels, read from a yaml file. // |loop|: Libuv event loop. // |channel|: Underline channel connecting agent and server. // |heartbeat_interval|: How often a heartbeat signal is sent back to server. // |flush_cb|: Callback to flush any downstream buffer. ConnectionCaretaker( std::string_view hostname, ClientType client_type, config_labels_t const &config_labels, uv_loop_t *loop, ebpf_net::ingest::Writer &writer, std::chrono::milliseconds metadata_timeout, std::chrono::milliseconds heartbeat_interval, std::function flush_cb, std::function set_compression_cb, std::function on_connected_cb); ~ConnectionCaretaker(); // Note, this function triggers metadata to be sent back, and starts // heartbeat signal. void set_connected(); // Note, this function stops heartbeat timer implicitly void set_disconnected(); // Sends one heartbeat. It is public so that timer callback can use it. void send_heartbeat(); private: // Sends following information: // agent version and any config labels. // TODO: Send agent type as well. void send_metadata_header(); void start_heartbeat(); void stop_heartbeat(); void flush(); std::string_view const hostname_; ClientType const client_type_; const config_labels_t config_labels_; uv_loop_t *loop_ = nullptr; // not owned std::optional aws_metadata_; std::optional gcp_metadata_; const std::chrono::milliseconds heartbeat_interval_; std::function flush_cb_; std::function set_compression_cb_; std::function on_connected_cb_; uv_timer_t heartbeat_timer_; ebpf_net::ingest::Writer &writer_; }; } // namespace channel ================================================ FILE: channel/double_write_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include namespace channel { DoubleWriteChannel::DoubleWriteChannel(Channel &first, Channel &second) : first_(first), second_(second) {} std::error_code DoubleWriteChannel::send(const u8 *data, int size) { if (auto error = first_.send(data, size)) { return error; } if (second_.is_open()) { if (auto error = second_.send(data, size)) { return error; } } return {}; } void DoubleWriteChannel::close() { first_.close(); second_.close(); } std::error_code DoubleWriteChannel::flush() { if (auto error = first_.flush()) { return error; } if (second_.is_open()) { if (auto error = second_.flush()) { return error; } } return {}; } } // namespace channel ================================================ FILE: channel/double_write_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include namespace channel { class DoubleWriteChannel : public Channel { public: DoubleWriteChannel(Channel &first, Channel &second); std::error_code send(const u8 *data, int size) override; void close() override; std::error_code flush() override; bool is_open() const override { return first_.is_open() && second_.is_open(); } private: Channel &first_; Channel &second_; }; } // namespace channel ================================================ FILE: channel/file_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include namespace channel { FileChannel::FileChannel(FileDescriptor fd) : fd_(std::move(fd)) {} std::error_code FileChannel::send(const u8 *data, int size) { std::string_view const buffer{reinterpret_cast(data), static_cast(size)}; if (auto const error = fd_.write_all(buffer)) { const std::string error_message = error.message(); LOG::error("error while writing {} bytes into file channel: {} ({})", size, error_message, error.value()); return error; } return {}; } void FileChannel::close() { fd_.close(); } std::error_code FileChannel::flush() { auto const error = fd_.flush_data(); if (error) { const std::string error_message = error.message(); LOG::error("error while flushing data for file channel: {} ({})", error_message, error.value()); } return error; } } // namespace channel ================================================ FILE: channel/file_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include namespace channel { class FileChannel : public Channel { public: FileChannel(FileDescriptor fd); std::error_code send(const u8 *data, int size) override; void close() override; std::error_code flush() override; bool valid() const { return fd_.valid(); } explicit operator bool() const { return valid(); } bool operator!() const { return !valid(); } bool is_open() const override { return fd_.valid(); } private: FileDescriptor fd_; }; } // namespace channel ================================================ FILE: channel/ibuffered_writer.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include // Interface for classes that write through a buffer. // class IBufferedWriter { public: virtual ~IBufferedWriter() {} // Starts a new write. // // Returns the memory where caller should write the data, or nullptr in case // of an error. // virtual Expected start_write(u32 length) = 0; // Finishes the current write. // virtual void finish_write() = 0; // Writes the given payload in smaller batches to fit the internal buffer Expected write_as_chunks(std::string_view payload) { for (auto const max = buf_size(); !payload.empty();) { auto const size = (max <= payload.size()) ? max : static_cast(payload.size()); auto const allocated = start_write(size); if (!allocated) { return {unexpected, allocated.error()}; } memcpy(*allocated, payload.data(), size); finish_write(); payload.remove_prefix(size); } return true; } // Flushes the buffer. // virtual std::error_code flush() = 0; // Returns the buffer size. // virtual u32 buf_size() const = 0; virtual bool is_writable() const = 0; }; ================================================ FILE: channel/lz4_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "channel/lz4_channel.h" #include namespace channel { Lz4Channel::Lz4Channel(Channel &channel, u32 max_data_length) : compression_enabled_(false), channel_(channel), buffer_(LZ4F_compressBound(max_data_length, NULL) + LZ4F_HEADER_SIZE_MAX) { if (LZ4F_cctx *lz4_context = nullptr; LZ4F_isError(LZ4F_createCompressionContext(&lz4_context, LZ4F_VERSION))) { throw std::runtime_error("Lz4Channel: Failed to create LZ4 context."); } else { lz4_ctx_.reset(lz4_context); } } void Lz4Channel::set_compression(bool enabled) { compression_enabled_ = enabled; } #define _CHECK_LZ4_ERROR(code) \ if (LZ4F_isError(code)) { \ throw std::runtime_error(std::string("Lz4Channel: compression failed: ") + std::string(LZ4F_getErrorName(code))); \ } std::error_code Lz4Channel::send(const u8 *data, int data_len) { if (!compression_enabled_) { return channel_.send(data, data_len); } // Reference: https://github.com/lz4/lz4/blob/dev/lib/lz4frame.h#L248 size_t tail = 0; size_t res = LZ4F_compressBegin(lz4_ctx_.get(), (void *)buffer_.data(), buffer_.size(), NULL); _CHECK_LZ4_ERROR(res); tail += res; res = LZ4F_compressUpdate(lz4_ctx_.get(), (void *)(buffer_.data() + tail), buffer_.size() - tail, (void *)data, data_len, NULL); _CHECK_LZ4_ERROR(res); tail += res; res = LZ4F_compressEnd(lz4_ctx_.get(), (void *)(buffer_.data() + tail), buffer_.size() - tail, NULL); _CHECK_LZ4_ERROR(res); tail += res; return channel_.send(buffer_.data(), tail); } void Lz4Channel::close() { channel_.close(); } std::error_code Lz4Channel::flush() { return channel_.flush(); } } // namespace channel ================================================ FILE: channel/lz4_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include namespace channel { // Lz4Channel serves as an adapter between upstream data source and downstream // channel. // // When the compression is disabled, the Lz4Channel will pass any incoming // data packets to downstream channel directly. // // When the compression is enabled, the Lz4Channel will compress the incoming // data packets before relay then. class Lz4Channel : public Channel { public: // |channel|: the downstream channel which will actually send out the data. // |max_data_length|: max number of bytes of any incoming data packet sent // via send() function. Note that it's caller's // responsibility to honor this constraint. Lz4Channel(Channel &channel, u32 max_data_length); std::error_code send(const u8 *data, int data_len) override; void set_compression(bool enabled); void close() override; std::error_code flush() override; bool is_open() const override { return channel_.is_open(); } private: bool compression_enabled_; Channel &channel_; std::vector buffer_; pod_unique_ptr lz4_ctx_; }; } // namespace channel ================================================ FILE: channel/mock_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #include #include namespace channel { class MockChannel : public Channel { public: MockChannel() = default; ~MockChannel() override = default; MOCK_METHOD2(send, std::error_code(const u8 *, int)); MOCK_CONST_METHOD0(is_open, bool()); }; // class MockChannel } // namespace channel ================================================ FILE: channel/network_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include namespace channel { /** * An interface that allows reading and writing data to a pipe/socket/etc */ class NetworkChannel : public Channel { public: /** * Connects to an endpoint and starts negotiating * @param callbacks: the callbacks to use during this connection */ virtual void connect(Callbacks &callbacks) = 0; /** * Returns the address (in binary format) that this channel is connected to, * if available. `nullptr` otherwise. */ virtual in_addr_t const *connected_address() const = 0; }; } /* namespace channel */ ================================================ FILE: channel/reconnecting_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include namespace channel { namespace { // Callbacks passed to libuv void connection_timer_cb(uv_timer_t *timer) { ReconnectingChannel *channel = (ReconnectingChannel *)(timer->data); LOG::error("ReconnectingChannel: Connection timeout."); channel->on_error(UV_ETIMEDOUT); } } // namespace void start_timer_cb(uv_timer_t *timer) { ReconnectingChannel *channel = (ReconnectingChannel *)(timer->data); channel->to_connecting_state(); } ReconnectingChannel::ReconnectingChannel(config::IntakeConfig intake_config, uv_loop_t &loop, std::size_t buffer_size) : loop_(loop), intake_config_(std::move(intake_config)), network_channel_(intake_config_.make_channel(loop)), upstream_connection_(buffer_size, intake_config_.allow_compression(), *network_channel_), state_(State::INACTIVE) { int res = uv_timer_init(&loop_, &start_timer_); if (res != 0) { LOG::error("ReconnectingChannel: Cannot init start_timer"); } start_timer_.data = this; res = uv_timer_init(&loop_, &connection_timer_); if (res != 0) { LOG::error("ReconnectingChannel: Cannot init connection_timer"); } connection_timer_.data = this; } ReconnectingChannel::~ReconnectingChannel() { close(); uv_close((uv_handle_t *)&connection_timer_, NULL); uv_close((uv_handle_t *)&start_timer_, NULL); } void ReconnectingChannel::register_pipeline_observer(Callbacks *observer) { pipeline_observers_.insert(observer); } void ReconnectingChannel::unregister_pipeline_observer(Callbacks *observer) { pipeline_observers_.erase(observer); } //// Callbacks interface //// u32 ReconnectingChannel::received_data(const u8 *data, int data_len) { // Do nothing for now. return data_len; } void ReconnectingChannel::on_error(int err) { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: on_error(). prev_state: {}", state_string()); LOG::warn("ReconnectingChannel: Connection error: {}", uv_err_name(err)); for (auto *observer : pipeline_observers_) { observer->on_error(err); } to_closing_state(); } void ReconnectingChannel::on_closed() { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: on_closed(). State: {}", state_string()); for (auto *observer : pipeline_observers_) { observer->on_closed(); } to_backoff_state(); } void ReconnectingChannel::on_connect() { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: on_connect(). State: {}", state_string()); LOG::info("ReconnectingChannel: Remote connection established."); num_bytes_sent_ = 0; set_compression(false); stop_all_timers(); assert(state_ == State::CONNECTING); state_ = State::CONNECTED; for (auto *observer : pipeline_observers_) { observer->on_connect(); } } void ReconnectingChannel::set_compression(bool enabled) { upstream_connection_.set_compression(enabled); } //// Channel interface //// std::error_code ReconnectingChannel::send(const u8 *data, int data_len) { if (state_ != State::CONNECTED) { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: Attempt to send when the channel is NOT connected."); return std::make_error_code(std::errc::not_connected); } auto &buffered_writer = upstream_connection_.buffered_writer(); num_bytes_sent_ += data_len; LOG::trace_in( Component::reconnecting_channel, "Sending ReconnectingChannel: {} bytes. {} bytes sent in total", data_len, num_bytes_sent_); auto buffer = buffered_writer.start_write(data_len); if (!buffer) { LOG::error("ReconnectingChannel: buffered writer overflow: {}", buffer.error()); return buffer.error(); } memcpy(*buffer, data, data_len); buffered_writer.finish_write(); return {}; } BufferedWriter &ReconnectingChannel::buffered_writer() { return upstream_connection_.buffered_writer(); } void ReconnectingChannel::close() { state_ = State::INACTIVE; stop_all_timers(); upstream_connection_.close(); } std::error_code ReconnectingChannel::flush() { return upstream_connection_.flush(); } u64 ReconnectingChannel::get_start_wait_time() const { // TODO: better back-off mechanism here. return 1000; } void ReconnectingChannel::to_connecting_state() { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: to_connecting_state(). State: {}", state_string()); state_ = State::CONNECTING; try { upstream_connection_.connect(*this); } catch (std::exception &e) { LOG::warn( "ReconnectingChannel: Connection attempt failed; will backoff " "and retry. Error: {}", e.what()); to_backoff_state(); return; } stop_all_timers(); int res = uv_timer_start(&connection_timer_, connection_timer_cb, connection_timeout_ms_, 0); if (res != 0) { LOG::error("ReconnectingChannel: Cannot start connection_timer {}", uv_err_name(res)); } } void ReconnectingChannel::to_backoff_state() { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: start_timer(). State: {}", state_string()); state_ = State::BACKOFF; stop_all_timers(); int res = uv_timer_start(&start_timer_, start_timer_cb, get_start_wait_time(), 0); if (res != 0) { LOG::error("ReconnectingChannel: Cannot start start_timer {}", uv_err_name(res)); } } void ReconnectingChannel::start_connect() { LOG::trace_in(Component::reconnecting_channel, "ReconnectingChannel: start_connect(). State: {}", state_string()); assert(state_ == State::INACTIVE); to_connecting_state(); } void ReconnectingChannel::to_closing_state() { state_ = State::CLOSING; stop_all_timers(); try { upstream_connection_.close(); } catch (std::exception &e) { LOG::warn("ReconnectingChannel: Cannot close connection: {}", e.what()); } // to_backoff_state(); } const char *ReconnectingChannel::state_string() const { switch (state_) { case State::INACTIVE: return "INACTIVE"; case State::CONNECTING: return "CONNECTING"; case State::CONNECTED: return "CONNECTED"; case State::CLOSING: return "CLOSING"; case State::BACKOFF: return "BACKOFF"; } // Make compiler happy. // TODO: remove this line if we switch to clang++ return "UNKNOWN"; } ReconnectingChannel::State ReconnectingChannel::state() const { return state_; } void ReconnectingChannel::stop_all_timers() { uv_timer_stop(&connection_timer_); uv_timer_stop(&start_timer_); } } // namespace channel ================================================ FILE: channel/reconnecting_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include namespace channel { // ReconnectingChannel manages the connection to a remote server. // // Retries connection when network error occurs. // // Note that this class is NOT thread safe. class ReconnectingChannel : public Channel, public Callbacks { public: ReconnectingChannel(config::IntakeConfig intake_config, uv_loop_t &loop, std::size_t buffer_size); ~ReconnectingChannel() final; // Registers/unregisters a observer. // // They are expected to use during initialization or clean-up phase. void register_pipeline_observer(Callbacks *cb); void unregister_pipeline_observer(Callbacks *cb); // Enables or disables compression. void set_compression(bool enabled); // Channel interface std::error_code send(const u8 *data, int data_len) override; // Callbacks interface. u32 received_data(const u8 *data, int data_len) override; void on_error(int err) override; void on_closed() override; void on_connect() override; // Starts the connection to remote server. // // It can only be called once. void start_connect(); BufferedWriter &buffered_writer(); void close() override; // Flushes and sends out any remaining messages in the send buffer. std::error_code flush() override; enum class State : int { INACTIVE, CONNECTING, CONNECTED, CLOSING, BACKOFF }; const char *state_string() const; State state() const; config::IntakeConfig const &intake_config() const { return intake_config_; } bool is_open() const override { return upstream_connection_.is_open(); } private: friend void start_timer_cb(uv_timer_t *timer); // How long we should wait for the connection to be established, before // it times out and reconnects again. static constexpr u64 connection_timeout_ms_ = 10000; // Starts the connection_timer_ to track if the TCP connection is // established within |connection_timeout_ms_| void to_connecting_state(); // Starts the start_timer_, to let the service sleep for certain period // of time before start a new connection. void to_backoff_state(); // Closes the TCP connection, and timers. void to_closing_state(); // Stops all active timers void stop_all_timers(); // Returns how much time the system should wait, in microsecond, // before it tries to start a new connection. u64 get_start_wait_time() const; // UV loop that this object runs on. uv_loop_t &loop_; // Intake endpoint config config::IntakeConfig const intake_config_; // Handles low-level TLS, TCP connection. std::unique_ptr network_channel_; UpstreamConnection upstream_connection_; // Current state of the pipeline. State state_; // The timer to clock the waiting period before TCP connection restarts. // ([INACTIVE | BACKOFF] -> CONNECTING) uv_timer_t start_timer_; // The timer to track if connection is establish successfully. // (CONNECTING -> CONNECTED) uv_timer_t connection_timer_; // Observers interested in the status of the pipeline. std::set pipeline_observers_; // Number of bytes this channel has sent, or is about to send. u64 num_bytes_sent_ = 0; }; } // namespace channel ================================================ FILE: channel/tcp_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include //for disabling Nagle's #include #include #include #include #define INVALID_FD -1 namespace channel { static constexpr std::string_view CONNECTED_DISCONNECTED[2] = {"disconnected", "connected"}; /** * Callback passed to uv_read_start that allocates memory for the read callback */ void TCPChannel::conn_read_alloc_cb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { uv_tcp_t *tcp_conn = (uv_tcp_t *)handle; TCPChannel *conn = (TCPChannel *)tcp_conn->data; if (conn->allocated_) { buf->base = nullptr; return; } /* assign the free buffer, reserving bytes for overflow */ buf->base = (char *)conn->rx_buffer_ + conn->rx_len_; buf->len = TCPChannel::rx_buffer_size - conn->rx_len_; conn->allocated_ = true; } /** * Read callback */ void TCPChannel::conn_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { uv_tcp_t *tcp_conn = (uv_tcp_t *)stream; TCPChannel *conn = (TCPChannel *)tcp_conn->data; if (nread < 0) { /* oh-oh, read error */ /* free buffer if it is valid */ if (buf->base) { conn->allocated_ = false; } /* need to notify */ conn->connected_ = false; conn->callbacks_->on_error(nread); return; } /* "merge" the read data into the buffer */ conn->rx_len_ += nread; conn->allocated_ = false; /* read all complete messages from buffer */ try { u32 res = conn->callbacks_->received_data((u8 *)conn->rx_buffer_, conn->rx_len_); if (res > 0) { ASSUME(res <= conn->rx_len_); conn->rx_len_ -= res; memmove(conn->rx_buffer_, (u8 *)conn->rx_buffer_ + res, conn->rx_len_); } } catch (const std::exception &e) { LOG::error("TCPChannel: error handling received data: '{}'", e.what()); conn->connected_ = false; conn->callbacks_->on_error(-EPROTO); return; } /* check that we don't exceed the buffer size */ if (conn->rx_len_ == TCPChannel::rx_buffer_size) { conn->connected_ = false; conn->callbacks_->on_error(-EOVERFLOW); return; } } void TCPChannel::conn_close_cb(uv_handle_t *handle) { TCPChannel *conn = (TCPChannel *)handle->data; LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: calling on_closed()", __func__); conn->callbacks_->on_closed(); } void TCPChannel::conn_close_and_reinit_cb(uv_handle_t *handle) { TCPChannel *conn = (TCPChannel *)handle->data; LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: calling on_closed()", __func__); conn->callbacks_->on_closed(); /* re-initialize so we can reuse the handle */ LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: calling reinit()", __func__); conn->reinit(handle->loop); } void TCPChannel::conn_connect_cb(uv_connect_t *req, int status) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); auto tcp = (TCPChannel *)req->handle->data; if (status < 0) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}(): error {}", __func__, uv_error_t{status}); /* error occurred */ tcp->connected_ = false; tcp->callbacks_->on_error(status); return; } tcp->connected_ = true; LOG::trace_in(channel::Component::tcp, "TCPChannel::{}(): calling callback::on_connect()", __func__); tcp->callbacks_->on_connect(); tcp->start_processing(); } void TCPChannel::conn_write_cb(uv_write_t *req, int status) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); auto tcp = (TCPChannel *)req->handle->data; if (status < 0) { /* no need to notify if close() was called, otherwise -- notify */ if (!uv_is_closing((uv_handle_t *)req->handle)) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: connection not closing, calling close on handle()", __func__); tcp->connected_ = false; tcp->callbacks_->on_error(status); } } /* assumes req is the first field in send_buffer_t */ free(req); } TCPChannel::TCPChannel(uv_loop_t &loop) { reinit(&loop); } TCPChannel::TCPChannel(uv_loop_t &loop, std::string addr, std::string port) : addr_(std::move(addr)), port_(std::move(port)) { reinit(&loop); } TCPChannel::~TCPChannel() { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: connection not closing, calling close on handle()", __func__); DEBUG_ASSUME(uv_is_closing((uv_handle_t *)&conn_)); } void TCPChannel::connect(Callbacks &callbacks) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); callbacks_ = callback_wrapper_ ? callback_wrapper_.get() : &callbacks; struct addrinfo hints; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; struct addrinfo *res = nullptr; LOG::debug("TCPChannel::{}: Connecting to intake @ {}:{}", __func__, addr_, port_); int status = getaddrinfo(addr_.c_str(), port_.c_str(), &hints, &res); if (status != 0) { const char *error_text = gai_strerror(status); LOG::critical("getaddrinfo failed: {} - calling abort", error_text ? error_text : "unknown error"); // TODO: gracefully handle getaddrinfo errors std::abort(); } Defer free_addrinfo([&res] { freeaddrinfo(res); }); if (res->ai_addr->sa_family == AF_INET) { struct sockaddr_in *sa = (struct sockaddr_in *)(res->ai_addr); connected_address_available_ = true; connected_address_ = sa->sin_addr.s_addr; } if (auto const error = ::uv_tcp_connect(&connect_req_, &conn_, res->ai_addr, &conn_connect_cb)) { LOG::error("TCPChannel::{}: failed to establish connection to {}:{}: {}", __func__, addr_, port_, uv_error_t{error}); callbacks_->on_error(error); } } void TCPChannel::accept(Callbacks &callbacks, uv_tcp_t *listener) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); callbacks_ = &callbacks; if (auto const error = ::uv_accept(reinterpret_cast(listener), reinterpret_cast(&conn_))) { // this is guaranteed to succeed when called from the connection callback: // http://docs.libuv.org/en/v1.x/stream.html#c.uv_accept LOG::error("TCPChannel::{}: failed to accept incoming connections: {}", __func__, uv_error_t{error}); CHECK_UV(error); // TODO: verify that users of `accept` properly handle `on_error` callbacks_->on_error(error); } else { connected_ = true; start_processing(); } } void TCPChannel::open_fd(Callbacks &callbacks, const uv_os_sock_t fd) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); callbacks_ = &callbacks; if (auto const error = ::uv_tcp_open(&conn_, fd)) { LOG::error("TCPChannel::{}: failed to open existing file descriptor as a TCP handle: {}", __func__, uv_error_t{error}); CHECK_UV(error); // TODO: verify that users of `open_fd` properly handle `on_error` callbacks_->on_error(error); } else { connected_ = true; start_processing(); } } void TCPChannel::close() { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); close_internal(&conn_close_and_reinit_cb); } void TCPChannel::close_permanently() { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); close_internal(&conn_close_cb); } std::error_code TCPChannel::send(const u8 *data, int data_len) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}(len:{})", __func__, data_len); auto send_buffer = allocate_send_buffer(data_len); if (!send_buffer) { // TODO: gracefully handle out-of-memory errors std::abort(); return std::make_error_code(std::errc::not_enough_memory); } memcpy(send_buffer->data, data, data_len); send_buffer->len = data_len; return send(send_buffer); } struct TCPChannel::send_buffer_t *TCPChannel::allocate_send_buffer(u32 size) { u32 mem_size = ((sizeof(struct send_buffer_t) + 7) & ~7) + size; struct send_buffer_t *ret = (struct send_buffer_t *)malloc(mem_size); if (ret == nullptr) { LOG::critical("Failed to allocate send buffer of size {} mem_size {}", size, mem_size); return nullptr; } // clear memory to ensure we don't exfiltrate uninitialized data memset(ret, 0, mem_size); return ret; } std::error_code TCPChannel::send(struct send_buffer_t *send_buffer) { uv_buf_t uv_buf = {.base = (char *)send_buffer->data, .len = send_buffer->len}; if (auto const error = ::uv_write(&send_buffer->req, reinterpret_cast(&conn_), &uv_buf, 1, conn_write_cb)) { LOG::error( "TCPChannel::{}: failed to write {} bytes into {} channel: {}", __func__, send_buffer->len, CONNECTED_DISCONNECTED[connected_], uv_error_t{error}); callbacks_->on_error(error); return {error, libuv_category()}; } return {}; } void TCPChannel::reinit(uv_loop_t *loop) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); /* reinit RX buffers */ rx_len_ = 0; allocated_ = false; /* re-init handle */ CHECK_UV(uv_tcp_init(loop, &conn_)); conn_.data = this; } void TCPChannel::start_processing() { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); if (auto const error = ::uv_tcp_nodelay(&conn_, true)) { LOG::error("TCPChannel::{}: failed to disable Nagle's algorithm: {}", __func__, uv_error_t{error}); // this error is not critical, we may continue } if (auto const error = ::uv_read_start(reinterpret_cast(&conn_), &conn_read_alloc_cb, conn_read_cb)) { LOG::error("TCPChannel::{}: failed to start read loop on channel: {}", __func__, uv_error_t{error}); callbacks_->on_error(error); } } in_addr_t const *channel::TCPChannel::connected_address() const { std::array choice = {nullptr, &connected_address_}; return choice[connected_address_available_]; } void channel::TCPChannel::close_internal(const uv_close_cb close_cb) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}()", __func__); connected_address_available_ = false; connected_ = false; if (!uv_is_closing((uv_handle_t *)&conn_)) { LOG::trace_in(channel::Component::tcp, "TCPChannel::{}: connection not closing, calling close on handle()", __func__); uv_close((uv_handle_t *)&conn_, close_cb); } } } // namespace channel ================================================ FILE: channel/tcp_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include namespace channel { struct buffer_t { char *base; u32 offset; u32 len; }; /** * A TCP channel * * Errors for on_error callback: * -EPROTO: handler threw exception * -EOVERFLOW: overflow occupies entire buffer SERVER_CONN_BUFFER_SIZE and not * handled by handler * libuv errors. */ class TCPChannel : public NetworkChannel { public: static constexpr u32 rx_buffer_size = (64 * 1024); struct send_buffer_t { uv_write_t req; /* must be first */ u32 len; u64 data[0]; }; /** * c'tor -- leaves socket ready for accept() */ TCPChannel(uv_loop_t &loop); /** * c'tor -- leaves socket ready for connect() */ TCPChannel(uv_loop_t &loop, std::string addr, std::string port); /** * d'tor */ virtual ~TCPChannel(); /** * Connects to an endpoint * @param callbacks: the callbacks to use during this connection * @param addr: ip address or hostname * @param port: string holding port number */ void connect(Callbacks &callbacks) override; /** * Accepts a connection * * @param callbacks: the callbacks to use during this connection * @param listener: the listening socket to accept on */ void accept(Callbacks &callbacks, uv_tcp_t *listener); /** * Opens a TCP connection from the file descriptor. */ void open_fd(Callbacks &callbacks, uv_os_sock_t fd); /** * closes the channel. Callbacks::on_close will be called */ void close() override; /** * Closes the channel, and does not try to reinitilize. */ void close_permanently(); /** * @see Channel::send */ std::error_code send(const u8 *data, int data_len) override; /** * Allocates a send buffer capable of holding @size bytes. * * It is the responsibility of the caller to call send() with the buffer. */ struct send_buffer_t *allocate_send_buffer(u32 size); /** * Sends the given TCPChannel::send_buffer_t allocated with * allocate_send_buffer(). */ std::error_code send(struct send_buffer_t *send_buffer); /** * Returns the address (in binary format) that this channel is connected to, * if available. `nullptr` otherwise. */ in_addr_t const *connected_address() const override; bool is_open() const override { return connected_; } private: static void conn_read_alloc_cb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); static void conn_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf); static void conn_close_cb(uv_handle_t *handle); static void conn_close_and_reinit_cb(uv_handle_t *handle); static void conn_connect_cb(uv_connect_t *req, int status); static void conn_write_cb(uv_write_t *req, int status); void close_internal(const uv_close_cb close_cb); /** * Inits the tcp handle (conn_) and buffers */ void reinit(uv_loop_t *loop); /** * Completes socket configuration and starts reading */ void start_processing(); Callbacks *callbacks_ = nullptr; std::string addr_; std::string port_; std::unique_ptr callback_wrapper_; uv_tcp_t conn_; uv_connect_t connect_req_; u64 rx_buffer_[(rx_buffer_size + 7) / 8]; /* number of bytes currently in the rx_buffer */ u32 rx_len_; bool allocated_ = false; bool connected_address_available_ = false; bool connected_ = false; in_addr_t connected_address_ = 0; }; } /* namespace channel */ ================================================ FILE: channel/test_channel.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include namespace channel { TestChannel::TestChannel(std::optional> loop, IntakeEncoder encoder) : loop_(loop), encoder_(encoder) {} std::error_code TestChannel::send(const u8 *data, int size) { try { ++num_sends_; LOG::trace("TestChannel::send() num_sends_ {}", num_sends_); switch (encoder_) { case IntakeEncoder::binary: { auto rpc = reinterpret_cast(data + sizeof(u64)); // + sizeof(u64) to skip past timestamp LOG::trace("TestChannel::send() rpc: rpc_id {} size {}", rpc->rpc_id, rpc->size); binary_messages_.emplace_back(data, data + size); std::stringstream ss; json_converter::WireToJsonConverter converter(ss); if (auto const handled = converter.process(reinterpret_cast(data), size); !handled) { if (handled.error().value() == EAGAIN) { LOG::error("TestChannel::send() converter.process() returned EAGAIN"); } else { LOG::error("TestChannel::send() error while handling message: {}", handled.error()); } ++num_failed_sends_; } else { LOG::trace("TestChannel::send() binary format msg converted to JSON {}", log_waive(ss.str())); std::string str = "[" + ss.str() + "]"; nlohmann::json const objects = nlohmann::json::parse(str); for (auto const &object : objects) { ++message_counts_[object["name"]]; json_messages_.push_back(object); if (sent_msg_cb_) { sent_msg_cb_(object); } } } } break; default: ++num_failed_sends_; LOG::error("unknown IntakeEncoder {}", encoder_); break; } } catch (std::exception &ex) { ++num_failed_sends_; LOG::error("exception caught in TestChannel::send() {}", ex.what()); } return {}; } void TestChannel::close() {} std::error_code TestChannel::flush() { return {}; } u64 TestChannel::get_num_sends() { return num_sends_; }; u64 TestChannel::get_num_failed_sends() { return num_failed_sends_; } std::stringstream &TestChannel::get_ss() { return ss_; } TestChannel::MessageCountsType &TestChannel::get_message_counts() { return message_counts_; } void TestChannel::binary_messages_for_each(std::function const &cb) { for (auto const &msg : binary_messages_) { cb(msg); } } TestChannel::JsonMessagesType &TestChannel::get_json_messages() { return json_messages_; } void TestChannel::json_messages_for_each(std::function const &cb) { for (auto const &msg : json_messages_) { cb(msg); } } void TestChannel::set_sent_msg_cb(std::function const &sent_msg_cb) { sent_msg_cb_ = sent_msg_cb; } void TestChannel::connect(Callbacks &callbacks) { auto fake_connected_cb = [&callbacks]() { LOG::trace("TestChannel::connect() fake_connected_cb() - calling callbacks.on_connect()"); callbacks.on_connect(); }; fake_connected_cb_timer_ = std::make_unique(*loop_, fake_connected_cb); auto delay_sec = 1; if (auto const result = fake_connected_cb_timer_->defer(std::chrono::seconds(delay_sec))) { LOG::trace("successfully scheduled fake_connected_cb() {} second(s) from now", delay_sec); } else { throw std::runtime_error("failed to schedule fake_connected_cb()"); } } in_addr_t const *TestChannel::connected_address() const { return 0; } } // namespace channel ================================================ FILE: channel/test_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include namespace channel { /** * A channel intended for use by unit tests. It implements all Channel methods. If loop is provided to the constructor then * it also implements the NetworkChannel methods. */ class TestChannel : public NetworkChannel { public: TestChannel( std::optional> loop = std::nullopt, IntakeEncoder encoder = IntakeEncoder::binary); std::error_code send(const u8 *data, int size) override; void close() override; std::error_code flush() override; bool is_open() const override { return true; } void connect(Callbacks &callbacks) override; in_addr_t const *connected_address() const override; u64 get_num_sends(); u64 get_num_failed_sends(); std::stringstream &get_ss(); using MessageCountsType = std::map; // map of message name to number sent MessageCountsType &get_message_counts(); using BinaryMessageType = std::vector; using BinaryMessagesType = std::vector; // vector of messages sent in binary format BinaryMessagesType &get_binary_messages(); void binary_messages_for_each(std::function const &cb); using JsonMessageType = nlohmann::json; using JsonMessagesType = std::vector; // vector of messages sent in JSON format JsonMessagesType &get_json_messages(); void json_messages_for_each(std::function const &cb); // Used to specify a function for the TestChannel to call for every render message processed by send(). void set_sent_msg_cb(std::function const &sent_msg_cb_); private: std::optional> loop_; std::unique_ptr fake_connected_cb_timer_; u64 num_sends_ = 0; u64 num_failed_sends_ = 0; IntakeEncoder encoder_; std::stringstream ss_; MessageCountsType message_counts_; BinaryMessagesType binary_messages_; JsonMessagesType json_messages_; std::function sent_msg_cb_; }; } // namespace channel ================================================ FILE: channel/tls_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include namespace channel { TLSHandler::TLSHandler( uv_loop_t &loop, std::string addr, std::string port, std::string agent_key, std::string agent_crt, std::string server_hostname, std::optional proxy) : creds_(std::move(agent_key), std::move(agent_crt)), tls_channel_(loop, std::move(addr), std::move(port), creds_, std::move(server_hostname), std::move(proxy)) {} void TLSHandler::connect(Callbacks &callbacks) { tls_channel_.connect(callbacks); } std::error_code TLSHandler::send(const u8 *data, int data_len) { return tls_channel_.send(data, data_len); } void TLSHandler::close() { tls_channel_.close(); } std::error_code TLSHandler::flush() { return tls_channel_.flush(); } in_addr_t const *TLSHandler::connected_address() const { return tls_channel_.get_tcp_channel().connected_address(); } } // namespace channel ================================================ FILE: channel/tls_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include namespace channel { class TLSHandler : public NetworkChannel { public: /** * c'tor * Throws if: * 1. connection error on tcp_channel_ * 2. call to connect_tls throws * */ TLSHandler( uv_loop_t &loop, std::string addr, std::string port, std::string agent_key = "", std::string agent_crt = "", std::string server_hostname = "", std::optional proxy = {}); /** * Connects to an endpoint and starts negotiating * @param callbacks: the callbacks to use during this connection */ void connect(Callbacks &callbacks) override; std::error_code send(const u8 *data, int data_len) override; /** * disconnects the channel */ void close() override; std::error_code flush() override; in_addr_t const *connected_address() const override; bool is_open() const override { return tls_channel_.is_open(); } private: TLSChannel::Credentials creds_; TlsOverTcpChannel tls_channel_; }; } // namespace channel ================================================ FILE: channel/upstream_connection.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include namespace channel { UpstreamConnection::UpstreamConnection( std::size_t buffer_size, bool allow_compression, NetworkChannel &primary_channel, Channel *secondary_channel) : primary_channel_(primary_channel), lz4_channel_(primary_channel_, buffer_size), allow_compression_(allow_compression), double_write_channel_(lz4_channel_, secondary_channel ? *secondary_channel : lz4_channel_), buffered_writer_(secondary_channel ? static_cast(double_write_channel_) : lz4_channel_, buffer_size) {} void UpstreamConnection::connect(Callbacks &callbacks) { buffered_writer_.reset(); primary_channel_.connect(callbacks); } std::error_code UpstreamConnection::send(const u8 *data, int data_len) { auto buffer = buffered_writer_.start_write(data_len); if (!buffer) { return buffer.error(); } memcpy(*buffer, data, data_len); buffered_writer_.finish_write(); return {}; } std::error_code UpstreamConnection::flush() { return buffered_writer_.flush(); } void UpstreamConnection::close() { buffered_writer_.reset(); primary_channel_.close(); } void UpstreamConnection::set_compression(bool enabled) { buffered_writer_.flush(); LOG::trace_in( Component::upstream, "UpstreamConnection: {} ({}allowed) LZ4 compression", enabled ? "enabling" : "disabling", allow_compression_ ? "" : "not "); lz4_channel_.set_compression(enabled && allow_compression_); } BufferedWriter &UpstreamConnection::buffered_writer() { return buffered_writer_; } in_addr_t const *UpstreamConnection::connected_address() const { return primary_channel_.connected_address(); } } // namespace channel ================================================ FILE: channel/upstream_connection.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include namespace channel { class UpstreamConnection : public NetworkChannel { public: UpstreamConnection( std::size_t buffer_size, bool allow_compression, NetworkChannel &primary_channel, Channel *secondary_channel = nullptr); /** * Connects to an endpoint and starts negotiating * @param callbacks: the callbacks to use during this connection * @param addr: ip address or hostname * @param port: string holding port number */ void connect(Callbacks &callbacks) override; std::error_code send(const u8 *data, int data_len) override; /** * Flushes the internal buffers. */ std::error_code flush() override; /** * disconnects the channel */ void close() override; /** * Enables/disables compression. */ void set_compression(bool enabled); BufferedWriter &buffered_writer(); in_addr_t const *connected_address() const override; bool is_open() const override { return primary_channel_.is_open(); } private: NetworkChannel &primary_channel_; Lz4Channel lz4_channel_; bool allow_compression_; DoubleWriteChannel double_write_channel_; BufferedWriter buffered_writer_; }; } // namespace channel ================================================ FILE: clang-format.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 CLANG_FORMAT="clang-format-19" if ! command -v ${CLANG_FORMAT} then echo "ERROR: requires ${CLANG_FORMAT}" exit 1 fi RC=0 CMD="${CLANG_FORMAT} -Werror -i -style=file" function format_file { if ! ${CMD} $1 then RC=1 fi } # Check that C and C++ source files are properly clang-formatted FILES=$(find ./geoip ./reducer ./test ./collector/kernel ./common ./tools \ -type f \ \( -name "*.c" \ -o -name "*.cc" \ -o -name "*.h" \ -o -name "*.inl" \) \ -print) for FILE in ${FILES} do format_file ${FILE} done exit ${RC} ================================================ FILE: cmake/abseil.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(absl REQUIRED) ================================================ FILE: cmake/aws-sdk.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(AWSSDK REQUIRED) set(AWS_SERVICES ec2 s3 core) AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWSSDK_LIBS) message(STATUS "Found AWS SDK Services: ${AWSSDK_SERVICES}") message(STATUS "Found AWS SDK Libraries: ${AWSSDK_LIBS}") add_library(aws-sdk-cpp INTERFACE) target_link_libraries( aws-sdk-cpp INTERFACE ${AWSSDK_LIBS} ${AWSSDK_LIBS} crypto s2n dl ) target_include_directories( aws-sdk-cpp INTERFACE ${AWSSDK_INCLUDE_DIR} ) ================================================ FILE: cmake/cargo-test.cmake ================================================ include_guard() # Adds a single target to run all Rust tests in the workspace. # Tests run from the source root, with artifacts under ${CMAKE_BINARY_DIR}/target. if(NOT TARGET cargo-test) add_custom_target( cargo-test COMMAND ${CMAKE_COMMAND} -E chdir ${PROJECT_SOURCE_DIR} ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target cargo test VERBATIM ) endif() ================================================ FILE: cmake/cargo_build_rust.cmake ================================================ cmake_minimum_required(VERSION 3.16) if(NOT DEFINED LINK_FILE) message(FATAL_ERROR "LINK_FILE not provided to cargo_build_rust.cmake") endif() if(NOT DEFINED BIN_DIR) message(FATAL_ERROR "BIN_DIR not provided to cargo_build_rust.cmake") endif() if(NOT DEFINED PROJ_DIR) message(FATAL_ERROR "PROJ_DIR not provided to cargo_build_rust.cmake") endif() if(NOT DEFINED RUST_BIN_TARGET_DIR) message(FATAL_ERROR "RUST_BIN_TARGET_DIR not provided to cargo_build_rust.cmake") endif() if(NOT DEFINED RUST_PACKAGE) message(FATAL_ERROR "RUST_PACKAGE not provided to cargo_build_rust.cmake") endif() # Read the CMake-generated link command for the dummy/existing C++ target file(READ "${LINK_FILE}" LINK_CONTENT) get_filename_component(LINK_DIR "${LINK_FILE}" DIRECTORY) # Derive the target binary directory (two levels up from CMakeFiles/.dir) get_filename_component(_cmakefiles_dir "${LINK_DIR}" DIRECTORY) get_filename_component(TARGET_BIN_DIR "${_cmakefiles_dir}" DIRECTORY) # Seed library search paths with known build output dirs and system dirs set(SEARCH_DIRS "${BIN_DIR}/collector/kernel" "${BIN_DIR}/collector" "${TARGET_BIN_DIR}" "${BIN_DIR}/render" "${BIN_DIR}/channel" "${BIN_DIR}/config" "${BIN_DIR}/platform" "${BIN_DIR}/scheduling" "${BIN_DIR}/util" "${BIN_DIR}/otlp" "${BIN_DIR}/geoip" "${BIN_DIR}/reducer" "${BIN_DIR}/reducer/util" "/install/lib" "/install/usr/lib64" "/usr/lib/x86_64-linux-gnu" ) # Extract -L entries from link line string(REGEX MATCHALL "-L([^ \t\n]+)" LFLAGS "${LINK_CONTENT}") foreach(LF IN LISTS LFLAGS) string(REGEX REPLACE "^-L" "" LF_PATH "${LF}") list(APPEND SEARCH_DIRS "${LF_PATH}") endforeach() # Extract directories of static/shared libraries present on the link line # Include both absolute and relative paths; resolve relatives against LINK_DIR. string(REGEX MATCHALL "([^ \t\n]*lib[^ \t\n]+\\.(a|so)(\\.[0-9.]+)?)" ALL_LIB_PATHS "${LINK_CONTENT}") foreach(LP IN LISTS ALL_LIB_PATHS) set(LIB_DIR "${LP}") if(NOT IS_ABSOLUTE "${LIB_DIR}") get_filename_component(LIB_DIR "${LINK_DIR}/${LIB_DIR}" DIRECTORY) else() get_filename_component(LIB_DIR "${LIB_DIR}" DIRECTORY) endif() list(APPEND SEARCH_DIRS "${LIB_DIR}") endforeach() list(REMOVE_DUPLICATES SEARCH_DIRS) # Build library list specs set(LIB_SPECS) # 1) Static libs by path (.a), absolute or relative string(REGEX MATCHALL "([^ \t\n]*lib[^ \t\n]+\\.a)" ANY_A "${LINK_CONTENT}") foreach(LIB IN LISTS ANY_A) get_filename_component(FNAME "${LIB}" NAME) string(REGEX REPLACE "^lib" "" NAME_NO_PREFIX "${FNAME}") string(REGEX REPLACE "\\.a$" "" NAME_NO_EXT "${NAME_NO_PREFIX}") if(NOT NAME_NO_EXT STREQUAL "encoder_ebpf_net_all") list(APPEND LIB_SPECS "static=${NAME_NO_EXT}") endif() endforeach() # 2) Shared libs by absolute path (.so) string(REGEX MATCHALL "(/[^ \t\n]*/lib[^ \t\n]+\\.so(\\.[0-9.]+)?)" ABS_SO "${LINK_CONTENT}") foreach(LIB IN LISTS ABS_SO) get_filename_component(FNAME "${LIB}" NAME) string(REGEX REPLACE "^lib" "" NAME_NO_PREFIX "${FNAME}") string(REGEX REPLACE "\\.so(\\..*)?$" "" NAME_NO_EXT "${NAME_NO_PREFIX}") list(APPEND LIB_SPECS "dylib=${NAME_NO_EXT}") endforeach() # 3) -l flags (match standalone tokens only, not substrings like -static-libgcc or paths) string(REGEX MATCHALL "(^|[ \t\n])-l[A-Za-z0-9_+\-]+" LFLAG_MATCHES "${LINK_CONTENT}") foreach(TOK IN LISTS LFLAG_MATCHES) string(STRIP "${TOK}" TOK_CLEAN) string(REGEX REPLACE "^(-l)" "" LNAME "${TOK_CLEAN}") list(APPEND LIB_SPECS "dylib=${LNAME}") endforeach() # Ensure C++ runtime bits list(APPEND LIB_SPECS "dylib=stdc++" "dylib=gcc_s") # Deduplicate while preserving first occurrence list(REMOVE_DUPLICATES LIB_SPECS) # Linker args for static group ordering set(LINK_ARGS "-Wl,--start-group;-Wl,--end-group") # Compose env var strings string(JOIN ":" OTN_LINK_SEARCH ${SEARCH_DIRS}) string(JOIN ";" OTN_LINK_LIBS ${LIB_SPECS}) set(OTN_LINK_ARGS "${LINK_ARGS}") message(STATUS "OTN_LINK_SEARCH=${OTN_LINK_SEARCH}") message(STATUS "OTN_LINK_LIBS=${OTN_LINK_LIBS}") # Escape semicolons for passing via CMake -E env string(REPLACE ";" "\\;" OTN_LINK_LIBS_ESC "${OTN_LINK_LIBS}") string(REPLACE ";" "\\;" OTN_LINK_ARGS_ESC "${OTN_LINK_ARGS}") execute_process( COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${RUST_BIN_TARGET_DIR} OTN_LINK_SEARCH=${OTN_LINK_SEARCH} OTN_LINK_LIBS=${OTN_LINK_LIBS_ESC} OTN_LINK_ARGS=${OTN_LINK_ARGS_ESC} cargo build --release --package ${RUST_PACKAGE} --manifest-path ${PROJ_DIR}/Cargo.toml WORKING_DIRECTORY ${PROJ_DIR} RESULT_VARIABLE CARGO_RES OUTPUT_VARIABLE CARGO_OUT ERROR_VARIABLE CARGO_ERR ) if(NOT CARGO_RES EQUAL 0) message(STATUS "Cargo stdout:\n${CARGO_OUT}") message(STATUS "Cargo stderr:\n${CARGO_ERR}") message(FATAL_ERROR "Cargo build failed with exit code ${CARGO_RES}") endif() ================================================ FILE: cmake/ccache.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() # use ccache if available. thanks to http://stackoverflow.com/a/24305849 find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) endif(CCACHE_FOUND) ================================================ FILE: cmake/civetweb.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() set(CIVETWEB_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/ext/civetweb/include) add_library( civetweb OBJECT ext/civetweb/include/CivetServer.h ext/civetweb/include/civetweb.h ext/civetweb/src/CivetServer.cpp ext/civetweb/src/civetweb.c ext/civetweb/src/handle_form.inl ext/civetweb/src/md5.inl ) target_compile_definitions( civetweb PRIVATE USE_IPV6 NDEBUG NO_CGI NO_CACHING NO_SSL NO_FILES ) target_include_directories(civetweb PUBLIC ${CIVETWEB_INCLUDE_DIR}) find_library(LIBDL dl) target_link_libraries(civetweb PRIVATE ${LIBDL}) add_library(civetweb-interface INTERFACE) target_include_directories(civetweb-interface INTERFACE ${CIVETWEB_INCLUDE_DIR}) ================================================ FILE: cmake/clang.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(CLANG REQUIRED CONFIG NAMES Clang) message(STATUS "Found Clang ${CLANG_VERSION}") message(STATUS "Using ClangConfig.cmake in: ${CLANG_CONFIG}") option(ENABLE_LLVM_SHARED "Enable linking LLVM as a shared library" OFF) if(NOT ENABLE_LLVM_SHARED) # # Overwrite libclang's INTERFACE_LINK_LIBRARIES property to link with static LLVM libraries. # set_target_properties(clangBasic PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMCore;LLVMMC;LLVMSupport" ) set_target_properties(clangLex PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;LLVMSupport" ) set_target_properties(clangParse PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangLex;clangSema;LLVMMC;LLVMMCParser;LLVMSupport" ) set_target_properties(clangAST PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangLex;LLVMBinaryFormat;LLVMSupport" ) set_target_properties(clangDynamicASTMatchers PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangASTMatchers;clangBasic;LLVMSupport" ) set_target_properties(clangASTMatchers PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;LLVMSupport" ) set_target_properties(clangCrossTU PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangFrontend;clangIndex;LLVMSupport" ) set_target_properties(clangSema PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangAnalysis;clangBasic;clangEdit;clangLex;LLVMSupport" ) set_target_properties(clangCodeGen PROPERTIES INTERFACE_LINK_LIBRARIES "clangAnalysis;clangAST;clangBasic;clangFrontend;clangLex;clangSerialization;LLVMAnalysis;LLVMBitReader;LLVMBitWriter;LLVMCore;LLVMCoroutines;LLVMCoverage;LLVMipo;LLVMIRReader;LLVMAggressiveInstCombine;LLVMInstCombine;LLVMInstrumentation;LLVMLTO;LLVMLinker;LLVMMC;LLVMObjCARCOpts;LLVMObject;LLVMPasses;LLVMProfileData;LLVMScalarOpts;LLVMSupport;LLVMTarget;LLVMTransformUtils" ) set_target_properties(clangAnalysis PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangASTMatchers;clangBasic;clangLex;LLVMSupport" ) set_target_properties(clangEdit PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangLex;LLVMSupport" ) set_target_properties(clangRewrite PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangLex;LLVMSupport" ) set_target_properties(clangARCMigrate PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangAnalysis;clangBasic;clangEdit;clangFrontend;clangLex;clangRewrite;clangSema;clangSerialization;clangStaticAnalyzerCheckers;clangStaticAnalyzerCore;LLVMSupport" ) set_target_properties(clangDriver PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;LLVMBinaryFormat;LLVMOption;LLVMSupport" ) set_target_properties(clangSerialization PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangLex;clangSema;LLVMBitReader;LLVMSupport" ) set_target_properties(clangRewriteFrontend PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangEdit;clangFrontend;clangLex;clangRewrite;clangSerialization;LLVMSupport" ) set_target_properties(clangFrontend PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangDriver;clangEdit;clangLex;clangParse;clangSema;clangSerialization;LLVMBitReader;LLVMOption;LLVMProfileData;LLVMSupport" ) set_target_properties(clangFrontendTool PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangCodeGen;clangDriver;clangFrontend;clangRewriteFrontend;clangARCMigrate;clangStaticAnalyzerFrontend;LLVMOption;LLVMSupport" ) set_target_properties(clangToolingCore PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangLex;clangRewrite;LLVMSupport" ) set_target_properties(clangToolingInclusions PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangLex;clangRewrite;clangToolingCore;LLVMSupport" ) set_target_properties(clangToolingASTDiff PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangAST;clangLex;LLVMSupport" ) set_target_properties(clangTooling PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangASTMatchers;clangBasic;clangDriver;clangFormat;clangFrontend;clangLex;clangRewrite;clangSerialization;clangToolingCore;LLVMOption;LLVMSupport" ) set_target_properties(clangIndex PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangBasic;clangFormat;clangFrontend;clangLex;clangRewrite;clangSerialization;clangToolingCore;LLVMCore;LLVMSupport" ) set_target_properties(clangStaticAnalyzerCore PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangASTMatchers;clangAnalysis;clangBasic;clangCrossTU;clangLex;clangRewrite;LLVMSupport" ) set_target_properties(clangStaticAnalyzerCheckers PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangASTMatchers;clangAnalysis;clangBasic;clangLex;clangStaticAnalyzerCore;LLVMSupport" ) set_target_properties(clangStaticAnalyzerFrontend PROPERTIES INTERFACE_LINK_LIBRARIES "clangAST;clangAnalysis;clangBasic;clangCrossTU;clangFrontend;clangLex;clangStaticAnalyzerCheckers;clangStaticAnalyzerCore;LLVMSupport" ) set_target_properties(clangFormat PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangLex;clangToolingCore;clangToolingInclusions;LLVMSupport" ) set_target_properties(clangHandleCXX PROPERTIES INTERFACE_LINK_LIBRARIES "clangBasic;clangCodeGen;clangFrontend;clangLex;clangSerialization;clangTooling;LLVMBPFCodeGen;LLVMBPFAsmParser;LLVMBPFAsmPrinter;LLVMBPFDesc;LLVMBPFDisassembler;LLVMBPFInfo;LLVMX86CodeGen;LLVMX86AsmParser;LLVMX86AsmPrinter;LLVMX86Desc;LLVMX86Disassembler;LLVMX86Info;LLVMX86Utils;LLVMSupport" ) set_target_properties(clangHandleLLVM PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMAnalysis;LLVMCodeGen;LLVMCore;LLVMExecutionEngine;LLVMipo;LLVMIRReader;LLVMMC;LLVMMCJIT;LLVMObject;LLVMRuntimeDyld;LLVMSelectionDAG;LLVMSupport;LLVMTarget;LLVMTransformUtils;LLVMX86CodeGen;LLVMX86AsmParser;LLVMX86AsmPrinter;LLVMX86Desc;LLVMX86Disassembler;LLVMX86Info;LLVMX86Utils" ) endif(NOT ENABLE_LLVM_SHARED) ================================================ FILE: cmake/cpp-compiler.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() # resolves the command line option for limiting error output # this is extremely useful for debugging compilation errors if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(CXX_ERROR_LIMIT_FLAG "-ferror-limit") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") set(CXX_ERROR_LIMIT_FLAG "-fmax-errors") endif() message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER}") message(STATUS "C++ compiler version: ${CMAKE_CXX_COMPILER_VERSION}") execute_process( COMMAND ${CMAKE_CXX_COMPILER} --version WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CXX_COMPILER_NATIVE_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ) message(STATUS "C++ compiler native version string: ${CXX_COMPILER_NATIVE_VERSION}") # most expressive debugging, and some optimization # also, disabling -Wno-stringop-truncation where necessary given that it warns about # behavior we intend to get out of strncpy set(EBPF_NET_COMMON_COMPILE_FLAGS "-ggdb3 -Wall -Werror -fno-omit-frame-pointer -Wno-stringop-truncation ${CXX_ERROR_LIMIT_FLAG}=1 -pthread") set(EBPF_NET_COMMON_C_FLAGS "${EBPF_NET_COMMON_COMPILE_FLAGS}") set(EBPF_NET_COMMON_CXX_FLAGS "${EBPF_NET_COMMON_COMPILE_FLAGS}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EBPF_NET_COMMON_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EBPF_NET_COMMON_CXX_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${EBPF_NET_COMMON_LINKER_FLAGS} -static-libgcc -static-libstdc++ -pthread") if(OPTIMIZE) set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O2") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O2") endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_EXPORT_COMPILE_COMMANDS "ON") function (harden_executable TARGET) target_compile_options( ${TARGET} PUBLIC -Wl,-z,relro,-z,now -fstack-protector -static-pie -fpie -fPIE ) endfunction() ================================================ FILE: cmake/curl.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(CURL) if(NOT CURL_FOUND) message(FATAL_ERROR "cURL not found! Please install cURL development files.") endif() message(STATUS "Found cURL: ${CURL_LIBRARIES}") ================================================ FILE: cmake/curlpp.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_path(CURLPP_INCLUDE_DIR NAMES curlpp/cURLpp.hpp) find_library(CURLPP_LIBRARY NAMES curlpp) if (NOT CURLPP_INCLUDE_DIR OR NOT CURLPP_LIBRARY) message(FATAL_ERROR "cURLpp not found! Please install cURLpp development files.") endif() message(STATUS "curlpp library: ${CURLPP_LIBRARY}") add_library(curl-cpp INTERFACE) target_include_directories( curl-cpp INTERFACE "${CURLPP_INCLUDE_DIR}" ) target_link_libraries( curl-cpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl ) ================================================ FILE: cmake/debug.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() function(strip_binary TARGET) cmake_parse_arguments(ARG "" "" "DEPENDS" ${ARGN}) add_custom_target( ${TARGET}-stripped DEPENDS ${ARG_DEPENDS} COMMAND ${CMAKE_SOURCE_DIR}/dev/strip-symbols.sh $ ) set_property( TARGET ${TARGET}-stripped PROPERTY "OUTPUT" $-stripped ) endfunction() ================================================ FILE: cmake/defaults.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() # Default build type is debug for fast iterations if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Build type" FORCE) endif() # If building inside the container, help CMake find dependencies in /install if(EXISTS "/install") if(NOT DEFINED CMAKE_INSTALL_PREFIX OR CMAKE_INSTALL_PREFIX STREQUAL "/usr/local") set(CMAKE_INSTALL_PREFIX "/install" CACHE PATH "Install prefix" FORCE) endif() if(NOT CMAKE_PREFIX_PATH) set(CMAKE_PREFIX_PATH "/install" CACHE STRING "Prefix search path" FORCE) elseif(NOT CMAKE_PREFIX_PATH MATCHES "(^|;)/install(;|$)") set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};/install" CACHE STRING "Prefix search path" FORCE) endif() if(EXISTS "/install/openssl" AND (NOT DEFINED OPENSSL_ROOT_DIR OR OPENSSL_ROOT_DIR STREQUAL "")) set(OPENSSL_ROOT_DIR "/install/openssl" CACHE PATH "OpenSSL root" FORCE) endif() # Ensure pkg-config sees libraries installed under /install set(_pkg_paths "/install/lib64/pkgconfig:/install/lib/pkgconfig:/install/usr/lib64/pkgconfig:/install/usr/lib/pkgconfig:/install/share/pkgconfig:/install/usr/share/pkgconfig") if(DEFINED ENV{PKG_CONFIG_PATH} AND NOT "$ENV{PKG_CONFIG_PATH}" STREQUAL "") set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${_pkg_paths}") else() set(ENV{PKG_CONFIG_PATH} "${_pkg_paths}") endif() endif() # Provide explicit package locations commonly installed in this image. # These hints short-circuit expensive probing inside find_package. if(NOT DEFINED LLVM_DIR AND EXISTS "/usr/lib/llvm-19/cmake") set(LLVM_DIR "/usr/lib/llvm-19/cmake" CACHE PATH "LLVM config dir" FORCE) endif() if(NOT DEFINED gRPC_DIR AND EXISTS "/usr/lib/x86_64-linux-gnu/cmake/grpc") set(gRPC_DIR "/usr/lib/x86_64-linux-gnu/cmake/grpc" CACHE PATH "gRPC config dir" FORCE) endif() if(NOT DEFINED absl_DIR AND EXISTS "/usr/lib/x86_64-linux-gnu/cmake/absl") set(absl_DIR "/usr/lib/x86_64-linux-gnu/cmake/absl" CACHE PATH "abseil config dir" FORCE) endif() # Make git tolerant to container bind-mount ownership (mirrors build script behavior) execute_process( COMMAND git config --global --add safe.directory ${CMAKE_SOURCE_DIR} ERROR_QUIET OUTPUT_QUIET ) # Derive version defaults from version.sh if not provided set(_need_version OFF) foreach(v IN ITEMS EBPF_NET_MAJOR_VERSION EBPF_NET_MINOR_VERSION EBPF_NET_PATCH_VERSION EBPF_NET_COLLECTOR_BUILD_NUMBER) if(NOT DEFINED ${v} OR ${v} STREQUAL "") set(_need_version ON) endif() endforeach() if(_need_version) find_program(BASH_EXECUTABLE bash) if(BASH_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/version.sh") execute_process( COMMAND ${BASH_EXECUTABLE} -c "source '${CMAKE_SOURCE_DIR}/version.sh' >/dev/null 2>&1; printf '%s %s %s %s' \"$EBPF_NET_MAJOR_VERSION\" \"$EBPF_NET_MINOR_VERSION\" \"$EBPF_NET_PATCH_VERSION\" \"$EBPF_NET_COLLECTOR_BUILD_NUMBER\"" OUTPUT_VARIABLE _ver OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) endif() if(DEFINED _ver AND NOT _ver STREQUAL "") string(REGEX REPLACE " +" ";" _ver_list "${_ver}") list(LENGTH _ver_list _len) if(_len GREATER_EQUAL 4) list(GET _ver_list 0 _maj) list(GET _ver_list 1 _min) list(GET _ver_list 2 _pat) list(GET _ver_list 3 _bld) if(NOT DEFINED EBPF_NET_MAJOR_VERSION OR EBPF_NET_MAJOR_VERSION STREQUAL "") set(EBPF_NET_MAJOR_VERSION "${_maj}" CACHE STRING "Major version" FORCE) endif() if(NOT DEFINED EBPF_NET_MINOR_VERSION OR EBPF_NET_MINOR_VERSION STREQUAL "") set(EBPF_NET_MINOR_VERSION "${_min}" CACHE STRING "Minor version" FORCE) endif() if(NOT DEFINED EBPF_NET_PATCH_VERSION OR EBPF_NET_PATCH_VERSION STREQUAL "") set(EBPF_NET_PATCH_VERSION "${_pat}" CACHE STRING "Patch version" FORCE) endif() if(NOT DEFINED EBPF_NET_COLLECTOR_BUILD_NUMBER OR EBPF_NET_COLLECTOR_BUILD_NUMBER STREQUAL "") set(EBPF_NET_COLLECTOR_BUILD_NUMBER "${_bld}" CACHE STRING "Collector build number" FORCE) endif() endif() endif() endif() # Final fallback to keep project(VERSION ...) valid if(NOT DEFINED EBPF_NET_MAJOR_VERSION OR EBPF_NET_MAJOR_VERSION STREQUAL "") set(EBPF_NET_MAJOR_VERSION "0" CACHE STRING "Major version" FORCE) endif() if(NOT DEFINED EBPF_NET_MINOR_VERSION OR EBPF_NET_MINOR_VERSION STREQUAL "") set(EBPF_NET_MINOR_VERSION "0" CACHE STRING "Minor version" FORCE) endif() if(NOT DEFINED EBPF_NET_PATCH_VERSION OR EBPF_NET_PATCH_VERSION STREQUAL "") set(EBPF_NET_PATCH_VERSION "0" CACHE STRING "Patch version" FORCE) endif() if(NOT DEFINED EBPF_NET_COLLECTOR_BUILD_NUMBER OR EBPF_NET_COLLECTOR_BUILD_NUMBER STREQUAL "") set(EBPF_NET_COLLECTOR_BUILD_NUMBER "0" CACHE STRING "Collector build number" FORCE) endif() ================================================ FILE: cmake/docker-utils.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() option(RUN_DOCKER_COMMANDS "when disabled, prepares docker images to be built but stop short of running `docker` commands" ON) add_custom_target(docker) add_custom_target(docker-registry) ################ # DOCKER IMAGE # ################ function(build_custom_docker_image IMAGE_NAME) cmake_parse_arguments(ARG "" "DOCKERFILE_PATH;DOCKERFILE_NAME;OUT_DIR" "ARGS;IMAGE_TAGS;FILES;BINARIES;DIRECTORIES;DEPENDS;OUTPUT_OF;ARTIFACTS_OF;DOCKER_REGISTRY;DEPENDENCY_OF" ${ARGN}) # Dockerfile's directory defaults to the one containing CMakeLists.txt if (NOT DEFINED ARG_DOCKERFILE_PATH) set(ARG_DOCKERFILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") endif() if (NOT DEFINED ARG_DOCKERFILE_NAME) set(ARG_DOCKERFILE_NAME "Dockerfile") endif() if (NOT DEFINED ARG_IMAGE_TAGS) if (DEFINED ENV{IMAGE_TAGS}) set(ARG_IMAGE_TAGS "$ENV{IMAGE_TAGS}") else() set(ARG_IMAGE_TAGS "latest") endif() endif() if (NOT DEFINED ARG_DOCKER_REGISTRY) if (DEFINED ENV{DOCKER_REGISTRY}) set(ARG_DOCKER_REGISTRY "$ENV{DOCKER_REGISTRY}") else() set(ARG_DOCKER_REGISTRY "localhost:5000") endif() endif() foreach (FILE ${ARG_FILES}) if (FILE MATCHES "^[/~]") list(APPEND FILES_LIST "${FILE}") else() list(APPEND FILES_LIST "${CMAKE_CURRENT_SOURCE_DIR}/${FILE}") endif() endforeach() foreach (OUTPUT_OF ${ARG_OUTPUT_OF}) get_target_property(OUTPUT ${OUTPUT_OF} OUTPUT) list(APPEND FILES_LIST "${OUTPUT}") endforeach() # everything in BINARIES is relative to current binary dir foreach (BINARY ${ARG_BINARIES}) list(APPEND BINARIES_LIST "${CMAKE_CURRENT_BINARY_DIR}/${BINARY}") endforeach() foreach (ARTIFACTS_OF ${ARG_ARTIFACTS_OF}) list(APPEND FILES_LIST $) endforeach() set(DOCKER_ARGS "") foreach (ARG ${ARG_ARGS}) list(APPEND DOCKER_ARGS "--build-arg" "${ARG}") endforeach() set(out_path "${CMAKE_BINARY_DIR}/docker.out/${IMAGE_NAME}") if (NOT DEFINED ARG_OUT_DIR) set(files_path "${out_path}") else() set(files_path "${out_path}/${ARG_OUT_DIR}") endif() ################ # docker build # ################ add_custom_target( "${IMAGE_NAME}-docker" DEPENDS ${ARG_DEPENDS} ${ARG_OUTPUT_OF} ${ARG_ARTIFACTS_OF} ) add_dependencies( docker "${IMAGE_NAME}-docker" ) foreach (DEPENDENT_TARGET ${ARG_DEPENDENCY_OF}) add_dependencies("${DEPENDENT_TARGET}-docker" "${IMAGE_NAME}-docker") endforeach() add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "${out_path}" COMMAND ${CMAKE_COMMAND} -E make_directory "${files_path}" ) add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD WORKING_DIRECTORY "${out_path}" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ARG_DOCKERFILE_PATH}/${ARG_DOCKERFILE_NAME} ${out_path} ) if (DEFINED FILES_LIST) add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD WORKING_DIRECTORY "${out_path}" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FILES_LIST} ${files_path} ) endif() if (DEFINED BINARIES_LIST) add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD WORKING_DIRECTORY "${out_path}" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${BINARIES_LIST} ${files_path} ) endif() foreach (DIRECTORY ${ARG_DIRECTORIES}) get_filename_component(DIR_NAME ${DIRECTORY} NAME) add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD WORKING_DIRECTORY "${out_path}" COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}" "${files_path}/${DIR_NAME}" ) endforeach() if (RUN_DOCKER_COMMANDS) add_custom_command( TARGET "${IMAGE_NAME}-docker" POST_BUILD WORKING_DIRECTORY "${out_path}" # Intentionally build with host networking to avoid relying on # rootless networking helpers (pasta/slirp4netns) inside nested CI # containers. This improves reliability of podman builds in GitHub # Actions and similar environments. COMMAND podman build --network host -t "${IMAGE_NAME}" ${DOCKER_ARGS} -f "${ARG_DOCKERFILE_NAME}" . ) endif() ########################### # push to docker registry # ########################### add_custom_target( "${IMAGE_NAME}-docker-registry" DEPENDS docker-registry-login "${IMAGE_NAME}-docker" ${ARG_DEPENDS} ) add_dependencies( docker-registry "${IMAGE_NAME}-docker-registry" ) foreach (DEPENDENT_TARGET ${ARG_DEPENDENCY_OF}) add_dependencies( "${DEPENDENT_TARGET}-docker-registry" "${IMAGE_NAME}-docker-registry" ) endforeach() if (RUN_DOCKER_COMMANDS) # TODO: merge docker-registry-push.sh and push_docker_image.sh foreach (IMAGE_TAG ${ARG_IMAGE_TAGS}) add_custom_command( TARGET "${IMAGE_NAME}-docker-registry" POST_BUILD COMMAND podman tag "${IMAGE_NAME}" "${IMAGE_NAME}:${IMAGE_TAG}" COMMAND "${CMAKE_SOURCE_DIR}/dev/docker-registry-push.sh" "${IMAGE_NAME}" "${IMAGE_TAG}" --no-login "${ARG_DOCKER_REGISTRY}" ) endforeach() endif() endfunction() ################### # DOCKER REGISRY ## ################### add_custom_target( docker-registry-login COMMAND "${CMAKE_SOURCE_DIR}/dev/docker-registry-login.sh" --no-vault env ) ================================================ FILE: cmake/executable.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() ########################### # static compilation target add_library(static-executable INTERFACE) target_link_libraries( static-executable INTERFACE address_sanitizer-static -static-libstdc++ ) ########################### # shared compilation target add_library(shared-executable INTERFACE) target_link_libraries( shared-executable INTERFACE address_sanitizer-shared ) ================================================ FILE: cmake/geoip.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_path(LIBMAXMINDDB_INCLUDE_DIR NAMES maxminddb.h) find_library(LIBMAXMINDDB_LIBRARIES_C NAMES "libmaxminddb.a") find_package_handle_standard_args(LIBMAXMINDDB DEFAULT_MSG LIBMAXMINDDB_LIBRARIES_C LIBMAXMINDDB_INCLUDE_DIR) if(NOT LIBMAXMINDDB_FOUND) message(FATAL_ERROR "Could not find libmaxminddb. Build container should already have that set up") endif() message(STATUS "libmaxminddb INCLUDE_DIR: ${LIBMAXMINDDB_INCLUDE_DIR}") message(STATUS "libmaxminddb LIBRARY: ${LIBMAXMINDDB_LIBRARIES_C}") add_library(libmaxminddb INTERFACE) target_include_directories(libmaxminddb INTERFACE "${LIBMAXMINDDB_INCLUDE_DIR}") target_link_libraries(libmaxminddb INTERFACE "${LIBMAXMINDDB_LIBRARIES_C}") ================================================ FILE: cmake/libbpf.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # Find libbpf library and create target # Since the .pc file has wrong prefix, let's find libbpf manually # Prefer static library over shared library find_library(LIBBPF_LIBRARY NAMES libbpf.a bpf PATHS ${CMAKE_INSTALL_PREFIX}/usr/lib64 ${CMAKE_INSTALL_PREFIX}/lib64 NO_DEFAULT_PATH REQUIRED ) find_path(LIBBPF_INCLUDE_DIR NAMES bpf/libbpf.h PATHS ${CMAKE_INSTALL_PREFIX}/usr/include ${CMAKE_INSTALL_PREFIX}/include NO_DEFAULT_PATH REQUIRED ) if(LIBBPF_LIBRARY AND LIBBPF_INCLUDE_DIR) message(STATUS "Found libbpf: ${LIBBPF_LIBRARY}") message(STATUS "Found libbpf headers: ${LIBBPF_INCLUDE_DIR}") set(LIBBPF_FOUND TRUE) else() message(FATAL_ERROR "libbpf not found in ${CMAKE_INSTALL_PREFIX}") endif() # Create imported target for libbpf if(NOT TARGET libbpf::libbpf) add_library(libbpf::libbpf UNKNOWN IMPORTED) set_target_properties(libbpf::libbpf PROPERTIES IMPORTED_LOCATION "${LIBBPF_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LIBBPF_INCLUDE_DIR}" ) # Add dependencies (libelf and zlib as mentioned in .pc file) find_package(PkgConfig REQUIRED) pkg_check_modules(LIBELF REQUIRED libelf) pkg_check_modules(ZLIB REQUIRED zlib) set_property(TARGET libbpf::libbpf APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${LIBELF_LIBRARIES}" "${ZLIB_LIBRARIES}" ) endif() ================================================ FILE: cmake/libelf.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() pkg_check_modules(LIBELF REQUIRED libelf) add_library(libelf INTERFACE) target_compile_options(libelf INTERFACE "${LIBELF_CFLAGS}") target_link_libraries(libelf INTERFACE "${LIBELF_LINK_LIBRARIES}") message(STATUS "libelf CFLAGS: ${LIBELF_CFLAGS}") message(STATUS "libelf LIBRARIES: ${LIBELF_LINK_LIBRARIES}") ================================================ FILE: cmake/llvm.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") add_library(llvm INTERFACE) add_library(llvm-shared INTERFACE) add_library(llvm-interface INTERFACE) target_include_directories( llvm-interface INTERFACE ${LLVM_INCLUDE_DIRS} ) llvm_map_components_to_libnames( LLVM_LIBS core mcjit native executionengine scalaropts ) target_compile_definitions(llvm-interface INTERFACE ${LLVM_DEFINITIONS}) target_link_libraries(llvm INTERFACE llvm-interface ${LLVM_LIBS}) target_link_libraries(llvm-shared INTERFACE llvm-interface -L${LLVM_LIBRARY_DIRS} -lLLVM) ================================================ FILE: cmake/lz4.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_path(LZ4_INCLUDE_DIR lz4.h) find_library(LZ4_LIBRARY NAMES "liblz4.a") find_package_handle_standard_args(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) if(NOT LZ4_FOUND) message(FATAL_ERROR "Could not find lz4. Build container should already have that set up") endif() message(STATUS "lz4 INCLUDE_DIR: ${LZ4_INCLUDE_DIR}") message(STATUS "lz4 LIBRARY: ${LZ4_LIBRARY}") add_library(lz4 INTERFACE) target_include_directories(lz4 INTERFACE "${LZ4_INCLUDE_DIR}") target_link_libraries(lz4 INTERFACE "${LZ4_LIBRARY}") ================================================ FILE: cmake/openssl.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(OpenSSL REQUIRED) ================================================ FILE: cmake/protobuf.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_package(Protobuf REQUIRED) # Avoid the very slow pkg-config-based RE2 lookup inside gRPC's Findre2.cmake # by predefining the imported target when possible. This short-circuits the # module and prevents multiple costly pkg-config invocations. if(NOT TARGET re2::re2) find_library(_RE2_LIB NAMES re2) find_path(_RE2_INCLUDE_DIR NAMES re2/re2.h) if(_RE2_LIB AND _RE2_INCLUDE_DIR) add_library(re2::re2 INTERFACE IMPORTED) set_property(TARGET re2::re2 PROPERTY INTERFACE_LINK_LIBRARIES "${_RE2_LIB}") set_property(TARGET re2::re2 PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${_RE2_INCLUDE_DIR}") message(STATUS "Using preconfigured RE2 (no pkg-config): ${_RE2_LIB}") endif() endif() find_package(gRPC CONFIG REQUIRED) find_program(GRPC_CPP_PLUGIN_LOCATION grpc_cpp_plugin REQUIRED) if(GRPC_CPP_PLUGIN_LOCATION) message(STATUS "Found grpc_cpp_plugin: ${GRPC_CPP_PLUGIN_LOCATION}") else() message(FATAL_ERROR "grpc_cpp_plugin not found") endif() set(GO_PROTOBUF_ANNOTATIONS_DIR /usr/local/go/src/github.com/grpc-ecosystem/grpc-gateway) set(GO_PROTOBUF_GOOGLEAPIS_DIR /usr/local/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis) # GO should be given the go-module name associated with the built packages function (build_protobuf NAME) cmake_parse_arguments(ARG "CPP;GRPC;GRPC_GATEWAY" "GO" "DEPENDS;DEPENDENCY_OF" ${ARGN}) set(TARGET_PREPARE "${NAME}-protobuf-prepare") set(TARGET "${NAME}-protobuf") add_custom_target( "${TARGET_PREPARE}" DEPENDS ${ARG_DEPENDS} ) set( PROTOBUF_ARGS -I"${CMAKE_CURRENT_SOURCE_DIR}" ) if (ARG_CPP) list( APPEND PROTOBUF_ARGS -I/usr/local/include --cpp_out="${CMAKE_CURRENT_BINARY_DIR}/generated" ) add_custom_command( TARGET "${TARGET_PREPARE}" POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/generated" ) set( GEN_FILES_CPP "${CMAKE_CURRENT_BINARY_DIR}/generated/${NAME}.pb.h" "${CMAKE_CURRENT_BINARY_DIR}/generated/${NAME}.pb.cc" ) if (ARG_GRPC) list( APPEND PROTOBUF_ARGS --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_LOCATION}" --grpc_out="${CMAKE_CURRENT_BINARY_DIR}/generated" ) list( APPEND GEN_FILES_CPP "${CMAKE_CURRENT_BINARY_DIR}/generated/${NAME}.grpc.pb.h" "${CMAKE_CURRENT_BINARY_DIR}/generated/${NAME}.grpc.pb.cc" ) endif() endif() if (DEFINED ARG_GO) get_target_property(MOD_BUILD_DIR "${ARG_GO}-go-module" MOD_BUILD_DIR) list( APPEND PROTOBUF_ARGS --go_out="${GO_PATH_SRC}" --go-grpc_out="${GO_PATH_SRC}" ) add_dependencies( "${TARGET_PREPARE}" "${ARG_GO}-go-module" ) set( GEN_FILES_GO "${MOD_BUILD_DIR}/${NAME}.pb.go" "${MOD_BUILD_DIR}/${NAME}_grpc.pb.go" ) if (ARG_GRPC) list( APPEND PROTOBUF_ARGS -I"${GO_PROTOBUF_ANNOTATIONS_DIR}" -I"${GO_PROTOBUF_GOOGLEAPIS_DIR}" --grpc-gateway_out="logtostderr=true:${GO_PATH_SRC}" ) list( APPEND GEN_FILES_GO "${MOD_BUILD_DIR}/${NAME}.pb.gw.go" ) endif() endif() list( APPEND PROTOBUF_ARGS "${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.proto" ) set(GEN_FILES_ALL ${GEN_FILES_CPP}) if (DEFINED ARG_GO) list(APPEND GEN_FILES_ALL ${GEN_FILES_GO}) endif() add_custom_command( OUTPUT ${GEN_FILES_ALL} COMMAND protoc ${PROTOBUF_ARGS} DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.proto" ${TARGET_PREPARE} ) add_custom_target( "${TARGET}" DEPENDS ${GEN_FILES_ALL} ) if (ARG_CPP) set(CPP_TARGET "${NAME}-cpp-protobuf") set_source_files_properties( ${GEN_FILES_CPP} PROPERTIES GENERATED TRUE ) add_library( "${CPP_TARGET}" STATIC ${GEN_FILES_CPP} ) target_link_libraries( "${CPP_TARGET}" protobuf::libprotobuf gRPC::grpc++ ) target_include_directories( "${CPP_TARGET}" PUBLIC "${CMAKE_CURRENT_BINARY_DIR}" ) endif() if (DEFINED ARG_GO) set(GO_TARGET "${NAME}-go-protobuf") set_source_files_properties( ${GEN_FILES_GO} PROPERTIES GENERATED TRUE ) add_custom_target( "${GO_TARGET}" DEPENDS ${GEN_FILES_GO} ) endif() foreach(DEPENDENCY ${ARG_DEPENDENCY_OF}) add_dependencies(${DEPENDENCY} ${TARGET_PREPARE}) endforeach() endfunction() ================================================ FILE: cmake/render.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() # Aggregate target to copy all generated Rust crates (apps + aggregator) if(NOT TARGET render_source_dir) add_custom_target(render_source_dir) endif() function(render_compile INPUT_DIR) cmake_parse_arguments(ARG "" "OUTPUT_DIR;PACKAGE;COMPILER" "APPS;DEPENDS" ${ARGN}) if(DEFINED ARG_OUTPUT_DIR) set(OUTPUT_DIR ${ARG_OUTPUT_DIR}) else() set(OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated") endif() if(DEFINED ARG_PACKAGE) set(PACKAGE ${ARG_PACKAGE}) else() set(PACKAGE "default") endif() if(DEFINED ARG_COMPILER) set(RENDER_COMPILER ${ARG_COMPILER}) else() get_property( RENDER_COMPILER TARGET render_compiler PROPERTY RENDER_COMPILER_PATH ) endif() set(RENDER_${PACKAGE}_OUTPUTS "") foreach(APP ${ARG_APPS}) set( RENDER_${PACKAGE}_${APP}_DESCRIPTOR "${OUTPUT_DIR}/${PACKAGE}/${APP}/descriptor.cc" ) set( RENDER_${PACKAGE}_${APP}_HASH "${OUTPUT_DIR}/${PACKAGE}/${APP}/hash.c" ) set( RENDER_${PACKAGE}_${APP}_WRITER "${OUTPUT_DIR}/${PACKAGE}/${APP}/writer.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/writer.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/encoder.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/encoder.cc" ) set( RENDER_${PACKAGE}_${APP}_OUTPUTS ${RENDER_${PACKAGE}_${APP}_HASH} ${RENDER_${PACKAGE}_${APP}_DESCRIPTOR} ${RENDER_${PACKAGE}_${APP}_WRITER} "${OUTPUT_DIR}/${PACKAGE}/${APP}/index.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/index.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/containers.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/containers.inl" "${OUTPUT_DIR}/${PACKAGE}/${APP}/containers.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/keys.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/handles.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/handles.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/auto_handles.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/auto_handles.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/weak_refs.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/weak_refs.inl" "${OUTPUT_DIR}/${PACKAGE}/${APP}/weak_refs.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/modifiers.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/modifiers.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/spans.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/spans.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/span_base.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/connection.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/connection.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/protocol.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/protocol.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/transform_builder.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/transform_builder.cc" "${OUTPUT_DIR}/${PACKAGE}/${APP}/parsed_message.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/wire_message.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/meta.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/bpf.h" # Rust per-app crate files live under src/ "${OUTPUT_DIR}/${PACKAGE}/${APP}/src/wire_messages.rs" "${OUTPUT_DIR}/${PACKAGE}/${APP}/src/parsed_message.rs" "${OUTPUT_DIR}/${PACKAGE}/${APP}/src/encoder.rs" "${OUTPUT_DIR}/${PACKAGE}/${APP}/src/hash.rs" "${OUTPUT_DIR}/${PACKAGE}/${APP}/Cargo.toml" "${OUTPUT_DIR}/${PACKAGE}/${APP}/src/lib.rs" # Headers that are also generated alongside sources "${OUTPUT_DIR}/${PACKAGE}/${APP}/descriptor.h" "${OUTPUT_DIR}/${PACKAGE}/${APP}/hash.h" ) list( APPEND RENDER_${PACKAGE}_OUTPUTS ${RENDER_${PACKAGE}_${APP}_OUTPUTS} ) endforeach() # Per-package Rust aggregator crate outputs (generated by the render compiler) list(APPEND RENDER_${PACKAGE}_OUTPUTS "${OUTPUT_DIR}/${PACKAGE}/Cargo.toml") list(APPEND RENDER_${PACKAGE}_OUTPUTS "${OUTPUT_DIR}/${PACKAGE}/src/lib.rs") list( APPEND RENDER_${PACKAGE}_OUTPUTS "${OUTPUT_DIR}/${PACKAGE}/metrics.h" ) set_source_files_properties( ${RENDER_${PACKAGE}_OUTPUTS} PROPERTIES GENERATED TRUE ) file( GLOB RENDER_INPUTS "${INPUT_DIR}/*.render" ) # Generate sources # add_custom_command( OUTPUT ${RENDER_${PACKAGE}_OUTPUTS} WORKING_DIRECTORY ${INPUT_DIR} COMMAND ${CMAKE_COMMAND} -E make_directory "${OUTPUT_DIR}" COMMAND java -jar ${RENDER_COMPILER} -i . -o "${OUTPUT_DIR}" DEPENDS ${RENDER_INPUTS} ${RENDER_COMPILER} ${ARG_DEPENDS} ) add_custom_target( render_compile_${PACKAGE} DEPENDS ${RENDER_${PACKAGE}_OUTPUTS} ) # Generated sources interface library # add_library( render_${PACKAGE}_artifacts INTERFACE ) target_include_directories( render_${PACKAGE}_artifacts INTERFACE "${OUTPUT_DIR}" ) add_dependencies( render_${PACKAGE}_artifacts render_compile_${PACKAGE} ) # Generated sources app libraries # foreach(APP ${ARG_APPS}) add_library( render_${PACKAGE}_${APP} STATIC EXCLUDE_FROM_ALL ${RENDER_${PACKAGE}_${APP}_OUTPUTS} ) target_include_directories( render_${PACKAGE}_${APP} PUBLIC ${OUTPUT_DIR} ) target_link_libraries( render_${PACKAGE}_${APP} PUBLIC fixed_hash render_${PACKAGE}_artifacts ) add_library( render_${PACKAGE}_${APP}_hash STATIC EXCLUDE_FROM_ALL ${RENDER_${PACKAGE}_${APP}_HASH} ) target_include_directories( render_${PACKAGE}_${APP}_hash PUBLIC ${OUTPUT_DIR} ) target_link_libraries( render_${PACKAGE}_${APP}_hash PUBLIC render_${PACKAGE}_artifacts ) add_library( render_${PACKAGE}_${APP}_descriptor STATIC EXCLUDE_FROM_ALL ${RENDER_${PACKAGE}_${APP}_DESCRIPTOR} ) target_include_directories( render_${PACKAGE}_${APP}_descriptor PUBLIC ${OUTPUT_DIR} ) target_link_libraries( render_${PACKAGE}_${APP}_descriptor PUBLIC render_${PACKAGE}_artifacts ) add_library( render_${PACKAGE}_${APP}_writer STATIC EXCLUDE_FROM_ALL ${RENDER_${PACKAGE}_${APP}_WRITER} ) target_include_directories( render_${PACKAGE}_${APP}_writer PUBLIC ${OUTPUT_DIR} ) target_link_libraries( render_${PACKAGE}_${APP}_writer PUBLIC render_${PACKAGE}_artifacts ) endforeach() # Build a single Rust aggregator staticlib from the source workspace (crates/), # not from the generated/ tree. The aggregator crate is expected at # ${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE} and is updated manually via # the render_source_dir targets below. string(REPLACE ":" "_" _pkg_safe "${PACKAGE}") set(RUST_AGG_DIR "${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE}") # Build the aggregator staticlib set(RUST_AGG_TARGET_DIR "${CMAKE_BINARY_DIR}/cargo/${PACKAGE}-agg") set(RUST_AGG_CRATE_NAME "encoder_${_pkg_safe}_all") set(RUST_AGG_STATICLIB "${RUST_AGG_TARGET_DIR}/release/lib${RUST_AGG_CRATE_NAME}.a") add_custom_command( OUTPUT "${RUST_AGG_STATICLIB}" WORKING_DIRECTORY "${RUST_AGG_DIR}" COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${RUST_AGG_TARGET_DIR} RUSTFLAGS=-Cpanic=abort cargo build --release VERBATIM ) # Target `render_rust_${PACKAGE}_build` depends on the library being built add_custom_target(render_rust_${PACKAGE}_build DEPENDS "${RUST_AGG_STATICLIB}") # Users can add `render_rust_${PACKAGE}` to their link libraries to link the Rust staticlib add_library(render_rust_${PACKAGE} STATIC IMPORTED GLOBAL) set_target_properties(render_rust_${PACKAGE} PROPERTIES IMPORTED_LOCATION "${RUST_AGG_STATICLIB}") add_dependencies(render_rust_${PACKAGE} render_rust_${PACKAGE}_build) # Note: do not implicitly link the Rust staticlib into writer libs. # Consumers that need the Rust aggregator (e.g., reducer) should link # target `render_rust_${PACKAGE}` explicitly. # Developer-only: copy generated Rust crates into the source tree for review/commit # This target is safe in CI because it is not part of the default build. # Destination will be: ${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE}/${APP} set(_render_copy_targets) foreach(APP ${ARG_APPS}) set(_src_dir "${OUTPUT_DIR}/${PACKAGE}/${APP}") set(_dest_dir "${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE}/${APP}") add_custom_target( copy_render_crate_${PACKAGE}_${APP} COMMAND ${CMAKE_COMMAND} -E remove_directory "${_dest_dir}" COMMAND ${CMAKE_COMMAND} -E make_directory "${_dest_dir}/src" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/Cargo.toml" "${_dest_dir}/Cargo.toml" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/src/lib.rs" "${_dest_dir}/src/lib.rs" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/src/encoder.rs" "${_dest_dir}/src/encoder.rs" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/src/wire_messages.rs" "${_dest_dir}/src/wire_messages.rs" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/src/parsed_message.rs" "${_dest_dir}/src/parsed_message.rs" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_src_dir}/src/hash.rs" "${_dest_dir}/src/hash.rs" DEPENDS render_compile_${PACKAGE} VERBATIM ) list(APPEND _render_copy_targets copy_render_crate_${PACKAGE}_${APP}) endforeach() # Also copy the generated aggregator crate into the source tree at # ${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE} set(_agg_src_dir "${OUTPUT_DIR}/${PACKAGE}") set(_agg_dest_dir "${PROJECT_SOURCE_DIR}/crates/render/${PACKAGE}") add_custom_target( copy_render_crate_${PACKAGE}_aggregator COMMAND ${CMAKE_COMMAND} -E make_directory "${_agg_dest_dir}/src" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_agg_src_dir}/Cargo.toml" "${_agg_dest_dir}/Cargo.toml" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_agg_src_dir}/src/lib.rs" "${_agg_dest_dir}/src/lib.rs" DEPENDS render_compile_${PACKAGE} VERBATIM ) list(APPEND _render_copy_targets copy_render_crate_${PACKAGE}_aggregator) # Developer entrypoint per package to avoid name collisions with other projects add_custom_target(render_source_dir_${PACKAGE}) add_dependencies(render_source_dir_${PACKAGE} ${_render_copy_targets}) add_dependencies(render_source_dir render_source_dir_${PACKAGE}) endfunction() ================================================ FILE: cmake/rust_cxxbridge.cmake ================================================ ## Copyright The OpenTelemetry Authors ## SPDX-License-Identifier: Apache-2.0 include_guard() # add_rust_cxxbridge( ) # - Runs `cxxbridge` on the given Rust source to generate a header and C++ source. # - Generates: # - Header: ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge/.h # - Source: ${CMAKE_CURRENT_BINARY_DIR}/.cc # - Produces a STATIC library that compiles the generated C++ source # and exposes the header directory as a PUBLIC include dir so consumers can # include it via `#include <.h>`. function(add_rust_cxxbridge TARGET_NAME RUST_SRC) if(NOT TARGET_NAME) message(FATAL_ERROR "add_rust_cxxbridge requires a target name") endif() if(NOT RUST_SRC) message(FATAL_ERROR "add_rust_cxxbridge requires a Rust source path") endif() # Resolve Rust source to an absolute path for dependable dependency tracking. if(NOT IS_ABSOLUTE "${RUST_SRC}") set(RUST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/${RUST_SRC}") endif() # Find the cxxbridge CLI installed via `cargo install cxxbridge-cmd`. find_program(CXXBRIDGE_EXECUTABLE NAMES cxxbridge REQUIRED) # Output paths set(CXXBRIDGE_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/cxxbridge") set(CXXBRIDGE_HEADER "${CXXBRIDGE_OUT_DIR}/${TARGET_NAME}.h") set(CXXBRIDGE_CC "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.cc") # Ensure output directory exists at configure time file(MAKE_DIRECTORY "${CXXBRIDGE_OUT_DIR}") # Generate header add_custom_command( OUTPUT "${CXXBRIDGE_HEADER}" COMMAND "${CMAKE_COMMAND}" -E make_directory "${CXXBRIDGE_OUT_DIR}" COMMAND sh -lc "\"${CXXBRIDGE_EXECUTABLE}\" \"${RUST_SRC}\" --header > \"${CXXBRIDGE_HEADER}\"" DEPENDS "${RUST_SRC}" COMMENT "Generating cxxbridge header ${CXXBRIDGE_HEADER} from ${RUST_SRC}" VERBATIM ) # Generate C++ source (depends on header so include dir exists and header is fresh) add_custom_command( OUTPUT "${CXXBRIDGE_CC}" COMMAND sh -lc "\"${CXXBRIDGE_EXECUTABLE}\" \"${RUST_SRC}\" > \"${CXXBRIDGE_CC}\"" DEPENDS "${RUST_SRC}" "${CXXBRIDGE_HEADER}" COMMENT "Generating cxxbridge source ${CXXBRIDGE_CC} from ${RUST_SRC}" VERBATIM ) # Build a static library from the generated C++ and expose the header dir add_library(${TARGET_NAME} STATIC "${CXXBRIDGE_CC}") target_include_directories(${TARGET_NAME} PUBLIC "${CXXBRIDGE_OUT_DIR}") endfunction() ================================================ FILE: cmake/rust_main.cmake ================================================ include_guard() function(add_rust_main) cmake_parse_arguments(ARG "" "TARGET;STRIPPED_TARGET;PACKAGE;BIN_NAME;PROJ_DIR;RUST_BIN_TARGET_DIR;DUMMY_TARGET" "LINK_LIBS" ${ARGN}) if(NOT DEFINED ARG_TARGET) message(FATAL_ERROR "add_rust_main: TARGET is required") endif() if(NOT DEFINED ARG_STRIPPED_TARGET) message(FATAL_ERROR "add_rust_main: STRIPPED_TARGET is required") endif() if(NOT DEFINED ARG_PACKAGE) message(FATAL_ERROR "add_rust_main: PACKAGE is required (Cargo package)") endif() if(NOT DEFINED ARG_BIN_NAME) message(FATAL_ERROR "add_rust_main: BIN_NAME is required (expected executable name)") endif() if(NOT ARG_LINK_LIBS) message(FATAL_ERROR "add_rust_main: LINK_LIBS is required (C++ link libraries)") endif() if(NOT DEFINED ARG_PROJ_DIR) set(ARG_PROJ_DIR "${PROJECT_SOURCE_DIR}") endif() if(NOT DEFINED ARG_RUST_BIN_TARGET_DIR) set(ARG_RUST_BIN_TARGET_DIR "${CMAKE_BINARY_DIR}/target") endif() if(NOT DEFINED ARG_DUMMY_TARGET) set(ARG_DUMMY_TARGET "${ARG_TARGET}-dummy") endif() # 1) Create a tiny dummy main to force generation of a link.txt for the link line set(_dummy_main "${CMAKE_CURRENT_BINARY_DIR}/${ARG_DUMMY_TARGET}_main.cc") file(WRITE "${_dummy_main}" "int main(int, char**) { return 0; }\n") add_executable(${ARG_DUMMY_TARGET} "${_dummy_main}") target_link_libraries( ${ARG_DUMMY_TARGET} PUBLIC ${ARG_LINK_LIBS} static-executable ) # Path to dummy's link.txt produced by the generator set(_link_file "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${ARG_DUMMY_TARGET}.dir/link.txt") # 2) Build the Rust binary via cargo using link hints from the dummy set(_rust_bin_path "${ARG_RUST_BIN_TARGET_DIR}/release/${ARG_BIN_NAME}") add_custom_command( OUTPUT "${_rust_bin_path}" COMMAND ${CMAKE_COMMAND} -DLINK_FILE=${_link_file} -DBIN_DIR=${CMAKE_BINARY_DIR} -DPROJ_DIR=${ARG_PROJ_DIR} -DRUST_BIN_TARGET_DIR=${ARG_RUST_BIN_TARGET_DIR} -DRUST_PACKAGE=${ARG_PACKAGE} -P ${CMAKE_SOURCE_DIR}/cmake/cargo_build_rust.cmake WORKING_DIRECTORY ${ARG_PROJ_DIR} DEPENDS ${ARG_LINK_LIBS} VERBATIM ) add_custom_target( ${ARG_TARGET} DEPENDS "${_rust_bin_path}" ) # 3) Stripped variant add_custom_command( OUTPUT ${_rust_bin_path}-stripped COMMAND ${CMAKE_SOURCE_DIR}/dev/strip-symbols.sh ${_rust_bin_path} DEPENDS ${_rust_bin_path} VERBATIM ) add_custom_target( ${ARG_STRIPPED_TARGET} DEPENDS "${_rust_bin_path}-stripped" ) set_property( TARGET ${ARG_STRIPPED_TARGET} PROPERTY "OUTPUT" "${_rust_bin_path}-stripped" ) endfunction() ================================================ FILE: cmake/sanitizer.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() ###### # ASAN ###### option(USE_ADDRESS_SANITIZER "Use Address Sanitizer in compilation" OFF) add_library(address_sanitizer-static INTERFACE) add_library(address_sanitizer-shared INTERFACE) message(STATUS "Address Sanitizer is ${USE_ADDRESS_SANITIZER}") if (USE_ADDRESS_SANITIZER) target_compile_options( address_sanitizer-shared INTERFACE -fsanitize=address -U_FORTIFY_SOURCE -fno-stack-protector -fno-omit-frame-pointer -U__SANITIZE_ADDRESS__ ) target_compile_definitions( address_sanitizer-shared INTERFACE NDEBUG_SANITIZER ) target_compile_options( address_sanitizer-static INTERFACE -fsanitize=address -U_FORTIFY_SOURCE -fno-stack-protector -fno-omit-frame-pointer -U__SANITIZE_ADDRESS__ ) target_compile_definitions( address_sanitizer-static INTERFACE NDEBUG_SANITIZER ) target_link_libraries( address_sanitizer-shared INTERFACE -static-libasan -static-libstdc++ -fsanitize=address ) target_link_libraries( address_sanitizer-static INTERFACE -static-libasan -static-libstdc++ -fsanitize=address ) endif() ####### # UBSAN ####### option(USE_UNDEFINED_BEHAVIOR_SANITIZER "Use Undefined Behavior Sanitizer in compilation" OFF) add_library(undefined_behavior_sanitizer-static INTERFACE) add_library(undefined_behavior_sanitizer-shared INTERFACE) message(STATUS "Undefined Behavior Sanitizer is ${USE_UNDEFINED_BEHAVIOR_SANITIZER}") if (USE_UNDEFINED_BEHAVIOR_SANITIZER) target_compile_options( undefined_behavior_sanitizer-shared INTERFACE -fsanitize=undefined -U_FORTIFY_SOURCE -fno-stack-protector -fno-omit-frame-pointer ) target_compile_options( undefined_behavior_sanitizer-static INTERFACE -fsanitize=undefined -U_FORTIFY_SOURCE -fno-stack-protector -fno-omit-frame-pointer ) target_link_libraries( undefined_behavior_sanitizer-shared INTERFACE -static-libubsan -static-libstdc++ -fsanitize=undefined ) target_link_libraries( undefined_behavior_sanitizer-static INTERFACE -static-libubsan -static-libstdc++ -fsanitize=undefined ) endif() ================================================ FILE: cmake/shell.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() # Runs linters on the given set of shell scripts. # # The scripts are bundled as a single cmake target. # # Positional Arguments: # 1. TARGET: # The name of the target to create for the bundle. # # Named Arguments: # SOURCES: # The list of scripts to add to the bundle, relative to the current source directory. # # DEPENDS: # The list of targets this bundle explicitly depends on. # # Output: # The list of files linted (`SOURCES`) is exposed through the property `OUTPUT` and can be # accessed with the expression `$`, where `TARGET` is the # target name given to the bundle. E.g.: `SET(script_list $)`. # # Usage: # lint_shell_script_bundle( # my_target # SOURCES # my_script_1.sh # my_script_2.sh # DEPENDS # some_dependency_1 # some_dependency_2 # ) function(lint_shell_script_bundle TARGET) cmake_parse_arguments(ARG "" "" "SOURCES;DEPENDS" ${ARGN}) add_custom_target( ${TARGET} ALL DEPENDS ${ARG_DEPENDS} ) foreach(SOURCE ${ARG_SOURCES}) add_custom_command( TARGET ${TARGET} POST_BUILD WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMAND shellcheck -x "${SOURCE}" ) list(APPEND OUTPUT_LIST "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}") endforeach() set_property(TARGET ${TARGET} PROPERTY "OUTPUT" ${OUTPUT_LIST}) endfunction() ================================================ FILE: cmake/spdlog.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() add_library(spdlog INTERFACE) target_include_directories(spdlog INTERFACE "${CMAKE_SOURCE_DIR}/ext/spdlog/include") target_link_libraries(spdlog INTERFACE) ================================================ FILE: cmake/test.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() enable_testing() link_directories("${CMAKE_INSTALL_PREFIX}/lib") # This is for tests that aren't run as part of `make test`. These are useful # for manual or component tests that you don't want to necessarily run as # part of the unit test suite. function(add_standalone_gtest testName) set(options "") set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(add_standalone_gtest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${testName} ${add_standalone_gtest_SRCS}) target_link_libraries(${testName} gmock_main gtest gmock "-pthread" ${add_standalone_gtest_DEPS}) endfunction (add_standalone_gtest) # This function is use for gtest/gmock-dependent libraries for use in other # unit tests (i.e. does not link a `main` symbol). function(add_gtest_lib testName) set(options "") set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(add_gtest_lib "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_library(${testName} ${add_gtest_lib_SRCS}) target_link_libraries(${testName} gtest gmock "-pthread" ${add_gtest_lib_DEPS}) endfunction (add_gtest_lib) # Adds a unit test named `${NAME}_test`. # # The file `${NAME}_test.cc` is implicitly added as a source. # Additional source files can be declared with the `SRCS` parameter. # # Test executable is implicitly linked with `gtest` and `gmock`. # Additional libraries can be linked with the `LIBS` parameter. # # Additional dependencies can be declares with the `DEPS` parameter. function(add_cpp_test NAME) set(TEST_NAME "${NAME}_test") cmake_parse_arguments(ARG "" "" "SRCS;LIBS;DEPS" ${ARGN}) add_executable( ${TEST_NAME} "${TEST_NAME}.cc" ${ARG_SRCS} ) add_test(${TEST_NAME} ${TEST_NAME}) target_link_libraries( ${TEST_NAME} ${ARG_LIBS} gmock_main gtest gmock shared-executable ) endfunction(add_cpp_test) # Adds a unit test named `${NAME}_test`, which is part of the `unit_tests` target. # # The file `${NAME}_test.cc` is implicitly added as a source. # Additional source files can be declared with the `SRCS` parameter. # # Test executable is implicitly linked with `gtest` and `gmock`. # Additional libraries can be linked with the `LIBS` parameter. # # Additional dependencies can be declares with the `DEPS` parameter. add_custom_target(unit_tests) function(add_unit_test NAME) cmake_parse_arguments(ARG "" "" "SRCS;LIBS;DEPS" ${ARGN}) add_cpp_test(${NAME} SRCS ${ARG_SRCS} LIBS ${ARG_LIBS} DEPS ${ARG_DEPS}) add_dependencies(unit_tests "${NAME}_test") endfunction(add_unit_test) # For tests that use eBPF components and need to run in a container that supports eBPF. Does everything in add_unit_test() # and labels test with the eBPFContainer label. These tests will not be run by default, but only if the RUN_EBPF_TESTS # environment variable is set to TRUE. function(add_ebpf_unit_test NAME) set(TEST_NAME "${NAME}_test") cmake_parse_arguments(ARG "" "" "SRCS;LIBS;DEPS" ${ARGN}) add_executable( ${TEST_NAME} "${TEST_NAME}.cc" ${ARG_SRCS} ) # https://stackoverflow.com/questions/61572034/ctest-disable-a-set-of-labeled-tests-by-default-but-run-them-when-explicitly add_test(NAME ${TEST_NAME} COMMAND sh -c "if [ \"\${RUN_EBPF_TESTS:-}\" ]; then $; else exit 127; fi") set_tests_properties(${TEST_NAME} PROPERTIES SKIP_RETURN_CODE 127) target_link_libraries( ${TEST_NAME} ${ARG_LIBS} gmock_main gtest gmock shared-executable ) add_dependencies(unit_tests "${NAME}_test") set_property(TEST ${TEST_NAME} PROPERTY LABELS eBPFContainer) endfunction(add_ebpf_unit_test) # Ensure running `test` builds all unit test binaries first. # The `test` target is available when testing is enabled. if (TARGET test) add_dependencies(test unit_tests) endif () ================================================ FILE: cmake/tool.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() add_custom_target(tools) function(add_tool_executable EXEC_NAME) cmake_parse_arguments(P "" "" "SRCS;DEPS" ${ARGN}) add_executable("${EXEC_NAME}" "${P_SRCS}") add_dependencies(tools "${EXEC_NAME}") install( TARGETS "${EXEC_NAME}" RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT tools ) if(DEFINED P_DEPS) target_link_libraries("${EXEC_NAME}" "${P_DEPS}") endif() endfunction(add_tool_executable) ================================================ FILE: cmake/uv.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_path(LIBUV_INCLUDE_DIR uv.h) find_library(LIBUV_LIBS NAMES uv libuv) find_library(LIBUV_STATIC_LIBRARY NAMES libuv.a) find_package_handle_standard_args(LIBUV DEFAULT_MSG LIBUV_LIBS LIBUV_INCLUDE_DIR LIBUV_STATIC_LIBRARY) if((NOT LIBUV_INCLUDE_DIR) OR (NOT LIBUV_LIBS) OR (NOT LIBUV_STATIC_LIBRARY)) message(FATAL_ERROR "Could not find libuv. Build container should already have that set up") endif() message(STATUS "libuv INCLUDE_DIR: ${LIBUV_INCLUDE_DIR}") message(STATUS "libuv LIBS: ${LIBUV_LIBS}") message(STATUS "libuv STATIC_LIBRARY: ${LIBUV_STATIC_LIBRARY}") add_library(libuv-interface INTERFACE) target_include_directories( libuv-interface INTERFACE "${LIBUV_INCLUDE_DIR}" ) add_library(libuv-shared INTERFACE) target_link_libraries( libuv-shared INTERFACE ${LIBUV_LIBS} libuv-interface ) add_library(libuv-static INTERFACE) target_link_libraries( libuv-static INTERFACE ${LIBUV_STATIC_LIBRARY} ) target_include_directories( libuv-static INTERFACE "${LIBUV_INCLUDE_DIR}" libuv-interface ) ================================================ FILE: cmake/xxd.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() function (add_xxd FIL GENERATED_NAME) cmake_parse_arguments(MOD "" "OUTPUT" "DEPENDS" ${ARGN}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE BASENAME ${CMAKE_CURRENT_SOURCE_DIR}) get_filename_component(FIL_N ${FIL} NAME) if (MOD_OUTPUT) set(DST_FILENAME "${CMAKE_BINARY_DIR}/generated/${MOD_OUTPUT}") else () set(DST_FILENAME "${CMAKE_BINARY_DIR}/generated/${FIL_N}.xxd") endif() get_filename_component(FIL_D ${ABS_FIL} DIRECTORY) add_custom_command( OUTPUT "${DST_FILENAME}" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/generated" COMMAND xxd ARGS -i ${FIL_N} > "${DST_FILENAME}.generated" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${DST_FILENAME}.generated" "${DST_FILENAME}" DEPENDS ${FIL} "${MOD_DEPENDS}" WORKING_DIRECTORY ${FIL_D} COMMENT "Generating XXD from ${FIL} -> ${DST_FILENAME}" VERBATIM ) set_source_files_properties("${DST_FILENAME}" PROPERTIES GENERATED TRUE) set(${GENERATED_NAME} "${DST_FILENAME}" PARENT_SCOPE) endfunction() ================================================ FILE: cmake/yamlcpp.cmake ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 include_guard() find_path(YAMLCPP_INCLUDE_DIR yaml-cpp/yaml.h) find_library(YAMLCPP_LIBRARY NAMES "libyaml-cpp.a") find_package_handle_standard_args(YAMLCPP DEFAULT_MSG YAMLCPP_LIBRARY YAMLCPP_INCLUDE_DIR) if(NOT YAMLCPP_FOUND) message(FATAL_ERROR "Could not find yaml-cpp. Build container should already have that set up") endif() message(STATUS "yaml-cpp INCLUDE_DIR: ${YAMLCPP_INCLUDE_DIR}") message(STATUS "yamp-cpp LIBRARY: ${YAMLCPP_LIBRARY}") add_library(yamlcpp INTERFACE) target_include_directories(yamlcpp INTERFACE "${YAMLCPP_INCLUDE_DIR}") target_link_libraries(yamlcpp INTERFACE "${YAMLCPP_LIBRARY}") ================================================ FILE: collector/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 add_custom_target(collectors) add_custom_target(collectors-docker) add_custom_target(collectors-docker-registry) include(rust_main) add_rust_main( TARGET k8s-collector STRIPPED_TARGET k8s-collector-stripped PACKAGE k8s-collector-bin BIN_NAME k8s-collector LINK_LIBS versions ) add_dependencies(collectors k8s-collector k8s-collector-stripped) build_custom_docker_image( k8s-collector DOCKERFILE_PATH ${CMAKE_CURRENT_SOURCE_DIR} DOCKERFILE_NAME Dockerfile.k8s-collector OUT_DIR srv ARGS "EBPF_NET_MAJOR_VERSION=${EBPF_NET_MAJOR_VERSION}" "EBPF_NET_MINOR_VERSION=${EBPF_NET_MINOR_VERSION}" "EBPF_NET_PATCH_VERSION=${EBPF_NET_PATCH_VERSION}" OUTPUT_OF k8s-collector-stripped BINARIES debug-info.conf FILES ../NOTICE.txt ../LICENSE.txt DEPENDENCY_OF collectors ) add_subdirectory(cloud) add_subdirectory(kernel) ================================================ FILE: collector/Dockerfile.k8s-collector ================================================ FROM docker.io/bitnami/minideb:trixie@sha256:0766a3b76750541ae2c5f3b806e5201e1b2ca1549a0a036a057726dc9e5dfa4d LABEL org.label-schema.name="opentelemetry-network-k8s-collector" LABEL org.label-schema.description="OpenTelemetry Network Kubernetes metadata collector" LABEL org.label-schema.vcs-url="https://github.com/open-telemetry/opentelemetry-network" LABEL org.label-schema.schema-version="1.0" # Embed the collector version so the binary can report it in handshakes. ARG EBPF_NET_MAJOR_VERSION ARG EBPF_NET_MINOR_VERSION ARG EBPF_NET_PATCH_VERSION ENV EBPF_NET_MAJOR_VERSION=${EBPF_NET_MAJOR_VERSION} \ EBPF_NET_MINOR_VERSION=${EBPF_NET_MINOR_VERSION} \ EBPF_NET_PATCH_VERSION=${EBPF_NET_PATCH_VERSION} COPY srv /srv WORKDIR /srv # Transitional compatibility: allow overriding the watcher container image # by exposing the expected binary name used by the chart. RUN if [ ! -e /srv/k8s-collector ]; then \ ln -s /srv/k8s-collector-stripped /srv/k8s-collector; \ fi \ && ln -sf /srv/k8s-collector /srv/k8s-watcher ENTRYPOINT ["/srv/k8s-collector"] ================================================ FILE: collector/agent_log.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME AgentLogKind #define ENUM_TYPE uint32_t #define ENUM_ELEMENTS(X) \ X(BPF, 0, "") \ X(UDP, 1, "") \ X(DNS, 2, "") \ X(TCP, 3, "") \ X(HTTP, 4, "") \ X(NAT, 6, "") \ X(DOCKER, 7, "") \ X(FLOW, 8, "") \ X(CGROUPS, 9, "") \ X(PERF, 10, "") \ X(PID, 11, "") \ X(PROTOCOL, 12, "") \ X(TRACKED_PROCESS, 13, "") \ X(NOMAD, 14, "") \ X(SOCKET, 15, "") #define ENUM_DEFAULT BPF #include ================================================ FILE: collector/cloud/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 add_library( cloud_agentlib STATIC entrypoint.cc collector.cc enumerator.cc ingest_connection.cc ) target_include_directories( cloud_agentlib PRIVATE ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) target_link_libraries( cloud_agentlib PUBLIC signal_handler render_ebpf_net_cloud_collector ) set(CLOUD_COLLECTOR_LINK_LIBRARIES cloud_agentlib render_ebpf_net_cloud_collector render_ebpf_net_ingest_writer aws-sdk-cpp reconnecting_channel connection_caretaker resource_usage_reporter config_file ip_address scheduling libuv-static args_parser system_ops aws_instance_metadata spdlog static-executable protobuf::libprotobuf ZLIB::ZLIB versions ) lint_shell_script_bundle( cloud-collector-scripts SOURCES entrypoint.sh ) include(rust_main) add_rust_main( TARGET cloud-collector STRIPPED_TARGET cloud-collector-stripped PACKAGE cloud-collector-bin BIN_NAME cloud-collector LINK_LIBS ${CLOUD_COLLECTOR_LINK_LIBRARIES} ) add_dependencies(collectors cloud-collector cloud-collector-stripped) if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") build_custom_docker_image( cloud-collector OUT_DIR srv OUTPUT_OF cloud-collector-scripts cloud-collector-stripped BINARIES debug-info.conf FILES ../../NOTICE.txt ../../LICENSE.txt DEPENDENCY_OF collectors ) else() build_custom_docker_image( cloud-collector OUT_DIR srv OUTPUT_OF cloud-collector-scripts cloud-collector-stripped BINARIES debug-info.conf FILES ../../NOTICE.txt ../../LICENSE.txt DEPENDENCY_OF collectors ) endif() ================================================ FILE: collector/cloud/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 FROM docker.io/bitnami/minideb:trixie@sha256:0766a3b76750541ae2c5f3b806e5201e1b2ca1549a0a036a057726dc9e5dfa4d LABEL org.label-schema.name="opentelemetry-ebpf-cloud-collector" LABEL org.label-schema.description="OpenTelemetry eBPF cloud metadata collector" LABEL org.label-schema.vcs-url="https://github.com/open-telemetry/opentelemetry-ebpf" LABEL org.label-schema.schema-version="1.0" # ca-certificates are required by libcurl RUN install_packages ca-certificates libprotobuf32 libgrpc++1.51 libcurl4 libcurlpp0 ENV SSL_CERT_DIR=/etc/ssl/certs ENTRYPOINT [ "/srv/entrypoint.sh" ] COPY srv /srv WORKDIR /srv RUN if [ ! -e /srv/cloud-collector ]; then \ ln /srv/cloud-collector-stripped /srv/cloud-collector; \ fi ================================================ FILE: collector/cloud/README.md ================================================ # Cloud Collector The `cloud-collector` collects cloud information that cannot be otherwise collected by our kernel collector, and feeds that information to the reducer. The collector is comprised of two components: - the agent that periodically queries AWS and pushes the information to the pipeline server; - a span within the pipeline server that consumes messages sent by `cloud-collector` and enriches the flow. # Running: The `cloud-collector` requires two pieces of settings to run: auth config and AWS credentials. The TL;DR to build and run in a test environment within `benv` is this: ```bash cd ~/out/ make -j cloud-collector export AWS_ACCESS_KEY_ID="your_access_key" export AWS_SECRET_ACCESS_KEY="your_secret_access_key" # make sure the server is running, as well as stunnel src/collector/cloud/cloud-collector --auth-config=$HOME/src/misc/localhost_auth_config.yaml ``` ## AWS Credentials Check the AWS SDK Developer Guide for [recommended ways to provide credentials](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html). For testing, you can supply two environment variables: ```bash export AWS_ACCESS_KEY_ID=your_access_key_id export AWS_SECRET_ACCESS_KEY=your_secret_access_key ``` ================================================ FILE: collector/cloud/collector.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include namespace collector::cloud { namespace { constexpr auto RECONNECT_DELAY = 5s; // explicitly using milliseconds for a finer grained jitter constexpr std::chrono::milliseconds RECONNECT_JITTER = 1s; } // namespace CloudCollector::CloudCollector( ::uv_loop_t &loop, std::string_view hostname, std::chrono::milliseconds aws_metadata_timeout, std::chrono::milliseconds heartbeat_interval, std::size_t buffer_size, config::IntakeConfig intake_config, std::chrono::milliseconds poll_interval) : loop_(loop), connection_( hostname, loop_, aws_metadata_timeout, heartbeat_interval, std::move(intake_config), buffer_size, *this, std::bind(&CloudCollector::on_connected, this)), log_(connection_.writer()), enumerator_(log_, connection_.index(), connection_.writer()), scheduler_(loop_, std::bind(&CloudCollector::callback, this)), poll_interval_(poll_interval) {} CloudCollector::~CloudCollector() { ::uv_loop_close(&loop_); } void CloudCollector::run_loop() { connection_.connect(); while (::uv_run(&loop_, UV_RUN_DEFAULT)) ; scheduler_.stop(); } scheduling::JobFollowUp CloudCollector::callback() { auto result = enumerator_.enumerate(); connection_.flush(); return result; } void CloudCollector::on_error(int err) { scheduler_.stop(); enumerator_.free_handles(); std::this_thread::sleep_for(add_jitter(RECONNECT_DELAY, -RECONNECT_JITTER, RECONNECT_JITTER)); } void CloudCollector::on_connected() { scheduler_.start(poll_interval_, poll_interval_); } } // namespace collector::cloud ================================================ FILE: collector/cloud/collector.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include namespace collector::cloud { struct CloudCollector : channel::Callbacks { CloudCollector( ::uv_loop_t &loop, std::string_view hostname, std::chrono::milliseconds aws_metadata_timeout, std::chrono::milliseconds heartbeat_interval, std::size_t buffer_size, config::IntakeConfig intake_config, std::chrono::milliseconds poll_interval); ~CloudCollector(); void run_loop(); ::uv_loop_t &get_loop() { return loop_; } private: scheduling::JobFollowUp callback(); void on_error(int err); void on_connected(); ::uv_loop_t &loop_; IngestConnection connection_; logging::Logger log_; NetworkInterfacesEnumerator enumerator_; scheduling::IntervalScheduler scheduler_; std::chrono::milliseconds const poll_interval_; }; } // namespace collector::cloud ================================================ FILE: collector/cloud/entrypoint.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * Cloud Collector Agent * * Requires AWS Access Key ID and Secret Access Key to be set up in the * environment: * https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html * * In production, in an EC2 instance, this should work automagically. * * The easiest way to achieve that in a development environment is by setting up * environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. */ static int cloud_collector_run(int argc, char **argv) { ::uv_loop_t loop; if (auto const error = ::uv_loop_init(&loop)) { throw std::runtime_error(::uv_strerror(error)); } // args parsing cli::ArgsParser parser("Cloud Collector Agent"); args::HelpFlag help(*parser, "help", "Display this help menu", {'h', "help"}); args::ValueFlag conf_file(*parser, "config_file", "The location of the custom config file", {"config-file"}, ""); args::ValueFlag ec2_poll_interval_ms( *parser, "ec2_poll_interval_ms", "How often, in milliseconds, to enumerate interfaces in EC2.", {"ec2-poll-interval-ms"}, std::chrono::milliseconds(1s).count()); args::ValueFlag aws_metadata_timeout_ms( *parser, "milliseconds", "Milliseconds to wait for AWS instance metadata", {"aws-timeout"}, 1 * 1000); parser.new_handler>("channel"); parser.new_handler>("cloud-platform"); parser.new_handler>("utility"); auto &intake_config_handler = parser.new_handler(); SignalManager signal_manager(loop, "cloud-collector"); if (auto result = parser.process(argc, argv); !result.has_value()) { return result.error(); } // Initialize minimal signal handler behavior (ignore SIGPIPE, disable core dumps) signal_manager.handle(); if (ec2_poll_interval_ms.Get() == 0) { LOG::error("--ec2-poll-interval-ms cannot be 0"); return EXIT_FAILURE; } // resolve hostname std::string const hostname = get_host_name(MAX_HOSTNAME_LENGTH).recover([](auto &error) { LOG::error("Unable to retrieve host information from uname: {}", log_waive(error.message())); return "(unknown)"; }); auto curl_engine = CurlEngine::create(&loop); auto agent_id = gen_agent_id(); config::ConfigFile configuration_data(config::ConfigFile::YamlFormat(), conf_file.Get()); config::IntakeConfig intake_config = configuration_data.intake_config(); intake_config_handler.read_config(intake_config); LOG::info("Cloud Collector version {} ({}) started on host {}", versions::release, release_mode_string, hostname); LOG::info("Cloud Collector agent ID is {}", agent_id); // aws sdk init Aws::InitAPI({}); // main collector::cloud::CloudCollector collector{ loop, hostname, std::chrono::milliseconds(aws_metadata_timeout_ms.Get()), HEARTBEAT_INTERVAL, WRITE_BUFFER_SIZE, std::move(intake_config), std::chrono::milliseconds(ec2_poll_interval_ms.Get())}; signal_manager.handle_signals({SIGINT, SIGTERM} // TODO: close gracefully ); collector.run_loop(); // shutdown Aws::ShutdownAPI({}); return EXIT_SUCCESS; } extern "C" int otn_cloud_collector_main(int argc, const char **argv) { return cloud_collector_run(argc, const_cast(argv)); } ================================================ FILE: collector/cloud/entrypoint.sh ================================================ #!/bin/bash -e # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # shellcheck disable=SC1091 [[ ! -e ./debug-info.conf ]] || source ./debug-info.conf # to run the collector under gdb, set `EBPF_NET_RUN_UNDER_GDB` to the flavor of gdb # you want (e.g.: `cgdb` or `gdb`) - this is intended for development purposes if [[ -n "${EBPF_NET_RUN_UNDER_GDB}" ]]; then apt-get update -y apt-get install -y --no-install-recommends "${EBPF_NET_RUN_UNDER_GDB}" if [[ "${#EBPF_NET_GDB_COMMANDS[@]}" -lt 1 ]]; then # default behavior is to run the agent, print a stack trace after it exits # and exit gdb without confirmation EBPF_NET_GDB_COMMANDS=( \ 'set pagination off' 'handle SIGPIPE nostop pass' 'handle SIGUSR1 nostop pass' 'handle SIGUSR2 nostop pass' run bt 'server q' ) fi GDB_ARGS=() for gdb_cmd in "${EBPF_NET_GDB_COMMANDS[@]}"; do GDB_ARGS+=(-ex "${gdb_cmd}") done (set -x; exec "${EBPF_NET_RUN_UNDER_GDB}" -q "${GDB_ARGS[@]}" \ --args /srv/cloud-collector "$@" \ ) else (set -x; exec /srv/cloud-collector "$@") fi ================================================ FILE: collector/cloud/enumerator.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace collector::cloud { NetworkInterfacesEnumerator::NetworkInterfacesEnumerator( logging::Logger &log, ebpf_net::cloud_collector::Index &index, ebpf_net::ingest::Writer &writer) : index_(index), writer_(writer), log_(log) {} NetworkInterfacesEnumerator::~NetworkInterfacesEnumerator() { free_handles(); } void NetworkInterfacesEnumerator::set_handles(std::vector handles) { free_handles(); handles_ = std::move(handles); } void NetworkInterfacesEnumerator::free_handles() { for (auto &handle : handles_) { handle.put(index_); } handles_.clear(); } void translate_interfaces_to_spans( ebpf_net::cloud_collector::Index &index, Aws::Vector const &interfaces, std::vector &handles) { for (auto const &interface : interfaces) { auto const &attachment = interface.GetAttachment(); auto const &association = interface.GetAssociation(); auto const &ip_owner_id = association.GetIpOwnerId(); auto const &vpc_id = interface.GetVpcId(); auto const &az = interface.GetAvailabilityZone(); auto const &interface_id = interface.GetNetworkInterfaceId(); auto const raw_interface_type = static_cast(interface.GetInterfaceType()); auto const &instance_id = attachment.GetInstanceId(); auto const &instance_owner_id = attachment.GetInstanceOwnerId(); auto const &public_dns_name = association.GetPublicDnsName(); auto const &private_dns_name = interface.GetPrivateDnsName(); auto const &description = interface.GetDescription(); auto const add_entry = [&](IPv6Address const &ipv6) { auto handle = index.aws_network_interface.by_key(ebpf_net::cloud_collector::keys::aws_network_interface{ipv6.as_int()}); LOG::trace( "network_interface_info:" " ip={}" " ip_owner_id={}" " vpc_id={}" " az={}" " interface_id={} interface_type={} instance_id={} instance_owner_id={}" " public_dns_name={} private_dns_name={} description={}", ipv6, ip_owner_id, vpc_id, az, interface_id, raw_interface_type, instance_id, instance_owner_id, public_dns_name, private_dns_name, description); handle.network_interface_info( jb_blob{ip_owner_id}, jb_blob{vpc_id}, jb_blob{az}, jb_blob{interface_id}, raw_interface_type, jb_blob{instance_id}, jb_blob{instance_owner_id}, jb_blob{public_dns_name}, jb_blob{private_dns_name}, jb_blob{description}); handles.emplace_back(handle.to_handle()); }; if (auto const public_ip = IPv4Address::parse(association.GetPublicIp().c_str())) { add_entry(public_ip->to_ipv6()); } for (auto const &ipv4 : interface.GetPrivateIpAddresses()) { auto const private_ip = IPv4Address::parse(ipv4.GetPrivateIpAddress().c_str()); if (private_ip) { add_entry(private_ip->to_ipv6()); } } for (auto const &address : interface.GetIpv6Addresses()) { if (auto const ipv6 = IPv6Address::parse(address.GetIpv6Address().c_str())) { add_entry(*ipv6); } } } } void NetworkInterfacesEnumerator::handle_ec2_error( CollectorStatus status, Aws::Client::AWSError const &error) { if (error.GetErrorType() == Aws::EC2::EC2Errors::THROTTLING) { log_.error("{} - AWS API call throttled: {}", to_string(status), error.GetMessage()); return; } auto const http_status = static_cast>(error.GetResponseCode()); LOG::trace("reporting cloud collector as unhealthy (status={} detail={})", to_string(status), http_status); writer_.collector_health(integer_value(status), http_status); if (http_status >= 400 && http_status < 500) { log_.error( "{} - API call failed with http status {}. Double check that AWS credentials are" " properly set up for this pod. Check setup instructions for more information." " Error message from AWS: {}", to_string(status), http_status, error.GetMessage()); } else { log_.error( "{} - API call failed with http status {}. Error message from AWS: {}", to_string(status), http_status, error.GetMessage()); } } scheduling::JobFollowUp NetworkInterfacesEnumerator::enumerate() { ResourceUsageReporter::report(writer_); auto const regions_response = ec2_.DescribeRegions({}); if (!regions_response.IsSuccess()) { handle_ec2_error(CollectorStatus::aws_describe_regions_error, regions_response.GetError()); return scheduling::JobFollowUp::backoff; } std::vector handles; auto result = scheduling::JobFollowUp::ok; LOG::trace("starting AWS network interfaces enumeration"); StopWatch<> watch; for (auto const ®ion : regions_response.GetResult().GetRegions()) { LOG::trace("enumerating network interfaces in region '{}'", region.GetRegionName()); Aws::Client::ClientConfiguration client_config; client_config.region = region.GetRegionName(); Aws::EC2::EC2Client client(client_config); auto const interfaces_response = client.DescribeNetworkInterfaces({}); if (!interfaces_response.IsSuccess()) { handle_ec2_error(CollectorStatus::aws_describe_network_interfaces_error, interfaces_response.GetError()); result = scheduling::JobFollowUp::backoff; continue; } auto const &interfaces = interfaces_response.GetResult().GetNetworkInterfaces(); LOG::trace("found {} network interfaces in region '{}'", interfaces.size(), region.GetRegionName()); translate_interfaces_to_spans(index_, interfaces, handles); } LOG::trace("finished AWS network interfaces enumeration after {}", watch.elapsed()); set_handles(std::move(handles)); LOG::trace("network interface live span count: {}", handles_.size()); if (result == scheduling::JobFollowUp::ok) { LOG::trace("reporting cloud collector as healthy"); writer_.collector_health(integer_value(CollectorStatus::healthy), 0); } return result; } } // namespace collector::cloud ================================================ FILE: collector/cloud/enumerator.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include namespace collector::cloud { struct NetworkInterfacesEnumerator { NetworkInterfacesEnumerator(logging::Logger &log, ebpf_net::cloud_collector::Index &index, ebpf_net::ingest::Writer &writer); ~NetworkInterfacesEnumerator(); scheduling::JobFollowUp enumerate(); void free_handles(); private: void set_handles(std::vector handles); void handle_ec2_error(CollectorStatus status, Aws::Client::AWSError const &error); Aws::EC2::EC2Client ec2_; ebpf_net::cloud_collector::Index &index_; ebpf_net::ingest::Writer &writer_; logging::Logger &log_; std::vector handles_; }; } // namespace collector::cloud ================================================ FILE: collector/cloud/ingest_connection.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include namespace collector::cloud { IngestConnection::IngestConnection( std::string_view hostname, ::uv_loop_t &loop, std::chrono::milliseconds aws_metadata_timeout, std::chrono::milliseconds heartbeat_interval, config::IntakeConfig intake_config, std::size_t buffer_size, channel::Callbacks &connection_callback, std::function on_connected_cb) : curl_(CurlEngine::create(&loop)), channel_(std::move(intake_config), loop, buffer_size), connection_callback_(connection_callback), encoder_(channel_.intake_config().make_encoder()), writer_(channel_.buffered_writer(), monotonic, get_boot_time(), encoder_.get()), caretaker_( hostname, ClientType::cloud, {}, &loop, writer_, aws_metadata_timeout, heartbeat_interval, std::bind(&channel::ReconnectingChannel::flush, &channel_), std::bind(&channel::ReconnectingChannel::set_compression, &channel_, std::placeholders::_1), std::move(on_connected_cb)), index_({writer_}) { channel_.register_pipeline_observer(this); } void IngestConnection::connect() { channel_.start_connect(); } void IngestConnection::flush() { channel_.flush(); } u32 IngestConnection::received_data(const u8 *data, int data_len) { return connection_callback_.received_data(data, data_len); } void IngestConnection::on_error(int err) { caretaker_.set_disconnected(); connection_callback_.on_error(err); } void IngestConnection::on_closed() { connection_callback_.on_closed(); } void IngestConnection::on_connect() { caretaker_.set_connected(); connection_callback_.on_connect(); } } // namespace collector::cloud ================================================ FILE: collector/cloud/ingest_connection.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include namespace collector::cloud { class IngestConnection : channel::Callbacks { public: IngestConnection( std::string_view hostname, ::uv_loop_t &loop, std::chrono::milliseconds aws_metadata_timeout, std::chrono::milliseconds heartbeat_interval, config::IntakeConfig intake_config, std::size_t buffer_size, channel::Callbacks &connection_callback, std::function on_connected_cb); void connect(); void flush(); ebpf_net::ingest::Writer &writer() { return writer_; } ebpf_net::cloud_collector::Index &index() { return index_; } private: u32 received_data(const u8 *data, int data_len); void on_error(int err); void on_closed(); void on_connect(); std::unique_ptr curl_; channel::ReconnectingChannel channel_; channel::Callbacks &connection_callback_; std::unique_ptr<::ebpf_net::ingest::Encoder> encoder_; ebpf_net::ingest::Writer writer_; channel::ConnectionCaretaker caretaker_; ebpf_net::cloud_collector::Index index_; }; } // namespace collector::cloud ================================================ FILE: collector/constants.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include constexpr auto HEARTBEAT_INTERVAL = 2s; constexpr auto WRITE_BUFFER_SIZE = 16 * 1024; ================================================ FILE: collector/kernel/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # Kernel collector legacy executable # add_executable( kernel-collector-legacy main.cc ) set(KERNEL_COLLECTOR_LINK_LIBRARIES agentlib render_ebpf_net_ingest_writer render_rust_ebpf_net fastpass_util file_ops config_file libuv-static args_parser system_ops spdlog ) target_link_libraries( kernel-collector-legacy PUBLIC ${KERNEL_COLLECTOR_LINK_LIBRARIES} static-executable ) target_include_directories( kernel-collector-legacy PRIVATE ${PROJECT_SOURCE_DIR} ${CMAKE_BINARY_DIR} ) # DNS library # add_library( agentdnslib STATIC dns/ares_expand_name.c dns/ares_parse_a_aaaa_reply.c dns/ares_parse_query.c ) add_dependencies(agentdnslib render_ebpf_net_artifacts) target_compile_options(agentdnslib PRIVATE -fPIC) # Agent library # add_library( agentlib STATIC perf_reader.cc perf_poller.cc buffered_poller.cc dns_requests.cc proc_reader.cc process_prober.cc process_handler.cc socket_prober.cc fd_reader.cc proc_net_reader.cc proc_cmdline.cc probe_handler.cc kernel_collector.cc kernel_collector_restarter.cc bpf_handler.cc cgroup_prober.cc cgroup_handler.cc nat_prober.cc nat_handler.cc troubleshooting.cc tcp_data_handler.cc kernel_symbols.cc protocols/protocol_handler_base.cc protocols/protocol_handler_unknown.cc protocols/protocol_handler_http.cc entrypoint.cc ) target_link_libraries( agentlib PUBLIC render_ebpf_net_agent_internal_hash render_ebpf_net_ingest_writer render_ebpf_net_kernel_collector agentdnslib yamlcpp curl_engine agent_id resource_usage_reporter scheduling libuv-interface element_queue_writer file_channel upstream_connection aws_instance_metadata gcp_instance_metadata docker_host_config_metadata k8s_metadata nomad_metadata ip_address intake_config proc_ops system_ops time absl::flat_hash_map absl::flat_hash_set stdc++fs libbpf::libbpf versions signal_handler ) target_compile_options(agentlib PRIVATE -fPIC) # some agentlib sources include ${BPF_DEBUG_INFO} add_dependencies(agentlib generate_bpf_skeleton) ################################################################################ # Rust binary integration (generalized) # include(rust_main) add_rust_main( TARGET kernel-collector STRIPPED_TARGET kernel-collector-stripped PACKAGE kernel-collector-bin BIN_NAME kernel-collector LINK_LIBS ${KERNEL_COLLECTOR_LINK_LIBRARIES} ) add_dependencies(collectors kernel-collector kernel-collector-stripped) ################################################################################ # libbpf skeleton # set(BPF_DEPENDENCIES "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/render_bpf.c" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/render_bpf.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_debug.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_http_protocol.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_inet_csk_accept.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_memory.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_tcp_events.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_tcp_processor.c" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_tcp_send_recv.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_tcp_socket.h" "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/tcp-processor/bpf_types.h" "${CMAKE_BINARY_DIR}/generated/ebpf_net/agent_internal/wire_message.h" "${CMAKE_BINARY_DIR}/generated/ebpf_net/agent_internal/bpf.h" "${PROJECT_SOURCE_DIR}/jitbuf/jb.h" render_compile_ebpf_net ) set(BPF_SKEL_HEADER "${CMAKE_BINARY_DIR}/generated/render_bpf.skel.h") set(BPF_OBJECT_FILE "${CMAKE_BINARY_DIR}/generated/render_bpf.bpf.o") set(BPFTOOL "${CMAKE_INSTALL_PREFIX}/usr/local/sbin/bpftool") set(CLANG clang) # Get Clang's system includes for BPF compilation execute_process( COMMAND ${CLANG} -v -E - INPUT_FILE /dev/null ERROR_VARIABLE CLANG_VERBOSE_OUTPUT OUTPUT_QUIET ) string(REGEX MATCH "#include <\\.\\.\\.> search starts here:(.*)End of search list\\." _ "${CLANG_VERBOSE_OUTPUT}") string(REGEX REPLACE "\n" ";" CLANG_INCLUDES "${CMAKE_MATCH_1}") set(CLANG_BPF_SYS_INCLUDES "") foreach(INCLUDE_PATH ${CLANG_INCLUDES}) string(STRIP "${INCLUDE_PATH}" INCLUDE_PATH) if(NOT "${INCLUDE_PATH}" STREQUAL "") list(APPEND CLANG_BPF_SYS_INCLUDES "-idirafter" "${INCLUDE_PATH}") endif() endforeach() # Determine architecture for vmlinux.h set(ARCH_RAW "${CMAKE_HOST_SYSTEM_PROCESSOR}") string(REGEX REPLACE "x86_64" "x86" ARCH "${ARCH_RAW}") string(REGEX REPLACE "arm.*" "arm" ARCH "${ARCH}") string(REGEX REPLACE "aarch64" "arm64" ARCH "${ARCH}") string(REGEX REPLACE "ppc64le" "powerpc" ARCH "${ARCH}") string(REGEX REPLACE "mips.*" "mips" ARCH "${ARCH}") string(REGEX REPLACE "riscv64" "riscv" ARCH "${ARCH}") string(REGEX REPLACE "loongarch64" "loongarch" ARCH "${ARCH}") set(VMLINUX_H "${PROJECT_SOURCE_DIR}/ext/vmlinux.h/include/${ARCH}/vmlinux.h") get_filename_component(VMLINUX_DIR "${VMLINUX_H}" DIRECTORY) # Compile BPF source to object file for libbpf skeleton add_custom_command( OUTPUT "${BPF_OBJECT_FILE}" COMMAND "${CLANG}" -g -O2 -target bpf -D__TARGET_ARCH_${ARCH} -D_PROCESSING_BPF=1 -I "${CMAKE_BINARY_DIR}/generated" -I "${CMAKE_BINARY_DIR}" -I "${CMAKE_INSTALL_PREFIX}/usr/lib64" -I "${CMAKE_INSTALL_PREFIX}/usr/include" -I "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src" -I "${PROJECT_SOURCE_DIR}" -I "${VMLINUX_DIR}" ${CLANG_BPF_SYS_INCLUDES} -c "${CMAKE_CURRENT_SOURCE_DIR}/bpf_src/render_bpf.c" -o "${BPF_OBJECT_FILE}.tmp" COMMAND "${BPFTOOL}" gen object "${BPF_OBJECT_FILE}" "${BPF_OBJECT_FILE}.tmp" DEPENDS "${BPF_DEPENDENCIES}" "${VMLINUX_H}" COMMENT "Compiling BPF source to object file for skeleton generation" VERBATIM ) # Generate BPF skeleton header add_custom_command( OUTPUT "${BPF_SKEL_HEADER}" COMMAND "${BPFTOOL}" gen skeleton "${BPF_OBJECT_FILE}" > "${BPF_SKEL_HEADER}" DEPENDS "${BPF_OBJECT_FILE}" COMMENT "Generating BPF skeleton header" VERBATIM ) set_source_files_properties("${BPF_SKEL_HEADER}" PROPERTIES GENERATED TRUE) # Target for libbpf skeleton generation add_custom_target( generate_bpf_skeleton DEPENDS "${BPF_SKEL_HEADER}" ) ################################################################################ # Shell scripts # lint_shell_script_bundle( kernel-collector-scripts SOURCES entrypoint.sh entrypoint-kct.sh ) # Docker image # if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") build_custom_docker_image( kernel-collector OUT_DIR srv OUTPUT_OF kernel-collector-scripts kernel-collector-stripped BINARIES debug-info.conf FILES ../../NOTICE.txt ../../LICENSE.txt DEPENDENCY_OF collectors ARGS BUILD_TYPE="Debug" ) build_custom_docker_image( kernel-collector-test DOCKERFILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/kernel_collector_test_docker" OUT_DIR srv ARTIFACTS_OF bpf_wire_to_json intake_wire_to_json kernel_collector_test OUTPUT_OF kernel-collector-scripts FILES ../../NOTICE.txt ../../LICENSE.txt ARGS BUILD_TYPE="Debug" ) else() build_custom_docker_image( kernel-collector OUT_DIR srv OUTPUT_OF kernel-collector-scripts kernel-collector-stripped BINARIES debug-info.conf FILES ../../NOTICE.txt ../../LICENSE.txt DEPENDENCY_OF collectors ) build_custom_docker_image( kernel-collector-test DOCKERFILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/kernel_collector_test_docker" OUT_DIR srv ARTIFACTS_OF bpf_wire_to_json intake_wire_to_json kernel_collector_test OUTPUT_OF kernel-collector-scripts FILES ../../NOTICE.txt ../../LICENSE.txt ) endif() # Unit Tests # add_unit_test(cgroup_handler LIBS agentlib test_channel render_ebpf_net_ingest_writer render_rust_ebpf_net) add_unit_test(kernel_symbols LIBS agentlib) add_ebpf_unit_test(kernel_collector LIBS signal_handler agentlib fastpass_util file_ops config_file libuv-static system_ops static-executable test_channel render_ebpf_net_ingest_writer render_rust_ebpf_net) ================================================ FILE: collector/kernel/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 FROM docker.io/bitnami/minideb:trixie@sha256:0766a3b76750541ae2c5f3b806e5201e1b2ca1549a0a036a057726dc9e5dfa4d LABEL org.label-schema.name="opentelemetry-ebpf-kernel-collector" LABEL org.label-schema.description="OpenTelemetry eBPF kernel information collector" LABEL org.label-schema.vcs-url="https://github.com/open-telemetry/opentelemetry-ebpf" LABEL org.label-schema.schema-version="1.0" # ca-certificates are required by libcurl # libllvm is not required at runtime; drop libllvm16 to avoid apt failures on bookworm RUN install_packages ca-certificates libcurlpp0 libgrpc++1.51 libelf1 ENV SSL_CERT_DIR=/etc/ssl/certs ENV EBPF_NET_INSTALL_DIR=/srv ENV EBPF_NET_HOST_DIR=/hostfs ENV EBPF_NET_DATA_DIR=/var/run/ebpf_net ENTRYPOINT [ "/srv/entrypoint.sh" ] ARG BUILD_TYPE RUN if [ "$BUILD_TYPE" = "Debug" ]; then \ install_packages bc cgdb gawk gdb gzip iputils-ping jq netcat-openbsd procps ripgrep vim valgrind; \ fi COPY srv /srv WORKDIR /srv RUN if [ ! -e /srv/kernel-collector ]; then \ ln /srv/kernel-collector-stripped /srv/kernel-collector; \ fi ================================================ FILE: collector/kernel/bpf_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include BPFHandler::BPFHandler( uv_loop_t &loop, const BpfConfiguration &bpf_config, bool enable_http_metrics, FileDescriptor &bpf_dump_file, logging::Logger &log, ::ebpf_net::ingest::Encoder *encoder, HostInfo const &host_info) : loop_(loop), probe_handler_(log), bpf_skel_(nullptr), perf_(), encoder_(encoder), buf_poller_(nullptr), enable_http_metrics_(enable_http_metrics), bpf_dump_file_(bpf_dump_file), log_(log), last_lost_count_(0), host_info_(host_info) { // Open the BPF skeleton (doesn't load yet) bpf_skel_ = probe_handler_.open_bpf_skeleton(); if (!bpf_skel_) { throw std::system_error(errno, std::generic_category(), "ProbeHandler couldn't open BPF skeleton"); } // Configure global variables before loading probe_handler_.configure_bpf_skeleton(bpf_skel_, bpf_config); // Now load the skeleton and set up perf rings int res = probe_handler_.load_bpf_skeleton(bpf_skel_, perf_); if (res != 0) { probe_handler_.destroy_bpf_skeleton(bpf_skel_); bpf_skel_ = nullptr; throw std::system_error(errno, std::generic_category(), "ProbeHandler couldn't load BPF skeleton"); } } BPFHandler::~BPFHandler() { probe_handler_.cleanup_probes(); probe_handler_.cleanup_tail_calls(bpf_skel_); buf_poller_.reset(); if (bpf_skel_) { probe_handler_.destroy_bpf_skeleton(bpf_skel_); bpf_skel_ = nullptr; } } void BPFHandler::load_buffered_poller( IBufferedWriter &buffered_writer, u64 boot_time_adjustment, CurlEngine &curl_engine, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings const &cgroup_settings, KernelCollectorRestarter &kernel_collector_restarter) { LOG::trace("--- Starting BufferedPoller ---"); buf_poller_ = std::make_unique( loop_, perf_, buffered_writer, boot_time_adjustment, curl_engine, bpf_dump_file_, log_, probe_handler_, bpf_skel_, socket_stats_interval_sec, cgroup_settings, encoder_, kernel_collector_restarter); last_lost_count_ = serv_lost_count(); } void BPFHandler::load_probes(::ebpf_net::ingest::Writer &writer) { probe_handler_.load_kernel_symbols(); CgroupProber cgroup_prober( probe_handler_, bpf_skel_, host_info_, [this]() { buf_poller_->start(1, 1); }, [this](std::string error_loc) { check_cb(error_loc); }); if (cgroup_prober.error_count() > 0) { log_.warn("load_probes could not close {} directories", cgroup_prober.error_count()); } /* Start instrumentation for processes */ ProcessProber process_prober( probe_handler_, bpf_skel_, [this]() { buf_poller_->start(1, 1); }, [this](std::string error_loc) { check_cb(error_loc); }); NatProber nat_prober(probe_handler_, bpf_skel_, [this]() { buf_poller_->start(1, 1); }); // one more poll to make sure the perf rings are clear buf_poller_->start(1, 1); // send a process_steady_state message writer.process_steady_state(0); // UDP (v4+v6) DNS Replies // And receive udp statistics ProbeAlternatives dns_probe_alternatives{ "DNS", { // skb_consume_udp was introduced in f970bd9e3a06f, i.e., // v4.10-rc1~202^2~423^2~1 {"on_skb_consume_udp", "skb_consume_udp"}, // __skb_free_datagram_locked was introduced in 627d2d6b55009, i.e., // v4.7-rc1~154^2~349^2 {"on___skb_free_datagram_locked", "__skb_free_datagram_locked"}, // skb_free_datagram_locked exists in udp_recvmsg since 9d410c7960676, i.e., // v2.6.32-rc6~9^2~3 {"on_skb_free_datagram_locked", "skb_free_datagram_locked"}, }}; probe_handler_.start_probe(bpf_skel_, dns_probe_alternatives); // Start instrumentation for sockets SocketProber socket_prober( probe_handler_, bpf_skel_, [this]() { buf_poller_->start(1, 1); }, [this](std::string error_loc) { check_cb(error_loc); }, log_); // one more poll to make sure the perf rings are clear buf_poller_->start(1, 1); // send a socket_steady_state message writer.socket_steady_state(0); // probe for steady-state data ProbeAlternatives probe_alternatives{ "tcp rtt estimator", { {"on_tcp_rtt_estimator", "tcp_rtt_estimator"}, {"on_tcp_rtt_estimator", "tcp_update_pacing_rate"}, {"on_tcp_rtt_estimator", "tcp_ack"}, }}; probe_handler_.start_probe(bpf_skel_, probe_alternatives); // SYN timeouts probe_handler_.start_probe(bpf_skel_, "on_tcp_retransmit_timer", "tcp_retransmit_timer"); // SYN-ACK timeouts probe_handler_.start_probe(bpf_skel_, "on_tcp_syn_ack_timeout", "tcp_syn_ack_timeout"); // TCP resets probe_handler_.start_probe(bpf_skel_, "on_tcp_reset", "tcp_reset"); probe_handler_.start_probe(bpf_skel_, "on_tcp_send_active_reset", "tcp_send_active_reset"); buf_poller_->start(1, 1); check_cb("loading rtt probes"); /** * TODO: is it possible we're missing events from the new sock notification * to the point we start the instrumentation below? we read some socket * recv state when opening the socket */ probe_handler_.start_probe(bpf_skel_, "on_tcp_event_data_recv", "tcp_event_data_recv"); probe_handler_.start_probe(bpf_skel_, "on_tcp_rcv_established", "tcp_rcv_established"); buf_poller_->start(1, 1); check_cb("loading tcp steady-state probes"); // set up tail calls table probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_ON_UDP_SEND_SKB__2, "on_udp_send_skb__2"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_ON_UDP_V6_SEND_SKB__2, "on_udp_v6_send_skb__2"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_ON_IP_SEND_SKB__2, "on_ip_send_skb__2"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_ON_IP6_SEND_SKB__2, "on_ip6_send_skb__2"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_HANDLE_RECEIVE_UDP_SKB, "handle_receive_udp_skb"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_HANDLE_RECEIVE_UDP_SKB__2, "handle_receive_udp_skb__2"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_CONTINUE_TCP_SENDMSG, "continue_tcp_sendmsg"); probe_handler_.register_tail_call(bpf_skel_, "tail_calls", TAIL_CALL_CONTINUE_TCP_RECVMSG, "continue_tcp_recvmsg"); // udp v4 send statistics and dns requests ProbeAlternatives udp_v4_alternatives{ "udp v4 send skb", { {"on_udp_send_skb", "udp_send_skb"}, {"on_ip_send_skb", "ip_send_skb"}, }}; probe_handler_.start_probe(bpf_skel_, udp_v4_alternatives); // udp v6 send statistics and dns requests ProbeAlternatives udp_v6_alternatives{ "udp v6 send skb", { {"on_udp_v6_send_skb", "udp_v6_send_skb"}, {"on_ip6_send_skb", "ip6_send_skb"}, }}; probe_handler_.start_probe(bpf_skel_, udp_v6_alternatives); // start the udp probes buf_poller_->start(1, 1); check_cb("loading udp steady-state probes"); // Probes for tcp-processor if (enable_http_metrics_ /*|| any other tcp-processor flags eventually */) { // Using _tcpproc as the event suffix to ensure we don't collide our kprobes // with anything in 'render_bpf' // // Keep kretprobes registered before kprobes to the same function to avoid // races LOG::debug("Adding TCP processor probes"); probe_handler_.start_probe(bpf_skel_, "handle_kprobe__tcp_init_sock", "tcp_init_sock", "_tcpproc"); probe_handler_.start_probe(bpf_skel_, "handle_kprobe__security_sk_free", "security_sk_free", "_tcpproc"); probe_handler_.start_kretprobe(bpf_skel_, "handle_kretprobe__inet_csk_accept", "inet_csk_accept", "_tcpproc"); probe_handler_.start_probe(bpf_skel_, "handle_kprobe__inet_csk_accept", "inet_csk_accept", "_tcpproc"); probe_handler_.start_kretprobe(bpf_skel_, "handle_kretprobe__tcp_sendmsg", "tcp_sendmsg", "_tcpproc"); probe_handler_.start_probe(bpf_skel_, "handle_kprobe__tcp_sendmsg", "tcp_sendmsg", "_tcpproc"); probe_handler_.start_kretprobe(bpf_skel_, "handle_kretprobe__tcp_recvmsg", "tcp_recvmsg", "_tcpproc"); probe_handler_.start_probe(bpf_skel_, "handle_kprobe__tcp_recvmsg", "tcp_recvmsg", "_tcpproc"); buf_poller_->start(1, 1); check_cb("tcp processor probes"); } buf_poller_->start(1, 1); check_cb("end of load_probes()"); buf_poller_->set_all_probes_loaded(); probe_handler_.clear_kernel_symbols(); } void BPFHandler::start_poll(u64 interval_useconds, u64 n_intervals) { buf_poller_->start(interval_useconds, n_intervals); } void BPFHandler::slow_poll() { buf_poller_->slow_poll(); } u64 BPFHandler::serv_lost_count() { return buf_poller_->serv_lost_count(); } void BPFHandler::check_cb(std::string error_loc) { u64 lost_count = serv_lost_count(); if (lost_count > last_lost_count_) { u64 diff_lost_count = lost_count - last_lost_count_; last_lost_count_ = lost_count; log_.warn("after {}, cumulative lost count non-zero: {} new, {} total", error_loc, diff_lost_count, lost_count); } } #ifndef NDEBUG void BPFHandler::debug_bpf_lost_samples() { if (buf_poller_) { buf_poller_->debug_bpf_lost_samples(); } } #endif ================================================ FILE: collector/kernel/bpf_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include // Forward declaration for the skeleton struct render_bpf_bpf; class BPFHandler { friend class KernelCollectorTest; public: /** * c'tor * * Will throw if: * 1. PerfContainer cannot be allocated * 2. ProbeHandler can't load BPF skeleton */ BPFHandler( uv_loop_t &loop, const BpfConfiguration &bpf_config, bool enable_http_metrics, FileDescriptor &bpf_dump_file, logging::Logger &log, ::ebpf_net::ingest::Encoder *encoder, HostInfo const &host_info_); /** * d'tor */ virtual ~BPFHandler(); /** * Loads the buffered poller */ void load_buffered_poller( IBufferedWriter &buffered_writer, u64 boot_time_adjustment, CurlEngine &curl_engine, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings const &cgroup_settings, KernelCollectorRestarter &kernel_collector_restarter); /** * Loads BPF probes. Takes writer to send out steady_state msgs * where necessary. */ void load_probes(::ebpf_net::ingest::Writer &writer); /** * Calls start(interval_useconds, n_intervals) on buf_poller_ */ void start_poll(u64 interval_useconds, u64 n_intervals); /** * Calls less frequent cleanup operations on buf_poller_ */ void slow_poll(); /** * Calls serv_lost_count() on buf_poller_ */ u64 serv_lost_count(); /** * Callback passed to probers for checking lost count */ void check_cb(std::string error_loc); #ifndef NDEBUG /** * Debug code for internal development to simulate lost BPF samples (PERF_RECORD_LOST) in BufferedPoller. */ void debug_bpf_lost_samples(); #endif private: uv_loop_t &loop_; ProbeHandler probe_handler_; struct render_bpf_bpf *bpf_skel_; PerfContainer perf_; ::ebpf_net::ingest::Encoder *encoder_; std::unique_ptr buf_poller_; bool enable_http_metrics_; FileDescriptor &bpf_dump_file_; logging::Logger &log_; u64 last_lost_count_; HostInfo const host_info_; }; ================================================ FILE: collector/kernel/bpf_src/render_bpf.c ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // Global variables that can be set from userspace volatile const long boot_time_adjustment = 0; volatile const long filter_ns = 1000000000; // Default 1 second in nanoseconds volatile const int enable_tcp_data_stream = 0; // Set to 1 to enable TCP data stream processing #include #include #include #include #include extern int LINUX_KERNEL_VERSION __kconfig; // Networking macros #define tcp_sk(ptr) container_of(ptr, struct tcp_sock, inet_conn.icsk_inet.sk) #define inet_csk(ptr) container_of(ptr, struct inet_connection_sock, icsk_inet.sk) #define inet_sk(ptr) container_of(ptr, struct inet_sock, sk) #define sk_num __sk_common.skc_num #define sk_dport __sk_common.skc_dport #define sk_v6_daddr __sk_common.skc_v6_daddr #define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr #define sk_daddr __sk_common.skc_daddr #define sk_rcv_saddr __sk_common.skc_rcv_saddr #define sk_family __sk_common.skc_family #define sk_state __sk_common.skc_state #define AF_INET 2 /* Internet IP Protocol */ #define AF_INET6 10 /* IP version 6 */ #define s6_addr16 in6_u.u6_addr16 #define s6_addr32 in6_u.u6_addr32 #define inet_num sk.__sk_common.skc_num #define fl4_sport uli.ports.sport #define fl4_dport uli.ports.dport #define fl6_sport uli.ports.sport #define fl6_dport uli.ports.dport #define rsk_listener __req_common.skc_listener // pointer error handling #define MAX_ERRNO 4095 #define IS_ERR_VALUE(x) unlikely((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO) #include "vmlinux_extensions.h" #include "vmlinux_compat.h" // Configuration #include "config.h" #include "render_bpf.h" // Perf events - per-CPU perf ring buffer struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(u32)); } events SEC(".maps"); #include "ebpf_net/agent_internal/bpf.h" // Common utility functions #include "tcp-processor/bpf_debug.h" #include "tcp-processor/bpf_types.h" static u64 abs_val(int val) { return val < 0 ? -val : val; } /* HELPERS FOR FILLERS */ struct pkts_if_t { u32 packets_out; u32 sacked_out; u32 lost_out; u32 retrans_out; }; static inline u32 packets_in_flight_helper(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); struct pkts_if_t t = {}; bpf_probe_read(&t.packets_out, sizeof(u32), &tp->packets_out); bpf_probe_read(&t.sacked_out, sizeof(u32), &tp->sacked_out); bpf_probe_read(&t.lost_out, sizeof(u32), &tp->lost_out); bpf_probe_read(&t.retrans_out, sizeof(u32), &tp->retrans_out); return t.packets_out - (t.sacked_out + t.lost_out) + t.retrans_out; } /** * Tracking open sockets */ struct tcp_open_socket_t { u32 tgid; u32 rcv_holes; u64 last_output; u64 bytes_received; /* last observed */ u32 rcv_delivered; u32 padding; #if TCP_STATS_ON_PARENT struct sock *parent; /* parent listen socket if accepted, null otherwise */ #endif }; struct udp_stats_t { u64 last_output; u32 laddr6[4]; u32 raddr6[4]; u16 lport; u16 rport; u8 addr_changed; u32 bytes; /* bytes seen on socket since last submit */ u32 packets; /* packets seen since last submit */ u32 drops; /* drops total for socket as of last submit (receive side only) */ }; /** * Tracking open sockets */ struct udp_open_socket_t { u32 tgid; struct udp_stats_t stats[2]; }; struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, TGID); __type(value, TGID); __uint(max_entries, TABLE_SIZE__TGID_INFO); } tgid_info_table SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, struct task_struct *); __type(value, struct task_struct *); __uint(max_entries, TABLE_SIZE__DEAD_GROUP_TASKS); } dead_group_tasks SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, u32); __type(value, u32); __uint(max_entries, TABLE_SIZE__SEEN_INODES); __uint(map_flags, BPF_F_NO_PREALLOC); } seen_inodes SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, struct nf_conn *); __type(value, struct nf_conn *); __uint(max_entries, TABLE_SIZE__SEEN_CONNTRACKS); } seen_conntracks SEC(".maps"); // Conntracks that we've reported to userspace // Stash skb per-CPU for __nf_conntrack_confirm entry to kretprobe path. // __nf_conntrack_confirm runs in softirq/non-sleepable context; not re-entrant per-CPU. struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); __type(key, u32); __type(value, struct sk_buff *); } nfct_confirm_skb SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, struct sock *); __type(value, struct tcp_open_socket_t); __uint(max_entries, TABLE_SIZE__TCP_OPEN_SOCKETS); } tcp_open_sockets SEC(".maps"); /* information on live sks */ struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, struct sock *); __type(value, struct udp_open_socket_t); __uint(max_entries, TABLE_SIZE__UDP_OPEN_SOCKETS); } udp_open_sockets SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, u64); __type(value, struct sock *); __uint(max_entries, TABLE_SIZE__UDP_GET_PORT_HASH); } udp_get_port_hash SEC(".maps"); BEGIN_DECLARE_SAVED_ARGS(cgroup_exit) pid_t tgid; END_DECLARE_SAVED_ARGS(cgroup_exit) /* * cgroups subsystem used for cgroup probing * This is used by cgroup related probes to filter out cgroups that aren't in the memory hierarchy. * See SUBSYS macro in /linux_kernel/kernel/cgroup/cgroup.c. */ static __always_inline int get_flow_cgroup_subsys() { return bpf_core_enum_value(enum cgroup_subsys_id, memory_cgrp_id); } #define FLOW_CGROUP_SUBSYS (get_flow_cgroup_subsys()) /* forward declarations */ static __always_inline void perf_check_and_submit_dns(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb, u8 proto, u16 sport, u16 dport, int is_rx); //////////////////////////////////////////////////////////////////////////////////// /* PROCESSES */ static int report_pid_exit(TIMESTAMP timestamp, struct pt_regs *ctx, struct task_struct *task) { perf_submit_agent_internal__pid_exit(ctx, timestamp, task->tgid, task->pid, task->exit_code); return 1; } static int set_task_group_dead(struct pt_regs *ctx, struct task_struct *tsk) { int ret = bpf_map_update_elem(&dead_group_tasks, &tsk, &tsk, BPF_NOEXIST); if (ret != 0) { // Check if key already exists to distinguish duplicate vs table full void *existing = bpf_map_lookup_elem(&dead_group_tasks, &tsk); if (existing) { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("set_task_group_dead: set_task_group_dead duplicate insert, dropping tsk tsk=%llx\n", tsk); #endif bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_DEAD_GROUP_TASKS, 0, (u64)tsk); return 0; } else { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("set_task_group_dead: set_task_group_dead table is full, dropping tsk tsk=%llx\n", tsk); #endif bpf_log(ctx, BPF_LOG_TABLE_FULL, BPF_TABLE_DEAD_GROUP_TASKS, 0, (u64)tsk); return 0; } } return 1; } static int task_group_check_dead_and_remove(struct pt_regs *ctx, struct task_struct *tsk) { int ret = bpf_map_delete_elem(&dead_group_tasks, &tsk); if (ret != 0) { // wasn't in map return 0; } return 1; } static int task_is_group_leader(struct pt_regs *ctx, struct task_struct *tsk) { int ret; if (tsk == NULL) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return 0; } struct task_struct *group_leader = NULL; ret = bpf_probe_read(&group_leader, sizeof(group_leader), &tsk->group_leader); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } if (group_leader == NULL) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return 0; } if (tsk == group_leader) { return 1; } return 0; } static u64 get_task_cgroup(struct pt_regs *ctx, struct task_struct *tsk) { int ret; if (tsk == NULL) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return 0; } struct cgroup *cgrp = (struct cgroup *)BPF_CORE_READ((struct task_struct___with_css_set *)tsk, cgroups, subsys[FLOW_CGROUP_SUBSYS], cgroup); if (cgrp == NULL) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return 0; } return (u64)cgrp; } static pid_t get_task_parent(struct pt_regs *ctx, struct task_struct *tsk) { int ret = 0; pid_t parent_tgid = BPF_CORE_READ(tsk, parent, tgid); ret = 0; if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return -1; } return parent_tgid; } // insert_tgid_info: // adds a task to the 'tgid_info' table // returns 0 if the task was not inserted because it was a duplicate or table full // returns 1 if a task is new and was inserted static int insert_tgid_info(struct pt_regs *ctx, TGID tgid) { int ret = bpf_map_update_elem(&tgid_info_table, &tgid, &tgid, BPF_NOEXIST); if (ret != 0) { // Check if key already exists to distinguish duplicate vs table full void *existing = bpf_map_lookup_elem(&tgid_info_table, &tgid); if (existing) { // if we've already seen it, then don't make a duplicate msg // could arise if a new task shows up during the initial scan // or this is a thread inside an existing group, so we're seeing the tgid again return 0; } else { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("insert_tgid_info: tgid_info table is full, dropping task tgid=%u\n", tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_FULL, BPF_TABLE_TGID_INFO, tgid, tgid); return 0; } } return 1; } // remove_tgid_info: // removes a task to the 'tgid_info' table // returns 0 if the tgid was not found in the table or if there was an error // returns 1 if a task was removed successfully static int remove_tgid_info(struct pt_regs *ctx, TGID tgid) { int ret = bpf_map_delete_elem(&tgid_info_table, &tgid); if (ret != 0) { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("remove_tgid_info: can't remove missing tgid=%u\n", tgid); #endif return 0; } return 1; } // note is the tgid is dead or not // used later by on_cgroup_exit handling SEC("kprobe/taskstats_exit") int on_taskstats_exit(struct pt_regs *ctx) { struct task_struct *tsk = (struct task_struct *)PT_REGS_PARM1(ctx); int group_dead = (int)PT_REGS_PARM2(ctx); if (group_dead) { set_task_group_dead(ctx, tsk); } return 0; } // end // this routine is called by 'do_exit' near the end of the destruction of a task // but after all of the resources has been cleaned up, including file descriptor references SEC("kprobe/cgroup_exit") int on_cgroup_exit(struct pt_regs *ctx) { struct task_struct *tsk = (struct task_struct *)PT_REGS_PARM1(ctx); int ret; pid_t tgid = 0; ret = bpf_probe_read(&tgid, sizeof(tgid), &(tsk->tgid)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } // only consider tasks that are the last task in the thread group // since the process will not be terminating if there are more threads left if (!task_group_check_dead_and_remove(ctx, tsk)) { return 0; } GET_PID_TGID; BEGIN_SAVE_ARGS(cgroup_exit) SAVE_ARG(tgid) END_SAVE_ARGS(cgroup_exit) return 0; } SEC("kretprobe/cgroup_exit") int onret_cgroup_exit(struct pt_regs *ctx) { GET_PID_TGID; GET_ARGS_MISSING_OK(cgroup_exit, args) if (args == NULL) { return 0; } pid_t tgid = args->tgid; DELETE_ARGS(cgroup_exit); // do some cleanup from our hashmap // if we didn't record this tgid before, a log message will have // fired, and won't send this along to the server if (!remove_tgid_info(ctx, tgid)) { return 0; } u64 now = get_timestamp(); char comm[16] = {}; bpf_get_current_comm(comm, sizeof(comm)); perf_submit_agent_internal__pid_close(ctx, now, tgid, (u8 *)comm); return 0; } // set_task_comm: notice when command line is set for a process SEC("kprobe/__set_task_comm") int on_set_task_comm(struct pt_regs *ctx) { struct task_struct *tsk = (struct task_struct *)PT_REGS_PARM1(ctx); const char *buf = (const char *)PT_REGS_PARM2(ctx); int ret; // only interested tasks considered the 'leader' of the group, // effectively userland processes, not threads if (!task_is_group_leader(ctx, tsk)) { return 0; } u64 now = get_timestamp(); pid_t tgid = 0; ret = bpf_probe_read(&tgid, sizeof(tgid), &tsk->tgid); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } perf_submit_agent_internal__pid_set_comm(ctx, now, tgid, (uint8_t *)buf); return 0; } // start SEC("kprobe/wake_up_new_task") int on_wake_up_new_task(struct pt_regs *ctx) { struct task_struct *tsk = (struct task_struct *)PT_REGS_PARM1(ctx); int ret; pid_t tgid = 0; ret = bpf_probe_read(&tgid, sizeof(tgid), &tsk->tgid); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } // mark the thread group id as seen, and if we've already seen it, return if (!insert_tgid_info(ctx, tgid)) { return 0; } u64 now = get_timestamp(); u64 cgroup = get_task_cgroup(ctx, tsk); pid_t parent_tgid = get_task_parent(ctx, tsk); u8 comm[16] = {}; ret = bpf_probe_read(comm, sizeof(comm), tsk->comm); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } perf_submit_agent_internal__pid_info(ctx, now, tgid, comm, cgroup, parent_tgid); return 0; } // existing SEC("kretprobe/get_pid_task") int onret_get_pid_task(struct pt_regs *ctx) { int ret; struct task_struct *tsk = (struct task_struct *)PT_REGS_RC(ctx); pid_t tgid = 0; ret = bpf_probe_read(&tgid, sizeof(tgid), &tsk->tgid); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), (u64)tsk, 0); return 0; } // mark the task as seen, and if we've already seen it, return if (!insert_tgid_info(ctx, tgid)) { return 0; } u64 now = get_timestamp(); u64 cgroup = get_task_cgroup(ctx, tsk); pid_t parent_tgid = get_task_parent(ctx, tsk); u8 comm[16] = {}; ret = bpf_probe_read(comm, sizeof(comm), tsk->comm); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } perf_submit_agent_internal__pid_info(ctx, now, tgid, comm, cgroup, parent_tgid); return 0; } //////////////////////////////////////////////////////////////////////////////////// /* TCP SOCKETS */ static inline u32 tcp_get_delivered(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); /* delivered accounting was introduced in ddf1af6fa00e77, i.e., * v4.6-rc1~91^2~316^2~3 */ if (LINUX_KERNEL_VERSION < KERNEL_VERSION(4, 6, 0)) { u32 packets_out = 0; u32 sacked_out = 0; bpf_probe_read(&packets_out, sizeof(packets_out), &tp->packets_out); bpf_probe_read(&sacked_out, sizeof(sacked_out), &tp->sacked_out); return packets_out - sacked_out; } else { u32 delivered = 0; bpf_probe_read(&delivered, sizeof(delivered), &tp->delivered); return delivered; } } static inline void report_rtt_estimator(struct pt_regs *ctx, struct sock *sk, struct tcp_open_socket_t *sk_info, u64 now, bool adjust) { int ret; sk_info->last_output = now; // update the time in place u32 rcv_rtt_us = 0; if (LINUX_KERNEL_VERSION < KERNEL_VERSION(4, 12, 0)) { bpf_probe_read(&rcv_rtt_us, sizeof(rcv_rtt_us), &((struct tcp_sock___rcv_rtt_est_rtt *)tcp_sk(sk))->rcv_rtt_est.rtt); } else { bpf_probe_read(&rcv_rtt_us, sizeof(rcv_rtt_us), &tcp_sk(sk)->rcv_rtt_est.rtt_us); } // These values need to be taken from bpf_probe_read u32 srtt = 0; u32 snd_cwnd = 0; u64 bytes_acked = 0; u8 ca_state = 0; u32 packets_retrans = 0; u64 bytes_received = 0; ret = bpf_probe_read(&srtt, sizeof(srtt), &(tcp_sk(sk)->srtt_us)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } ret = bpf_probe_read(&snd_cwnd, sizeof(snd_cwnd), &(tcp_sk(sk)->snd_cwnd)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } ret = bpf_probe_read(&bytes_acked, sizeof(bytes_acked), &(tcp_sk(sk)->bytes_acked)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } ret = bpf_probe_read(&ca_state, sizeof(ca_state), &(*(&(inet_csk(sk)->icsk_sync_mss) + 1))); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } ret = bpf_probe_read(&packets_retrans, sizeof(packets_retrans), &(tcp_sk(sk)->total_retrans)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } ret = bpf_probe_read(&bytes_received, sizeof(bytes_received), &(tcp_sk(sk)->bytes_received)); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return; } // adjustment for bytes reported when the connection is closed // zero out last two bits to account for this if (adjust) { bytes_acked &= ~3ull; bytes_received &= ~3ull; } perf_submit_agent_internal__rtt_estimator( ctx, now, srtt, snd_cwnd, bytes_acked, ca_state, (__u64)sk, packets_in_flight_helper(sk), tcp_get_delivered(sk), packets_retrans, sk_info->rcv_holes, bytes_received, sk_info->rcv_delivered, rcv_rtt_us); } // add tcp_open_socket // // should be paired with a `perf_submit_agent_internal__new_sock_created` // in most circumstances, however we don't do this in the case of already // existing sockets. // // returns 1 if it added the socket, 0 if it already existed, // and -1 if the table is full, broken, or out of memory // this operation is atomic and not susceptible to race conditions, because // map.insert is atomic static int add_tcp_open_socket(struct pt_regs *ctx, struct sock *sk, u32 tgid, u64 now, struct sock *parent) { int ret; #if TRACE_TCP_SOCKETS bpf_trace_printk("add_tcp_open_socket: %llx (tgid=%u)\n", sk, tgid); #endif struct tcp_open_socket_t sk_info = { .tgid = tgid, .last_output = 0, // always do the first reporting .rcv_holes = 0, .rcv_delivered = 0, #if TCP_STATS_ON_PARENT .parent = parent #endif }; ret = bpf_probe_read(&sk_info.bytes_received, sizeof(sk_info.bytes_received), &tcp_sk(sk)->bytes_received); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return -1; } ret = bpf_map_update_elem(&tcp_open_sockets, &sk, &sk_info, BPF_NOEXIST); if (ret != 0) { // Check if key already exists to distinguish duplicate vs table full void *existing = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (existing) { return 0; } else { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("add_tcp_open_socket: tcp_open_sockets table is full, dropping socket sk=%llx (tgid=%u)\n", sk, tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_FULL, BPF_TABLE_TCP_OPEN_SOCKETS, tgid, (u64)sk); return -1; } } return 1; } static void remove_tcp_open_socket(struct pt_regs *ctx, struct sock *sk) { struct tcp_open_socket_t *sk_info_p; sk_info_p = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info_p) { return; } #if TRACE_TCP_SOCKETS bpf_trace_printk("remove_tcp_open_socket: %llx\n", sk); #endif // The lookup was successful. Make a copy before deleting the entry, and only send related telemetry if the delete is // successful. struct tcp_open_socket_t sk_info = *sk_info_p; // do some cleanup from our hashmap int ret = bpf_map_delete_elem(&tcp_open_sockets, &sk); if (ret != 0) { // Another thread must have already deleted the entry for this sk. return; } // always report last rtt estimator before close // for short-lived connections, we won't see any data otherwise u64 now = get_timestamp(); report_rtt_estimator(ctx, sk, &sk_info, now, true); perf_submit_agent_internal__close_sock_info(ctx, now, (__u64)sk); } static inline void submit_set_state_ipv6(struct pt_regs *ctx, u64 now, int tx_rx, struct sock *sk) { struct sock *skp = NULL; bpf_probe_read(&skp, sizeof(skp), &sk); if (!skp) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return; } u16 dport = 0; u16 sport = 0; uint8_t daddr[16] = {}; uint8_t saddr[16] = {}; // These values need to be taken from bpf_probe_read bpf_probe_read(&dport, sizeof(dport), &(skp->sk_dport)); bpf_probe_read(&sport, sizeof(sport), &(skp->sk_num)); bpf_probe_read(daddr, sizeof(daddr), (uint8_t *)(sk->sk_v6_daddr.in6_u.u6_addr32)); bpf_probe_read(saddr, sizeof(saddr), (uint8_t *)(sk->sk_v6_rcv_saddr.in6_u.u6_addr32)); perf_submit_agent_internal__set_state_ipv6(ctx, now, daddr, saddr, bpf_ntohs(dport), sport, (__u64)sk, tx_rx); } // state - we want to get the 5-tuple as early as possible. static inline void submit_set_state_ipv4(struct pt_regs *ctx, u64 now, int tx_rx, struct sock *sk) { struct sock *skp = NULL; bpf_probe_read(&skp, sizeof(skp), &sk); if (!skp) { bpf_log(ctx, BPF_LOG_INVALID_POINTER, 0, 0, 0); return; } u16 dport = 0; u16 sport = 0; u32 daddr = 0; u32 saddr = 0; // These values need to be taken from bpf_probe_read bpf_probe_read(&dport, sizeof(dport), &(skp->sk_dport)); bpf_probe_read(&sport, sizeof(sport), &(skp->sk_num)); bpf_probe_read_kernel(&daddr, sizeof(daddr), &sk->sk_daddr); bpf_probe_read_kernel(&saddr, sizeof(saddr), &sk->sk_rcv_saddr); perf_submit_agent_internal__set_state_ipv4(ctx, now, daddr, saddr, bpf_ntohs(dport), sport, (__u64)sk, tx_rx); } static inline void submit_reset_tcp_counters(struct pt_regs *ctx, u64 now, u64 pid, struct sock *sk) { int ret; u64 bytes_acked = 0; u32 packets_retrans = 0; u64 bytes_received = 0; bytes_acked = BPF_CORE_READ(tcp_sk(sk), bytes_acked); packets_retrans = BPF_CORE_READ(tcp_sk(sk), total_retrans); bytes_received = BPF_CORE_READ(tcp_sk(sk), bytes_received); perf_submit_agent_internal__reset_tcp_counters( ctx, now, (__u64)sk, bytes_acked, tcp_get_delivered(sk), packets_retrans, bytes_received, pid); } // common logic for handling existing sockets static int ensure_tcp_existing(struct pt_regs *ctx, struct sock *sk, u32 pid) { if (!sk) { return -1; } u16 family = BPF_CORE_READ(sk, sk_family); if (family != AF_INET && family != AF_INET6) { return -1; } u64 now = get_timestamp(); int tx_rx = 0; u8 state = BPF_CORE_READ(sk, sk_state); if (state == TCP_LISTEN) { tx_rx = 2; } int ret = add_tcp_open_socket(ctx, sk, pid, now, NULL); if (ret == 1) { submit_reset_tcp_counters(ctx, now, pid, sk); if (family == AF_INET6) { submit_set_state_ipv6(ctx, now, tx_rx, sk); } else { submit_set_state_ipv4(ctx, now, tx_rx, sk); } } return ret; } // // HACK: Add tcp sockets that should exist to the table but don't // #if TCP_EXISTING_HACK static struct tcp_open_socket_t *tcp_existing_hack(struct pt_regs *ctx, struct sock *sk) { // Can only do this hack if we are in a userland process context PID_TGID _pid_tgid = bpf_get_current_pid_tgid(); TGID _tgid = TGID_FROM_PID_TGID(_pid_tgid); PID _pid = PID_FROM_PID_TGID(_pid_tgid); if (_tgid == 0 || _pid == 0) { return NULL; } // ensure the tcp socket exists int ret = ensure_tcp_existing(ctx, sk, _tgid); #if DEBUG_TCP_SOCKET_ERRORS if (ret == 1) { bpf_trace_printk( "tcp_update_stats: add_tcp_open_socket of missing socket: %llx (tgid=%u, cpu=%u)\n", sk, _tgid, bpf_get_smp_processor_id()); bpf_log(ctx, BPF_LOG_EXISTING_HACK, BPF_TABLE_TCP_OPEN_SOCKETS, (u64)sk, 0); } else if (ret == 0) { // already existing, okay } else if (ret < 0) { bpf_trace_printk("tcp_update_stats: add_tcp_open_socket failed: %llx (tgid=%u)\n", sk, _tgid); } #endif struct tcp_open_socket_t *sk_info_p = bpf_map_lookup_elem(&tcp_open_sockets, &sk); return sk_info_p; } #endif // Ends a tcp socket lifetime and starts a new one // Prep for Ticket #1166 static void restart_tcp_socket(struct pt_regs *ctx, TIMESTAMP now, struct sock *sk) { GET_PID_TGID; // Connect to af_unspec starts a new span remove_tcp_open_socket(ctx, sk); // add an entry to our hash map int ret = add_tcp_open_socket(ctx, sk, _tgid, now, NULL); if (ret == 1) { perf_submit_agent_internal__new_sock_created(ctx, now, _tgid, (__u64)sk); } else if (ret == 0) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("tcp_init_sock: add_tcp_open_socket of existing socket: %llx (tgid=%u)\n", sk, _tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, _tgid, (u64)sk); } else { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("tcp_init_sock: add_tcp_open_socket failed: %llx (tgid=%u)\n", sk, _tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, _tgid, abs_val(ret)); } } // connectors SEC("kprobe/tcp_connect") int on_tcp_connect(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { #if TCP_EXISTING_HACK sk_info = tcp_existing_hack(ctx, sk); if (sk_info == NULL) { return 0; } #else bpf_log(ctx, BPF_LOG_TABLE_MISSING_KEY, BPF_TABLE_TCP_OPEN_SOCKETS, (u64)sk, 0); return 0; #endif } u64 now = get_timestamp(); int tx_rx = 1; u16 family = BPF_CORE_READ(sk, sk_family); if (family == AF_INET) { submit_set_state_ipv4(ctx, now, tx_rx, sk); } else if (family == AF_INET6) { submit_set_state_ipv6(ctx, now, tx_rx, sk); } else { bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); } return 0; } // listeners (acceptors) // XXX: this function might fail so maybe we want to do some sort of kretprobe? // I assume that if it fails, the user will handle things in a clever way // and destroy the socket. But who knows? SEC("kprobe/inet_csk_listen_start") int on_inet_csk_listen_start(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); // filter out non-tcp connections u16 family = BPF_CORE_READ(sk, sk_family); if (family != AF_INET && family != AF_INET6) return 0; struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { #if TCP_EXISTING_HACK sk_info = tcp_existing_hack(ctx, sk); if (sk_info == NULL) { return 0; } #else bpf_log(ctx, BPF_LOG_TABLE_MISSING_KEY, BPF_TABLE_TCP_OPEN_SOCKETS, (u64)sk, 0); return 0; #endif } u64 now = get_timestamp(); int tx_rx = 2; // this is the listener if (family == AF_INET) { submit_set_state_ipv4(ctx, now, tx_rx, sk); } else if (family == AF_INET6) { submit_set_state_ipv6(ctx, now, tx_rx, sk); } else { bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); } return 0; } #if TCP_LIFETIME_HACK // // HACK: Remove open socket from table if it exists // static void tcp_lifetime_hack(struct pt_regs *ctx, struct sock *sk) { struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (sk_info) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("tcp_lifetime_hack: tcp_lifetime_hack, sk=%llx, cpu=%u\n", sk, bpf_get_smp_processor_id()); bpf_log(ctx, BPF_LOG_LIFETIME_HACK, BPF_TABLE_TCP_OPEN_SOCKETS, (u64)sk, 0); #endif remove_tcp_open_socket(ctx, sk); } } #endif // --- tcp_init_sock ---------------------------------------------------- // Where the start of TCP socket lifetimes is for IPv4 and IPv6 SEC("kprobe/tcp_init_sock") int on_tcp_init_sock(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); GET_PID_TGID; u64 now = get_timestamp(); #if TCP_LIFETIME_HACK // remove the entry from our hash map if it's there tcp_lifetime_hack(ctx, sk); #endif // add an entry to our hash map int ret = add_tcp_open_socket(ctx, sk, _tgid, now, NULL); if (ret == 1) { perf_submit_agent_internal__new_sock_created(ctx, now, _tgid, (__u64)sk); } else if (ret == 0) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("tcp_init_sock: add_tcp_open_socket of existing socket: %llx (tgid=%u)\n", sk, _tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, _tgid, (u64)sk); } else { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("tcp_init_sock: add_tcp_open_socket failed: %llx (tgid=%u)\n", sk, _tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, _tgid, abs_val(ret)); } return 0; } // --- inet_csk_accept -------------------------------------------------- // Called when a listen socket accepts gets a connection BEGIN_DECLARE_SAVED_ARGS(on_inet_csk_accept) struct sock *sk; int flags; u32 _pad_0; // required alignment int *err; END_DECLARE_SAVED_ARGS(on_inet_csk_accept) SEC("kprobe/inet_csk_accept") int on_inet_csk_accept(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); int flags = (int)PT_REGS_PARM2(ctx); int *err = (int *)PT_REGS_PARM3(ctx); // In kernels before 4.11, there's no bool kern parameter // The kern parameter doesn't exist, so we ignore it // bool kern = (bool)PT_REGS_PARM4(ctx); GET_PID_TGID; #if TCP_STATS_ON_PARENT #if TCP_EXISTING_HACK // Only need to do this if we are using the parent stats reporting and need the socket to exist struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { sk_info = tcp_existing_hack(ctx, sk); if (sk_info == NULL) { return 0; } } #endif #endif // Link us to our kretprobe BEGIN_SAVE_ARGS(on_inet_csk_accept) SAVE_ARG(sk) SAVE_ARG(flags) SAVE_ARG(err) END_SAVE_ARGS(on_inet_csk_accept) return 0; } SEC("kretprobe/inet_csk_accept") int onret_inet_csk_accept(struct pt_regs *ctx) { GET_PID_TGID; // Ensure the accept succeeded struct sock *newsk = (struct sock *)PT_REGS_RC(ctx); if (!newsk) { #if TRACE_TCP_SOCKETS DEBUG_PRINTK("onret_inet_csk_accept: accept failed, tgid=%u\n", _tgid); #endif // Unlink the kprobe/kretprobe DELETE_ARGS(on_inet_csk_accept); return 0; } // filter out non-tcp connections u16 family = 0; bpf_probe_read(&family, sizeof(family), &newsk->sk_family); if (family != AF_INET && family != AF_INET6) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("onret_inet_csk_accept: family is not ipv4 or ipv6 sk=%llx\n", newsk); #endif bpf_log(ctx, BPF_LOG_UNEXPECTED_TYPE, (u64)newsk, (u64)family, 0); DELETE_ARGS(on_inet_csk_accept); return 0; } // Link us from our kprobe GET_ARGS(on_inet_csk_accept, args); if (args == NULL) { // Race condition where we might have been inside an accept when the probe // was inserted, ignore return 0; } // Set up the child socket u64 now = get_timestamp(); #if TCP_LIFETIME_HACK // remove the entry from our hash map if it's there tcp_lifetime_hack(ctx, newsk); #endif #if TCP_STATS_ON_PARENT int ret = add_tcp_open_socket(ctx, newsk, _tgid, now, args->sk); #else int ret = add_tcp_open_socket(ctx, newsk, _tgid, now, NULL); #endif if (ret == 1) { // Socket doesn't exist yet, create it in our table perf_submit_agent_internal__new_sock_created(ctx, now, _tgid, (__u64)newsk); } else if (ret == 0) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("onret_inet_csk_accept: add_tcp_open_socket of existing socket: %llx (tgid=%u)\n", newsk, _tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, _tgid, (u64)newsk); DELETE_ARGS(on_inet_csk_accept); return 0; } else { // no log here because we already had another log inside add_tcp_open_socket for this DELETE_ARGS(on_inet_csk_accept); return 0; } // Set the state u16 dport = 0; u16 sport = 0; bpf_probe_read(&dport, sizeof(dport), &newsk->sk_dport); bpf_probe_read(&sport, sizeof(sport), &newsk->sk_num); if (family == AF_INET) { u32 daddr = 0; u32 saddr = 0; bpf_probe_read_kernel(&daddr, sizeof(daddr), &newsk->sk_daddr); bpf_probe_read_kernel(&saddr, sizeof(saddr), &newsk->sk_rcv_saddr); perf_submit_agent_internal__set_state_ipv4(ctx, now, daddr, saddr, bpf_ntohs(dport), sport, (__u64)newsk, 2); } else if (family == AF_INET6) { uint8_t daddr[16] = {}; uint8_t saddr[16] = {}; bpf_probe_read(daddr, sizeof(daddr), (uint8_t *)(newsk->sk_v6_daddr.in6_u.u6_addr32)); bpf_probe_read(saddr, sizeof(saddr), (uint8_t *)(newsk->sk_v6_rcv_saddr.in6_u.u6_addr32)); perf_submit_agent_internal__set_state_ipv6(ctx, now, daddr, saddr, bpf_ntohs(dport), sport, (__u64)newsk, 2); } #if TRACE_TCP_SOCKETS DEBUG_PRINTK("on_inet_csk_accept exit: accepted socket %llx, tgid=%u\n", newsk, _tgid); #endif DELETE_ARGS(on_inet_csk_accept); return 0; } // existing static int tcp46_seq_show_impl(struct pt_regs *ctx, struct seq_file *seq, void *v) { struct sock *sk = v; u32 ino = BPF_CORE_READ(sk, sk_socket, file, f_inode, i_ino); u32 *lookup_tgid = bpf_map_lookup_elem(&seen_inodes, &ino); if (!lookup_tgid) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("on_tcp46_seq_show: ignoring socket %llx because of missing inode %u\n", sk, ino); #endif return 0; } u32 tgid = *lookup_tgid; #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("on_tcp46_seq_show: creating socket %llx with inode %u\n", sk, ino); #endif /* we don't want to explore this inode again */ bpf_map_delete_elem(&seen_inodes, &ino); int ret = ensure_tcp_existing(ctx, sk, tgid); if (ret == 0) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("on_tcp46_seq_show: add_tcp_open_socket of existing socket: %llx (tgid=%u)\n", sk, tgid); bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, tgid, (u64)sk); #endif } else if (ret < 0) { #if DEBUG_TCP_SOCKET_ERRORS bpf_trace_printk("on_tcp46_seq_show: add_tcp_open_socket failed: %llx (tgid=%u)\n", sk, tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_OPEN_SOCKETS, tgid, abs_val(ret)); } return 0; } SEC("kprobe/tcp4_seq_show") int on_tcp4_seq_show(struct pt_regs *ctx) { struct seq_file *seq = (struct seq_file *)PT_REGS_PARM1(ctx); void *v = (void *)PT_REGS_PARM2(ctx); return tcp46_seq_show_impl(ctx, seq, v); } SEC("kprobe/tcp6_seq_show") int on_tcp6_seq_show(struct pt_regs *ctx) { struct seq_file *seq = (struct seq_file *)PT_REGS_PARM1(ctx); void *v = (void *)PT_REGS_PARM2(ctx); return tcp46_seq_show_impl(ctx, seq, v); } //////////////////////////////////////////////////////////////////////////////////// /* UDP SOCKETS */ // add udp_open_socket // // should be paired with a `perf_submit_agent_internal__udp_new_socket` // in most circumstances, however we don't do this in the case of already // existing sockets. // // returns 1 if it added the socket, 0 if it already existed, // and -1 if the table is full, broken, or out of memory // this operation is atomic and not susceptible to race conditions, because // map.insert is atomic static __always_inline int add_udp_open_socket(struct pt_regs *ctx, struct sock *sk, u32 tgid, u64 now) { #if TRACE_UDP_SOCKETS bpf_trace_printk("add_udp_open_socket: %llx (tgid=%u, cpu=%u)\n", sk, tgid, bpf_get_smp_processor_id()); #endif struct udp_open_socket_t sk_info = { .tgid = tgid, }; int ret = bpf_map_update_elem(&udp_open_sockets, &sk, &sk_info, BPF_NOEXIST); if (ret != 0) { // Check if key already exists to distinguish duplicate vs table full void *existing = bpf_map_lookup_elem(&udp_open_sockets, &sk); if (existing) { return 0; } else { #if DEBUG_UDP_SOCKET_ERRORS bpf_trace_printk("add_udp_open_socket: udp_open_sockets table is full, dropping socket sk=%llx (tgid=%u)\n", sk, tgid); bpf_log(ctx, BPF_LOG_TABLE_FULL, BPF_TABLE_UDP_OPEN_SOCKETS, (u64)tgid, (u64)sk); #endif return -1; } } return 1; } static __always_inline int ensure_udp_existing(struct pt_regs *ctx, struct sock *sk, u32 tgid) { u16 family = BPF_CORE_READ(sk, sk_family); if (family != AF_INET && family != AF_INET6) { return -1; } u64 now = get_timestamp(); int ret = add_udp_open_socket(ctx, sk, tgid, now); if (ret == 1) { struct in6_addr addr; if (family == AF_INET6) { addr = BPF_CORE_READ(sk, sk_v6_rcv_saddr); } else { addr.s6_addr32[0] = addr.s6_addr32[1] = 0; addr.s6_addr16[4] = 0; addr.s6_addr16[5] = 0xffff; addr.s6_addr32[3] = BPF_CORE_READ(sk, sk_rcv_saddr); } u16 lport = 0; bpf_probe_read(&lport, sizeof(lport), &(inet_sk(sk)->inet_num)); perf_submit_agent_internal__udp_new_socket(ctx, now, tgid, (__u64)sk, (uint8_t *)(&addr), lport); } return ret; } // // HACK: Add tcp sockets that should exist to the table but don't // #if UDP_EXISTING_HACK static __always_inline struct udp_open_socket_t *udp_existing_hack(struct pt_regs *ctx, struct sock *sk) { // Can only do this hack if we are in a userland process context PID_TGID _pid_tgid = bpf_get_current_pid_tgid(); TGID _tgid = TGID_FROM_PID_TGID(_pid_tgid); PID _pid = PID_FROM_PID_TGID(_pid_tgid); if (_tgid == 0 || _pid == 0) { return NULL; } // Ensure the udp socket exists int ret = ensure_udp_existing(ctx, sk, _tgid); #if DEBUG_UDP_SOCKET_ERRORS if (ret == 1) { bpf_trace_printk( "udp_existing_hack: add_udp_open_socket of missing socket: %llx (tgid=%u, cpu=%u)\n", sk, _tgid, bpf_get_smp_processor_id()); bpf_log(ctx, BPF_LOG_EXISTING_HACK, BPF_TABLE_UDP_OPEN_SOCKETS, (u64)sk, 0); } else if (ret == 0) { // already existed, okay } else if (ret < 0) { bpf_trace_printk("udp_existing_hack: add_udp_open_socket failed: %llx (tgid=%u)\n", sk, _tgid); } #endif struct udp_open_socket_t *sk_info_p = bpf_map_lookup_elem(&udp_open_sockets, &sk); return sk_info_p; } #endif static __always_inline void udp_send_stats_if_nonempty(struct pt_regs *ctx, u64 now, struct sock *sk, struct udp_stats_t *stats, u8 is_rx) { /* is no data to send, return */ u32 sk_drops_counter = BPF_CORE_READ(sk, sk_drops.counter); if (!stats->addr_changed && stats->packets == 0 && sk_drops_counter == stats->drops) return; // bpf_trace_printk("is_rx: %d sk->sk_drops.counter: %d stats->drops: // %d\n", is_rx, sk->sk_drops.counter, stats->drops); /* there is data, send a report */ /* only send drops on receive side, since udp sends never drop packets */ perf_submit_agent_internal__udp_stats( ctx, now, (__u64)sk, (uint8_t *)(stats->raddr6), stats->packets, stats->bytes, stats->addr_changed, stats->rport, is_rx, (uint8_t *)(stats->laddr6), stats->lport, is_rx ? (sk_drops_counter - stats->drops) : 0); } static void remove_udp_open_socket(struct pt_regs *ctx, struct sock *sk) { struct udp_open_socket_t *sk_info_p; sk_info_p = bpf_map_lookup_elem(&udp_open_sockets, &sk); if (!sk_info_p) { return; } #if TRACE_UDP_SOCKETS bpf_trace_printk("remove_udp_open_socket: %llx (cpu=%u)\n", sk, bpf_get_smp_processor_id()); #endif // The lookup was successful. Make a copy before deleting the entry, and only send related telemetry if the delete is // successful. struct udp_open_socket_t sk_info = *sk_info_p; int ret = bpf_map_delete_elem(&udp_open_sockets, &sk); if (ret != 0) { // Another thread must have already deleted the entry for this sk. return; } u64 now = get_timestamp(); udp_send_stats_if_nonempty(ctx, now, sk, &sk_info.stats[0], 0); udp_send_stats_if_nonempty(ctx, now, sk, &sk_info.stats[1], 1); perf_submit_agent_internal__udp_destroy_socket(ctx, now, (__u64)sk); } #if UDP_LIFETIME_HACK // // HACK: Remove open socket from table if it exists // static void udp_lifetime_hack(struct pt_regs *ctx, struct sock *sk) { struct udp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&udp_open_sockets, &sk); if (sk_info) { #if DEBUG_UDP_SOCKET_ERRORS bpf_trace_printk("udp_lifetime_hack: udp_lifetime_hack sk=%llx cpu=%u\n", sk, bpf_get_smp_processor_id()); bpf_log(ctx, BPF_LOG_LIFETIME_HACK, BPF_TABLE_UDP_OPEN_SOCKETS, (u64)sk, 0); #endif remove_udp_open_socket(ctx, sk); } } #endif // EXISTING static int udp46_seq_show_impl(struct pt_regs *ctx, struct seq_file *seq, void *v) { struct sock *sk = v; u32 ino = BPF_CORE_READ(sk, sk_socket, file, f_inode, i_ino); u32 *lookup_tgid = bpf_map_lookup_elem(&seen_inodes, &ino); if (!lookup_tgid) { #if DEBUG_UDP_SOCKET_ERRORS bpf_trace_printk("on_udp46_seq_show: ignoring socket %llx because of missing inode %u\n", sk, ino); #endif return 0; } u32 tgid = *lookup_tgid; /* we don't want to explore this inode again */ bpf_map_delete_elem(&seen_inodes, &ino); int ret = ensure_udp_existing(ctx, sk, tgid); if (ret == 0) { #if DEBUG_UDP_SOCKET_ERRORS bpf_trace_printk("on_udp46_seq_show: add_udp_open_socket of existing socket: %llx (tgid=%u)\n", sk, tgid); bpf_log(ctx, BPF_LOG_TABLE_DUPLICATE_INSERT, BPF_TABLE_UDP_OPEN_SOCKETS, tgid, (u64)sk); #endif } else if (ret < 0) { #if DEBUG_UDP_SOCKET_ERRORS bpf_trace_printk("on_udp46_seq_show: add_udp_open_socket failed: %llx (tgid=%u)\n", sk, tgid); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_UDP_OPEN_SOCKETS, tgid, abs_val(ret)); } return 0; } SEC("kprobe/udp4_seq_show") int on_udp4_seq_show(struct pt_regs *ctx) { struct seq_file *seq = (struct seq_file *)PT_REGS_PARM1(ctx); void *v = (void *)PT_REGS_PARM2(ctx); return udp46_seq_show_impl(ctx, seq, v); } SEC("kprobe/udp6_seq_show") int on_udp6_seq_show(struct pt_regs *ctx) { struct seq_file *seq = (struct seq_file *)PT_REGS_PARM1(ctx); void *v = (void *)PT_REGS_PARM2(ctx); return udp46_seq_show_impl(ctx, seq, v); } // NEW static int udp_v46_get_port_impl(struct pt_regs *ctx, struct sock *sk) { GET_PID_TGID; int ret = bpf_map_update_elem(&udp_get_port_hash, &_cpu, &sk, BPF_NOEXIST); if (ret != 0) { // Check if key already exists to distinguish duplicate vs table full void *existing = bpf_map_lookup_elem(&udp_get_port_hash, &_cpu); if (existing) { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("on_udp_v46_get_port: should not see existing hash entry (tgid=%u, cpu=%u)\n", _tgid, _cpu); #endif return 0; } else { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("on_udp_v46_get_port: udp_get_port_hash table is full, dropping call (tgid=%u, cpu=%u)\n", _tgid, _cpu); #endif return 0; } } return 0; } SEC("kprobe/udp_v4_get_port") int on_udp_v4_get_port(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); return udp_v46_get_port_impl(ctx, sk); } SEC("kprobe/udp_v6_get_port") int on_udp_v6_get_port(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); return udp_v46_get_port_impl(ctx, sk); } static int onret_udp_get_port_impl(struct pt_regs *ctx) { struct sock **found = NULL; struct sock *sk = NULL; GET_PID_TGID; int retval = (int)PT_REGS_RC(ctx); found = bpf_map_lookup_elem(&udp_get_port_hash, &_cpu); if (!found) return 0; sk = *found; /** * on success, udp_lib_get_port sets inet_sk(sk)->inet_num. * * get_port is called from: * - inet_autobind; does not set inet->inet_rcv_saddr * - inet_bind, which sets inet->inet_rcv_saddr and inet->inet_saddr * just before calling get_port * */ if (retval == 0) { #if UDP_LIFETIME_HACK // remove the entry from our hash map if it's there udp_lifetime_hack(ctx, sk); #endif int ret = ensure_udp_existing(ctx, sk, _tgid); #if DEBUG_UDP_SOCKET_ERRORS if (ret == 0) { bpf_trace_printk( "common_ret_udp_v46_get_port: add_udp_open_socket of existing socket: %llx (tgid=%u, cpu=%u)\n", sk, _tgid, _cpu); } else if (ret < 0) { bpf_trace_printk("common_ret_udp_v46_get_port: add_udp_open_socket failed: %llx (tgid=%u, cpu=%u)\n", sk, _tgid, _cpu); } #endif } bpf_map_delete_elem(&udp_get_port_hash, &_cpu); return 0; } SEC("kretprobe/udp_v4_get_port") int onret_udp_v4_get_port(struct pt_regs *ctx) { return onret_udp_get_port_impl(ctx); } SEC("kretprobe/udp_v6_get_port") int onret_udp_v6_get_port(struct pt_regs *ctx) { return onret_udp_get_port_impl(ctx); } /* * socket lifetime end * Note: This function is called from on_security_sk_free() and on_inet_release(). Historically, only the security_sk_free() * kprobe was used, but there were some unexplained missing socket close events that seemed to be due to security_sk_free() * probes not being triggered from certain kernel thread states/contexts. The inet_release() kprobe was added to make sure all * socket close events were captured. inet_release() calls security_sk_free(), synchronously in some cases and asynchronously * via call_rcu() in other cases. Asynchronous calls can result in multiple threads executing this and subsequent functions * concurrently, so they need to deal with tcp/udp_open_sockets map lookup and delete failures appropriately to make sure only * one of the calls sends the final socket telemetry. (i.e. If a lookup or delete fails, assume that the other thread already * successfully deleted the entry, and only send the final telemetry if the delete is successful.) */ static inline void remove_open_socket(struct pt_regs *ctx, struct sock *sk) { { // TCP(v4/v6) Socket? struct tcp_open_socket_t *tcp_sk_info; tcp_sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (tcp_sk_info) { remove_tcp_open_socket(ctx, sk); return; } } { // UDP(v4/v6) Socket? struct udp_open_socket_t *udp_sk_info; udp_sk_info = bpf_map_lookup_elem(&udp_open_sockets, &sk); if (udp_sk_info) { remove_udp_open_socket(ctx, sk); return; } } // Must be some other kind of socket we don't yet care about } // --- security_sk_free ---------------------------------------------------- // This is where final socket destruction happens for all socket types SEC("kprobe/security_sk_free") int on_security_sk_free(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); remove_open_socket(ctx, sk); return 0; } BEGIN_DECLARE_SAVED_ARGS(inet_release) struct sock *sk; END_DECLARE_SAVED_ARGS(inet_release) SEC("kprobe/inet_release") int on_inet_release(struct pt_regs *ctx) { struct socket *sock = (struct socket *)PT_REGS_PARM1(ctx); GET_PID_TGID; struct sock *sk = NULL; bpf_probe_read(&sk, sizeof(sk), &sock->sk); if (!sk) { return 0; } BEGIN_SAVE_ARGS(inet_release) SAVE_ARG(sk) END_SAVE_ARGS(inet_release) return 0; } SEC("kretprobe/inet_release") int onret_inet_release(struct pt_regs *ctx) { GET_PID_TGID; GET_ARGS_MISSING_OK(inet_release, args); if (args == NULL) { return 0; } struct sock *sk = args->sk; DELETE_ARGS(inet_release); if (sk) { remove_open_socket(ctx, sk); } return 0; } // TCP reset static void handle_tcp_reset(struct pt_regs *ctx, struct sock *sk, u8 is_rx) { if (!bpf_map_lookup_elem(&tcp_open_sockets, &sk)) { // don't send an event on sockets we never notified userspace about // bpf_trace_printk("handle_tcp_reset: no socket\n"); return; } // bpf_trace_printk("handle_tcp_reset: perf_submit_agent_internal__tcp_reset\n"); u64 now = get_timestamp(); perf_submit_agent_internal__tcp_reset(ctx, now, (__u64)sk, is_rx); } // receive TCP RST SEC("kprobe/tcp_reset") int on_tcp_reset(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); // bpf_trace_printk("on_tcp_reset\n"); // receive RST (is_rx = 1) handle_tcp_reset(ctx, sk, 1); return 0; } //////////////////////////////////////////////////////////////////////////////////// /* LIVE UDP */ // laddr, raddr is in big endian format, lport and rport are in little endian // format static __always_inline void udp_update_stats( struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb, struct in6_addr *laddr, u16 lport, struct in6_addr *raddr, u16 rport, u8 is_rx) { u16 family = BPF_CORE_READ(sk, sk_family); struct udp_open_socket_t *sk_info_p; struct udp_stats_t *stats; sk_info_p = bpf_map_lookup_elem(&udp_open_sockets, &sk); if (sk_info_p == NULL) { #if UDP_EXISTING_HACK sk_info_p = udp_existing_hack(ctx, sk); if (sk_info_p == NULL) { return; } #else bpf_log(ctx, BPF_LOG_TABLE_MISSING_KEY, BPF_TABLE_UDP_OPEN_SOCKETS, (u64)sk, 0); return; #endif } stats = &sk_info_p->stats[is_rx]; /* compare sk_info_p's address and the one in fl4 */ u32 lchanged = (stats->laddr6[0] ^ laddr->s6_addr32[0]) | (stats->laddr6[1] ^ laddr->s6_addr32[1]) | (stats->laddr6[2] ^ laddr->s6_addr32[2]) | (stats->laddr6[3] ^ laddr->s6_addr32[3]) | (stats->lport ^ lport); u32 rchanged = (stats->raddr6[0] ^ raddr->s6_addr32[0]) | (stats->raddr6[1] ^ raddr->s6_addr32[1]) | (stats->raddr6[2] ^ raddr->s6_addr32[2]) | (stats->raddr6[3] ^ raddr->s6_addr32[3]) | (stats->rport ^ rport); u32 sk_drops_counter = BPF_CORE_READ(sk, sk_drops.counter); u32 dchanged = is_rx ? (sk_drops_counter != stats->drops) : 0; u32 changed = lchanged | rchanged | dchanged; u64 now = get_timestamp(); if (changed || ((now - stats->last_output) >= filter_ns)) { /* set the address */ if (lchanged) { stats->laddr6[0] = laddr->s6_addr32[0]; stats->laddr6[1] = laddr->s6_addr32[1]; stats->laddr6[2] = laddr->s6_addr32[2]; stats->laddr6[3] = laddr->s6_addr32[3]; stats->lport = lport; } if (rchanged) { stats->raddr6[0] = raddr->s6_addr32[0]; stats->raddr6[1] = raddr->s6_addr32[1]; stats->raddr6[2] = raddr->s6_addr32[2]; stats->raddr6[3] = raddr->s6_addr32[3]; stats->rport = rport; } stats->addr_changed = (lchanged || rchanged) ? (u8)family : 0; /* send the update if anything has changed, or if we have statistics */ udp_send_stats_if_nonempty(ctx, now, sk, stats, is_rx); /* reset statistics */ stats->packets = 1; stats->drops = is_rx ? sk_drops_counter : 0; stats->bytes = BPF_CORE_READ(skb, len); /* schedule next update */ stats->last_output = now; return; } /* address is the same and too early to send a notification, just update */ stats->packets++; stats->bytes += BPF_CORE_READ(skb, len); /** don't update 'drops' here, because it's not cumulative, * it's a total since last reset */ return; } /* Tail calls used by kprobes below, so we can have enough stack space */ struct { __uint(type, BPF_MAP_TYPE_PROG_ARRAY); __uint(max_entries, NUM_TAIL_CALLS); __uint(key_size, sizeof(__u32)); __uint(value_size, sizeof(__u32)); } tail_calls SEC(".maps"); SEC("kprobe") int on_udp_send_skb__2(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); struct flowi4 *fl4 = (struct flowi4 *)PT_REGS_PARM2(ctx); if (!skb || !fl4) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); perf_check_and_submit_dns(ctx, sk, skb, IPPROTO_UDP, BPF_CORE_READ(fl4, fl4_sport), BPF_CORE_READ(fl4, fl4_dport), 0); return 0; } SEC("kprobe") int on_udp_v6_send_skb__2(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); struct flowi6 *fl6 = (struct flowi6 *)PT_REGS_PARM2(ctx); if (!skb || !fl6) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); u16 sport = BPF_CORE_READ(fl6, fl6_sport); u16 dport = BPF_CORE_READ(fl6, fl6_dport); perf_check_and_submit_dns(ctx, sk, skb, IPPROTO_UDP, sport, dport, 0); return 0; } SEC("kprobe") int on_ip_send_skb__2(struct pt_regs *ctx) { struct net *net = (struct net *)PT_REGS_PARM1(ctx); struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); if (!skb) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); u16 network_header = BPF_CORE_READ(skb, network_header); u16 transport_header = BPF_CORE_READ(skb, transport_header); unsigned char *skb_head = BPF_CORE_READ(skb, head); struct iphdr *ip_hdr = (struct iphdr *)(skb_head + network_header); u8 protocol = BPF_CORE_READ(ip_hdr, protocol); if (protocol == IPPROTO_UDP) { struct udphdr *udp_hdr = (struct udphdr *)(skb_head + transport_header); u16 source = BPF_CORE_READ(udp_hdr, source); u16 dest = BPF_CORE_READ(udp_hdr, dest); perf_check_and_submit_dns(ctx, sk, skb, IPPROTO_UDP, source, dest, 0); } return 0; } SEC("kprobe") int on_ip6_send_skb__2(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); if (!skb) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); unsigned char *skb_head = BPF_CORE_READ(skb, head); struct ipv6hdr *ipv6_hdr = (struct ipv6hdr *)(skb_head + BPF_CORE_READ(skb, network_header)); if (BPF_CORE_READ(ipv6_hdr, nexthdr) == IPPROTO_UDP) { struct udphdr *udp_hdr = (struct udphdr *)(skb_head + BPF_CORE_READ(skb, transport_header)); perf_check_and_submit_dns(ctx, sk, skb, IPPROTO_UDP, BPF_CORE_READ(udp_hdr, source), BPF_CORE_READ(udp_hdr, dest), 0); } return 0; } /* udp kprobes, reference the tail calls above */ // may be optimized out on some kernels, if hook fails, we will use // on_ip_send_skb SEC("kprobe/udp_send_skb") int on_udp_send_skb(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); struct flowi4 *fl4 = (struct flowi4 *)PT_REGS_PARM2(ctx); if (!skb || !fl4) return 0; struct in6_addr laddr = make_ipv6_address(BPF_CORE_READ(fl4, saddr)); struct in6_addr raddr = make_ipv6_address(BPF_CORE_READ(fl4, daddr)); struct sock *sk_ptr = BPF_CORE_READ(skb, sk); if (!sk_ptr) return 0; udp_update_stats(ctx, sk_ptr, skb, &laddr, BPF_CORE_READ(fl4, fl4_sport), &raddr, BPF_CORE_READ(fl4, fl4_dport), 0); // Call on_udp_send_skb__2 bpf_tail_call(ctx, &tail_calls, TAIL_CALL_ON_UDP_SEND_SKB__2); return 0; } // may be optimized out on some kernels, if hook fails, we will use // on_ip6_send_skb SEC("kprobe/udp_v6_send_skb") int on_udp_v6_send_skb(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); struct flowi6 *fl6 = (struct flowi6 *)PT_REGS_PARM2(ctx); if (!skb || !fl6) return 0; GET_PID_TGID; struct in6_addr laddr = BPF_CORE_READ(fl6, saddr); struct in6_addr raddr = BPF_CORE_READ(fl6, daddr); __be16 sport = BPF_CORE_READ(fl6, fl6_sport); __be16 dport = BPF_CORE_READ(fl6, fl6_dport); #if DEBUG_UDP_SOCKET_ERRORS __check_broken_in6_addr(&laddr, __LINE__); __check_broken_in6_addr(&raddr, __LINE__); #endif struct sock *sk_ptr = BPF_CORE_READ(skb, sk); if (!sk_ptr) return 0; udp_update_stats(ctx, sk_ptr, skb, &laddr, sport, &raddr, dport, 0); // Call on_udp_v6_send_skb__2 bpf_tail_call(ctx, &tail_calls, TAIL_CALL_ON_UDP_V6_SEND_SKB__2); return 0; } // send TCP RST SEC("kprobe/tcp_send_active_reset") int on_tcp_send_active_reset(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); // gfp_t priority = (gfp_t)PT_REGS_PARM2(ctx); -- unused // bpf_trace_printk("on_tcp_send_active_reset\n"); // send RST (is_rx = 0) handle_tcp_reset(ctx, sk, 0); return 0; } SEC("kprobe/ip_send_skb") int on_ip_send_skb(struct pt_regs *ctx) { // struct net *net = (struct net *)PT_REGS_PARM1(ctx); -- unused struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); if (!skb) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); unsigned char *head = BPF_CORE_READ(skb, head); struct iphdr *ip_hdr = (struct iphdr *)(head + BPF_CORE_READ(skb, network_header)); __u8 protocol = BPF_CORE_READ(ip_hdr, protocol); if (protocol == IPPROTO_UDP) { struct udphdr *udp_hdr = (struct udphdr *)(head + BPF_CORE_READ(skb, transport_header)); struct in6_addr laddr = make_ipv6_address(BPF_CORE_READ(ip_hdr, saddr)); struct in6_addr raddr = make_ipv6_address(BPF_CORE_READ(ip_hdr, daddr)); udp_update_stats(ctx, sk, skb, &laddr, BPF_CORE_READ(udp_hdr, source), &raddr, BPF_CORE_READ(udp_hdr, dest), 0); } if (protocol == IPPROTO_TCP) { struct tcphdr *tcp_hdr = (struct tcphdr *)(head + BPF_CORE_READ(skb, transport_header)); u16 flags = 0; bpf_probe_read(&flags, 2, ((u8 *)tcp_hdr) + 12); if (flags & TCP_FLAG_RST) { // bpf_trace_printk("on_ip_send_skb: tcp rst is set\n"); // send RST (is_rx = 0) handle_tcp_reset(ctx, sk, 0); } } // Call on_ip_send_skb__2 bpf_tail_call(ctx, &tail_calls, TAIL_CALL_ON_IP_SEND_SKB__2); return 0; } SEC("kprobe/ip6_send_skb") int on_ip6_send_skb(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); if (!skb) return 0; struct sock *sk = BPF_CORE_READ(skb, sk); unsigned char *head = BPF_CORE_READ(skb, head); __u16 network_header = BPF_CORE_READ(skb, network_header); __u16 transport_header = BPF_CORE_READ(skb, transport_header); struct ipv6hdr *ipv6_hdr = (struct ipv6hdr *)(head + network_header); __u8 nexthdr = BPF_CORE_READ(ipv6_hdr, nexthdr); GET_PID_TGID; if (nexthdr == IPPROTO_UDP) { struct udphdr *udp_hdr = (struct udphdr *)(head + transport_header); struct in6_addr laddr = BPF_CORE_READ(ipv6_hdr, saddr); struct in6_addr raddr = BPF_CORE_READ(ipv6_hdr, daddr); __be16 source = BPF_CORE_READ(udp_hdr, source); __be16 dest = BPF_CORE_READ(udp_hdr, dest); #if DEBUG_UDP_SOCKET_ERRORS __check_broken_in6_addr(&laddr, __LINE__); __check_broken_in6_addr(&raddr, __LINE__); #endif udp_update_stats(ctx, sk, skb, &laddr, source, &raddr, dest, 0); } if (nexthdr == IPPROTO_TCP) { struct tcphdr *tcp_hdr = (struct tcphdr *)(head + transport_header); u16 flags = 0; bpf_probe_read(&flags, 2, ((u8 *)tcp_hdr) + 12); if (flags & TCP_FLAG_RST) { // bpf_trace_printk("on_ip6_send_skb: tcp rst is set\n"); // send RST (is_rx = 0) handle_tcp_reset(ctx, sk, 0); } } // Call on_ip6_send_skb__2 bpf_tail_call(ctx, &tail_calls, TAIL_CALL_ON_IP6_SEND_SKB__2); return 0; } // Common handler -tail call- for receiving udp skb's // step one, update stats, make sure udp socket exists SEC("kprobe") int handle_receive_udp_skb(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); if (!sk || !skb) return 0; // find offsets for ip and udp headers unsigned char *skb_head = BPF_CORE_READ(skb, head); __u16 transport_header = BPF_CORE_READ(skb, transport_header); __u16 network_header = BPF_CORE_READ(skb, network_header); // get the ip header struct iphdr *ip_hdr = (struct iphdr *)(skb_head + network_header); struct udphdr *udp_hdr = (struct udphdr *)(skb_head + transport_header); // get the version from the ip packet (common location for ipv4 and ipv6) u8 version; bpf_probe_read(&version, 1, (const u8 *)ip_hdr); version &= 0xF0; // Parse the addresses out of the header struct in6_addr laddr, raddr; if (version == 0x40) { // IPv4 __be32 daddr = BPF_CORE_READ(ip_hdr, daddr); __be32 saddr = BPF_CORE_READ(ip_hdr, saddr); laddr = make_ipv6_address(daddr); raddr = make_ipv6_address(saddr); } else if (version & 0x60) { // IPV6 laddr = BPF_CORE_READ((struct ipv6hdr *)ip_hdr, daddr); raddr = BPF_CORE_READ((struct ipv6hdr *)ip_hdr, saddr); } else { // Unknown IP Protocol version, possibly malformed packet received? bpf_log(ctx, BPF_LOG_UNREACHABLE, (u64)version, 0, 0); return 0; } #if DEBUG_UDP_SOCKET_ERRORS if (__check_broken_in6_addr(&laddr, __LINE__) || __check_broken_in6_addr(&raddr, __LINE__)) { u16 family = BPF_CORE_READ(sk, sk_family); bpf_trace_printk("sk_family = %d, version = %u\n", family, (unsigned int)version); stack_trace(ctx); } #endif __be16 dest = BPF_CORE_READ(udp_hdr, dest); __be16 source = BPF_CORE_READ(udp_hdr, source); udp_update_stats(ctx, sk, skb, &laddr, dest, &raddr, source, 1); bpf_tail_call(ctx, &tail_calls, TAIL_CALL_HANDLE_RECEIVE_UDP_SKB__2); return 0; } // Common handler -tail call- for receiving udp skb's // step two, check for receiving dns packets SEC("kprobe") int handle_receive_udp_skb__2(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); if (!sk || !skb) return 0; GET_PID_TGID; unsigned char *skb_head = BPF_CORE_READ(skb, head); __u16 transport_header = BPF_CORE_READ(skb, transport_header); const struct udphdr *hdr = (const struct udphdr *)(skb_head + transport_header); __be16 source = BPF_CORE_READ(hdr, source); __be16 dest = BPF_CORE_READ(hdr, dest); perf_check_and_submit_dns(ctx, sk, skb, IPPROTO_UDP, source, dest, 1); return 0; } //////////////////////////////////////////////////////////////////////////////////// /* LIVE TCP */ static void report_rtt_estimator_if_time(struct pt_regs *ctx, struct sock *sk, struct tcp_open_socket_t *sk_info) { u64 now = get_timestamp(); if ((now - sk_info->last_output) < filter_ns) return; // too little time passed report_rtt_estimator(ctx, sk, sk_info, now, false); } SEC("kprobe/tcp_rtt_estimator") int on_tcp_rtt_estimator(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_open_socket_t *sk_info; /* for filtering */ if (BPF_CORE_READ(sk, sk_state) != TCP_ESTABLISHED) { return 0; } sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { // Okay if socket is not yet tracked because it could be in kernel accept // queue but not returned by inet_csk_accept yet. return 0; } report_rtt_estimator_if_time(ctx, sk, sk_info); return 0; } SEC("kprobe/tcp_rcv_established") int on_tcp_rcv_established(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { // Okay if socket is not yet tracked because it could be in kernel accept // queue but not returned by inet_csk_accept yet. return 0; } int ret; u64 bytes_received = 0; ret = bpf_probe_read(&bytes_received, sizeof(bytes_received), &tcp_sk(sk)->bytes_received); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } if (bytes_received != sk_info->bytes_received) { /* update statistic for next report */ sk_info->rcv_holes++; sk_info->rcv_delivered++; /* update the bytes_received so we won't report this occurrence again */ sk_info->bytes_received = bytes_received; report_rtt_estimator_if_time(ctx, sk, sk_info); } return 0; } SEC("kprobe/tcp_event_data_recv") int on_tcp_event_data_recv(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { // Okay if socket is not yet tracked because it could be in kernel accept // queue but not returned by inet_csk_accept yet. return 0; } int ret; u64 bytes_received = 0; ret = bpf_probe_read(&bytes_received, sizeof(bytes_received), &tcp_sk(sk)->bytes_received); if (ret != 0) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, abs_val(ret), 0, 0); return 0; } sk_info->rcv_delivered++; sk_info->bytes_received = bytes_received; report_rtt_estimator_if_time(ctx, sk, sk_info); return 0; } // SYN timeouts. static void handle_syn_timeout(struct pt_regs *ctx, struct sock *sk) { u8 state = BPF_CORE_READ(sk, sk_state); // is this a SYN timeout? if (((1 << state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) == 0) { // no, return. return; } if (!bpf_map_lookup_elem(&tcp_open_sockets, &sk)) { // don't send a retransmit event on sockets we never notified userspace about return; } // okay, can send an event u64 now = get_timestamp(); perf_submit_agent_internal__tcp_syn_timeout(ctx, now, (__u64)sk); } // tcp_retransmit_timer is an ancient function which retained basically the // same structure for a long time (since the initial git repo at 1da177e4c3f4, // "Linux-2.6.12-rc2"). The static qualifier was removed in f1ecd5d9e7366 // (v2.6.32-rc1~703^2~172) SEC("kprobe/tcp_retransmit_timer") int on_tcp_retransmit_timer(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); handle_syn_timeout(ctx, sk); return 0; } // NOTE: tcp_syn_ack_timeout's signature was different pre v4.1 kernels. // See kernel commit 42cb80a2353f4, (v4.1-rc1~128^2~175^2~6). // the function seems to have been around since 72659ecce6858 // (v2.6.34-rc1~233^2~563). SEC("kprobe/tcp_syn_ack_timeout") int on_tcp_syn_ack_timeout(struct pt_regs *ctx) { // the request_sock parameter is the first parameter since 4.1 kernels const struct request_sock *req = (const struct request_sock *)PT_REGS_PARM1(ctx); if (!req) { return 0; } #if TCP_STATS_ON_PARENT if (LINUX_KERNEL_VERSION < KERNEL_VERSION(4, 4, 0)) { /* Linux<4.4 does not have req->rsk_listener */ struct sock *sk = BPF_CORE_READ(req, sk); struct tcp_open_socket_t *sk_info; sk_info = bpf_map_lookup_elem(&tcp_open_sockets, &sk); if (!sk_info) { // don't send a retransmit event on sockets we never notified userspace about return 0; } handle_syn_timeout(ctx, sk_info->parent); } else { struct sock *listener = BPF_CORE_READ(req, rsk_listener); handle_syn_timeout(ctx, listener); } #else struct sock *sk = BPF_CORE_READ(req, sk); if (sk == NULL) { bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); return 0; } handle_syn_timeout(ctx, sk); #endif return 0; } //////////////////////////////////////////////////////////////////////////////////// /* DNS */ #define BACKWARDS_COMPATIBLE_DNS_BUFFER_SIZE 280 #define DNS_MAX_PACKET_LEN 512 // For newer kernels, define the structure and per-CPU array struct dns_message_data { char data[DNS_MAX_PACKET_LEN + sizeof(struct bpf_agent_internal__dns_packet) + 16]; }; // use per-CPU array to overcome eBPF stack size limit struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); __type(key, __u32); __type(value, struct dns_message_data); } dns_message_array SEC(".maps"); #pragma passthrough off // Depending on when the skb is inspected, the header may or may not be filled // in yet so we are passing in protocol and port components as parameters here // sport and dport are in -network byte order- static __always_inline void perf_check_and_submit_dns(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb, u8 proto, u16 sport, u16 dport, int is_rx) { unsigned int len = BPF_CORE_READ(skb, len); // Filter for DNS requests and responses if (!((proto == IPPROTO_UDP) && ((sport == bpf_htons(53)) || (dport == bpf_htons(53))) && (len > 0))) { return; } // Read skb fields directly to avoid bpf verifier errors unsigned int skb_data_len = BPF_CORE_READ(skb, data_len); unsigned char *from = BPF_CORE_READ(skb, data); if (from == NULL) { return; } unsigned char *skb_head = BPF_CORE_READ(skb, head); if (skb_head == NULL) { return; } u16 skb_transport_header = BPF_CORE_READ(skb, transport_header); u16 skb_network_header = BPF_CORE_READ(skb, network_header); /* we only take the linear part of the skb, not the paged part * 'valid_len' here is the amount of -non paged- data in the skb, * since 'data_len' refers to just the amount of -paged- data, and 'len' * is the total amount of skb data */ unsigned int valid_len = len - skb_data_len; /* in e6afc8ace6dd5 (i.e., v4.7-rc1~154^2~349^2~1), udp_recvmsg started pulling the udp header before our kprobe. before then, skb->data pointed to the udp header. make sure we don't copy the udp header on pre-4.7 kernels */ /* if data points to transport header (udp) then advance to the packet body */ if (from == (skb_head + skb_transport_header)) { if (valid_len < sizeof(struct udphdr)) return; from += sizeof(struct udphdr); valid_len -= sizeof(struct udphdr); } /* if data points to network header (ip) then advance to the packet body */ else if (from == skb_head + skb_network_header) { from = skb_head + skb_transport_header + sizeof(struct udphdr); unsigned int diff = (from - (skb_head + skb_network_header)); if (valid_len < diff) return; valid_len -= diff; } /* only enable this if we need to detect if (valid_len < len) { // we are facing a paged or fragmented skb. only copy headlen bytes bpf_log(ctx, BPF_LOG_DATA_TRUNCATED, (u64)len, (u64)valid_len, 0); } */ /* allocate buffer for event */ char *buf; char stack_buf[BACKWARDS_COMPATIBLE_DNS_BUFFER_SIZE + sizeof(struct bpf_agent_internal__dns_packet) + 16] = {}; if (LINUX_KERNEL_VERSION < KERNEL_VERSION(4, 15, 0)) { // Use stack-based approach on kernels below 4.15 (chosen using empirical testing). // This is limited by max eBPF stack size (512 bytes). On newer kernels we can // use per-CPU arrays for copying data which allows processing larger packets. buf = stack_buf; if (valid_len > BACKWARDS_COMPATIBLE_DNS_BUFFER_SIZE) { bpf_log(ctx, BPF_LOG_DATA_TRUNCATED, (u64)valid_len, (u64)BACKWARDS_COMPATIBLE_DNS_BUFFER_SIZE, 0); valid_len = BACKWARDS_COMPATIBLE_DNS_BUFFER_SIZE; } /* the actual offset into buf has to start from buf's start */ char *to = buf + bpf_agent_internal__dns_packet__data_size; bpf_probe_read_kernel(to, valid_len, from); } else { // use bigger per-CPU array based buffer on newer kernels __u32 zero = 0; struct dns_message_data *pkt = bpf_map_lookup_elem(&dns_message_array, &zero); if (pkt == NULL) { bpf_log(ctx, BPF_LOG_BPF_CALL_FAILED, 0, 0, 0); return; } buf = pkt->data; // truncate dns packets at our maximum packet length right now valid_len &= (DNS_MAX_PACKET_LEN - 1); /* the actual offset into buf has to start from buf's start */ char *to = buf + bpf_agent_internal__dns_packet__data_size; bpf_probe_read_kernel(to, valid_len, from); } char *to = buf + bpf_agent_internal__dns_packet__data_size; struct bpf_agent_internal__dns_packet *const msg = (struct bpf_agent_internal__dns_packet *)&buf[0]; struct jb_blob blob = {to, valid_len}; bpf_fill_agent_internal__dns_packet(msg, get_timestamp(), (u64)sk, blob, len, is_rx); bpf_perf_event_output( ctx, &events, BPF_F_CURRENT_CPU, &msg->unpadded_size, ((DNS_MAX_PACKET_LEN + sizeof(struct jb_agent_internal__dns_packet) + 8 + 7) / 8) * 8 + 4); } // - Receive UDP packets --------------------------------------- SEC("kprobe/skb_consume_udp") int on_skb_consume_udp(struct pt_regs *ctx, struct sock *sk, struct sk_buff *skb, int len) { // Call handle_receive_udp_skb bpf_tail_call(ctx, &tail_calls, TAIL_CALL_HANDLE_RECEIVE_UDP_SKB); return 0; } SEC("kprobe/__skb_free_datagram_locked") int on___skb_free_datagram_locked(struct pt_regs *ctx) { bpf_tail_call(ctx, &tail_calls, TAIL_CALL_HANDLE_RECEIVE_UDP_SKB); return 0; } SEC("kprobe/skb_free_datagram_locked") int on_skb_free_datagram_locked(struct pt_regs *ctx) { bpf_tail_call(ctx, &tail_calls, TAIL_CALL_HANDLE_RECEIVE_UDP_SKB); return 0; } //////////////////////////////////////////////////////////////////////////////////// /* CGROUPS */ static __always_inline u32 get_css_id(struct cgroup_subsys_state *css) { return (u32)BPF_CORE_READ(css, ss, id); } static __always_inline struct cgroup *get_css_parent_cgroup(struct cgroup_subsys_state *css) { if (!css) { return NULL; } return BPF_CORE_READ(css, parent, cgroup); } static const char *get_cgroup_name(struct cgroup *cg) { if (!cg) { return NULL; } if (bpf_core_field_exists(cg->kn->name)) { return BPF_CORE_READ(cg, kn, name); } else { // < 3.15 struct cgroup___3_11 *cg = cg; struct cgroup_name___3_11 *name = BPF_CORE_READ(cg, name); if (!name) { return NULL; } return (const char *)&(name->name[0]); } } static struct cgroup *get_cgroup_parent(struct cgroup *cgrp) { if (bpf_core_field_exists(cgrp->self)) { // introduced in kernel 3.16 return get_css_parent_cgroup(&cgrp->self); } else { struct cgroup___3_11 *cg = cg; return cg->parent; } } // close // For Kernel >= 3.12 SEC("kprobe/kill_css") int on_kill_css(struct pt_regs *ctx) { struct cgroup_subsys_state *css = (struct cgroup_subsys_state *)PT_REGS_PARM1(ctx); u32 ssid = get_css_id(css); if (ssid != FLOW_CGROUP_SUBSYS) return 0; struct cgroup *parent_cgroup = get_css_parent_cgroup(css); u64 now = get_timestamp(); perf_submit_agent_internal__kill_css( ctx, now, (__u64)BPF_CORE_READ(css, cgroup), (__u64)parent_cgroup, (void *)get_cgroup_name(BPF_CORE_READ(css, cgroup))); return 0; } // For Kernel < 3.12 SEC("kprobe/cgroup_destroy_locked") int on_cgroup_destroy_locked(struct pt_regs *ctx) { struct cgroup *cgrp = (struct cgroup *)PT_REGS_PARM1(ctx); struct cgroup_subsys_state *css = NULL; bpf_probe_read(&css, sizeof(css), &(cgrp->subsys[FLOW_CGROUP_SUBSYS])); if (css == NULL) return 0; u64 now = get_timestamp(); perf_submit_agent_internal__kill_css(ctx, now, (__u64)cgrp, (__u64)get_cgroup_parent(cgrp), (void *)get_cgroup_name(cgrp)); return 0; } // start // For Kernel >= 4.4 SEC("kprobe/css_populate_dir") int on_css_populate_dir(struct pt_regs *ctx) { struct cgroup_subsys_state *css = (struct cgroup_subsys_state *)PT_REGS_PARM1(ctx); u32 ssid = get_css_id(css); if (ssid != FLOW_CGROUP_SUBSYS) return 0; struct cgroup *parent_cgroup = get_css_parent_cgroup(css); u64 now = get_timestamp(); perf_submit_agent_internal__css_populate_dir( ctx, now, (__u64)BPF_CORE_READ(css, cgroup), (__u64)parent_cgroup, (void *)get_cgroup_name(BPF_CORE_READ(css, cgroup))); return 0; } // For Kernel < 4.4 SEC("kprobe/cgroup_populate_dir") int on_cgroup_populate_dir(struct pt_regs *ctx) { struct cgroup *cgrp = (struct cgroup *)PT_REGS_PARM1(ctx); unsigned long subsys_mask = (unsigned long)PT_REGS_PARM2(ctx); struct cgroup_subsys_state *css = NULL; bpf_probe_read(&css, sizeof(css), &(cgrp->subsys[FLOW_CGROUP_SUBSYS])); if (css == NULL) return 0; u64 now = get_timestamp(); perf_submit_agent_internal__css_populate_dir( ctx, now, (__u64)cgrp, (__u64)get_cgroup_parent(cgrp), (void *)get_cgroup_name(cgrp)); return 0; } // existing cgroups v2 BEGIN_DECLARE_SAVED_ARGS(cgroup_control) struct cgroup *cgrp; END_DECLARE_SAVED_ARGS(cgroup_control) // Only available for kernel >= 4.6.0 SEC("kprobe/cgroup_control") int on_cgroup_control(struct pt_regs *ctx) { struct cgroup *cgrp = (struct cgroup *)PT_REGS_PARM1(ctx); GET_PID_TGID; BEGIN_SAVE_ARGS(cgroup_control) SAVE_ARG(cgrp) END_SAVE_ARGS(cgroup_control) return 0; } // Common function to handle cgroup processing static inline int handle_existing_cgroup(struct pt_regs *ctx, struct cgroup *cgrp, u16 subsys_mask) { if (!(subsys_mask & 1 << FLOW_CGROUP_SUBSYS)) return 0; u64 now = get_timestamp(); struct cgroup *parent_cgroup = get_css_parent_cgroup(&cgrp->self); perf_submit_agent_internal__existing_cgroup_probe(ctx, now, (__u64)cgrp, (__u64)parent_cgroup, (void *)get_cgroup_name(cgrp)); return 0; } // For kernels >= 4.6.0 SEC("kretprobe/cgroup_control") int onret_cgroup_control(struct pt_regs *ctx) { GET_PID_TGID; GET_ARGS_MISSING_OK(cgroup_control, args) if (args == NULL) { return 0; } struct cgroup *cgrp = args->cgrp; DELETE_ARGS(cgroup_control); u16 subsys_mask = (u16)PT_REGS_RC(ctx); return handle_existing_cgroup(ctx, cgrp, subsys_mask); } SEC("kretprobe/cgroup_get_from_fd") int onret_cgroup_get_from_fd(struct pt_regs *ctx) { // cgroup_get_from_fd returns the cgroup pointer or an error struct cgroup *cgrp = (struct cgroup *)PT_REGS_RC(ctx); if (cgrp == NULL || IS_ERR_VALUE((unsigned long)cgrp)) { return 0; } // For cgroup_get_from_fd, we assume all subsystems are relevant // since this is called when accessing cgroup via fd u16 subsys_mask = 1 << FLOW_CGROUP_SUBSYS; return handle_existing_cgroup(ctx, cgrp, subsys_mask); } // existing cgroups v1 // Function that handles both kernel versions for clone children read int on_cgroup_clone_children_read_css(struct pt_regs *ctx, struct cgroup_subsys_state *css, struct cftype *cft) { // For Kernel >= 3.12.0 if (LINUX_KERNEL_VERSION < KERNEL_VERSION(3, 12, 0)) { return 0; // Should use the cgroup version instead } u32 subsys_mask = (u32)BPF_CORE_READ(css, cgroup, root, subsys_mask); if (subsys_mask != 1 << FLOW_CGROUP_SUBSYS) return 0; u64 now = get_timestamp(); struct cgroup *parent_cgroup = get_css_parent_cgroup(css); perf_submit_agent_internal__existing_cgroup_probe( ctx, now, (__u64)BPF_CORE_READ(css, cgroup), (__u64)parent_cgroup, (void *)get_cgroup_name(BPF_CORE_READ(css, cgroup))); return 0; } // For Kernel < 3.12.0 SEC("kprobe/cgroup_clone_children_read") int on_cgroup_clone_children_read(struct pt_regs *ctx) { struct cgroup *cgrp = (struct cgroup *)PT_REGS_PARM1(ctx); u32 subsys_mask = BPF_CORE_READ(cgrp, root, subsys_mask); if (subsys_mask != 1 << FLOW_CGROUP_SUBSYS) return 0; u64 now = get_timestamp(); struct cgroup *parent_cgroup = get_cgroup_parent(cgrp); perf_submit_agent_internal__existing_cgroup_probe(ctx, now, (__u64)cgrp, (__u64)parent_cgroup, (void *)get_cgroup_name(cgrp)); return 0; } // modify SEC("kprobe/cgroup_attach_task") int on_cgroup_attach_task(struct pt_regs *ctx) { struct cgroup *dst_cgrp = (struct cgroup *)PT_REGS_PARM1(ctx); struct task_struct *leader = (struct task_struct *)PT_REGS_PARM2(ctx); u32 subsys_mask = BPF_CORE_READ(dst_cgrp, root, subsys_mask); if (subsys_mask != 1 << FLOW_CGROUP_SUBSYS) return 0; pid_t pid = BPF_CORE_READ(leader, pid); u8 comm[TASK_COMM_LEN]; if (bpf_probe_read_kernel_str(comm, sizeof(comm), leader->comm) <= 0) { return 0; } u64 now = get_timestamp(); perf_submit_agent_internal__cgroup_attach_task(ctx, now, (__u64)dst_cgrp, pid, comm); return 0; } //////////////////////////////////////////////////////////////////////////////////// /* NAT */ /* end */ // Cleanup: nf_ct_delete is invoked for all teardown paths SEC("kprobe/nf_ct_delete") int on_nf_ct_delete(struct pt_regs *ctx) { struct nf_conn *ct = (struct nf_conn *)PT_REGS_PARM1(ctx); if (!ct) { return 0; } // Filter to IPv4 TCP/UDP only if ((u16)BPF_CORE_READ(ct, tuplehash[0].tuple.src.l3num) != AF_INET) { return 0; } u8 proto = (u8)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.protonum); if (proto != IPPROTO_TCP && proto != IPPROTO_UDP) { return 0; } // Only report for conntracks we've seen during confirm path (NAT-ed) struct nf_conn **seen_conntrack = bpf_map_lookup_elem(&seen_conntracks, &ct); if (!seen_conntrack) { return 0; } u64 now = get_timestamp(); perf_submit_agent_internal__nf_nat_cleanup_conntrack( ctx, now, (u64)ct, (u32)BPF_CORE_READ(ct, tuplehash[0].tuple.src.u3.ip), (u16)BPF_CORE_READ(ct, tuplehash[0].tuple.src.u.all), (u32)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.u3.ip), (u16)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.u.all), (u8)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.protonum)); // Remove from seen table bpf_map_delete_elem(&seen_conntracks, &ct); return 0; } /* start */ // Create: confirmation path via __nf_conntrack_confirm (entry + ret) SEC("kprobe/__nf_conntrack_confirm") int on___nf_conntrack_confirm(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); if (!skb) { return 0; } __u32 zero = 0; struct sk_buff **slot = bpf_map_lookup_elem(&nfct_confirm_skb, &zero); if (slot) *slot = skb; return 0; } SEC("kretprobe/__nf_conntrack_confirm") int onret___nf_conntrack_confirm(struct pt_regs *ctx) { int ret = (int)PT_REGS_RC(ctx); // Only proceed on successful confirm (NF_ACCEPT == 1) if (ret != 1) { __u32 zero = 0; struct sk_buff **slot = bpf_map_lookup_elem(&nfct_confirm_skb, &zero); if (slot) *slot = NULL; return 0; } __u32 zero = 0; struct sk_buff **skb_pp = bpf_map_lookup_elem(&nfct_confirm_skb, &zero); if (!skb_pp) { return 0; } struct sk_buff *skb = *skb_pp; *skb_pp = NULL; if (!skb) { return 0; } // Extract ct from skb->_nfct with NFCT_PTRMASK (~1UL). Use CO-RE flavor cast. if (!bpf_core_field_exists(((struct sk_buff___with_nfct *)skb)->_nfct)) { return 0; } unsigned long nfct = BPF_CORE_READ((struct sk_buff___with_nfct *)skb, _nfct); struct nf_conn *ct = (struct nf_conn *)(nfct & ~7UL); if (!ct) { return 0; } // IPv4 only if ((u16)BPF_CORE_READ(ct, tuplehash[0].tuple.src.l3num) != AF_INET) { return 0; } // TCP/UDP only u8 proto = (u8)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.protonum); if (proto != IPPROTO_TCP && proto != IPPROTO_UDP) { return 0; } // Build ORIGINAL tuple (pre-NAT, original direction) u32 orig_src_ip = (u32)BPF_CORE_READ(ct, tuplehash[0].tuple.src.u3.ip); u16 orig_src_port = (u16)BPF_CORE_READ(ct, tuplehash[0].tuple.src.u.all); u32 orig_dst_ip = (u32)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.u3.ip); u16 orig_dst_port = (u16)BPF_CORE_READ(ct, tuplehash[0].tuple.dst.u.all); // Build POST-NAT tuple in ORIGINAL direction from REPLY by flipping src/dst u32 nat_src_ip = (u32)BPF_CORE_READ(ct, tuplehash[1].tuple.dst.u3.ip); u16 nat_src_port = (u16)BPF_CORE_READ(ct, tuplehash[1].tuple.dst.u.all); u32 nat_dst_ip = (u32)BPF_CORE_READ(ct, tuplehash[1].tuple.src.u3.ip); u16 nat_dst_port = (u16)BPF_CORE_READ(ct, tuplehash[1].tuple.src.u.all); // Filter out non-NAT flows by comparing tuples if (orig_src_ip == nat_src_ip && orig_src_port == nat_src_port && orig_dst_ip == nat_dst_ip && orig_dst_port == nat_dst_port) { return 0; } // Record in seen_conntracks (permit duplicates; BPF_ANY) int up_ret = bpf_map_update_elem(&seen_conntracks, &ct, &ct, BPF_ANY); if (up_ret != 0) { #if DEBUG_OTHER_MAP_ERRORS bpf_trace_printk("onret___nf_conntrack_confirm: seen_conntracks update failed ret=%d\n", up_ret); #endif } u64 now = get_timestamp(); perf_submit_agent_internal__nf_conntrack_alter_reply( ctx, now, (u64)ct, orig_src_ip, orig_src_port, orig_dst_ip, orig_dst_port, proto, nat_src_ip, nat_src_port, nat_dst_ip, nat_dst_port, proto); return 0; } SEC("kprobe/ctnetlink_dump_tuples") int on_ctnetlink_dump_tuples(struct pt_regs *ctx) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); const struct nf_conntrack_tuple *ct = (const struct nf_conntrack_tuple *)PT_REGS_PARM2(ctx); if (!ct) { return 0; } u64 now = get_timestamp(); // "struct nf_conn" contains two "struct nf_conntrack_tuple_hash". On the // dir=1 case, we want to report the addr of the dir=0, which we expected to // see first. this is addr is sizeof(struct nf_conntrack_tuple_hash) ahead of // the start of our current "ct" u64 ct_addr = (u64)ct; u8 dir = BPF_CORE_READ(ct, dst.dir); if (dir == 0) { perf_submit_agent_internal__existing_conntrack_tuple( ctx, now, (u64)ct_addr, (u32)BPF_CORE_READ(ct, src.u3.ip), (u16)BPF_CORE_READ(ct, src.u.all), (u32)BPF_CORE_READ(ct, dst.u3.ip), (u16)BPF_CORE_READ(ct, dst.u.all), (u8)BPF_CORE_READ(ct, dst.protonum), (u8)dir); } else { ct_addr = ct_addr - sizeof(struct nf_conntrack_tuple_hash); // NOTE: in the dir=1 case, we flip src/dst to preserve four-tuple order. perf_submit_agent_internal__existing_conntrack_tuple( ctx, now, (u64)ct_addr, (u32)BPF_CORE_READ(ct, dst.u3.ip), (u16)BPF_CORE_READ(ct, dst.u.all), (u32)BPF_CORE_READ(ct, src.u3.ip), (u16)BPF_CORE_READ(ct, src.u.all), (u8)BPF_CORE_READ(ct, dst.protonum), (u8)dir); } return 0; } // // Include other modules // #include "tcp-processor/bpf_tcp_processor.c" char _license[] SEC("license") = "Dual MIT/GPL"; ================================================ FILE: collector/kernel/bpf_src/render_bpf.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once // This file is included both in BPF (render_bpf and tcp-processor) as well as by userland agent. //////////////////////////////////////////////////////////////////////////// // Config ///////// render_bpf.c config #define BPF_MAX_CPUS 1024 // Maximum number of CPUs to support #define TABLE_SIZE__TGID_INFO 65536 // Task (TGID) information - compile-time constant needed for map definition #define TABLE_SIZE__SEEN_INODES \ 70000 // XXX: Is this even necessary? could this tracking be done in userland with non-limited tables? #define TABLE_SIZE__TCP_OPEN_SOCKETS (256 * 1024) // Was 4096, but should be larger to accommodate high traffic systems #define TABLE_SIZE__UDP_OPEN_SOCKETS (256 * 1024) // Was 4096, but should be larger to accommodate high traffic systems #define TABLE_SIZE__UDP_GET_PORT_HASH 512 // Should be no more than the number of cores, in theory #define TABLE_SIZE__SEEN_CONNTRACKS \ (TABLE_SIZE__TCP_OPEN_SOCKETS + \ TABLE_SIZE__UDP_OPEN_SOCKETS) // Worst case scenario, we have a conntrack for every open socket #define TABLE_SIZE__DEAD_GROUP_TASKS 512 // Should be no more than the number of cores, in theory #define TABLE_SIZE__STACK_TRACES 16384 // Number of stack traces to keep in the table #define TABLE_SIZE__NIC_INFO_TABLE 128 // Info per network interface #define WATERMARK_STACK_TRACES \ (TABLE_SIZE__STACK_TRACES - 256) // When to clear the table (unfortunately non-atomic, but that's a lot of stack traces...) // Maximum number of bytes in a tcp_data message block #define DATA_CHANNEL_CHUNK_MAX 16383 // Shouldn't be necessary, be we aren't getting the lifetime of tcp // sockets right in some edge case // Forces socket lifetimes to not overlap due missing or out-of-order destroy events #define TCP_LIFETIME_HACK 1 #define UDP_LIFETIME_HACK 1 // Adds sockets that were missed during our initial 'existing' walk #define TCP_EXISTING_HACK 1 #define UDP_EXISTING_HACK 1 // When sockets are accepted, do we report the statistics on the child or parent socket? #define TCP_STATS_ON_PARENT 1 ///////// bpf_tcp_processor.c config #define TCP_CONNECTION_HASH_SIZE TABLE_SIZE__TCP_OPEN_SOCKETS // Same for now to ensure we can always handle all http requests //////////////////////////////////////////////////////////////////////////// // Debugging flags ///////// render_bpf.c debugging switches // #define TRACE_TCP_SOCKETS 1 #define TRACE_UDP_SOCKETS 0 // #define DEBUG_TCP_SOCKET_ERRORS 1 #define DEBUG_UDP_SOCKET_ERRORS 0 // #define DEBUG_OTHER_MAP_ERRORS 1 // #define DEBUG_ENABLE_STACKTRACE 1 ///////// bpf_tcp_processor.c debugging switches // #define TRACE_TCP_CONNECTION 1 // #define DEBUG_TCP_CONNECTION 1 // #define DEBUG_MEMORY 1 // #define TRACE_SOCKET_ACCEPT 1 // #define DEBUG_TCP_RECEIVE 1 // #define TRACE_TCP_RECEIVE 1 // #define DEBUG_TCP_SEND 1 // #define TRACE_TCP_SEND 1 // #define TRACE_HTTP_PROTOCOL 1 // #define DEBUG_TCP_DATA 1 // #define DEBUG_DATA_CHANNEL 1 //////////////////////////////////////////////////////////////////////////// // BPF error report codes enum BPF_LOG_CODE { BPF_LOG_UNKNOWN = 0, BPF_LOG_TABLE_FULL = 1, // arg0 = table id, arg1 = tgid, arg2 = item BPF_LOG_TABLE_BAD_INSERT = 2, // arg0 = table id, arg1 = tgid, arg2 = return code BPF_LOG_TABLE_BAD_REMOVE = 3, // arg0 = table id, arg1 = item being removed, arg2 = return code BPF_LOG_TABLE_MISSING_KEY = 4, // arg0 = table id, arg1 = item looked up BPF_LOG_LIFETIME_HACK = 5, // arg0 = table id, arg1 = item BPF_LOG_TABLE_DUPLICATE_INSERT = 6, // arg0 = table id, arg1 = tgid, arg2 = item BPF_LOG_UNEXPECTED_TYPE = 7, // arg0 = key, arg1 = type BPF_LOG_DATA_TRUNCATED = 8, // arg0 = total len, arg1 = truncated len BPF_LOG_INVALID_POINTER = 9, // none BPF_LOG_UNSUPPORTED_IO = 10, // arg0 = ST_SEND/ST_RECV, arg1 = sk, arg2 = type BPF_LOG_MISSING_ARGUMENTS = 11, // none BPF_LOG_UNREACHABLE = 12, // none BPF_LOG_THROTTLED = 13, // arg0 = cpu number BPF_LOG_UNEXPECTED_VALUE = 14, // arg0 = value BPF_LOG_INVALID_PID_TGID = 15, // arg0 = pid_tgid BPF_LOG_EXISTING_HACK = 16, // arg0 = table id, arg1 = item BPF_LOG_BPF_CALL_FAILED = 17, // arg0 = ret val }; enum BPF_TABLE_ID { BPF_TABLE_TGID_INFO = 0, BPF_TABLE_SEEN_INODES = 1, BPF_TABLE_TCP_OPEN_SOCKETS = 2, BPF_TABLE_TCP_OPENREQ_CHILD_HASH = 3, BPF_TABLE_UDP_OPEN_SOCKETS = 4, BPF_TABLE_UDP_GET_PORT_HASH = 5, BPF_TABLE_SEEN_CONNTRACKS = 6, BPF_TABLE_TCP_CONNECTIONS = 7, BPF_TABLE_DEAD_GROUP_TASKS = 8, BPF_TABLE_PID_INFO = 9, }; // Log throttling, currently set to a max of 20 messages per second #define BPF_LOG_THROTTLE_MAX_PER_PERIOD 20 // maximum number of log messages from bpf per period #define BPF_LOG_THROTTLE_PERIOD_LENGTH_MS 1000 // length of the log throttle period in milliseconds // Tail Calls #define TAIL_CALL_ON_UDP_SEND_SKB__2 0 #define TAIL_CALL_ON_UDP_V6_SEND_SKB__2 1 #define TAIL_CALL_ON_IP_SEND_SKB__2 2 #define TAIL_CALL_ON_IP6_SEND_SKB__2 3 #define TAIL_CALL_HANDLE_RECEIVE_UDP_SKB 4 #define TAIL_CALL_HANDLE_RECEIVE_UDP_SKB__2 5 #define TAIL_CALL_CONTINUE_TCP_SENDMSG 6 #define TAIL_CALL_CONTINUE_TCP_RECVMSG 7 #define NUM_TAIL_CALLS 8 #if _PROCESSING_BPF // Include this until we merge the tcp-processor code into render_bpf more closely // Note, this is included by bpf and userland c++, so it -must- be an include with "" not <> // as this distinction determines which includes are processed during BPF compilation #include "tcp-processor/tcp_processor.h" #else #include #endif ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_data_channel.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once // TCP Data sent to userland - perf event array map for libbpf struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(__u32)); __uint(value_size, sizeof(__u32)); } data_channel SEC(".maps"); // Slightly more compact fore than COPY_BIT(2) + COPY_BIT(1) #define COPY_LAST_BITS \ const int last_bits = len & 3; \ msg.hdr.length = last_bits; \ switch (last_bits) { \ default: \ break; \ case 1: \ bpf_probe_read(&msg.data, 1, in); \ bpf_perf_event_output(ctx, &data_channel, BPF_F_CURRENT_CPU, &msg, sizeof(struct data_channel_header_t) + 1); \ break; \ case 2: \ bpf_probe_read(&msg.data, 2, in); \ bpf_perf_event_output(ctx, &data_channel, BPF_F_CURRENT_CPU, &msg, sizeof(struct data_channel_header_t) + 2); \ break; \ case 3: \ bpf_probe_read(&msg.data, 3, in); \ bpf_perf_event_output(ctx, &data_channel, BPF_F_CURRENT_CPU, &msg, sizeof(struct data_channel_header_t) + 3); \ break; \ } #define COPY_BIT(B) \ if (len & B) { \ msg.hdr.length = B; \ bpf_probe_read(&msg.data, B, in); \ bpf_perf_event_output(ctx, &data_channel, BPF_F_CURRENT_CPU, &msg, sizeof(struct data_channel_header_t) + B); \ in += B; \ } #define COPY_CHUNK_256 \ bpf_probe_read(&msg.data, 256, in); \ bpf_perf_event_output(ctx, &data_channel, BPF_F_CURRENT_CPU, &msg, sizeof(struct data_channel_header_t) + 256); \ in += 256; #define COPY_BIT_256 \ if (len & 256) { \ COPY_CHUNK_256; \ } #define COPY_BIT_512 \ if (len & 512) { \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ } #define COPY_BIT_1024 \ if (len & 1024) { \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ } #define COPY_BIT_2048 \ if (len & 2048) { \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ } #define COPY_BIT_4096 \ if (len & 4096) { \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ } #define COPY_BIT_8192 \ if (len & 8192) { \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ COPY_CHUNK_256; \ } // Submit contents of TCP data stream to the perf ring to userland static __always_inline void data_channel_submit(struct pt_regs *ctx, struct tcp_connection_t *pconn, const void *data, size_t data_len, size_t *actual_len) { // Clip the length to the most we can copy unsigned int len = data_len; if (len > DATA_CHANNEL_CHUNK_MAX) { #if DEBUG_DATA_CHANNEL bpf_log(ctx, BPF_LOG_DATA_TRUNCATED, (u64)len, (u64)DATA_CHANNEL_CHUNK_MAX, 0); #endif len = DATA_CHANNEL_CHUNK_MAX; } *actual_len = len; #if DEBUG_DATA_CHANNEL bpf_trace_printk("tcp_events_submit_tcp_data: sk=%llx, len=%u, data_len=%u\n", pconn->sk, len, data_len); #endif // Copy the data to the data stream const u8 *in = (const u8 *)data; struct { struct data_channel_header_t hdr; char data[256]; } msg = {.hdr.length = 256, .data = {}}; // Copy 256 byte chunks, because we can't have a very large on-stack buffer // and early ebpf can't bpf_probe_read to anywhere but the stack, and also // early ebpf can not perf_submit from anything other than the stack #if DATA_CHANNEL_CHUNK_MAX >= 8192 COPY_BIT_8192; #endif #if DATA_CHANNEL_CHUNK_MAX >= 4096 COPY_BIT_4096; #endif #if DATA_CHANNEL_CHUNK_MAX >= 2048 COPY_BIT_2048; #endif #if DATA_CHANNEL_CHUNK_MAX >= 1024 COPY_BIT_1024; #endif #if DATA_CHANNEL_CHUNK_MAX >= 512 COPY_BIT_512; #endif #if DATA_CHANNEL_CHUNK_MAX >= 256 COPY_BIT_256; #endif #if DATA_CHANNEL_CHUNK_MAX < 256 #error "Invalid data channel chunk size" #endif // Copy < 256 byte variable length chunks COPY_BIT(128); COPY_BIT(64); COPY_BIT(32); COPY_BIT(16); COPY_BIT(8); COPY_BIT(4); // Copy last bits using a slightly more compact form so this fits in the 4k instruction limit COPY_LAST_BITS; // COPY_BIT(2); // COPY_BIT(1); } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_debug.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // // bpf_debug.h - BPF Debugging Tools // #pragma once #include "tcp-processor/bpf_types.h" #if DEBUG_LOG #define DEBUG_PRINTK bpf_trace_printk #else #define DEBUG_PRINTK(...) #endif static void s_print_bpf_assert(int cond, const char *condstr) { DEBUG_PRINTK("BPF_ASSERT failed: %s\n", condstr); } #define BPF_ASSERT_RET(COND, RET) \ if (!(COND)) { \ char __condstr[] = #COND; \ s_print_bpf_assert((COND), __condstr); \ return (RET); \ } #if DEBUG_ENABLE_STACKTRACE #include #include BPF_STACK_TRACE(stack_traces, TABLE_SIZE__STACK_TRACES); static __always_inline void stack_trace(struct pt_regs *ctx) { char comm[16]; u64 now = get_timestamp(); s32 kernel_stack_id = stack_traces.get_stackid(ctx, 0); s32 user_stack_id = stack_traces.get_stackid(ctx, BPF_F_USER_STACK); u32 tgid = bpf_get_current_pid_tgid() >> 32; bpf_get_current_comm(comm, sizeof(comm)); perf_submit_agent_internal__stack_trace(ctx, now, kernel_stack_id, user_stack_id, tgid, comm); } #endif static __always_inline int __check_broken_in6_addr(struct in6_addr *addr, int line) { if (addr->in6_u.u6_addr16[0] == 0x1140 && addr->in6_u.u6_addr16[2] == 0x007f) { bpf_trace_printk("broken in6_addr on line %d\n", line); return 1; } return 0; } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_http_protocol.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // // bpf_http_request.h - BPF HTTP Request Parsing // #pragma once struct http_protocol_state_data_t { u64 request_timestamp; u64 __unused; // Keep this to TCP_SOCKET_PROTOCOL_STATE_SIZE }; static __always_inline enum TCP_PROTOCOL_DETECT_RESULT http_detect( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const char *data, size_t data_len) { #if TRACE_HTTP_PROTOCOL DEBUG_PRINTK( "http_detect: start=%llu, total=%llu\n", (u32)pctrl->streams[(int)streamtype].start, pconn->streams[(int)streamtype].total); DEBUG_PRINTK(" data=%s, data_len=%d\n", data, (int)data_len); #endif enum TCP_PROTOCOL_DETECT_RESULT res = TPD_UNKNOWN; // Does the buffer start with an HTTP verb? if (data_len >= 3) { char c = 0; bpf_probe_read(&c, 1, data); switch (c) { case 'G': res = string_starts_with(data + 1, (size_t)(data_len - 1), "ET", (sizeof("ET") - 1)) ? TPD_SUCCESS : TPD_FAILED; break; case 'H': if (data_len >= 4) { res = string_starts_with(data + 1, (size_t)(data_len - 1), "EAD", (sizeof("EAD") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; case 'D': if (data_len >= 6) { res = string_starts_with(data + 1, (size_t)(data_len - 1), "ELETE", (sizeof("ELETE") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; case 'C': if (data_len >= 7) { res = string_starts_with(data + 1, (size_t)(data_len - 1), "ONNECT", (sizeof("ONNECT") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; case 'O': if (data_len >= 7) { res = string_starts_with(data + 1, (size_t)(data_len - 1), "PTIONS", (sizeof("PTIONS") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; case 'T': if (data_len >= 5) { res = string_starts_with(data + 1, (size_t)(data_len - 1), "RACE", (sizeof("RACE") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; case 'P': bpf_probe_read(&c, 1, data + 1); switch (c) { case 'U': res = string_starts_with(data + 2, (size_t)(data_len - 2), "T", (sizeof("T") - 1)) ? TPD_SUCCESS : TPD_FAILED; break; case 'O': res = string_starts_with(data + 2, (size_t)(data_len - 2), "ST", (sizeof("ST") - 1)) ? TPD_SUCCESS : TPD_FAILED; break; case 'A': if (data_len >= 5) { res = string_starts_with(data + 2, (size_t)(data_len - 2), "TCH", (sizeof("TCH") - 1)) ? TPD_SUCCESS : TPD_FAILED; } break; default: res = TPD_FAILED; break; } break; default: res = TPD_FAILED; break; } } return res; } static __always_inline void http_process_request( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const u8 *data, size_t data_len) { #if TRACE_HTTP_PROTOCOL DEBUG_PRINTK( "http_process_request: start=%llu, total=%llu\n", (u32)pctrl->streams[(int)streamtype].start, pconn->streams[(int)streamtype].total); DEBUG_PRINTK(" data=%s, data_len=%d\n", data, (int)data_len); #endif // Keep the request timestamp for latency calculation struct http_protocol_state_data_t *state_data = (struct http_protocol_state_data_t *)(pconn->protocol_state.data); state_data->request_timestamp = get_timestamp(); // Enable getting the response, and turn off the request side until we get it if (streamtype == ST_RECV) { enable_tcp_connection(pctrl, -1, 1); } else { enable_tcp_connection(pctrl, 1, -1); } } static __always_inline void http_process_response( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const u8 *data, size_t data_len) { #if TRACE_HTTP_PROTOCOL DEBUG_PRINTK( "http_process_response: start=%llu, total=%llu\n", (u32)pctrl->streams[(int)streamtype].start, pconn->streams[(int)streamtype].total); DEBUG_PRINTK(" data=%s, data_len=%d\n", data, (int)data_len); #endif // HTTP response can't be less than 12 characters long, if we haven't gotten // 12 characters yet, just wait until more bytes show up if (data_len < 12) { goto finished_response; } char localdata[12] = {}; bpf_probe_read(localdata, 12, data); // Pattern match against HTTP/x.y. localdata is on BPF stack, so compare directly // instead of using string_starts_with (which reads via helpers). if (!(localdata[0] == 'H' && localdata[1] == 'T' && localdata[2] == 'T' && localdata[3] == 'P' && localdata[4] == '/') || localdata[6] != '.' || localdata[8] != ' ') { goto finished_response; } int http_version_major = char_to_number(localdata[5]); int http_version_minor = char_to_number(localdata[7]); #if TRACE_HTTP_PROTOCOL DEBUG_PRINTK("HTTP version: %d.%d\n", http_version_major, http_version_minor); #endif // Check for invalid HTTP version if (http_version_major == -1 || http_version_minor == -1) { goto finished_response; } // Convert HTTP response code to integer int http_code_2 = char_to_number(localdata[9]); int http_code_1 = char_to_number(localdata[10]); int http_code_0 = char_to_number(localdata[11]); #if TRACE_HTTP_PROTOCOL DEBUG_PRINTK("HTTP response code: %d%d%d\n", http_code_2, http_code_1, http_code_0); #endif // Ensure each digit of HTTP response code is valid if (http_code_2 == -1 || http_code_1 == -1 || http_code_0 == -1) { goto finished_response; } u16 code = (u16)(http_code_2 * 100 + http_code_1 * 10 + http_code_0); // Submit the http response code, and latency struct http_protocol_state_data_t *state_data = (struct http_protocol_state_data_t *)(pconn->protocol_state.data); u64 latency = get_timestamp() - state_data->request_timestamp; u8 client_server = (streamtype == ST_SEND) ? SC_SERVER : SC_CLIENT; tcp_events_submit_http_response(ctx, pconn->sk, code, latency, client_server); // Enable getting the request, and turn off the response side until we get it finished_response: if (streamtype == ST_RECV) { enable_tcp_connection(pctrl, -1, 1); } else { enable_tcp_connection(pctrl, 1, -1); } } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_inet_csk_accept.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // // bpf_inet_csk_accept.h - BPF handler for connection-based socket accepts // #pragma once #include "bpf_debug.h" #include "bpf_memory.h" #include "bpf_tcp_socket.h" #include "bpf_types.h" BEGIN_DECLARE_SAVED_ARGS(inet_csk_accept) struct sock *sk; int flags; u32 _pad_0; // required alignment int *err; END_DECLARE_SAVED_ARGS(inet_csk_accept) // --- inet_csk_accept -------------------------------------------------- // Called when a listen socket accepts gets a connection SEC("kprobe/inet_csk_accept") int handle_kprobe__inet_csk_accept(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); int flags = (int)PT_REGS_PARM2(ctx); int *err = (int *)PT_REGS_PARM3(ctx); // bool kern = (bool)PT_REGS_PARM4(ctx); // not used GET_PID_TGID // Ensure the parent socket has a tcp_connection struct tcp_connection_t *psk_info; psk_info = lookup_tcp_connection(sk); if (!psk_info) { psk_info = create_tcp_connection(ctx, sk); if (!psk_info) { #if DEBUG_TCP_CONNECTION DEBUG_PRINTK("inet_csk_accept: couldn't create tcp_connection"); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_CONNECTIONS, _tgid, (u64)sk); return 0; } } // Link us to our kretprobe BEGIN_SAVE_ARGS(inet_csk_accept) SAVE_ARG(sk) SAVE_ARG(flags) SAVE_ARG(err) END_SAVE_ARGS(inet_csk_accept) #if TRACE_SOCKET_ACCEPT DEBUG_PRINTK("inet_csk_accept enter: pid %u accepting socket on %llx\n", psk_info->upid, sk); DEBUG_PRINTK(" pid_tgid %llu added\n", _pid_tgid); #endif return 0; } SEC("kretprobe/inet_csk_accept") int handle_kretprobe__inet_csk_accept(struct pt_regs *ctx) { GET_PID_TGID // Ensure the accept succeeded struct sock *newsk = (struct sock *)PT_REGS_RC(ctx); if (!newsk) { #if TRACE_SOCKET_ACCEPT DEBUG_PRINTK(" accept failed\n"); DEBUG_PRINTK("inet_csk_accept exit: pid_tgid %llu removed\n", _pid_tgid); #endif // Unlink the kprobe/kretprobe DELETE_ARGS(inet_csk_accept); return 0; } // Link us from our kprobe GET_ARGS(inet_csk_accept, args); if (args == NULL) { // Race condition where we might have been inside an accept when the probe // was inserted, ignore return 0; } // Set up the child socket struct tcp_connection_t *pconn = create_tcp_connection(ctx, newsk); if (!pconn) { #if DEBUG_TCP_CONNECTION DEBUG_PRINTK("inet_csk_accept(ret): couldn't create tcp_connection"); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_CONNECTIONS, _tgid, (u64)newsk); // Unlink the kprobe/kretprobe DELETE_ARGS(inet_csk_accept); return 0; } // Note that this is a listening socket pconn->parent_sk = args->sk; #if TRACE_SOCKET_ACCEPT DEBUG_PRINTK(" accepted socket %llx\n", newsk); DEBUG_PRINTK("inet_csk_accept exit: pid_tgid %llu removed\n", _pid_tgid); #endif // Unlink the kprobe/kretprobe DELETE_ARGS(inet_csk_accept); return 0; } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_memory.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // // bpf_memory.h - BPF memory related utility functions // #pragma once #include "bpf_debug.h" #include "bpf_types.h" /* * Safe prefix compare for potentially untrusted pointers (kernel or user). * Copies up to 16 bytes from s1 via helper before comparing to s2 literal. * Intended for 5.4-era verifier quirks and mixed user/kernel buffers. */ static __always_inline int string_starts_with(const char *s1, const size_t s1_len, const char *s2, const size_t s2_len) { if (s2_len == 0) { return 1; } if (s2_len > 16) { return 0; } if (s1_len < s2_len) { return 0; } char local[16] = {}; if (bpf_probe_read(local, (u32)s2_len, s1) != 0) { return 0; } int n = (int)s2_len; // s2_len <= 16 guaranteed for (int i = 0; i < n; i++) { if (local[i] != s2[i]) { return 0; } } return 1; } static __always_inline int char_to_number(char x) { if (x < '0' || x > '9') return -1; return (int)(x - '0'); } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_tcp_events.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #ifdef STANDALONE_TCP_PROCESSOR // Only used by standalone debugging (tcp-processor.py) /////////////////////////////////////////////////////// // TCP Event Types #define TCP_EVENT_TYPE u32 #define TCP_EVENT_TYPE_HTTP_RESPONSE ((TCP_EVENT_TYPE)0) #define TCP_EVENT_TYPE_TCP_DATA ((TCP_EVENT_TYPE)1) // Format of tcp_events perf buffer messages struct tcp_events_t { TCP_EVENT_TYPE type; // TCP_EVENT_TYPE TGID pid; // Userland process id related to this event TIMESTAMP ts; // Timestamp u64 sk; // Socket pointer related to this event union { struct { // HTTP Reponse events data u16 code; // Response status code u8 dir; // client/server direction (0=client) u8 __pad0[5]; // 64 bit align u64 latency; // request/response latency in ns } http_response; struct { // TCP Data events data u32 length; // Length of chunk u8 streamtype; // STREAM_TYPE u8 is_server; // CLIENT_SERVER_TYPE u16 __pad0; // 64 bit align u64 offset; // Position in the stream } tcp_data; struct { u64 __pad0; // Padding to ensure length is multiple of 8 bytes u64 __pad1; // Padding to ensure length is multiple of 8 bytes } __align; }; }; // The perf event array map for libbpf struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(__u32)); __uint(value_size, sizeof(__u32)); } tcp_events SEC(".maps"); #endif // Utility functions static __always_inline void tcp_events_submit_http_response(struct pt_regs *ctx, struct sock *sk, u16 code, u64 latency, enum CLIENT_SERVER_TYPE dir) { GET_PID_TGID u64 now = get_timestamp(); #ifdef STANDALONE_TCP_PROCESSOR struct tcp_events_t event = { .type = TCP_EVENT_TYPE_HTTP_RESPONSE, .pid = _tgid, .ts = now, .sk = (u64)sk, .__align.__pad0 = 0, .__align.__pad1 = 0}; event.http_response.code = code; event.http_response.dir = (u8)dir; event.http_response.latency = latency; bpf_perf_event_output(ctx, &tcp_events, BPF_F_CURRENT_CPU, &event, sizeof(event)); #else perf_submit_agent_internal__http_response(ctx, now, (u64)sk, _tgid, code, latency, dir); #endif } static __always_inline void tcp_events_submit_tcp_data( struct pt_regs *ctx, struct tcp_connection_t *pconn, enum STREAM_TYPE streamtype, enum CLIENT_SERVER_TYPE is_server, const void *data, size_t data_len) { // submit data to data channel size_t actual_len; data_channel_submit(ctx, pconn, data, data_len, &actual_len); // now send render message announcing its existence u64 now = get_timestamp(); #if DEBUG_TCP_DATA bpf_trace_printk("tcp_events_submit_tcp_data: tstamp=%u sk=%llx streamtype=%d\n", now, pconn->sk, streamtype); bpf_trace_printk(" is_server=%d, data_len=%u\n", is_server, data_len); #endif GET_PID_TGID; #ifdef STANDALONE_TCP_PROCESSOR struct tcp_events_t event = { .type = TCP_EVENT_TYPE_TCP_DATA, .pid = _tgid, .ts = now, .sk = (u64)pconn->sk, .__align.__pad0 = 0, .__align.__pad1 = 0}; event.tcp_data.length = actual_len; event.tcp_data.streamtype = (u8)streamtype; event.tcp_data.is_server = (u8)is_server; event.tcp_data.offset = pconn->streams[streamtype].total; bpf_perf_event_output(ctx, &tcp_events, BPF_F_CURRENT_CPU, &event, sizeof(event)); #else // Submit the control perf ring tcp data message perf_submit_agent_internal__tcp_data( ctx, now, (u64)pconn->sk, _tgid, actual_len, pconn->streams[streamtype].total, streamtype, is_server); #endif } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_tcp_processor.c ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #ifdef STANDALONE_TCP_PROCESSOR #define KBUILD_MODNAME "bpf_http" #include "../render_bpf.h" #include // Include this from render_bpf.c so we can use tail calls in tcp processor test code struct { __uint(type, BPF_MAP_TYPE_PROG_ARRAY); __uint(max_entries, NUM_TAIL_CALLS); __uint(key_size, sizeof(__u32)); __uint(value_size, sizeof(__u32)); } tail_calls SEC(".maps"); #endif //////////////////////////////////////////////////////////////////////////// // Utilities #include "bpf_debug.h" #include "bpf_memory.h" #include "bpf_types.h" //////////////////////////////////////////////////////////////////////////// // Subcomponents #include "tcp_processor.h" // TCP socket handling #include "bpf_tcp_socket.h" // Connection accept handling #include "bpf_inet_csk_accept.h" // TCP send/receive handling static __always_inline void tcp_client_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len); static __always_inline void tcp_server_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len); #define TCP_CLIENT_HANDLER(CTX, CONN, CTRL, ST, DATA, LEN) tcp_client_handler(CTX, CONN, CTRL, ST, DATA, LEN) #define TCP_SERVER_HANDLER(CTX, CONN, CTRL, ST, DATA, LEN) tcp_server_handler(CTX, CONN, CTRL, ST, DATA, LEN) #include "bpf_tcp_send_recv.h" // Data channel output #include "bpf_data_channel.h" // TCP events output #include "bpf_tcp_events.h" #ifndef ENABLE_TCP_DATA_STREAM // HTTP protocol handling #include "bpf_http_protocol.h" #endif ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// static __always_inline void tcp_client_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len) { #pragma passthrough on #if ENABLE_TCP_DATA_STREAM #pragma passthrough off tcp_events_submit_tcp_data(ctx, pconn, streamtype, 0, data, data_len); #pragma passthrough on #else #pragma passthrough off struct tcp_protocol_state_t *state = &(pconn->protocol_state); struct tcp_stream_info_t *strm = pconn->streams + (int)streamtype; // Perform protocol detection if (state->protocol == TCPPROTO_UNKNOWN) { // Detect HTTP if (state->candidates & TCP_PROTOCOL_BIT(TCPPROTO_HTTP)) { enum TCP_PROTOCOL_DETECT_RESULT res = http_detect(ctx, pconn, pctrl, streamtype, data, data_len); if (res == TPD_FAILED) { state->candidates &= ~TCP_PROTOCOL_BIT(TCPPROTO_HTTP); } else if (res == TPD_SUCCESS) { state->protocol = TCPPROTO_HTTP; } } // If no candidates left, no need to process anything else if (state->candidates == 0) { enable_tcp_connection(pctrl, -1, -1); return; } } switch (state->protocol) { case TCPPROTO_UNKNOWN: // If we don't know the protocol, don't process this at all return; case TCPPROTO_HTTP: http_process_request(ctx, pconn, pctrl, streamtype, data, data_len); break; default: bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); break; } #pragma passthrough on #endif #pragma passthrough off } static __always_inline void tcp_server_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len) { #pragma passthrough on #if ENABLE_TCP_DATA_STREAM #pragma passthrough off tcp_events_submit_tcp_data(ctx, pconn, streamtype, 1, data, data_len); #pragma passthrough on #else #pragma passthrough off struct tcp_protocol_state_t *state = &(pconn->protocol_state); // Parse the protocol switch (state->protocol) { case TCPPROTO_UNKNOWN: // If we don't know the protocol, don't process this at all return; case TCPPROTO_HTTP: http_process_response(ctx, pconn, pctrl, streamtype, data, data_len); break; default: bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); break; } #pragma passthrough on #endif #pragma passthrough off } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_tcp_send_recv.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // // bpf_tcp_send_recv.h - BPF TCP socket streaming receive/send buffering // // Requires: // To declare a tcp client handler, #define TCP_CLIENT_HANDLER(...) // To declare a tcp server handler, #define TCP_SERVER_HANDLER(...) #pragma once #include "bpf_debug.h" #include "bpf_memory.h" #include "bpf_tcp_socket.h" #include "bpf_types.h" #ifndef TCP_CLIENT_HANDLER #error "Must define TCP_CLIENT_HANDLER" #endif #ifndef TCP_SERVER_HANDLER #error "Must define TCP_SERVER_HANDLER" #endif //////////////////////////////////////////////////////////////////////////// // TCP Receive BEGIN_DECLARE_SAVED_ARGS(tcp_recvmsg) struct sock *sk; struct msghdr *msg; size_t len; int nonblock; int flags; // int *addr_len; // decoded msg void *iov_base; size_t iov_len; int written; int depth; END_DECLARE_SAVED_ARGS(tcp_recvmsg) //////////////////////////////////////////////////////////////////////////// // TCP Send BEGIN_DECLARE_SAVED_ARGS(tcp_sendmsg) struct sock *sk; struct msghdr *msg; size_t size; // decoded msg struct iovec *iov; void *iov_ptr; size_t iov_len; size_t iov_offset; unsigned long nr_segs; int written; int depth; END_DECLARE_SAVED_ARGS(tcp_sendmsg) //////////////////////////////////////////////////////////////////////////// // Send / Receive Stream Handlers // Processes data that ends up in the send and receive streams // to determine protocols and deal with them. //////////////////////////////////////////////////////////////////////////// // Helpers: CO-RE safe iov_iter checks and resolution /* * Returns true if msg->msg_iter designates ITER_IOVEC or ITER_KVEC across * kernel versions. Uses enum CO-RE relocation to avoid numeric constants. * Optionally returns the iter type value for logging (if type_out!=NULL). */ static __always_inline bool msg_iter_is_iov_or_kvec(const struct msghdr *msg, unsigned int *type_out) { // Default iter type value for logging unsigned int iter_type = 0; // Require enum values to exist; otherwise, return false (non-permissive) bool has_iov = bpf_core_enum_value_exists(enum iter_type, ITER_IOVEC); bool has_kvec = bpf_core_enum_value_exists(enum iter_type, ITER_KVEC); if (!(has_iov && has_kvec)) { if (type_out) *type_out = iter_type; return false; } __u64 ev_iov = bpf_core_enum_value(enum iter_type, ITER_IOVEC); __u64 ev_kvec = bpf_core_enum_value(enum iter_type, ITER_KVEC); // Prefer ordinal-era field (>= 5.14): iter_type is an enum-ordinal (u8) if (bpf_core_field_exists(((struct msghdr *)0)->msg_iter.iter_type)) { __u8 it = BPF_CORE_READ(msg, msg_iter.iter_type); iter_type = it; if (type_out) *type_out = iter_type; return it == ev_iov || it == ev_kvec; } // Bitmask-era (<= 5.13): use compat msghdr to read iov_iter.type struct msghdr___5_13_19 *msg_compat = (void *)msg; if (msg_compat) { if (bpf_probe_read_kernel(&iter_type, sizeof(iter_type), &msg_compat->msg_iter.type) == 0) { if (type_out) *type_out = iter_type; unsigned int mask = (unsigned int)(ev_iov | ev_kvec); return (iter_type & mask) != 0; } } // If all else fails, return false and pass through iter_type (0) if (type_out) *type_out = iter_type; return false; } // Resolve iovec pointer across kernel versions: use __iov if present (6.4+), // else fall back to iov (<= 6.3). Layout matches kvec, so it's valid either way. static __always_inline struct iovec *msg_iter_get_iov(struct msghdr *msg) { if (bpf_core_field_exists(((struct msghdr *)0)->msg_iter.__iov)) { return (struct iovec *)BPF_CORE_READ(msg, msg_iter.__iov); } return (struct iovec *)BPF_CORE_READ((struct msghdr___5_13_19 *)msg, msg_iter.iov); } /* * Returns true if msg->msg_iter designates ITER_UBUF on kernels that support it. * Uses CO-RE field/enum relocation to avoid relying on numeric enum values. * * Behavior across eras: * - <=5.13: no iter_type field and no UBUF; returns false. * - 5.14–5.19: iter_type exists but UBUF not present; returns false. * - >=6.0: iter_type exists and ITER_UBUF may exist; compare ordinal safely. */ static __always_inline bool iter_is_ubuf(const struct msghdr *msg) { if (!bpf_core_field_exists(((struct msghdr *)0)->msg_iter.iter_type)) return false; // bitmask-era: no UBUF if (!bpf_core_enum_value_exists(enum iter_type, ITER_UBUF)) return false; // target kernel lacks UBUF support __u8 it = BPF_CORE_READ(msg, msg_iter.iter_type); __u64 ev_ubuf = bpf_core_enum_value(enum iter_type, ITER_UBUF); return it == ev_ubuf; } /* * Resolve UBUF as an iovec-like (base,len) pair across kernel versions. * - 6.4+: prefer __ubuf_iovec overlay (base/len). iov_offset is outside overlay. * - 6.0–6.3: use ubuf pointer and count fields. Caller may apply iov_offset. */ static __always_inline void ubuf_as_iovec(const struct msghdr *msg, void **out_base, size_t *out_len) { // 6.4+ overlay: exposes base/len as struct iovec if (bpf_core_field_exists(((struct msghdr *)0)->msg_iter.__ubuf_iovec)) { void *base = BPF_CORE_READ(msg, msg_iter.__ubuf_iovec.iov_base); size_t len = BPF_CORE_READ(msg, msg_iter.__ubuf_iovec.iov_len); *out_base = base; *out_len = len; return; } // 6.0–6.3: plain ubuf pointer + count void *ubuf = BPF_CORE_READ(msg, msg_iter.ubuf); size_t cnt = BPF_CORE_READ(msg, msg_iter.count); *out_base = ubuf; *out_len = cnt; } static __always_inline void tcp_send_stream_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len) { // Are we an accepting socket, or an originating socket? if (pconn->parent_sk) { // Accepting sockets send from server TCP_SERVER_HANDLER(ctx, pconn, pctrl, streamtype, data, data_len); } else { // Originating sockets send from clients TCP_CLIENT_HANDLER(ctx, pconn, pctrl, streamtype, data, data_len); } } static __always_inline void tcp_recv_stream_handler( struct pt_regs *ctx, struct tcp_connection_t *pconn, struct tcp_control_value_t *pctrl, enum STREAM_TYPE streamtype, const void *data, size_t data_len) { // Are we an accepting socket, or an originating socket? if (pconn->parent_sk) { // Accepting sockets receive from clients TCP_CLIENT_HANDLER(ctx, pconn, pctrl, streamtype, data, data_len); } else { // Originating sockets receive from servers TCP_SERVER_HANDLER(ctx, pconn, pctrl, streamtype, data, data_len); } } //////////////////////////////////////////////////////////////////////////// // --- tcp_sendmsg ---------------------------------------------------- // Called when data is to be send to a TCP socket SEC("kprobe/tcp_sendmsg") __attribute__((noinline)) int handle_kprobe__tcp_sendmsg(struct pt_regs *ctx) { // In post 4.1 kernels: struct sock *sk, struct msghdr *msg, size_t size struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct msghdr *msg = (struct msghdr *)PT_REGS_PARM2(ctx); size_t size = (size_t)PT_REGS_PARM3(ctx); GET_PID_TGID if (!msg || !sk) { return 0; } struct tcp_connection_t *pconn; pconn = lookup_tcp_connection(sk); if (!pconn) { // For now ignore sends on sockets we haven't seen the tcp_init_sock for // #if TRACE_TCP_SEND // DEBUG_PRINTK("tcp_sendmsg found no tcp connection in kprobe\n"); // #endif return 0; } struct tcp_control_value_t *pctrl = get_tcp_control(pconn); if (!pctrl) { return 0; } // Quick ignore for sockets we deem uninteresting #ifndef ENABLE_TCP_DATA_STREAM if (pconn->protocol_state.candidates == 0) { #if TRACE_TCP_SEND DEBUG_PRINTK("no candidates left on tcp_sendmsg\n"); #endif return 0; } if (pconn->streams[ST_SEND].protocol_count == 0) { #if TRACE_TCP_SEND DEBUG_PRINTK("no protocols interested in tcp_sendmsg\n"); #endif return 0; } #endif if (pctrl->streams[ST_SEND].enable == 0) { #if TRACE_TCP_SEND DEBUG_PRINTK("send stream is disabled for sk=%llx\n", (u64)sk); #endif return 0; } // decode msg struct iovec *iov = NULL; unsigned long nr_segs = 0; size_t iov_offset = 0; unsigned int iter_type = 0; void *iov_ptr = NULL; size_t iov_len = 0; int written = 0; int depth = 1; if (iter_is_ubuf(msg)) { // ITER_UBUF: treat as a single contiguous segment ubuf_as_iovec(msg, &iov_ptr, &iov_len); } else if (msg_iter_is_iov_or_kvec(msg, &iter_type)) { // IOVEC/KVEC: can access through iov since both share layout iov = (struct iovec *)msg_iter_get_iov(msg); if (iov) { iov_ptr = BPF_CORE_READ(iov, iov_base); iov_len = BPF_CORE_READ(iov, iov_len); } } else { #if DEBUG_TCP_SEND DEBUG_PRINTK("unsupported iov type: %u\n", iter_type); #endif bpf_log(ctx, BPF_LOG_UNSUPPORTED_IO, (u64)ST_SEND, (u64)sk, (u64)iter_type); return 0; } nr_segs = BPF_CORE_READ(msg, msg_iter.nr_segs); iov_offset = BPF_CORE_READ(msg, msg_iter.iov_offset); // Defer arguments to kretprobe BEGIN_SAVE_ARGS(tcp_sendmsg) SAVE_ARG(sk) SAVE_ARG(msg) SAVE_ARG(size) // decoded msg structure SAVE_ARG(iov) SAVE_ARG(iov_ptr) SAVE_ARG(iov_len) SAVE_ARG(iov_offset) SAVE_ARG(nr_segs) SAVE_ARG(written) SAVE_ARG(depth) END_SAVE_ARGS(tcp_sendmsg) #if TRACE_TCP_SEND DEBUG_PRINTK( "tcp_sendmsg enter: pid %u request to send up to %u bytes " "on socket %llx\n", pconn->upid, (unsigned int)size, sk); DEBUG_PRINTK(" nr_segs=%lu, iov_offset=%lu\n", nr_segs, iov_offset); #endif return 0; } SEC("kretprobe/tcp_sendmsg") int handle_kretprobe__tcp_sendmsg(struct pt_regs *ctx) { // This call recurses up to TCP_TAIL_CALL_MAX_DEPTH times, // writing up to DATA_CHANNEL_CHUNK_MAX each time bpf_tail_call(ctx, &tail_calls, TAIL_CALL_CONTINUE_TCP_SENDMSG); return 0; } SEC("kprobe") int continue_tcp_sendmsg(struct pt_regs *ctx) { GET_PID_TGID GET_ARGS_MISSING_OK(tcp_sendmsg, args) if (args == NULL) { return 0; } // Get copied byte count int copied = (int)PT_REGS_RC(ctx); if (copied <= 0) { DELETE_ARGS(tcp_sendmsg); return 0; } // Lookup our connection struct tcp_connection_t *pconn; pconn = lookup_tcp_connection(args->sk); if (!pconn) { // Socket was destroyed -during- tcp_sendmsg (happens on some control flow paths?) #if DEBUG_TCP_SEND DEBUG_PRINTK("tcp_sendmsg found no tcp connection in kretprobe\n"); bpf_log(ctx, BPF_LOG_TABLE_MISSING_KEY, BPF_TABLE_TCP_CONNECTIONS, (u64)args->sk, _tgid); #endif DELETE_ARGS(tcp_sendmsg); return 0; } if (args->iov_len == 0) { // Load next iovec if (args->nr_segs > 0) { void *iov_ptr = args->iov_ptr; size_t iov_len = args->iov_len; #if TRACE_TCP_SEND DEBUG_PRINTK("tcp_sendmsg continue: ptr=%llx, len=%d\n", iov_ptr, iov_len); #endif args->iov_ptr = iov_ptr; args->iov_len = iov_len; args->iov++; args->nr_segs--; if (args->iov_offset) { // Data in the first iovec can have an offset to the first byte args->iov_ptr += args->iov_offset; args->iov_offset = 0; } } else { DELETE_ARGS(tcp_sendmsg); return 0; } } const u8 *data = (const u8 *)args->iov_ptr; int remaining = copied - args->written; int to_copy = remaining > DATA_CHANNEL_CHUNK_MAX ? DATA_CHANNEL_CHUNK_MAX : remaining; if (to_copy > args->iov_len) { to_copy = args->iov_len; } write_to_tcp_stream(ctx, pconn, ST_SEND, data, to_copy, tcp_send_stream_handler); args->written += to_copy; args->iov_ptr += to_copy; args->iov_len -= to_copy; if (to_copy == remaining || args->depth == TCP_TAIL_CALL_MAX_DEPTH) { DELETE_ARGS(tcp_sendmsg); } else { args->depth++; bpf_tail_call(ctx, &tail_calls, TAIL_CALL_CONTINUE_TCP_SENDMSG); } return 0; } // --- tcp_recvmsg ---------------------------------------------------- // Called when data is to be received by a TCP socket SEC("kprobe/tcp_recvmsg") int handle_kprobe__tcp_recvmsg(struct pt_regs *ctx) { // In kernels 4.1 onwards: struct sock *sk, struct msghdr *msg, size_t len, int nonblock, int flags, int *addr_len struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct msghdr *msg = (struct msghdr *)PT_REGS_PARM2(ctx); size_t len = (size_t)PT_REGS_PARM3(ctx); int nonblock = (int)PT_REGS_PARM4(ctx); int flags = (int)PT_REGS_PARM5(ctx); // int *addr_len = NULL; -- unused GET_PID_TGID struct tcp_connection_t *pconn; pconn = lookup_tcp_connection(sk); if (!pconn) { // Ignore receives on sockets we haven't seen the tcp_init_sock for // #if TRACE_TCP_RECEIVE // DEBUG_PRINTK("tcp_recvmsg found no tcp connection in kprobe\n"); // #endif return 0; } struct tcp_control_value_t *pctrl = get_tcp_control(pconn); if (!pctrl) { return 0; } // Quick ignore for sockets we deem uninteresting #ifndef ENABLE_TCP_DATA_STREAM if (pconn->protocol_state.candidates == 0) { #if TRACE_TCP_RECEIVE DEBUG_PRINTK("no candidates left on tcp_recvmsg\n"); #endif return 0; } if (pconn->streams[ST_RECV].protocol_count == 0) { #if TRACE_TCP_RECEIVE DEBUG_PRINTK("no protocols interested in tcp_recvmsg\n"); #endif return 0; } #endif if (pctrl->streams[ST_RECV].enable == 0) { #if TRACE_TCP_RECEIVE DEBUG_PRINTK("recv stream is disabled for sk=%llx\n", (u64)sk); #endif return 0; } // decode msg struct iovec *iov = NULL; void *iov_base = NULL; size_t iov_len = 0; int written = 0; int depth = 1; unsigned int iter_type = 0; if (iter_is_ubuf(msg)) { // ITER_UBUF: read as iovec-like values ubuf_as_iovec(msg, &iov_base, &iov_len); } else if (msg_iter_is_iov_or_kvec(msg, &iter_type)) { // IOVEC/KVEC: can access through iov since both share layout iov = (struct iovec *)msg_iter_get_iov(msg); iov_base = BPF_CORE_READ(iov, iov_base); iov_len = BPF_CORE_READ(iov, iov_len); } else { #if DEBUG_TCP_RECEIVE DEBUG_PRINTK("unsupported iov type: %u\n", iter_type); #endif bpf_log(ctx, BPF_LOG_UNSUPPORTED_IO, (u64)ST_RECV, (u64)sk, (u64)iter_type); return 0; } // Add to receiver table BEGIN_SAVE_ARGS(tcp_recvmsg) SAVE_ARG(sk) SAVE_ARG(msg) SAVE_ARG(len) SAVE_ARG(nonblock) SAVE_ARG(flags) // SAVE_ARG(addr_len) // decoded msg structure SAVE_ARG(iov_base) SAVE_ARG(iov_len) SAVE_ARG(written) SAVE_ARG(depth) END_SAVE_ARGS(tcp_recvmsg) #if TRACE_TCP_RECEIVE DEBUG_PRINTK( "tcp_recvmsg enter: pid %u request to receive up to %u " "bytes on socket %llx\n", pconn->upid, (unsigned int)len, sk); DEBUG_PRINTK(" iov_base=%llx, iov_len=%d\n", iov_base, iov_len); #endif return 0; } SEC("kretprobe/tcp_recvmsg") int handle_kretprobe__tcp_recvmsg(struct pt_regs *ctx) { // This call recurses up to TCP_TAIL_CALL_MAX_DEPTH times, // writing up to DATA_CHANNEL_CHUNK_MAX each time bpf_tail_call(ctx, &tail_calls, TAIL_CALL_CONTINUE_TCP_RECVMSG); return 0; } SEC("kprobe") int continue_tcp_recvmsg(struct pt_regs *ctx) { GET_PID_TGID GET_ARGS_MISSING_OK(tcp_recvmsg, args) if (args == NULL) { return 0; } // Get copied byte count int copied = (int)PT_REGS_RC(ctx); if (copied <= 0) { DELETE_ARGS(tcp_recvmsg); return 0; } // Lookup our connection struct tcp_connection_t *pconn; pconn = lookup_tcp_connection(args->sk); if (!pconn) { // Socket was destroyed -during- tcp_recvmsg (happens on some control flow paths?) #if DEBUG_TCP_RECEIVE DEBUG_PRINTK("tcp_recvmsg found no tcp connection in kretprobe\n"); bpf_log(ctx, BPF_LOG_TABLE_MISSING_KEY, BPF_TABLE_TCP_CONNECTIONS, (u64)args->sk, _tgid); #endif DELETE_ARGS(tcp_recvmsg); return 0; } const u8 *data = ((const u8 *)args->iov_base) + args->written; int remaining = copied - args->written; int to_copy = remaining > DATA_CHANNEL_CHUNK_MAX ? DATA_CHANNEL_CHUNK_MAX : remaining; write_to_tcp_stream(ctx, pconn, ST_RECV, data, to_copy, tcp_recv_stream_handler); args->written += to_copy; if (to_copy == remaining || args->depth == TCP_TAIL_CALL_MAX_DEPTH) { DELETE_ARGS(tcp_recvmsg); } else { args->depth++; bpf_tail_call(ctx, &tail_calls, TAIL_CALL_CONTINUE_TCP_RECVMSG); } return 0; } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_tcp_socket.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once // // bpf_tcp_connection.h - TCP socket tracking // // Requires: // Define list of protocols prior to including this file as well and #define // index type to TCP_PROTOCOL_TYPE and #define the number of protocols to // TCP_PROTOCOL_COUNT, and TCP_PROTOCOL_MASK to (1<sk}; struct tcp_control_value_t *pvalue = bpf_map_lookup_elem(&_tcp_control, &key); return pvalue; } // Call this when we don't want to bother processing a tcp // connection any longer to minimize overhead // recv/send = -1 (ignore), 0 (unchanged), 1 (don't ignore) static __always_inline void enable_tcp_connection(struct tcp_control_value_t *pctrl, int recv, int send) { // DEBUG_PRINTK("enable_tcp_connection: recv=%d, send=%d\n", recv, send); if (send != 0) { pctrl->streams[ST_SEND].enable = send < 0 ? 0 : 1; } if (recv != 0) { pctrl->streams[ST_RECV].enable = recv < 0 ? 0 : 1; } } // Call this when we're completely done with a tcp connection and want to // release the socket map static void delete_tcp_connection(struct pt_regs *ctx, struct tcp_connection_t *pconn, struct sock *sk) { #if TRACE_TCP_CONNECTION DEBUG_PRINTK("delete_tcp_connection(%llx)\n", sk); #endif // remove from kernel data structures struct tcp_control_key_t key = {.sk = (u64)pconn->sk}; bpf_map_delete_elem(&_tcp_control, &key); int ret = bpf_map_delete_elem(&_tcp_connections, &sk); if (ret != 0) { #if DEBUG_TCP_CONNECTION DEBUG_PRINTK("delete_tcp_connection: delete on non-existent socket sk=%llx\n", sk); #endif bpf_log(ctx, BPF_LOG_TABLE_BAD_REMOVE, BPF_TABLE_TCP_CONNECTIONS, (u64)sk, 0); } } static __always_inline void write_to_tcp_stream( struct pt_regs *ctx, struct tcp_connection_t *pconn, enum STREAM_TYPE streamtype, const void *src_data, size_t src_bytes, void (*tcp_stream_handler)( struct pt_regs *ctx, struct tcp_connection_t *, struct tcp_control_value_t *, enum STREAM_TYPE, const void *, size_t)) { struct tcp_stream_info_t *strm = pconn->streams + (int)streamtype; struct tcp_control_value_t *pctrl = get_tcp_control(pconn); if (!pctrl) { bpf_log(ctx, BPF_LOG_UNREACHABLE, 0, 0, 0); return; } u64 start = pctrl->streams[streamtype].start; // If the entirety of what we are going to write is // before the window then we can skip this data if ((strm->total + src_bytes) <= start) { // just add the stream total without calling the callback strm->total += src_bytes; return; } u64 data_len = src_bytes; u8 *data = (u8 *)src_data; // If what we are about to write starts before the window, clip to the front // end of the window if (strm->total < start) { size_t trim_len = (start - strm->total); data_len -= trim_len; data += trim_len; strm->total = start; } // Inspect the buffer tcp_stream_handler(ctx, pconn, pctrl, streamtype, data, data_len); // Advance total strm->total += data_len; } // // BPF Probes for intercepting the creation and destruction of TCP sockets // // --- tcp_init_sock ---------------------------------------------------- // Where the start of TCP socket lifetimes is for IPv4 and IPv6 SEC("kprobe/tcp_init_sock") int handle_kprobe__tcp_init_sock(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_connection_t *pconn; pconn = create_tcp_connection(ctx, sk); if (!pconn) { #if DEBUG_TCP_CONNECTION DEBUG_PRINTK("tcp_init_sock: couldn't create tcp_connection"); #endif GET_PID_TGID; bpf_log(ctx, BPF_LOG_TABLE_BAD_INSERT, BPF_TABLE_TCP_CONNECTIONS, _tgid, (u64)sk); return 0; } return 0; } // --- security_sk_free ---------------------------------------------- // This is where final socket destruction happens for all socket types SEC("kprobe/security_sk_free") int handle_kprobe__security_sk_free(struct pt_regs *ctx) { struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); struct tcp_connection_t *pconn = lookup_tcp_connection(sk); if (pconn) { delete_tcp_connection(ctx, pconn, sk); } return 0; } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/bpf_types.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once typedef u64 PID_TGID; typedef u32 TGID; typedef u32 PID; // global variables for error counts // TODO: expose error counts to userspace volatile long save_args_update_existing_error = 0; volatile long save_args_update_full_error = 0; // tgid is upper 32 bits of pid_tgid, lower 32 bits is pid #define TGID_FROM_PID_TGID(X) ((TGID)((X) >> 32)) #define PID_FROM_PID_TGID(X) ((PID)((X))) // common operation, use this macro to define _pid and _tgid for kprobes #define GET_PID_TGID \ PID_TGID _pid_tgid = bpf_get_current_pid_tgid(); \ _pid_tgid &= 0xFFFFFFFFFFFFFFFF; \ TGID _tgid = TGID_FROM_PID_TGID(_pid_tgid); \ PID _pid = PID_FROM_PID_TGID(_pid_tgid); \ if (_tgid == 0 || _pid == 0) { \ bpf_log(ctx, BPF_LOG_INVALID_PID_TGID, _pid_tgid, 0, 0); \ } \ u64 _cpu = bpf_get_smp_processor_id(); typedef u64 TIMESTAMP; // // probe->retprobe argument deferral // this stuff is a cut-and-paste-error reduction mechanism // because we often need to pass arguments received in a 'kprobe' // into its corresponding 'kretprobe', where they are no longer available. // #define SAVED_ARGS_TABLE_KEY _pid_tgid // #define SAVED_ARGS_TABLE_KEY _cpu #define SAVED_ARGS_TABLE(FUNC) FUNC##_active #define SAVED_ARGS_TYPE(FUNC) struct FUNC##_active_args_t #define BEGIN_DECLARE_SAVED_ARGS(FUNC) \ SAVED_ARGS_TYPE(FUNC) \ { #define END_DECLARE_SAVED_ARGS(FUNC) \ } \ ; \ \ struct { \ __uint(type, BPF_MAP_TYPE_HASH); \ __type(key, u64); \ __type(value, SAVED_ARGS_TYPE(FUNC)); \ __uint(max_entries, 1024); \ } SAVED_ARGS_TABLE(FUNC) SEC(".maps"); #define BEGIN_SAVE_ARGS(FUNC) \ { \ SAVED_ARGS_TYPE(FUNC) __saved_args = { #define SAVE_ARG(ARG) .ARG = ARG, // don't bpf_trace_printk if we have debugging turned off #if DEBUG_OTHER_MAP_ERRORS #define __DEBUG_OTHER_MAP_ERRORS_PRINTK false #else #define __DEBUG_OTHER_MAP_ERRORS_PRINTK true #endif #define END_SAVE_ARGS(FUNC) \ } \ ; \ int __ret = bpf_map_update_elem(&SAVED_ARGS_TABLE(FUNC), &SAVED_ARGS_TABLE_KEY, &__saved_args, BPF_NOEXIST); \ if (__ret != 0 && __DEBUG_OTHER_MAP_ERRORS_PRINTK) { \ /* Check if key already exists to distinguish duplicate vs table full */ \ void *existing = bpf_map_lookup_elem(&SAVED_ARGS_TABLE(FUNC), &SAVED_ARGS_TABLE_KEY); \ if (existing) { \ save_args_update_existing_error++; \ } else { \ save_args_update_full_error++; \ } \ } \ } #define GET_ARGS(FUNC, NAME) \ SAVED_ARGS_TYPE(FUNC) *NAME = bpf_map_lookup_elem(&SAVED_ARGS_TABLE(FUNC), &SAVED_ARGS_TABLE_KEY); \ if (NAME == NULL) { \ __DEBUG_OTHER_MAP_ERRORS_PRINTK || bpf_trace_printk(#FUNC ": args table missing key. tgid=%u\n", _tgid); \ } #define GET_ARGS_MISSING_OK(FUNC, NAME) \ SAVED_ARGS_TYPE(FUNC) *NAME = bpf_map_lookup_elem(&SAVED_ARGS_TABLE(FUNC), &SAVED_ARGS_TABLE_KEY); #define DELETE_ARGS(FUNC) bpf_map_delete_elem(&SAVED_ARGS_TABLE(FUNC), &SAVED_ARGS_TABLE_KEY); // Timestamp abstraction // We do this so that eventually we could test the BPF outside // of the agent (in python or whatever). could replace this to simulate tight // timing tests eventually too static __always_inline TIMESTAMP get_timestamp(void) { #ifdef STANDALONE_TCP_PROCESSOR return (TIMESTAMP)bpf_ktime_get_ns(); #else // Get time, adjust for boot return bpf_ktime_get_ns() + boot_time_adjustment; #endif } // Error reporting abstraction struct BPF_LOG_GLOBALS { u32 period_start_ms; u32 count; }; struct { __uint(type, BPF_MAP_TYPE_ARRAY); __type(key, u32); __type(value, struct BPF_LOG_GLOBALS); __uint(max_entries, BPF_MAX_CPUS); } bpf_log_globals_per_cpu SEC(".maps"); #define bpf_log(ctx, _code, _arg0, _arg1, _arg2) \ do { \ _bpf_log(__LINE__, ctx, _code, _arg0, _arg1, _arg2); \ } while (0) static void __always_inline _bpf_log(int filelineid, struct pt_regs *ctx, enum BPF_LOG_CODE code, u64 arg0, u64 arg1, u64 arg2) { // Get timestamp in milliseconds TIMESTAMP now = get_timestamp(); u32 now_ms = (u32)(now / 1000000ull); // Get per-cpu globals int cpu = bpf_get_smp_processor_id(); if (cpu < 0 || cpu >= BPF_MAX_CPUS) { // what to do when 'log' itself fails? return; } struct BPF_LOG_GLOBALS *globals = bpf_map_lookup_elem(&bpf_log_globals_per_cpu, &cpu); if (globals == NULL) { // what to do when 'log' itself fails? return; } // Have we exceeded the previous period? // if so, start a new period if (now_ms >= globals->period_start_ms + BPF_LOG_THROTTLE_PERIOD_LENGTH_MS) { globals->period_start_ms = now_ms; globals->count = 0; } // Increment the count for this period globals->count += 1; // If the count has exceeded the max per period, log if it's the first one, else drop if (globals->count == (BPF_LOG_THROTTLE_MAX_PER_PERIOD + 1)) { #ifdef STANDALONE_TCP_PROCESSOR bpf_trace_printk("bpf_log: throttled on cpu %u\n", cpu); #else perf_submit_agent_internal__bpf_log(ctx, now, (u64)filelineid, (u64)BPF_LOG_THROTTLED, cpu, 0, 0); #endif } else if (globals->count <= BPF_LOG_THROTTLE_MAX_PER_PERIOD) { #ifdef STANDALONE_TCP_PROCESSOR bpf_trace_printk("bpf_log: filelineid=%d, code=%u, now=%llu\n", filelineid, code, now); bpf_trace_printk(" args=%llu, %llu, %llu\n", arg0, arg1, arg2); #else perf_submit_agent_internal__bpf_log(ctx, now, (u64)filelineid, (u64)code, arg0, arg1, arg2); #endif } } // Useful conversion from ip4 space to ip6 space for addresses static __always_inline struct in6_addr make_ipv6_address(__be32 addr) { struct in6_addr addr6 = {.in6_u.u6_addr32 = {0, 0, 0xffff0000, addr}}; return addr6; } ================================================ FILE: collector/kernel/bpf_src/tcp-processor/tcp-processor.py ================================================ #!/usr/bin/env python3 # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # DEPRECATED: This BCC-based debugging tool is no longer maintained # since the project has migrated to libbpf. The code is kept for # historical reference but may not work with current BPF code. # # Standalone debugging code for tcp-processor subtree of bpf_src # To use, sudo python tcp-processor.py # from bcc import BPF #, DEBUG_PREPROCESSOR import io, os, sys from pcpp.preprocessor import Preprocessor, OutputDirective, Action import ctypes as ct if "--debug" in sys.argv: import ptvsd print("Waiting for debugger attach") ptvsd.enable_attach(address=('localhost', 5678), redirect_output=True) ptvsd.wait_for_attach() breakpoint() # # Wrapper for BCC that prints useful diagnostics # class BPFWrapper: def __init__(self, bpf): self._bpf = bpf def attach_kprobe(self, event=b"", event_off=0, fn_name=b"", event_re=b""): print("attach_kprobe: event={} fn_name={}".format(event, fn_name)) try: self._bpf.attach_kprobe(event,event_off,fn_name,event_re) except: print(" failed for {}".format(event)) return print(" succeeded for {}".format(event)) def attach_kprobe_all(self, events, event_off=0, fn_name=b"", event_re=b""): print("attach_kprobe_all: events={} fn_name={}".format(str(events), fn_name)) for event in events: try: self._bpf.attach_kprobe(event,event_off,fn_name,event_re) except: print(" failed for {}".format(event)) continue print(" succeeded for {}".format(event)) def attach_kretprobe(self, event=b"", fn_name=b"", event_re=b"", maxactive=0): print("attach_kretprobe: event={} fn_name={}".format(event, fn_name)) try: self._bpf.attach_kretprobe(event,fn_name,event_re,maxactive) except: print(" failed for {}".format(event)) return print(" succeeded for {}".format(event)) def attach_kretprobe_all(self, events, fn_name=b"", event_re=b"", maxactive=0): print("attach_kretprobe_all: events={} fn_name={}".format(str(events), fn_name)) for event in events: try: self._bpf.attach_kretprobe(event,fn_name,event_re, maxactive) except: print(" failed for {}".format(event)) continue print(" succeeded for {}".format(event)) def __getattr__(self, name): if name in self.__dict__: return self.__dict__[name] if name in self.__class__.__dict__: return self.__class__.__dict__[name] return getattr(self._bpf, name) def __getitem__(self, key): return self._bpf[key] # # Pass-through preprocessor to improve BCC's parsing # class PassThruPreprocessor(Preprocessor): def __init__(self,lexer=None): super(PassThruPreprocessor, self).__init__(lexer) self.passthrough = False self.current_file_line_id = 0 self.file_line_table = [] def write_debug_info(self, debugfile): lineid = 0 debugfile.write("int g_bpf_debug_line_info[] = {\n") for x in self.file_line_table: debugfile.write(" {0},\n".format(x["line"])) lineid += 1 debugfile.write("};\n") lineid = 0 debugfile.write("const char *g_bpf_debug_file_info[] = {\n") for x in self.file_line_table: debugfile.write(" {0},\n".format(x["file"])) lineid += 1 debugfile.write("};\n") def token(self): """Method to return individual tokens, overriding custom macros""" tok = super(PassThruPreprocessor, self).token() if tok and tok.value=="__FILELINEID__": tok.value=str(self.current_file_line_id) self.file_line_table.append({ "file": self.macros['__FILE__'].value[0].value, "line": tok.lineno }) self.current_file_line_id += 1 return tok def on_include_not_found(self, is_system_include, curdir, includepath): # If includes are not found, complain sys.stderr.write("Unable to find include file: "+str(includepath)+"\n") sys.exit(1) def on_unknown_macro_in_defined_expr(self, tok): # Pass through as expanded as possible, unexpanded without complaining if not possible return None def on_unknown_macro_in_expr(self, tok): # Pass through as expanded as possible, unexpanded without complaining if not possible return None def on_directive_handle(self, directive, toks, ifpassthru, precedingtoks): if directive.value=="pragma": if len(toks)>=1 and toks[0].type=="CPP_ID" and toks[0].value=="passthrough": if len(toks)==3 and toks[1].type=="CPP_WS" and toks[2].type=="CPP_ID" and toks[2].value=="on": # Turn on passthrough self.passthrough = True raise OutputDirective(Action.IgnoreAndRemove) elif len(toks)==3 and toks[1].type=="CPP_WS" and toks[2].type=="CPP_ID" and toks[2].value=="off": # Turn on passthrough self.passthrough = False raise OutputDirective(Action.IgnoreAndRemove) sys.stderr.write("Invalid passthrough pragma\n") sys.exit(1) if self.passthrough: # Pass through without execution EVERYTHING if we have used pragma passthrough raise OutputDirective(Action.IgnoreAndPassThrough) if directive.value=="define": # Process and ALSO pass through as well return None if directive.value=="include": if toks[0].type=="CPP_STRING": # Process #include "" normally return True else: # Always pass through #include<> raise OutputDirective(Action.IgnoreAndPassThrough) # Attempt to process all other directives return True def on_directive_unknown(self, directive, toks, ifpassthru, precedingtoks): raise OutputDirective(Action.IgnoreAndPassThrough) # # Parse the BPF # pp = PassThruPreprocessor() instr = "#define DEBUG_LOG 1\n#define ENABLE_TCP_DATA_STREAM 1\n#define _PROCESSING_BPF 1\n#define STANDALONE_TCP_PROCESSOR 1\n" + open("./bpf_tcp_processor.c","rt").read() outfile = io.StringIO() pp.add_path("../../../../src/") pp.add_path("../../../../") pp.parse(instr, source = "./bpf_tcp_processor.c") pp.write(outfile) preprocessed_bpf_text = outfile.getvalue() # print("preproc: "+preprocessed_bpf_text) print("debug info:") pp.write_debug_info(sys.stdout) b = BPFWrapper(BPF(text=preprocessed_bpf_text)) # Set up tail calls (mirroring bpf_handler.cc) tail_calls = b.get_table("tail_calls") #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_ON_UDP_SEND_SKB__2"].value[0].value))] = b.load_func("on_udp_send_skb__2", BPF.KPROBE) #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_ON_UDP_V6_SEND_SKB__2"].value[0].value))] = b.load_func("on_udp_v6_send_skb__2", BPF.KPROBE) #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_ON_IP_SEND_SKB__2"].value[0].value))] = b.load_func("on_ip_send_skb__2", BPF.KPROBE) #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_ON_IP6_SEND_SKB__2"].value[0].value))] = b.load_func("on_ip6_send_skb__2", BPF.KPROBE) #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_HANDLE_RECEIVE_UDP_SKB"].value[0].value))] = b.load_func("handle_receive_udp_skb", BPF.KPROBE) #tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_HANDLE_RECEIVE_UDP_SKB__2"].value[0].value))] = b.load_func("handle_receive_udp_skb__2", BPF.KPROBE) tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_CONTINUE_TCP_SENDMSG"].value[0].value))] = b.load_func("continue_tcp_sendmsg", BPF.KPROBE) tail_calls[ct.c_int(int(pp.macros["TAIL_CALL_CONTINUE_TCP_RECVMSG"].value[0].value))] = b.load_func("continue_tcp_recvmsg", BPF.KPROBE) # # Attach probes # # Create/destroy b.attach_kprobe(event="tcp_init_sock", fn_name="handle_kprobe__tcp_init_sock") b.attach_kprobe(event="security_sk_free", fn_name="handle_kprobe__security_sk_free") # Accept b.attach_kprobe(event="inet_csk_accept", fn_name="handle_kprobe__inet_csk_accept") b.attach_kretprobe(event="inet_csk_accept", fn_name="handle_kretprobe__inet_csk_accept") # Send b.attach_kprobe(event="tcp_sendmsg", fn_name="handle_kprobe__tcp_sendmsg") b.attach_kretprobe(event="tcp_sendmsg", fn_name="handle_kretprobe__tcp_sendmsg") # Receive b.attach_kprobe(event="tcp_recvmsg", fn_name="handle_kprobe__tcp_recvmsg") b.attach_kretprobe(event="tcp_recvmsg", fn_name="handle_kretprobe__tcp_recvmsg") # # Print trace output # class TCPEventHTTPResponse(ct.Structure): _fields_ = [ ("code", ct.c_ushort), ("__pad0", ct.c_uint8*6), ("latency", ct.c_ulonglong) ] class TCPEventTCPData(ct.Structure): _fields_ = [ ("length", ct.c_uint), ("streamtype", ct.c_uint8), ("is_server", ct.c_uint8), ("__pad0", ct.c_uint16), ("offset", ct.c_ulonglong) ] class TCPEventData(ct.Union): _fields_ = [ ("http_response", TCPEventHTTPResponse), ("tcp_data", TCPEventTCPData), ("__pad0", ct.c_ulonglong), ("__pad1", ct.c_ulonglong) ] class TCPEvent(ct.Structure): _anonymous_ = ("u",) _fields_ = [ ("type", ct.c_uint), ("pid", ct.c_uint), ("ts", ct.c_ulonglong), ("sk", ct.c_ulonglong), ("u", TCPEventData) ] class TCPDataHeader(ct.Structure): _pack_ = 1 _fields_ = [ ("length", ct.c_ulonglong), ] class TCPDataMessage(ct.Structure): _pack_ = 1 _fields_ = [ ("hdr", TCPDataHeader), ("data", ct.c_ubyte * 256) ] print("sizeof(TCPDataHeader) = {}".format(ct.sizeof(TCPDataHeader))) print("sizeof(TCPDataMessage) = {}".format(ct.sizeof(TCPDataMessage))) def print_tcp_event(cpu, data, size): assert size >= ct.sizeof(TCPEvent) event = ct.cast(data, ct.POINTER(TCPEvent)).contents if event.type == 0: print(">>> TCP_EVENT_TYPE_HTTP_RESPONSE(pid=%u, ts=%u, sk=0x%X, code=%u, latency=%u)" % (event.pid, event.ts, event.sk, event.http_response.code, event.http_response.latency)) elif event.type == 1: print(">>> TCP_EVENT_TYPE_TCP_DATA(pid=%u, ts=%u, sk=0x%X, length=%u, streamtype=%u, is_server=%u, offset=%u)" % (event.pid, event.ts, event.sk, event.tcp_data.length, event.tcp_data.streamtype, event.tcp_data.is_server, event.tcp_data.offset)) else: print(">>> UNKNOWN TCP EVENT") def process_data_channel(cpu, data, size): assert size >= ct.sizeof(TCPDataHeader) header = ct.cast(data, ct.POINTER(TCPDataHeader)).contents datalen = header.length out = "### DATA(size: {}, datalen: {}): ".format(size, datalen) print(out) assert size >= ct.sizeof(TCPDataHeader) + datalen message = ct.cast(data, ct.POINTER(TCPDataMessage)).contents out = ct.cast(message.data, ct.c_char_p).value[0:datalen] print(out) b[b"tcp_events"].open_perf_buffer(print_tcp_event) b[b"data_channel"].open_perf_buffer(process_data_channel) while 1: try: had_message = False fields = b.trace_fields(True) if fields: (task, pid, cpu, flags, ts, msg) = fields if msg: had_message=True print(msg.decode('latin-1')) if not had_message: # Prefer to print messages instead of polling to speed through them b.perf_buffer_poll(10) except ValueError: continue ================================================ FILE: collector/kernel/bpf_src/tcp-processor/tcp_processor.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once // Protocols #define TCP_PROTOCOL_TYPE u32 #define TCP_PROTOCOL_COUNT 1 #define TCPPROTO_UNKNOWN 0 #define TCPPROTO_HTTP 1 #define TCP_PROTOCOL_BIT(X) (1 << ((X) - 1)) #define TCP_PROTOCOL_MASK ((1 << TCP_PROTOCOL_COUNT) - 1) #define TCP_TAIL_CALL_MAX_DEPTH 8 // Stream types enum STREAM_TYPE { ST_SEND = 0, ST_RECV = 1 }; #ifndef _PROCESSING_BPF inline const char *stream_type_to_string(enum STREAM_TYPE stream_type) { switch (stream_type) { case ST_SEND: return "SEND"; case ST_RECV: return "RECV"; } throw std::runtime_error("invalid STREAM_TYPE value"); } #endif // TCP Data sent to userland #define TCP_DATA_SIZE 256 // TCP Control channel map struct tcp_control_key_t { u64 sk; // the socket to control }; struct tcp_control_stream_t { u64 enable; // 0 = disable this side of the stream, 1 = enable u64 start; // start offset of stream to start watching }; struct tcp_control_value_t { struct tcp_control_stream_t streams[2]; // control for send and receive streams }; // this header could probably be removed if we are // ever allowed to perf_submit directly from kernel memory struct data_channel_header_t { u64 length; }; // Protocol handling enum TCP_PROTOCOL_DETECT_RESULT { TPD_FAILED = -1, TPD_UNKNOWN = 0, TPD_SUCCESS = 1 }; #include "common/client_server_type.h" ================================================ FILE: collector/kernel/bpf_src/vmlinux_compat.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // This file contains old struct definitions for compatibility #pragma once // required for btf relocations #pragma clang attribute push(__attribute__((preserve_access_index)), apply_to = record) struct iov_iter___5_13_19 { /* * Bit 0 is the read/write bit, set if we're writing. * Bit 1 is the BVEC_FLAG_NO_REF bit, set if type is a bvec and * the caller isn't expecting to drop a page reference when done. */ unsigned int type; size_t iov_offset; size_t count; union { const struct iovec *iov; const struct kvec *kvec; const struct bio_vec *bvec; struct xarray *xarray; struct pipe_inode_info *pipe; }; union { unsigned long nr_segs; struct { unsigned int head; unsigned int start_head; }; loff_t xarray_start; }; }; struct msghdr___5_13_19 { void *msg_name; /* ptr to socket address structure */ int msg_namelen; /* size of socket address structure */ struct iov_iter___5_13_19 msg_iter; /* data */ /* * Ancillary data. msg_control_user is the user buffer used for the * recv* side when msg_control_is_user is set, msg_control is the kernel * buffer used for all other cases. */ union { void *msg_control; // void __user *msg_control_user; }; bool msg_control_is_user : 1; __kernel_size_t msg_controllen; /* ancillary data buffer length */ unsigned int msg_flags; /* flags on received message */ struct kiocb *msg_iocb; /* ptr to iocb for async requests */ }; struct css_id___3_11 { /* * The css to which this ID points. This pointer is set to valid value * after cgroup is populated. If cgroup is removed, this will be NULL. * This pointer is expected to be RCU-safe because destroy() * is called after synchronize_rcu(). But for safe use, css_tryget() * should be used for avoiding race. */ struct cgroup_subsys_state *css; /* * ID of this css. */ unsigned short id; /* * Depth in hierarchy which this ID belongs to. */ unsigned short depth; /* * ID is freed by RCU. (and lookup routine is RCU safe.) */ struct rcu_head rcu_head; /* * Hierarchy of CSS ID belongs to. */ unsigned short stack[0]; /* Array of Length (depth+1) */ }; struct cgroup_name___3_11 { struct rcu_head rcu_head; char name[]; }; struct cgroup___3_11 { unsigned long flags; /* "unsigned long" so bitops work */ int id; /* ida allocated in-hierarchy ID */ /* * We link our 'sibling' struct into our parent's 'children'. * Our children link their 'sibling' into our 'children'. */ struct list_head sibling; /* my parent's children */ struct list_head children; /* my children */ struct list_head files; /* my files */ struct cgroup *parent; /* my parent */ struct dentry *dentry; /* cgroup fs entry, RCU protected */ /* * Monotonically increasing unique serial number which defines a * uniform order among all cgroups. It's guaranteed that all * ->children lists are in the ascending order of ->serial_nr. * It's used to allow interrupting and resuming iterations. */ u64 serial_nr; /* * This is a copy of dentry->d_name, and it's needed because * we can't use dentry->d_name in cgroup_path(). * * You must acquire rcu_read_lock() to access cgrp->name, and * the only place that can change it is rename(), which is * protected by parent dir's i_mutex. * * Normally you should use cgroup_name() wrapper rather than * access it directly. */ struct cgroup_name___3_11 *name; /* Private pointers for each registered subsystem */ // struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT]; struct cgroupfs_root *root; /* * List of cgrp_cset_links pointing at css_sets with tasks in this * cgroup. Protected by css_set_lock. */ struct list_head cset_links; /* * Linked list running through all cgroups that can * potentially be reaped by the release agent. Protected by * release_list_lock */ struct list_head release_list; /* * list of pidlists, up to two for each namespace (one for procs, one * for tasks); created on demand. */ struct list_head pidlists; struct mutex pidlist_mutex; /* For css percpu_ref killing and RCU-protected deletion */ struct rcu_head rcu_head; struct work_struct destroy_work; atomic_t css_kill_cnt; /* List of events which userspace want to receive */ struct list_head event_list; spinlock_t event_list_lock; /* directory xattrs */ struct simple_xattrs xattrs; }; struct cgroup_subsys_state___3_11 { /* * The cgroup that this subsystem is attached to. Useful * for subsystems that want to know about the cgroup * hierarchy structure */ struct cgroup___3_11 *cgroup; /* reference count - access via css_[try]get() and css_put() */ struct percpu_ref refcnt; unsigned long flags; /* ID for this css, if possible */ struct css_id___3_11 *id; /* Used to put @cgroup->dentry on the last css_put() */ struct work_struct dput_work; }; struct tcp_sock___rcv_rtt_est_rtt { struct { u32 rtt; u32 seq; u32 time; } rcv_rtt_est; }; // remove the instruction to add preserve_access_index #pragma clang attribute pop ================================================ FILE: collector/kernel/bpf_src/vmlinux_extensions.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // This file contains kernel data structures that are not in libbpf's regular vmlinux.h repository // (https://github.com/libbpf/vmlinux.h). // // libbpf's vmlinux.h is prepared with their own config, not from distros: // - https://github.com/libbpf/vmlinux.h/blob/main/.github/workflows/vmlinux.h.yml // - https://github.com/libbpf/vmlinux.h/blob/main/scripts/gen-vmlinux-header.sh // - https://github.com/libbpf/vmlinux.h/blob/main/kconfigs/config.x86_64 // and so contains partial structures. // // This file adds more that is required by the project. These structs must have __attribute__((preserve_access_index)); // See "Defining own CO-RE-relocatable type definitions" in https://nakryiko.com/posts/bpf-core-reference-guide/ // // You can generate an h file from a live system using: // bpftool btf dump file /sys/kernel/btf/vmlinux format c // (command extracted from https://github.com/aquasecurity/btfhub/blob/main/docs/btfgen-internals.md#btfhub) #pragma once // this is used to apply the required attribute to all structs #pragma clang attribute push(__attribute__((preserve_access_index)), apply_to = record) #define rcu_head callback_head union nf_inet_addr { __u32 all[4]; __be32 ip; __be32 ip6[4]; struct in_addr in; struct in6_addr in6; }; union nf_conntrack_man_proto { __be16 all; }; struct nf_conntrack_man { union nf_inet_addr u3; union nf_conntrack_man_proto u; uint16_t l3num; }; struct nf_conntrack_tuple { struct nf_conntrack_man src; struct { union nf_inet_addr u3; union { __be16 all; } u; u_int8_t protonum; u_int8_t dir; } dst; }; struct nf_conntrack_tuple_hash { struct hlist_nulls_node hnnode; struct nf_conntrack_tuple tuple; }; struct nf_conn { struct nf_conntrack_tuple_hash tuplehash[2]; }; // Optional sk_buff field flavor for kernels with NF_CONNTRACK configured. struct sk_buff___with_nfct { unsigned long _nfct; }; struct cgroup; struct cgroup_subsys; struct cftype { char name[64]; long unsigned int private; size_t max_write_len; unsigned int flags; unsigned int file_offset; struct cgroup_subsys *ss; struct list_head node; struct kernfs_ops *kf_ops; }; struct cgroup_subsys { struct cgroup_subsys_state *(*css_alloc)(struct cgroup_subsys_state *); bool early_init : 1; bool implicit_on_dfl : 1; bool threaded : 1; int id; const char *name; const char *legacy_name; struct cgroup_root *root; struct idr css_idr; struct list_head cfts; struct cftype *dfl_cftypes; struct cftype *legacy_cftypes; unsigned int depends_on; }; struct cgroup_subsys_state { struct cgroup *cgroup; struct cgroup_subsys *ss; struct percpu_ref refcnt; struct list_head sibling; struct list_head children; struct list_head rstat_css_node; int id; unsigned int flags; u64 serial_nr; atomic_t online_cnt; struct work_struct destroy_work; struct rcu_work destroy_rwork; struct cgroup_subsys_state *parent; }; struct cgroup { struct cgroup_subsys_state self; long unsigned int flags; int level; int max_depth; int nr_descendants; int nr_dying_descendants; int max_descendants; int nr_populated_csets; int nr_populated_domain_children; int nr_populated_threaded_children; int nr_threaded_children; struct kernfs_node *kn; // struct cgroup_file procs_file; // struct cgroup_file events_file; // struct cgroup_file psi_files[3]; u16 subtree_control; u16 subtree_ss_mask; u16 old_subtree_control; u16 old_subtree_ss_mask; struct cgroup_subsys_state *subsys[14]; struct cgroup_root *root; struct list_head cset_links; struct list_head e_csets[14]; struct cgroup *dom_cgrp; struct cgroup *old_dom_cgrp; // struct cgroup_rstat_cpu *rstat_cpu; struct list_head rstat_css_list; long : 64; long : 64; // struct cacheline_padding _pad_; struct cgroup *rstat_flush_next; // struct cgroup_base_stat last_bstat; // struct cgroup_base_stat bstat; struct prev_cputime prev_cputime; struct list_head pidlists; struct mutex pidlist_mutex; wait_queue_head_t offline_waitq; struct work_struct release_agent_work; struct psi_group *psi; // struct cgroup_bpf bpf; atomic_t congestion_count; // struct cgroup_freezer_state freezer; struct bpf_local_storage *bpf_cgrp_storage; struct cgroup *ancestors[0]; }; struct cgroup_root { struct kernfs_root *kf_root; unsigned int subsys_mask; int hierarchy_id; struct list_head root_list; struct callback_head rcu; long : 64; long : 64; struct cgroup cgrp; struct cgroup *cgrp_ancestor_storage; atomic_t nr_cgrps; unsigned int flags; char release_agent_path[4096]; char name[64]; long : 64; long : 64; long : 64; long : 64; long : 64; long : 64; }; struct css_set { struct cgroup_subsys_state *subsys[14]; refcount_t refcount; struct css_set *dom_cset; struct cgroup *dfl_cgrp; int nr_tasks; struct list_head tasks; struct list_head mg_tasks; struct list_head dying_tasks; struct list_head task_iters; struct list_head e_cset_node[14]; struct list_head threaded_csets; struct list_head threaded_csets_node; struct hlist_node hlist; struct list_head cgrp_links; struct list_head mg_src_preload_node; struct list_head mg_dst_preload_node; struct list_head mg_node; struct cgroup *mg_src_cgrp; struct cgroup *mg_dst_cgrp; struct css_set *mg_dst_cset; bool dead; struct callback_head callback_head; }; struct task_struct___with_css_set { struct css_set *cgroups; }; // dynamically generated by the kernel based on config variables, read with bpf_core_enum_value enum cgroup_subsys_id { mem_cgroup_subsys_id, memory_cgrp_id }; // remove the instruction to add preserve_access_index #pragma clang attribute pop ================================================ FILE: collector/kernel/buffered_poller.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include constexpr u16 DNS_MAX_PACKET_LEN = 512; #ifdef DEBUG_PID static BufferedPoller *singleton_ = nullptr; #endif // DEBUG_PID static constexpr u64 DNS_TIMEOUT_TIME_NS = 10'000'000'000ull; namespace { std::string_view comm_to_string(std::uint8_t const (&comm)[16]) { auto const length = strnlen(reinterpret_cast(comm), sizeof(comm)); return std::string_view(reinterpret_cast(comm), length); } } // namespace BufferedPoller::BufferedPoller( uv_loop_t &loop, PerfContainer &container, IBufferedWriter &writer, u64 time_adjustment, CurlEngine &curl_engine, FileDescriptor &bpf_dump_file, logging::Logger &log, ProbeHandler &probe_handler, struct render_bpf_bpf *skel, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings const &cgroup_settings, ::ebpf_net::ingest::Encoder *encoder, KernelCollectorRestarter &kernel_collector_restarter) : PerfPoller(container), loop_(loop), time_adjustment_(time_adjustment), bpf_dump_file_(bpf_dump_file), log_(log), buffered_writer_(writer), probe_handler_(probe_handler), skel_(skel), writer_(buffered_writer_, monotonic, time_adjustment, encoder), collector_index_({writer_}), process_handler_(writer_, collector_index_, log_), cgroup_handler_(writer_, curl_engine, std::move(cgroup_settings), log), nat_handler_(writer_, log), tcp_socket_table_ever_full_(false), tslot_(socket_stats_interval_sec * 1e9, 16), tcp_socket_stats_(tslot_), udp_socket_table_ever_full_(false), udp_socket_stats_{{{tslot_}, {tslot_}}}, all_probes_loaded_(false), kernel_collector_restarter_(kernel_collector_restarter) { if (buffered_writer_.buf_size() < MAX_ENCODED_DNS_MESSAGE) { throw std::runtime_error("BufferedPoller: buf size too small for DNS"); } { using namespace ebpf_net::agent_internal; memset(handlers_, 0, sizeof(handlers_)); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); add_handler(); } // Create a tcp data handler for the tcp_data message tcp_data_handler_ = std::make_unique(loop_, probe_handler_, skel, writer_, container, log_); // Set perf container callback for events container.set_callback(loop, this, [](void *ctx) { ((BufferedPoller *)ctx)->handle_event(); }); #ifdef DEBUG_PID singleton_ = this; signal(SIGUSR1, [](int) { singleton_->process_handler_.debug_pid_dump(); }); #endif // DEBUG_PID } void BufferedPoller::handle_event() { process_samples(true); } void BufferedPoller::poll(void) { process_samples(false); } void BufferedPoller::process_samples(bool is_event) { u64 t = monotonic() + time_adjustment_; PerfReader reader(container_, t); // in the case of event-driven poll, print debugging information to assist // with understanding the profile of the perf buffers if (is_event && is_log_whitelisted(AgentLogKind::PERF)) { std::string cstr = container_.inspect(); LOG::debug_in(AgentLogKind::PERF, "* Perf event triggered *\ncontainer_ is:\n{}\n\n", cstr); } // read the top contents of our container into our buffer while (!reader.empty()) { auto peek_type = reader.peek_type(); auto handle_bpf_lost_samples = [this]() { send_report_if_recent_loss(); log_.warn("Lost {} bpf samples - restarting kernel collector.", lost_count_); kernel_collector_restarter_.request_restart(); }; #ifndef NDEBUG if (debug_bpf_lost_samples_) { lost_count_ += 1; handle_bpf_lost_samples(); return; } #endif if (peek_type == PERF_RECORD_SAMPLE) { if (bpf_dump_file_) { auto const view = reader.peek_message(); bpf_dump_file_.write_all(view.first); bpf_dump_file_.write_all(view.second); } u16 length = reader.peek_unpadded_length(); /* special handling for DNS packets */ if (length < 10) throw std::runtime_error("got message < timestamp+rpc_id"); u16 rpc_id = reader.peek_rpc_id(); /* if rpc_id is in handlers list, call it. */ u32 handlers_idx = agent_internal_hash(rpc_id); if (handlers_[handlers_idx] != nullptr) { (this->*handlers_[handlers_idx])(reader, length); continue; } /* unknown message -- this is a bug */ throw std::runtime_error("unexpected bpf message\n"); } else if (peek_type == PERF_RECORD_LOST) { lost_count_ += reader.peek_n_lost(); reader.pop(); handle_bpf_lost_samples(); return; } else { throw std::runtime_error("Unexpected record type\n"); } } /* do we need to process stats? */ s16 relative = tcp_socket_stats_.relative_timeslot(t); if (relative != 0) { send_stats_from_queue(t); udp_send_stats_from_queue(t); send_report_if_recent_loss(); } // clear out buffer if there's still anything left if (auto error = buffered_writer_.flush(); error && buffered_writer_.is_writable()) { throw std::runtime_error(fmt::format("flush failed at end: {}", error)); } } void BufferedPoller::send_report_if_recent_loss() { if (lost_count_ == notified_lost_count_) { return; // no need to report } writer_.bpf_lost_samples(lost_count_ - notified_lost_count_); notified_lost_count_ = lost_count_; } u64 BufferedPoller::serv_lost_count() { return lost_count_; } template < typename MessageMetadata, BufferedPoller::message_handler_fn Handler, std::size_t MaxPadding, typename Alignment> void BufferedPoller::message_handler_entrypoint(PerfReader &reader, u16 length) { struct { u64 timestamp; typename MessageMetadata::wire_message msg; Alignment padding[(MaxPadding + (sizeof(Alignment) - 1)) / sizeof(Alignment)]; } in; if constexpr (MaxPadding) { if (length < sizeof(u64) + MessageMetadata::wire_message_size) { throw std::runtime_error(fmt::format( "message truncated (`{}`: {}/{})", MessageMetadata::name, length, sizeof(u64) + MessageMetadata::wire_message_size)); } else if (length > sizeof(u64) + MessageMetadata::wire_message_size + MaxPadding) { throw std::runtime_error(fmt::format( "message too long (`{}`: {}/{})", MessageMetadata::name, length, sizeof(u64) + MessageMetadata::wire_message_size + MaxPadding)); } } else if (length != sizeof(u64) + MessageMetadata::wire_message_size) { throw std::runtime_error(fmt::format( "invalid message length (`{}`: {}/{})", MessageMetadata::name, length, sizeof(u64) + MessageMetadata::wire_message_size)); } // Get the cpu index we're on so we can read the right data // do this before the pop, so we get the right index std::size_t const cpu_index = reader.peek_index(); reader.pop_and_copy_to(reinterpret_cast(&in)); // at this point it's ok to skip the handler since the message has been // consumed // don't handle the message when disconnected if (!buffered_writer_.is_writable()) { return; } (this->*Handler)( { .timestamp = in.timestamp, .cpu_index = cpu_index, .payload = {reinterpret_cast(&in), length}, .padding = {reinterpret_cast(&in.msg) + MessageMetadata::wire_message_size, MaxPadding}, }, in.msg); } template < typename MessageMetadata, BufferedPoller::message_handler_fn Handler, std::size_t MaxPadding, typename Alignment> void BufferedPoller::add_handler() { u32 idx = agent_internal_hash(MessageMetadata::rpc_id); if (handlers_[idx] != nullptr) { throw std::runtime_error("tried to add_handler to an occupied slot"); } handlers_[idx] = &BufferedPoller::message_handler_entrypoint; } void BufferedPoller::handle_dns_message(message_metadata const &metadata, jb_agent_internal__dns_packet &msg) { // we are assuming that struct id_addr is exactly 4 bytes and that struct // in6_addr is exactly 16 bytes // static_assert( sizeof(in_addr) == 4, "serialization protocol assumes IPv4 " "address structure is 4 bytes exactly"); static_assert( sizeof(in6_addr) == 16, "serialization protocol assumes IPv6 address structure is 16 " "bytes exactly"); LOG::debug_in(AgentLogKind::DNS, "handle_dns_message"); u64 const sk = msg.sk; auto const pkt_len = msg._len - jb_agent_internal__dns_packet__data_size; auto const &dns_packet = metadata.padding; /* sanity check the message */ if (dns_packet.data() + pkt_len > metadata.payload.data() + metadata.payload.size()) { throw std::runtime_error("dns: garbled message length"); } // Look up the udp socket table entry auto pos = udp_socket_table_.find(sk); if (pos.index == udp_socket_table_.invalid) { if (!udp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("ERROR: handle_dns_message - sk not found. sk={:x}", sk); } return; } u32 sk_id = pos.index; /* make variables to parse the DNS packet */ char hostname_out[DNS_NAME_MAX_LENGTH]; int hostname_len = 0; int num_ipv4_addrs = MAX_ENCODED_IP_ADDRS; struct in_addr ipv4_addrs[MAX_ENCODED_IP_ADDRS]; int num_ipv6_addrs = MAX_ENCODED_IP_ADDRS; struct in6_addr ipv6_addrs[MAX_ENCODED_IP_ADDRS]; /* see if this is a request */ uint16_t type_out; uint16_t qid_out; int is_response; int ret = dns_parse_query(dns_packet.data(), pkt_len, &is_response, &type_out, &qid_out, hostname_out, &hostname_len); if (ret != ARES_SUCCESS) { LOG::debug_in( AgentLogKind::DNS, "dns_parse_query returned {}, total len {} valid len {} packet {:n}", ret, msg.total_len, pkt_len, spdlog::to_hex(dns_packet.data(), dns_packet.data() + pkt_len)); return; } LOG::debug_in( AgentLogKind::DNS, "dns_parse_query successful, total len {} valid len " "{}\nis_response {} type_out {} qid_out {} hostname {}", msg.total_len, pkt_len, is_response, type_out, qid_out, std::string_view(hostname_out, hostname_len)); if (!is_response) { DnsRequests::dns_request_key key{ .qid = qid_out, .type = type_out, .name = std::string(hostname_out, hostname_len), .is_rx = (bool)msg.is_rx}; // Add request to table, for later processing when response shows up DnsRequests::dns_request_value value{.timestamp_ns = metadata.timestamp, .sk = sk}; dns_requests_.add(key, value); return; } // looking for requests in the other direction DnsRequests::dns_request_key key{ .qid = qid_out, .type = type_out, .name = std::string(hostname_out, hostname_len), .is_rx = !msg.is_rx}; // parse the reply int send_a_aaaa_response = 0; u16 sent_hostname_len = 0; char *sent_hostname = NULL; ret = dns_parse_a_aaaa_reply( dns_packet.data(), pkt_len, hostname_out, &hostname_len, ipv4_addrs, &num_ipv4_addrs, ipv6_addrs, &num_ipv6_addrs); if (hostname_len == 0 || (num_ipv4_addrs + num_ipv6_addrs) == 0) { LOG::debug_in( AgentLogKind::DNS, "dns_parse_a_aaaa_reply returned {}, total len {} valid len {} " "packet {:n}", ret, msg.total_len, pkt_len, spdlog::to_hex(dns_packet.data(), dns_packet.data() + pkt_len)); send_a_aaaa_response = 0; } else { /** * we continue here even if packet was partial or corrupt, as long as we * successfully parsed some IP addresses and the host name */ LOG::debug_in( AgentLogKind::DNS, "dns_parse_a_aaaa_reply successful, total len {} valid len " "{}\nnum_ipv4_addrs {} num_ipv6_addrs {}", msg.total_len, pkt_len, is_response, type_out, qid_out, std::string_view(hostname_out, hostname_len)); send_a_aaaa_response = 1; // truncate hostname */ sent_hostname_len = (hostname_len < DNS_NAME_MAX_LENGTH) ? (u16)hostname_len : DNS_NAME_MAX_LENGTH; sent_hostname = hostname_out + hostname_len - sent_hostname_len; } // Only process DNS replies have have a matching request // otherwise someone could be spoofing us std::list reqs; dns_requests_.lookup(key, reqs); if (reqs.size() > 0) { /* see if this response matches requests we have seen */ for (auto &req : reqs) { /* submit the dns response with latency information */ u64 request_timestamp = req->second.timestamp_ns; u64 latency_ns = metadata.timestamp - request_timestamp; if (send_a_aaaa_response) { LOG::debug_in( AgentLogKind::DNS, "sending DNS for hostname {} num_ipv4_addrs:{} " "num_ipv6_addrs:{} latency_ns:{}", sent_hostname, num_ipv4_addrs, num_ipv6_addrs, latency_ns); // if the socket is exactly the same, then we match bool matching = sk == req->second.sk; if (!matching) { // if it's not, but the port and address is exactly the same, then we // also match auto pos1 = udp_socket_table_.find(sk); if (pos1.index == udp_socket_table_.invalid) { if (!udp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("ERROR: handle_dns_message - sk not found. sk={:x}", sk); } } auto pos2 = udp_socket_table_.find(req->second.sk); if (pos2.index == udp_socket_table_.invalid) { if (!udp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("ERROR: handle_dns_message - sk2 not found. sk2={}", req->second.sk); } } // more strict would be this, thought i'm not sure port is necessarily // the same in k8s environments either. matching = // pos1.entry->pid==pos2.entry->pid && // pos1.entry->lport==pos2.entry->lport; matching = pos1.entry->pid == pos2.entry->pid; // why does the socket not match for request and response if (!matching) { LOG::debug_in( AgentLogKind::DNS, "dns socket mismatch {}@{}:{}(pid={}) != {}@{}:{}(pid={})\n" "hostname: {} num_ipv4_addrs:{} num_ipv6_addrs:{} latency: " "{}", sk, IPv6Address::from(pos1.entry->laddr), pos1.entry->lport, pos1.entry->pid, req->second.sk, IPv6Address::from(pos2.entry->laddr), pos2.entry->lport, pos2.entry->pid, sent_hostname, num_ipv4_addrs, num_ipv6_addrs, latency_ns); } } // if receiving a dns response, this is a client and 'total time' is the // appropriate metric if sending a dns response, this is a server and // 'processing time' is the appropriate metric writer_.dns_response_tstamp( metadata.timestamp, sk_id, hostname_len, /* domain_name */ jb_blob{sent_hostname, sent_hostname_len}, /* ipv4_addrs */ jb_blob{(char *)ipv4_addrs, (u16)(sizeof(u32) * num_ipv4_addrs)}, /* ipv6_addrs */ jb_blob{(char *)ipv6_addrs, (u16)(sizeof(struct in6_addr) * num_ipv6_addrs)}, latency_ns, msg.is_rx ? SC_CLIENT : SC_SERVER); } // else { // // someday add other dns responses, or dns resolution errors //} } /* remove request key */ dns_requests_.remove_all_with_key(key); } } void BufferedPoller::timeout_dns_request(u64 timestamp_ns, const DnsRequests::Request &req) { u64 t_req = req->second.timestamp_ns; u64 sk = req->second.sk; // Look up the udp socket table entry auto pos = udp_socket_table_.find(sk); if (pos.index == udp_socket_table_.invalid) { if (!udp_socket_table_ever_full_ && all_probes_loaded_) { // Just a debug message here, it's possible for a udp socket lifetime to // race with the timeout LOG::debug_in(AgentLogKind::DNS, "ERROR: timeout_dns_request - sk not found. sk={:x}", sk); } } else { u32 sk_id = pos.index; u64 duration_ns = (timestamp_ns - t_req); /* truncate hostname */ const char *hostname_out = req->first.name.c_str(); size_t hostname_len = req->first.name.size(); u16 sent_hostname_len = (hostname_len < DNS_NAME_MAX_LENGTH) ? (u16)hostname_len : DNS_NAME_MAX_LENGTH; const char *sent_hostname = hostname_out + hostname_len - sent_hostname_len; LOG::debug_in(AgentLogKind::DNS, "sending DNS timeout for hostname {} duration_ns {}", sent_hostname, duration_ns); writer_.dns_timeout_tstamp( timestamp_ns, sk_id, hostname_len, /* domain_name */ jb_blob{sent_hostname, sent_hostname_len}, duration_ns); } // drop this request from dns_requests_ dns_requests_.remove(req); } void BufferedPoller::slow_poll() { u64 const t = monotonic() + time_adjustment_; process_dns_timeouts(t); } void BufferedPoller::process_dns_timeouts(u64 t) { std::list old_reqs; dns_requests_.lookup_older_than(t - DNS_TIMEOUT_TIME_NS, old_reqs); for (auto &req : old_reqs) { timeout_dns_request(t, req); } } void BufferedPoller::handle_new_socket(message_metadata const &metadata, jb_agent_internal__new_sock_created &msg) { /** * NOTE: this is not the only handling of new sockets (contrary to name). * reset_tcp_counters also reports a new socket. */ LOG::debug_in(AgentLogKind::TCP, "handle_new_socket: sk={:x} pid={}", msg.sk, msg.pid); if (tcp_socket_table_.full()) { log_.warn("handle_new_socket: tcp socket table is full! dropping socket"); tcp_socket_table_ever_full_ = true; return; } auto pos = tcp_socket_table_.insert(msg.sk); if (pos.index != tcp_socket_table_.invalid) { tcp_index_to_sk_[pos.index] = msg.sk; } else { log_.error("handle_new_socket: duplicate tcp socket sk={:x} pid={}", msg.sk, msg.pid); return; } writer_.new_sock_info_tstamp(metadata.timestamp, msg.pid, msg.sk); } void BufferedPoller::handle_set_state_ipv4(message_metadata const &metadata, jb_agent_internal__set_state_ipv4 &msg) { LOG::trace_in( AgentLogKind::TCP, "handle_set_state_ipv4: sk:{:x}, {}:{} -> {}:{} (tx_rx={})", msg.sk, IPv4Address::from(msg.src), msg.sport, IPv4Address::from(msg.dest), msg.dport, msg.tx_rx); // Ensure that if we get a state_set_ipv4 that the socket is something that // exists in our table already auto pos = tcp_socket_table_.find(msg.sk); if (pos.index == tcp_socket_table_.invalid) { if (!tcp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("handle_set_state_ipv4: socket not tracked sk={:x}", msg.sk); } return; } writer_.set_state_ipv4_tstamp(metadata.timestamp, msg.dest, msg.src, msg.dport, msg.sport, msg.sk, msg.tx_rx); nat_handler_.handle_set_state_ipv4(metadata.timestamp, &msg); } void BufferedPoller::handle_set_state_ipv6(message_metadata const &metadata, jb_agent_internal__set_state_ipv6 &msg) { LOG::trace_in( AgentLogKind::TCP, "handle_set_state_ipv6: sk:{}, {}:{} -> {}:{} (tx_rx={})", msg.sk, IPv6Address::from(msg.src), msg.sport, IPv6Address::from(msg.dest), msg.dport, msg.tx_rx); // Ensure that if we get a state_set_ipv4 that the socket is something that // exists in our table already auto pos = tcp_socket_table_.find(msg.sk); if (pos.index == tcp_socket_table_.invalid) { if (!tcp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("handle_set_state_ipv6: socket not tracked sk={:x}", msg.sk); } return; } writer_.set_state_ipv6_tstamp(metadata.timestamp, msg.dest, msg.src, msg.dport, msg.sport, msg.sk, msg.tx_rx); nat_handler_.handle_set_state_ipv6(metadata.timestamp, &msg); } void BufferedPoller::handle_close_socket(message_metadata const &metadata, jb_agent_internal__close_sock_info &msg) { auto pos = tcp_socket_table_.find(msg.sk); if (pos.index == tcp_socket_table_.invalid) { // This should be prevented in BPF except when the socket table was full if (!tcp_socket_table_ever_full_ && all_probes_loaded_) { log_.error("handle_close_socket: socket not found sk={:x}", msg.sk); } return; } LOG::debug_in(AgentLogKind::TCP, "handle_close_socket: sk={:x}", msg.sk); // send out a statistics message if needed for (u32 epoch = 0; epoch < n_epochs; epoch++) { auto &stats = tcp_socket_stats_.lookup_relative(pos.index, epoch, false).second; if (stats.valid == true) { send_socket_stats(metadata.timestamp, msg.sk, stats); } } bool success = tcp_socket_table_.erase(msg.sk); if (!success) { throw std::runtime_error(fmt::format("handle_close_socket: removing socket from table failed sk={:x}", msg.sk)); } writer_.close_sock_info_tstamp(metadata.timestamp, msg.sk); nat_handler_.handle_close_socket(metadata.timestamp, &msg); // Also clean up any tcp data protocol handlers this socket may have // associated with it tcp_data_handler_->handle_close_socket(msg.sk); } void BufferedPoller::handle_rtt_estimator(message_metadata const &metadata, jb_agent_internal__rtt_estimator &msg) { auto pos = tcp_socket_table_.find(msg.sk); if (pos.index == tcp_socket_table_.invalid) { if (!tcp_socket_table_ever_full_ && all_probes_loaded_) { LOG::debug_in(AgentLogKind::TCP, "handle_rtt_estimator: rtt_estimator on unknown socket sk={:x}", msg.sk); } return; } auto entry = pos.entry; /* we have a valid position. will go on to compute statistics */ u64 diff_bytes_acked = msg.bytes_acked - entry->bytes_acked; u32 diff_delivered = msg.packets_delivered - entry->packets_delivered; u32 diff_retrans = msg.packets_retrans - entry->packets_retrans; u64 diff_bytes_received = msg.bytes_received - entry->bytes_received; u32 diff_rcv_holes = msg.rcv_holes - entry->rcv_holes; u32 diff_rcv_delivered = msg.rcv_delivered - entry->rcv_delivered; /* differences should be positive, if interpreted as signed numbers. avoid accumulating these huge diffs in stats below. */ if (((s64)diff_bytes_acked) < ((s64)0)) { diff_bytes_acked = 0; } if (((s32)diff_delivered) < ((s32)0)) { // on kernels < 4.6, packets_delivered is an estimate and can under-count // and even become negative. make sure we're reporting non-negative numbers diff_delivered = 0; } if (((s32)diff_retrans) < ((s32)0)) { diff_retrans = 0; } if (((s64)diff_bytes_received) < ((s64)0)) { diff_bytes_received = 0; } if (((s32)diff_rcv_holes) < ((s32)0)) { diff_rcv_holes = 0; } if (((s32)diff_rcv_delivered) < ((s32)0)) { diff_rcv_delivered = 0; } /* update the entry for next time */ entry->bytes_acked = msg.bytes_acked; entry->packets_delivered = msg.packets_delivered; entry->packets_retrans = msg.packets_retrans; entry->bytes_received = msg.bytes_received; entry->rcv_holes = msg.rcv_holes; entry->rcv_delivered = msg.rcv_delivered; /* find the statistics, and ask it to enqueue */ auto &stats = tcp_socket_stats_.lookup(pos.index, metadata.timestamp, true).second; /* if stats were invalid, reset the values */ if (!stats.valid) { stats.diff_bytes_acked = diff_bytes_acked; stats.diff_delivered = diff_delivered; stats.diff_retrans = diff_retrans; stats.max_srtt = msg.srtt; stats.diff_bytes_received = diff_bytes_received; stats.diff_rcv_holes = diff_rcv_holes; stats.diff_rcv_delivered = diff_rcv_delivered; stats.max_rcv_rtt = msg.rcv_rtt; stats.valid = true; } else { stats.diff_bytes_acked += diff_bytes_acked; stats.diff_delivered += diff_delivered; stats.diff_retrans += diff_retrans; stats.max_srtt = std::max(stats.max_srtt, msg.srtt); stats.diff_bytes_received += diff_bytes_received; stats.diff_rcv_holes += diff_rcv_holes; stats.diff_rcv_delivered += diff_rcv_delivered; stats.max_rcv_rtt = std::max(stats.max_rcv_rtt, msg.rcv_rtt); } } void BufferedPoller::handle_reset_tcp_counters(message_metadata const &metadata, jb_agent_internal__reset_tcp_counters &msg) { LOG::debug_in(AgentLogKind::TCP, "handle_reset_tcp_counters: sk={:x} pid={}", msg.sk, msg.pid); // first, write telemetry like handle_new_socket writer_.new_sock_info_tstamp(metadata.timestamp, msg.pid, msg.sk); if (tcp_socket_table_.full()) { log_.warn("handle_reset_tcp_counters: tcp socket table is full! dropping socket"); tcp_socket_table_ever_full_ = true; return; } auto pos = tcp_socket_table_.insert(msg.sk); if (pos.index != tcp_socket_table_.invalid) { tcp_index_to_sk_[pos.index] = msg.sk; } else { log_.error("handle_reset_tcp_counters: duplicate tcp socket sk={:x} pid={}", msg.sk, msg.pid); return; } auto entry = pos.entry; entry->bytes_acked = msg.bytes_acked; entry->packets_delivered = msg.packets_delivered; entry->packets_retrans = msg.packets_retrans; entry->bytes_received = msg.bytes_received; } void BufferedPoller::handle_tcp_syn_timeout(message_metadata const &metadata, jb_agent_internal__tcp_syn_timeout &msg) { writer_.syn_timeout_tstamp(metadata.timestamp, msg.sk); } void BufferedPoller::handle_tcp_reset(message_metadata const &metadata, jb_agent_internal__tcp_reset &msg) { writer_.tcp_reset_tstamp(metadata.timestamp, msg.sk, msg.is_rx); } void BufferedPoller::handle_http_response(message_metadata const &metadata, jb_agent_internal__http_response &msg) { LOG::debug_in( AgentLogKind::HTTP, "handle_http_response: timestamp={}, sk={:x}, pid={}, code={}, " "latency_ns={}, client_server={}", metadata.timestamp, msg.sk, msg.pid, msg.code, msg.latency_ns, client_server_type_to_string((enum CLIENT_SERVER_TYPE)msg.client_server)); writer_.http_response_tstamp(metadata.timestamp, msg.sk, msg.pid, msg.code, msg.latency_ns, msg.client_server); } void BufferedPoller::send_socket_stats(u64 t, u64 sk, tcp_statistics &stats) { if ((stats.diff_bytes_acked > 0) || (stats.diff_retrans > 0)) { writer_.socket_stats_tstamp(t, sk, stats.diff_bytes_acked, stats.diff_delivered, stats.diff_retrans, stats.max_srtt, 0); } if ((stats.diff_bytes_received > 0) || (stats.diff_rcv_holes > 0)) { writer_.socket_stats_tstamp( t, sk, stats.diff_bytes_received, stats.diff_rcv_delivered, stats.diff_rcv_holes, stats.max_rcv_rtt, 1); } stats.valid = false; } void BufferedPoller::send_stats_from_queue(u64 t) { if (tcp_socket_stats_.relative_timeslot(t) == 0) { /* not ready */ return; } auto &queue = tcp_socket_stats_.current_queue(); while (!queue.empty()) { /* get the next index */ u32 index = queue.peek(); /* get the stats entry for that index */ auto &stats = tcp_socket_stats_.lookup_relative(index, 0, false).second; if (stats.valid) { /* write the message */ send_socket_stats(t, tcp_index_to_sk_[index], stats); /* send_socket_stats sets stats.valid = false */ } queue.pop(); } /* done. advance the current timeslot */ tcp_socket_stats_.advance(); } void BufferedPoller::handle_udp_new_socket(message_metadata const &metadata, jb_agent_internal__udp_new_socket &msg) { LOG::debug_in( AgentLogKind::UDP, "handle_udp_new_socket: sk={:x} pid={} laddr={} lport={}", msg.sk, msg.pid, IPv6Address::from(msg.laddr), msg.lport); /* first, insert into table */ if (udp_socket_table_.full()) { log_.warn("handle_udp_new_socket: udp socket table full! dropping socket"); udp_socket_table_ever_full_ = true; return; } auto pos = udp_socket_table_.insert(msg.sk); if (pos.index == udp_socket_table_.invalid) { log_.error("handle_udp_new_socket: duplicate udp socket"); return; } pos.entry->pid = msg.pid; pos.entry->sk = msg.sk; memcpy(&pos.entry->laddr, msg.laddr, sizeof(struct in6_addr)); pos.entry->lport = msg.lport; udp_send_new_socket(metadata.timestamp, pos.entry, pos.index); // char ipaddr6_buf[INET6_ADDRSTRLEN]; // const char *addr_s = inet_ntop(AF_INET6, &msg.laddr, ipaddr6_buf, // INET6_ADDRSTRLEN); // // std::cout << (reported ? "MSG: ": "") // << "UDP existing socket pid " << msg.pid // << " sk " << msg.sk // << " lport " << msg.lport // << " laddr " << addr_s << std::endl; } void BufferedPoller::handle_udp_destroy_socket(message_metadata const &metadata, jb_agent_internal__udp_destroy_socket &msg) { LOG::debug_in(AgentLogKind::UDP, "handle_udp_destroy_socket: sk={:x}", msg.sk); auto pos = udp_socket_table_.find(msg.sk); if (pos.index == udp_socket_table_.invalid) { if (!udp_socket_table_ever_full_ && all_probes_loaded_) { LOG::debug_in(AgentLogKind::UDP, "handle_udp_destroy_socket: socket not found sk={:x}", msg.sk); } return; } // Ensure dns queries on this socket are timed out std::list reqs; dns_requests_.lookup_socket(msg.sk, reqs); for (auto &req : reqs) { timeout_dns_request(metadata.timestamp, req); } /* send out statistics message if available */ if (pos.entry->reported) { for (int is_rx = 0; is_rx < 2; is_rx++) { for (u32 epoch = 0; epoch < n_epochs; epoch++) { auto &stats = udp_socket_stats_[is_rx].lookup_relative(pos.index, epoch, false).second; if (stats.valid == true) udp_send_stats(metadata.timestamp, pos.index, is_rx, *pos.entry, stats); } } /* notify of the destruction */ writer_.udp_destroy_socket_tstamp(metadata.timestamp, pos.index); } bool success = udp_socket_table_.erase(msg.sk); if (!success) { log_.error("handle_udp_destroy_socket: removing socket from table failed sk={:x}", msg.sk); } } void BufferedPoller::handle_udp_stats(message_metadata const &metadata, jb_agent_internal__udp_stats &msg) { auto pos = udp_socket_table_.find(msg.sk); if (pos.index == udp_socket_table_.invalid) { // bpf should prevent this unless the socket table was ever full if (!udp_socket_table_ever_full_ && all_probes_loaded_) { // kprobe is inserted only after steady state, so this should never happen log_.error("handle_udp_stats: stats for missing socket sk={:x}", msg.sk); } return; } auto &entry = *pos.entry; /* find the statistics, and ask it to enqueue */ u8 is_rx = msg.is_rx; auto &stats = udp_socket_stats_[is_rx].lookup(pos.index, metadata.timestamp, true).second; /* if stats are valid and address changed, output the previous stat and update * address */ if (msg.changed_af != 0) { /* there might be statistics for a different address */ if (stats.valid) { /* send the stats. will clear the stats */ udp_send_stats(metadata.timestamp, pos.index, is_rx, entry, stats); } // Lookup whether this is a NAT-ed connection if this is ipv4 hostport_tuple *ft = nullptr; if (msg.changed_af == AF_INET) { // NOTE: the bpf code always has a changed_af event before // the first statistics are sent, since the remote_addr in the bpf // table is initialized to 0:0. u32 laddr = ((u32 *)msg.laddr)[3]; u32 raddr = ((u32 *)msg.raddr)[3]; ft = nat_handler_.get_nat_mapping(laddr, raddr, ntohs(msg.lport), ntohs(msg.rport), IPPROTO_UDP); } /* set the address */ auto &addr = entry.addrs[is_rx]; if (ft != nullptr) { u32 laddr[4] = {0, 0, 0xffff0000, ft->src_ip}; memcpy(&entry.laddr, laddr, sizeof(struct in6_addr)); entry.lport = ntohs(ft->src_port); u32 raddr[4] = {0, 0, 0xffff0000, ft->dst_ip}; memcpy(&addr.addr, raddr, sizeof(struct in6_addr)); addr.port = ntohs(ft->dst_port); } else { memcpy(&entry.laddr, msg.laddr, sizeof(struct in6_addr)); entry.lport = ntohs(msg.lport); memcpy(&addr.addr, msg.raddr, sizeof(struct in6_addr)); addr.port = ntohs(msg.rport); } addr.changed_af = msg.changed_af; // fast track stats for address changes. we can't delay pushing address // changes to the server, because the next messages (dns // responses/timeouts/etc for example) may require the address to be set udp_send_stats(metadata.timestamp, pos.index, is_rx, entry, stats); } if (!stats.valid) { /* if stats were invalid, reset the values */ stats.packets = msg.packets; stats.bytes = msg.bytes; stats.drops = msg.drops; stats.valid = true; } else { /* stats were valid, aggregate */ stats.packets += msg.packets; stats.bytes += msg.bytes; stats.drops += msg.drops; } char lipaddr6_buf[INET6_ADDRSTRLEN]; const char *laddr_s = inet_ntop(AF_INET6, &msg.laddr, lipaddr6_buf, INET6_ADDRSTRLEN); char ripaddr6_buf[INET6_ADDRSTRLEN]; const char *raddr_s = inet_ntop(AF_INET6, &msg.raddr, ripaddr6_buf, INET6_ADDRSTRLEN); LOG::trace_in( AgentLogKind::UDP, "UDP {} sk {:x} sk_id {} laddr {} lport {} raddr {} rport {} " "packets {} bytes {} changed_af {} drops {}", (msg.is_rx ? "RX" : "TX"), msg.sk, pos.index, laddr_s, ntohs(msg.lport), raddr_s, ntohs(msg.rport), msg.packets, msg.bytes, int(msg.changed_af), int(msg.drops)); } void BufferedPoller::handle_pid_info(message_metadata const &metadata, jb_agent_internal__pid_info &msg) { pid_count_++; const std::string_view comm = comm_to_string(msg.comm); LOG::debug_in( AgentLogKind::PID, "{}: pid={} parent={} cgroup=0x{:x} comm='{}' pid_count_={}", __func__, msg.pid, msg.parent_pid, msg.cgroup, comm, pid_count_); cgroup_handler_.handle_pid_info(msg.pid, msg.cgroup, msg.comm); process_handler_.on_new_process(std::chrono::nanoseconds{metadata.timestamp}, msg); // Read the process command-line from /proc/PID/cmdline. // By this time the process could have exited, so reading this entry can fail. // We are using the `try_read_proc_cmdline` function which ignores the // "no such file or directory" error and returns an empty string instead. std::string cmdline; if (auto r = try_read_proc_cmdline(msg.pid)) { cmdline = *r; } else { log_.error("handle_pid_info: error reading cmdline for pid={}: {}", msg.pid, r.error()); } writer_.pid_info_create_tstamp(metadata.timestamp, msg.pid, msg.comm, msg.cgroup, msg.parent_pid, jb_blob(cmdline)); } void BufferedPoller::handle_pid_close(message_metadata const &metadata, jb_agent_internal__pid_close &msg) { pid_count_--; const std::string_view comm = comm_to_string(msg.comm); LOG::debug_in(AgentLogKind::PID, "{}: pid={} comm='{}' pid_count_={}", __func__, msg.pid, comm, pid_count_); process_handler_.on_process_end(std::chrono::nanoseconds{metadata.timestamp}, msg); writer_.pid_close_info_tstamp(metadata.timestamp, msg.pid, msg.comm); } void BufferedPoller::handle_pid_set_comm(message_metadata const &metadata, jb_agent_internal__pid_set_comm &msg) { const std::string_view comm = comm_to_string(msg.comm); LOG::debug_in(AgentLogKind::PID, "{}: pid={} comm='{}'", __func__, msg.pid, comm); process_handler_.set_process_command(std::chrono::nanoseconds{metadata.timestamp}, msg); writer_.pid_set_comm_tstamp(metadata.timestamp, msg.pid, msg.comm); } void BufferedPoller::handle_pid_exit(message_metadata const &metadata, jb_agent_internal__pid_exit &msg) { LOG::debug_in(AgentLogKind::PID, "{}: tgid={} pid={} exit_code={}", __func__, msg.tgid, msg.pid, msg.exit_code); process_handler_.pid_exit(std::chrono::nanoseconds{metadata.timestamp}, msg); } void trace_print_udp_socket_entry(const std::string_view &location, udp_socket_entry *entry) { LOG::trace_in( AgentLogKind::UDP, "{}: udp_socket_entry\n" " laddr={} lport={}\n" " reported={} pid={} sk={:x}\n" " addrs[TX]: addr={} port={} changed_af={}\n" " addrs[RX]: addr={} port={} changed_af={}", location, IPv6Address::from(entry->laddr), entry->lport, entry->reported, entry->pid, entry->sk, IPv6Address::from(entry->addrs[0].addr), entry->addrs[0].port, entry->addrs[0].changed_af, IPv6Address::from(entry->addrs[1].addr), entry->addrs[1].port, entry->addrs[1].changed_af); } void BufferedPoller::udp_send_new_socket(u64 ts, udp_socket_entry *entry, u64 index) { trace_print_udp_socket_entry("udp_send_new_socket", entry); if (entry->lport != 0) { writer_.udp_new_socket_tstamp(ts, entry->pid, index, (uint8_t *)(entry->laddr.data()), entry->lport); entry->reported = true; } } void BufferedPoller::udp_send_stats(u64 t, u32 sk_id, u8 is_rx, udp_socket_entry &entry, udp_statistics &stats) { trace_print_udp_socket_entry("udp_send_stats", &entry); auto &addr = entry.addrs[is_rx]; if (entry.reported == false) { LOG::trace("BufferedPoller::udp_send_stats - entry.reported == false"); udp_send_new_socket(t, &entry, sk_id); } switch (addr.changed_af) { case AF_INET: writer_.udp_stats_addr_changed_v4_tstamp(t, sk_id, is_rx, stats.packets, stats.bytes, addr.addr[3], addr.port); // TODO: Can uncomment this once we decide to support laddr info // writer_.udp_stats_addr_changed_v4_tstamp( // t, sk_id, is_rx, stats.packets, stats.bytes, addr.addr[3], addr.port, // entry.laddr[3], entry.lport); break; case AF_INET6: writer_.udp_stats_addr_changed_v6_tstamp(t, sk_id, is_rx, stats.packets, stats.bytes, (u8 *)&addr.addr, addr.port); // TODO: Can uncomment this once we decide to support laddr info // writer_.udp_stats_addr_changed_v6_tstamp(t, sk_id, is_rx, stats.packets, // stats.bytes, (u8 *)&addr.addr, // addr.port, (u8*)&entry.laddr, // entry.lport); break; default: writer_.udp_stats_addr_unchanged_tstamp(t, sk_id, is_rx, stats.packets, stats.bytes); break; } // Send drops for receive side only, if we have drops if (is_rx && stats.drops > 0) { writer_.udp_stats_drops_changed_tstamp(t, sk_id, stats.drops); } if (is_log_whitelisted(AgentLogKind::UDP)) { LOG::trace_in( AgentLogKind::UDP, "BufferedPoller::udp_send_stats - sk_id: {}, raddr: {}, rport: {}, " "packets: {}, bytes: {}, changed_af: {}, drops: {}", sk_id, IPv6Address::from(addr.addr), addr.port, stats.packets, stats.bytes, int(addr.changed_af), stats.drops); } /* don't need to report the address next time if it doesn't change */ addr.changed_af = 0; /* the current statistic is invalid, don't try to send */ stats.valid = false; } void BufferedPoller::udp_send_stats_from_queue(u64 t) { for (int is_rx = 0; is_rx < 2; is_rx++) { auto &store = udp_socket_stats_[is_rx]; if (store.relative_timeslot(t) == 0) { /* not ready */ continue; } auto &queue = store.current_queue(); while (!queue.empty()) { /* get the next index */ u32 index = queue.peek(); /* get the stats entry for that index */ auto &stats = store.lookup_relative(index, 0, false).second; if (stats.valid) { /* write the message */ udp_send_stats(t, index, is_rx, udp_socket_table_[index], stats); /* udp_send_stats sets stats.valid = false */ } queue.pop(); } /* done. advance the current timeslot */ store.advance(); } } u32 BufferedPoller::u64_hasher::operator()(u64 const &s) const noexcept { return lookup3_hashword((u32 *)&s, sizeof(u64) / 4, 0x7AFBAF00); } void BufferedPoller::handle_kill_css(message_metadata const &metadata, jb_agent_internal__kill_css &msg) { cgroup_handler_.kill_css(metadata.timestamp, &msg); writer_.cgroup_close_tstamp(metadata.timestamp, msg.cgroup); } void BufferedPoller::handle_css_populate_dir(message_metadata const &metadata, jb_agent_internal__css_populate_dir &msg) { cgroup_handler_.css_populate_dir(metadata.timestamp, &msg); writer_.cgroup_create_tstamp(metadata.timestamp, msg.cgroup, msg.cgroup_parent, msg.name); } void BufferedPoller::handle_existing_cgroup_probe( message_metadata const &metadata, jb_agent_internal__existing_cgroup_probe &msg) { cgroup_handler_.existing_cgroup_probe(metadata.timestamp, &msg); writer_.cgroup_create_tstamp(metadata.timestamp, msg.cgroup, msg.cgroup_parent, msg.name); } void BufferedPoller::handle_cgroup_attach_task(message_metadata const &metadata, jb_agent_internal__cgroup_attach_task &msg) { cgroup_handler_.cgroup_attach_task(metadata.timestamp, &msg); process_handler_.on_cgroup_move(std::chrono::nanoseconds{metadata.timestamp}, msg); writer_.pid_cgroup_move_tstamp(metadata.timestamp, msg.pid, msg.cgroup); } void BufferedPoller::handle_nf_nat_cleanup_conntrack( message_metadata const &metadata, jb_agent_internal__nf_nat_cleanup_conntrack &msg) { nat_handler_.handle_nf_nat_cleanup_conntrack(metadata.timestamp, &msg); } void BufferedPoller::handle_nf_conntrack_alter_reply( message_metadata const &metadata, jb_agent_internal__nf_conntrack_alter_reply &msg) { nat_handler_.handle_nf_conntrack_alter_reply(metadata.timestamp, &msg); } void BufferedPoller::handle_existing_conntrack_tuple( message_metadata const &metadata, jb_agent_internal__existing_conntrack_tuple &msg) { nat_handler_.handle_existing_conntrack_tuple(metadata.timestamp, &msg); } void BufferedPoller::handle_bpf_log(message_metadata const &metadata, jb_agent_internal__bpf_log &msg) { // TODO: since removing the pre-processor, we have no line number information auto const linenumber = 0; std::string_view const filename = "TODO"; writer_.bpf_log(jb_blob{filename}, linenumber, msg.code, msg.arg0, msg.arg1, msg.arg2); } void BufferedPoller::handle_stack_trace(message_metadata const &metadata, jb_agent_internal__stack_trace &msg) { #if DEBUG_ENABLE_STACKTRACE std::string stacktrace = probe_handler_.get_stack_trace(skel_, msg.kernel_stack_id, msg.user_stack_id, msg.tgid); LOG::debug_in( AgentLogKind::BPF, "stack_trace: timestamp={}, kernel_stack_id={}, " "user_stack_id={}, tgid={}, comm={}\n{}\n", metadata.timestamp, msg.kernel_stack_id, msg.user_stack_id, msg.tgid, std::string_view((char *)msg.comm, strnlen((char *)msg.comm, sizeof(msg.comm))), stacktrace); #endif } void BufferedPoller::handle_tcp_data(message_metadata const &metadata, jb_agent_internal__tcp_data &msg) { LOG::debug_in( AgentLogKind::PROTOCOL, "tcp_data: idx={}, timestamp={}, sk={:x}, pid={}, length={}, offset={}, " "stream_type={}({}), client_server={}({})\n", metadata.cpu_index, metadata.timestamp, msg.sk, msg.pid, msg.length, msg.offset, msg.stream_type, stream_type_to_string((enum STREAM_TYPE)msg.stream_type), msg.client_server, client_server_type_to_string((enum CLIENT_SERVER_TYPE)msg.client_server)); tcp_data_handler_->process( metadata.cpu_index, metadata.timestamp, msg.sk, msg.pid, msg.length, msg.offset, (STREAM_TYPE)msg.stream_type, (CLIENT_SERVER_TYPE)msg.client_server); } void BufferedPoller::set_all_probes_loaded() { all_probes_loaded_ = true; } #ifndef NDEBUG void BufferedPoller::debug_bpf_lost_samples() { debug_bpf_lost_samples_ = true; } #endif ================================================ FILE: collector/kernel/buffered_poller.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // Forward declaration for the skeleton struct render_bpf_bpf; class KernelCollectorRestarter; /** * A BufferedPoller that empties the perf ring every poll * and sends the info to a specified channel */ class BufferedPoller : public PerfPoller { public: // The hash map needs to be some 20% larger than the max number of elements. // Otherwise you'd start getting hash failures before all the elements are // exhausted static constexpr u32 tcp_socket_table_max_sockets = TABLE_SIZE__TCP_OPEN_SOCKETS; static constexpr u32 udp_socket_table_max_sockets = TABLE_SIZE__UDP_OPEN_SOCKETS; static constexpr u32 n_epochs = 2; /** * c'tor * throws if buff_ can't be malloc-ed * @param loop: the libuv event loop on which to receive perf events * @param container: the perf container to extract messages from * @param writer: the writer using which to send messages * @param time_adjustment: how much to add to CLOCK_MONOTONIC when comparing * to ring timestamp */ BufferedPoller( uv_loop_t &loop, PerfContainer &container, IBufferedWriter &writer, u64 time_adjustment, CurlEngine &curl_engine, FileDescriptor &bpf_dump_file, logging::Logger &log, ProbeHandler &probe_handler, struct render_bpf_bpf *skel, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings const &cgroup_settings, ::ebpf_net::ingest::Encoder *encoder, KernelCollectorRestarter &kernel_collector_restarter); /** * batches as many entries into buffer as possible before calling * send_buffer(). always flushes the buffer at the end. * * there are four possible cases where it will throw: * 1) a record is larger than an empty buffer * 2) a record of unknown type was read * 3) call to send_buffer failed (there are two call locations) * */ void process_samples(bool is_event); /** * entrypoint for event-driven polling when the perf container announces * it is half full or whatever notification limit is set. * calls through to process_samples() */ void handle_event(); /** * entrypoint for manual polling. calls through to process_samples() * @see PerfPoller::poll */ virtual void poll(); /** * Sends a bpf loss notification to backend, if a loss happened since * the last notification call */ void send_report_if_recent_loss(); /** * accessor for lost_count_ */ u64 serv_lost_count(); void slow_poll(); /** * let us know when all the probes are loaded * and have achieved steady-state */ void set_all_probes_loaded(void); #ifndef NDEBUG /** * Debug code for internal development to simulate lost BPF samples (PERF_RECORD_LOST) in BufferedPoller. */ void debug_bpf_lost_samples(); #endif private: /** * polling point for dns timeout detection * called via slow poll */ void process_dns_timeouts(u64 t); /** * A message handler is a member function of this class * @returns true if message should be copied, false otherwise */ typedef void (BufferedPoller::*handler_fn)(PerfReader &reader, u16 length); struct message_metadata { u64 timestamp; std::size_t cpu_index; std::basic_string_view payload; std::basic_string_view padding; }; template using message_handler_fn = void (BufferedPoller::*)(message_metadata const &metadata, typename MessageMetadata::wire_message &); template , std::size_t MaxPadding, typename Alignment> void message_handler_entrypoint(PerfReader &reader, u16 length); /** * Adds a handler to the hash. Throws on collision. */ template , std::size_t MaxPadding = 0, typename Alignment = u64> void add_handler(); /** * Handler for DNS RPC messages */ void handle_dns_message(message_metadata const &metadata, jb_agent_internal__dns_packet &msg); /** * Handler a new socket message */ void handle_new_socket(message_metadata const &metadata, jb_agent_internal__new_sock_created &msg); /** * Handle a ipv4 address for a socket */ void handle_set_state_ipv4(message_metadata const &metadata, jb_agent_internal__set_state_ipv4 &msg); /** * Handle ipv6 address for a socket */ void handle_set_state_ipv6(message_metadata const &metadata, jb_agent_internal__set_state_ipv6 &msg); /** * Handler a socket close message */ void handle_close_socket(message_metadata const &metadata, jb_agent_internal__close_sock_info &msg); /** * Handler a rtt_estimator telemetry message */ void handle_rtt_estimator(message_metadata const &metadata, jb_agent_internal__rtt_estimator &msg); /** * Handler a rtt_estimator telemetry message */ void handle_reset_tcp_counters(message_metadata const &metadata, jb_agent_internal__reset_tcp_counters &msg); /** * Handle a TCP SYN timeout message */ void handle_tcp_syn_timeout(message_metadata const &metadata, jb_agent_internal__tcp_syn_timeout &msg); /** * Handle TCP RST */ void handle_tcp_reset(message_metadata const &metadata, jb_agent_internal__tcp_reset &msg); /** * Handle a http_response message */ void handle_http_response(message_metadata const &metadata, jb_agent_internal__http_response &msg); /** * Sends a message with statistics for the entry * * Also marks the entry as invalid. */ void send_socket_stats(u64 t, u64 sk, tcp_statistics &stats); /** * Processes the current queue in socket_stats_, sending out messages and * setting entries to stats.queued=false, stats.valid=false, then advances * the queue. */ void send_stats_from_queue(u64 t); /*** UDP ***/ /** * Handler a new or existing udp socket message */ void handle_udp_new_socket(message_metadata const &metadata, jb_agent_internal__udp_new_socket &msg); /** * Handler a UDP socket close message */ void handle_udp_destroy_socket(message_metadata const &metadata, jb_agent_internal__udp_destroy_socket &msg); /** * Handler for udp TX notification */ void handle_udp_stats(message_metadata const &metadata, jb_agent_internal__udp_stats &msg); /** * Handler for new process */ void handle_pid_info(message_metadata const &metadata, jb_agent_internal__pid_info &msg); /** * Handler for close process */ void handle_pid_close(message_metadata const &metadata, jb_agent_internal__pid_close &msg); /** * Handler for process comm change */ void handle_pid_set_comm(message_metadata const &metadata, jb_agent_internal__pid_set_comm &msg); /** * Handler for process exit */ void handle_pid_exit(message_metadata const &metadata, jb_agent_internal__pid_exit &msg); /** * Sends a udp socket message * * Only sends when lport is != 0 */ void udp_send_new_socket(u64 ts, udp_socket_entry *entry, u64 index); /** * Sends a message with statistics for the entry * * Also marks the entry as invalid. * @assumes entry is valid */ void udp_send_stats(u64 t, u32 sk_id, u8 is_rx, udp_socket_entry &entry, udp_statistics &stats); /** * Processes the current queue in socket_stats_, sending out messages and * setting entries to stats.queued=false, stats.valid=false, then advances * the queue. */ void udp_send_stats_from_queue(u64 t); /*** CONTAINERS ***/ /** * Handler for a new cgroup dir */ void handle_kill_css(message_metadata const &metadata, jb_agent_internal__kill_css &msg); void handle_css_populate_dir(message_metadata const &metadata, jb_agent_internal__css_populate_dir &msg); void handle_existing_cgroup_probe(message_metadata const &metadata, jb_agent_internal__existing_cgroup_probe &msg); void handle_cgroup_attach_task(message_metadata const &metadata, jb_agent_internal__cgroup_attach_task &msg); /*** NAT ***/ void handle_nf_nat_cleanup_conntrack(message_metadata const &metadata, jb_agent_internal__nf_nat_cleanup_conntrack &msg); void handle_nf_conntrack_alter_reply(message_metadata const &metadata, jb_agent_internal__nf_conntrack_alter_reply &msg); void handle_existing_conntrack_tuple(message_metadata const &metadata, jb_agent_internal__existing_conntrack_tuple &msg); /*** DNS ***/ void timeout_dns_request(u64 timestamp_ns, const DnsRequests::Request &req); /*** ERRORS ***/ void handle_bpf_log(message_metadata const &metadata, jb_agent_internal__bpf_log &msg); void handle_stack_trace(message_metadata const &metadata, jb_agent_internal__stack_trace &msg); /*** TCP DATA ***/ void handle_tcp_data(message_metadata const &metadata, jb_agent_internal__tcp_data &msg); ///////////////////////////////////////////////////////////////////////// uv_loop_t &loop_; u64 time_adjustment_; FileDescriptor &bpf_dump_file_; logging::Logger &log_; IBufferedWriter &buffered_writer_; ProbeHandler &probe_handler_; struct render_bpf_bpf *skel_; ::ebpf_net::ingest::Writer writer_; std::unique_ptr tcp_data_handler_; ::ebpf_net::kernel_collector::Index collector_index_; ProcessHandler process_handler_; u64 lost_count_ = 0; u32 pid_count_ = 0; /* the last lost count that a message was sent for */ u64 notified_lost_count_ = 0; handler_fn handlers_[AGENT_INTERNAL_HASH_SIZE]; /* u64 Hasher */ struct u64_hasher { u32 operator()(u64 const &s) const noexcept; }; CgroupHandler cgroup_handler_; NatHandler nat_handler_; /* TCP */ typedef FixedHash TcpSocketTable; typedef MetricStore TcpSocketStatistics; TcpSocketTable tcp_socket_table_; bool tcp_socket_table_ever_full_; fast_div tslot_; TcpSocketStatistics tcp_socket_stats_; u64 tcp_index_to_sk_[tcp_socket_table_max_sockets]; /* UDP */ typedef FixedHash UdpSocketTable; typedef MetricStore UdpSocketStatistics; UdpSocketTable udp_socket_table_; bool udp_socket_table_ever_full_; std::array udp_socket_stats_; /* 0: TX, 1: RX */ /* DNS */ DnsRequests dns_requests_; bool all_probes_loaded_; KernelCollectorRestarter &kernel_collector_restarter_; #ifndef NDEBUG bool debug_bpf_lost_samples_ = false; #endif }; ================================================ FILE: collector/kernel/cgroup_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include using json = nlohmann::json; inline std::string make_docker_query_url(std::string const &container_name) { static const std::string docker_query_base = "http://localhost/containers/"; return docker_query_base + container_name + "/json"; } std::string CgroupHandler::docker_ns_label_field; CgroupHandler::CgroupHandler( ::ebpf_net::ingest::Writer &writer, CurlEngine &curl_engine, CgroupSettings const &settings, logging::Logger &log) : writer_(writer), curl_engine_(curl_engine), settings_(settings), log_(log) {} CgroupHandler::~CgroupHandler() { // cancel all running queries for (auto &entry : queries_) { DockerQuery &query = entry.second; curl_engine_.cancel_fetch(*query.request); } } bool CgroupHandler::has_cgroup(u64 cgroup) { return (cgroup_table_.find(cgroup) != cgroup_table_.end()); } std::string_view CgroupHandler::get_name(u64 cgroup) { auto pos = cgroup_table_.find(cgroup); if (pos == cgroup_table_.end()) { return {}; } return pos->second.name; } /* END */ void CgroupHandler::kill_css(u64 timestamp, struct jb_agent_internal__kill_css *msg) { std::string name{(char *)msg->name, strnlen((char *)msg->name, sizeof(msg->name))}; LOG::debug_in( AgentLogKind::CGROUPS, "CgroupHandler::kill_css" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tcgroup_parent: 0x{:x}" "\n\tname: " "\n}}", msg->cgroup, msg->cgroup_parent, name); if (has_cgroup(msg->cgroup)) { LOG::debug_in(AgentLogKind::CGROUPS, "Success: cgroup found. \t{}", get_name(msg->cgroup)); } else { log_.warn("kill_css(): cgroup not found: 0x{:x}", msg->cgroup); } auto pos = cgroup_table_.find(msg->cgroup); if (pos == cgroup_table_.end()) { return; } cgroup_table_.erase(pos); } /* START */ void CgroupHandler::css_populate_dir(u64 timestamp, struct jb_agent_internal__css_populate_dir *msg) { std::string name{(char *)msg->name, strnlen((char *)msg->name, sizeof(msg->name))}; LOG::debug_in( AgentLogKind::CGROUPS, "CgroupHandler::css_populate_dir" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tcgroup_parent: 0x{:x}" "\n\tname: {}" "\n}}", msg->cgroup, msg->cgroup_parent, name); if (has_cgroup(msg->cgroup_parent)) { LOG::debug_in(AgentLogKind::CGROUPS, "Success: cgroup->parent found. \t{}", get_name(msg->cgroup_parent)); } else { log_.warn("cgroup->parent not found: cgroup_parent={:x}, cgroup={:x}, name={}", msg->cgroup_parent, msg->cgroup, name); } handle_cgroup(msg->cgroup, msg->cgroup_parent, name); } /* EXISTING */ void CgroupHandler::existing_cgroup_probe(u64 timestamp, struct jb_agent_internal__existing_cgroup_probe *msg) { std::string name{(char *)msg->name, strnlen((char *)msg->name, sizeof(msg->name))}; // msg->cgroup_parent, msg->name); LOG::debug_in( AgentLogKind::CGROUPS, "CgroupHandler::existing_cgroup_probe" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tcgroup_parent: 0x{:x}" "\n\tname: {}" "\n}}", msg->cgroup, msg->cgroup_parent, name); handle_cgroup(msg->cgroup, msg->cgroup_parent, name); } void CgroupHandler::cgroup_attach_task(u64 timestamp, struct jb_agent_internal__cgroup_attach_task *msg) { LOG::debug_in( AgentLogKind::CGROUPS, "CgroupHandler::cgroup_attach_task" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tpid: {}" "\n\tcomm: {}" "\n}}", msg->cgroup, msg->pid, std::string_view((char *)msg->comm, strnlen((char *)msg->comm, sizeof(msg->comm)))); if (has_cgroup(msg->cgroup)) { LOG::debug_in(AgentLogKind::CGROUPS, "Success: cgroup found. \t{}", get_name(msg->cgroup)); } else { log_.warn("cgroup_attach_task(): cgroup not found: 0x{:x}", msg->cgroup); } } void CgroupHandler::handle_pid_info(u32 pid, u64 cgroup, uint8_t comm[16]) { LOG::debug_in( AgentLogKind::CGROUPS, "CgroupHandler::handle_pid_info" "\n{{" "\n\tpid: {}" "\n\tcgroup: 0x{:x}" "\n\tcomm: {}" "\n}}", pid, cgroup, std::string_view((char *)comm, strnlen((char *)comm, 16))); if (has_cgroup(cgroup)) { LOG::debug_in(AgentLogKind::CGROUPS, "Success: cgroup found. \t{}", get_name(cgroup)); } else { log_.warn("handle_pid_info(): cgroup not found: 0x{:x}", cgroup); } } void CgroupHandler::handle_cgroup(u64 cgroup, u64 cgroup_parent, std::string const &name) { auto emp = cgroup_table_.emplace(cgroup, CgroupEntry{cgroup_parent, name}); if (emp.second == false) { return; } auto parent_pos = cgroup_table_.find(cgroup_parent); if (parent_pos == cgroup_table_.end()) { return; } bool is_docker = settings_.force_docker_metadata; if (!is_docker) { if (parent_pos->second.name == "docker") { // parent container's name is docker is_docker = true; } else { // grandparent auto gp_pos = cgroup_table_.find(parent_pos->second.cgroup_parent); if (gp_pos != cgroup_table_.end()) { if (gp_pos->second.name == "ecs") { // grandparent container's name is ecs is_docker = true; } } } } if (is_docker) { handle_docker_container(cgroup, name); } } void CgroupHandler::handle_docker_container(u64 cgroup, std::string const &name) { LOG::debug_in( AgentLogKind::DOCKER, "CgroupHandler::handle_docker_container:" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tname: {}" "\n}}", cgroup, name); auto request = std::make_unique( make_docker_query_url(name), [this, cgroup](const char *data, size_t data_length) { this->data_available_cb(data, data_length, cgroup); }, [this, cgroup](CurlEngineStatus status, int responseCode, std::string_view curlError) { this->fetch_done_cb(status, responseCode, curlError, cgroup); }); request->unix_socket(UNIX_SOCKET_PATH); // debug mode curl if debugging docker request->debug_mode(is_log_whitelisted(AgentLogKind::DOCKER)); auto emp = queries_.emplace(cgroup, DockerQuery{std::move(request)}); if (emp.second == false) { log_.error("query for cgroup:{} is already running", cgroup); return; } auto status = curl_engine_.schedule_fetch(*emp.first->second.request); if (status != CurlEngineStatus::OK) { // scheduling failure. curl engine will have called the done_fn, which // cleans up queries_ log_.error("failed to schedule a fetch request"); return; } LOG::debug_in(AgentLogKind::DOCKER, "\tqueries_.size(): {}", queries_.size()); } void CgroupHandler::data_available_cb(const char *data, size_t data_length, u64 cgroup) { std::string s(data, data_length); LOG::debug_in( AgentLogKind::DOCKER, "DataAvailableFn:" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tdata_length: {}" "\n\ts: {}" "\n}}", cgroup, data_length, s); auto pos = queries_.find(cgroup); if (pos == queries_.end()) { log_.error("query entry for cgroup: 0x{:x} not found", cgroup); return; } DockerQuery &query = pos->second; query.response.append(s); } void CgroupHandler::fetch_done_cb(CurlEngineStatus status, long responseCode, std::string_view curlError, u64 cgroup) { bool success = (status == CurlEngineStatus::OK); LOG::debug_in( AgentLogKind::DOCKER, "FetchDoneFn:" "\n{{" "\n\tcgroup: 0x{:x}" "\n\tsuccess: {}" "\n}}", cgroup, success); auto pos = queries_.find(cgroup); if (pos == queries_.end()) { log_.error("query entry for cgroup: 0x{:x} not found", cgroup); return; } std::string response_data(std::move(pos->second.response)); queries_.erase(pos); if (!success) { const std::string_view status_text = to_string(status); log_.error("docker fetch failed [{}:{}]: {}", status_text, responseCode, curlError); return; } if ((responseCode >= 200) && (responseCode <= 299)) { // success handle_docker_response(cgroup, response_data); } else if ((responseCode >= 500) && (responseCode <= 599)) { // server error log_.error("docker fetch failed with response {}", responseCode); } LOG::debug_in(AgentLogKind::DOCKER, "\tqueries_.size(): {}", queries_.size()); } inline std::string get_string(json const &j) { if (j.is_string()) { return j.get(); } else { return std::string(); } } inline std::string get_string(json const &object, char const *key) { if (!object.is_object()) { return std::string(); } auto pos = object.find(key); if (pos != object.end()) { return get_string(*pos); } else { return std::string(); } } inline jb_blob blob(std::string const &str) { return jb_blob{str.c_str(), static_cast(str.size())}; } void CgroupHandler::handle_docker_response(u64 cgroup, std::string const &response_data) { std::string id; std::string name; std::string image; std::string ip_addr; std::string cluster; std::string container; std::string task_family; std::string task_version; std::string ns; std::string pod_name; std::optional docker_host_config; std::optional nomad_metadata; std::optional k8s_metadata; if (settings_.docker_metadata_dump_dir) { auto const dump_filename = fmt::format( "{}/docker-inspect.{}.{}.json", *settings_.docker_metadata_dump_dir, cgroup, std::chrono::system_clock::now().time_since_epoch().count()); if (auto const error = write_file(dump_filename.c_str(), response_data)) { LOG::warn("failed to dump docker metadata to {}: {}", dump_filename, error); } } try { json root = json::parse(response_data); auto network = root["NetworkSettings"]; auto config = root["Config"]; auto host_config = root["HostConfig"]; auto labels = config["Labels"]; auto env = config["Env"]; id = get_string(root, "Id"); name = get_string(root, "Name"); image = get_string(config, "Image"); ip_addr = get_string(network, "IPAddress"); cluster = get_string(labels, "com.amazonaws.ecs.cluster"); container = get_string(labels, "com.amazonaws.ecs.container-name"); if (container.empty()) { // Try k8s if ECS container is missing container = get_string(labels, "io.kubernetes.container.name"); } pod_name = get_string(labels, "com.amazonaws.ecs.task-arn"); if (pod_name.empty()) { // k8s pod_name = get_string(labels, "io.kubernetes.pod.name"); } task_family = get_string(labels, "com.amazonaws.ecs.task-definition-family"); task_version = get_string(labels, "com.amazonaws.ecs.task-definition-version"); if (!docker_ns_label_field.empty()) { ns = get_string(labels, docker_ns_label_field.c_str()); } docker_host_config.emplace(host_config); nomad_metadata.emplace(NomadMetadata(env)); k8s_metadata.emplace(labels); for (auto const &item : labels.items()) { if (!item.value().is_string()) { continue; } std::string_view key = item.key(); std::string const value = item.value().get(); if ((key.size() + value.size() + sizeof(u64) + jb_ingest__container_annotation__data_size) > WRITE_BUFFER_SIZE) { // NOTE: we use a substring of the key to make sure it fits in the // warning message. log_.warn("Docker metadata label for key '{}' is too large to send", key.substr(0, 256)); continue; } writer_.container_annotation(cgroup, jb_blob{key}, jb_blob{value}); } } catch (json::exception &e) { log_.error("failed to parse response data: {}", e.what()); return; } if (docker_host_config.has_value()) { LOG::debug_in(AgentLogKind::DOCKER, "container resource limits: {}", fmt::streamed(*docker_host_config)); auto const cpu_period = docker_host_config->cpu_period(); auto const cpu_quota = docker_host_config->cpu_quota(); writer_.container_resource_limits( cgroup, std::max(kernel::MIN_CGROUP_CPU_SHARES, std::min(kernel::MAX_CGROUP_CPU_SHARES, docker_host_config->cpu_shares())), cpu_period <= 0 ? kernel::DEFAULT_CGROUP_QUOTA : cpu_period, cpu_quota < 0 ? kernel::DEFAULT_CGROUP_QUOTA : cpu_quota, docker_host_config->memory_swappiness(), docker_host_config->memory_limit(), docker_host_config->memory_soft_limit(), docker_host_config->total_memory_limit()); } LOG::debug_in( AgentLogKind::DOCKER, "container metadata:" "\n{{" "\n\tid: {}" "\n\tname: {}" "\n\timage: {}" "\n\tip_addr: {}" "\n\tcluster: {}" "\n\tcontainer: {}" "\n\ttask_family: {}" "\n\ttask_version: {}" "\n\tns: {}" "\n}}", id, name, image, ip_addr, cluster, container, task_family, task_version, ns); writer_.container_metadata( cgroup, blob(id), blob(name), blob(image), blob(ip_addr), blob(cluster), blob(container), blob(task_family), blob(task_version), blob(ns)); if (nomad_metadata.has_value() && *nomad_metadata) { writer_.nomad_metadata( cgroup, jb_blob{nomad_metadata->ns()}, jb_blob{nomad_metadata->group_name()}, jb_blob{nomad_metadata->task_name()}, jb_blob{nomad_metadata->job_name()}); } if (!pod_name.empty()) { LOG::debug_in( AgentLogKind::DOCKER, "pod_name:" "\n{{" "\n\tname: {}" "\n}}", pod_name); writer_.pod_name(cgroup, blob("") /* deprecated pod_uid */, blob(pod_name)); } if (k8s_metadata.has_value() && *k8s_metadata) { writer_.k8s_metadata( cgroup, jb_blob{k8s_metadata->container_name()}, jb_blob{k8s_metadata->pod_name()}, jb_blob{k8s_metadata->pod_ns()}, jb_blob{k8s_metadata->pod_uid()}, jb_blob{k8s_metadata->sandbox_uid()}); for (auto const &port : k8s_metadata->ports()) { writer_.k8s_metadata_port(cgroup, port.second.port, integer_value(port.second.protocol), jb_blob{port.second.name}); } }; writer_.flush(); } ================================================ FILE: collector/kernel/cgroup_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include static constexpr std::string_view UNIX_SOCKET_PATH = "/var/run/docker.sock"; class CgroupHandler { public: struct CgroupSettings { bool force_docker_metadata = false; std::optional docker_metadata_dump_dir; }; // When set, specifies the name of the docker label that will be used for // obtaining the 'namespace' value. static std::string docker_ns_label_field; CgroupHandler( ::ebpf_net::ingest::Writer &writer, CurlEngine &curl_engine, CgroupSettings const &settings, logging::Logger &log); ~CgroupHandler(); void kill_css(u64 timestamp, struct jb_agent_internal__kill_css *msg); void css_populate_dir(u64 timestamp, struct jb_agent_internal__css_populate_dir *msg); void existing_cgroup_probe(u64 timestamp, struct jb_agent_internal__existing_cgroup_probe *msg); void cgroup_attach_task(u64 timestamp, struct jb_agent_internal__cgroup_attach_task *msg); void handle_pid_info(u32 pid, u64 cgroup, uint8_t comm[16]); private: friend class CgroupHandlerTest_handle_docker_response_Test; struct CgroupEntry { u64 cgroup_parent; std::string name; }; struct DockerQuery { std::unique_ptr request; std::string response; }; ::ebpf_net::ingest::Writer &writer_; CurlEngine &curl_engine_; CgroupSettings const &settings_; logging::Logger &log_; std::unordered_map cgroup_table_; std::unordered_map queries_; bool has_cgroup(u64 cgroup); // returns empty string for unknown cgroups std::string_view get_name(u64 cgroup); void handle_cgroup(u64 cgroup, u64 cgroup_parent, std::string const &name); void handle_docker_container(u64 cgroup, std::string const &name); void data_available_cb(const char *data, size_t data_length, u64 cgroup); void fetch_done_cb(CurlEngineStatus status, long responseCode, std::string_view curlError, u64 cgroup); void handle_docker_response(u64 cgroup, std::string const &response_data); }; ================================================ FILE: collector/kernel/cgroup_handler_test.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include const char *dummy_json_response_data = R"delim( { "Id": "ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90", "Created": "2021-11-04T16:54:03.099687792Z", "Path": "/srv/entrypoint.sh", "Args": [ "--port=8000", "--internal-prom=0.0.0.0:7000", "--prom=0.0.0.0:7001", "--prom-queue-count=1", "--ingest_shard_count=1", "--matching_shard_count=1", "--aggregation_shard_count=1", "--enable-aws-enrichment", "--log-console", "--debug" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 4251, "ExitCode": 0, "Error": "", "StartedAt": "2021-11-04T16:54:05.666083454Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:5c9ce00b94a01bf23a44eaf995f068d1993baefccf45ee37e5448f03f8499ab0", "ResolvConfPath": "/var/lib/docker/containers/ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90/resolv.conf", "HostnamePath": "/var/lib/docker/containers/ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90/hostname", "HostsPath": "/var/lib/docker/containers/ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90/hosts", "LogPath": "/var/lib/docker/containers/ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90/ad87fc49c1389b0939d637ae700aa4eb4f51cf7da569551857e80789305a7d90-json.log", "Name": "/amazing_chatterjee", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "unconfined", "ExecIDs": null, "HostConfig": { "Binds": [ "/home/vagrant/src/:/root/src", "/home/vagrant/out/:/root/out" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": {} }, "NetworkMode": "default", "PortBindings": { "7000/tcp": [ { "HostIp": "", "HostPort": "7000" } ], "7001/tcp": [ { "HostIp": "", "HostPort": "7001" }, { "HostIp": "", "HostPort": "7002" }, { "HostIp": "", "HostPort": "7003" } ], "8000/tcp": [ { "HostIp": "", "HostPort": "8000" } ] }, "RestartPolicy": { "Name": "no", "MaximumRetryCount": 0 }, "AutoRemove": true, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "CgroupnsMode": "host", "Dns": [], "DnsOptions": [], "DnsSearch": [], "ExtraHosts": null, "GroupAdd": null, "IpcMode": "private", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": true, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "label=disable" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": [], "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DeviceCgroupRules": null, "DeviceRequests": null, "KernelMemory": 0, "KernelMemoryTCP": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": false, "PidsLimit": null, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0, "MaskedPaths": null, "ReadonlyPaths": null }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/925ab32b43b5e0b27aae80af7cf4edfe129b79e56aca5c41337094eb95b83fea-init/diff:/var/lib/docker/overlay2/835a05eea84b538ad3fa865e50895ce03c59517ea7db69ce7313a3b421321c2d/diff:/var/lib/docker/overlay2/3b05911a571c14d6965e32a2cbe998bf75ae287c5969cef9e9350958ed805b95/diff:/var/lib/docker/overlay2/15982a35f2ac7e3e362106b03e8ece88a0b959407147e483e09dc5674fa1566d/diff:/var/lib/docker/overlay2/1c7fb4b897d78db2bb5da88423bfafc92ea0df95fdce8b8c91316a985fe59738/diff:/var/lib/docker/overlay2/5a32678605742638e9c3f784e9f11eddf36f9c8a1436711feadc685d7f0800ec/diff:/var/lib/docker/overlay2/94cba6304ed8b8f0f3c71b192e100eb51d9fe663f6cbd909adbf71f2a23166a0/diff:/var/lib/docker/overlay2/531befd27476430f7b6df70dde3120cb16eedc9aef0ba86042b097655d3e90f4/diff:/var/lib/docker/overlay2/0c1ef5b3e5063760f7f87674f5957937ab46f5162b5bffaad5ead3df0b6b3657/diff:/var/lib/docker/overlay2/85ac7b1e505a2705a28d2a24d16e905035a851c60e09f8949d498dbe5590dcd0/diff", "MergedDir": "/var/lib/docker/overlay2/925ab32b43b5e0b27aae80af7cf4edfe129b79e56aca5c41337094eb95b83fea/merged", "UpperDir": "/var/lib/docker/overlay2/925ab32b43b5e0b27aae80af7cf4edfe129b79e56aca5c41337094eb95b83fea/diff", "WorkDir": "/var/lib/docker/overlay2/925ab32b43b5e0b27aae80af7cf4edfe129b79e56aca5c41337094eb95b83fea/work" }, "Name": "overlay2" }, "Mounts": [ { "Type": "bind", "Source": "/home/vagrant/src", "Destination": "/root/src", "Mode": "", "RW": true, "Propagation": "rprivate" }, { "Type": "bind", "Source": "/home/vagrant/out", "Destination": "/root/out", "Mode": "", "RW": true, "Propagation": "rprivate" } ], "Config": { "Hostname": "ad87fc49c138", "Domainname": "", "User": "", "AttachStdin": false, "AttachStdout": true, "AttachStderr": true, "ExposedPorts": { "7000/tcp": {}, "7001/tcp": {}, "7010/tcp": {}, "8000/tcp": {} }, "Tty": true, "OpenStdin": false, "StdinOnce": false, "Env": [ "EBPF_NET_RUN_UNDER_GDB=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "OPENSSL_CONF=/etc/openssl/openssl.cnf", "GEOIP_PATH=/usr/share/GeoIP" ], "Cmd": [ "--port=8000", "--internal-prom=0.0.0.0:7000", "--prom=0.0.0.0:7001", "--prom-queue-count=1", "--ingest_shard_count=1", "--matching_shard_count=1", "--aggregation_shard_count=1", "--enable-aws-enrichment", "--log-console", "--debug" ], "Image": "localhost:5000/reducer", "Volumes": null, "WorkingDir": "", "Entrypoint": [ "/srv/entrypoint.sh" ], "OnBuild": null, "Labels": { "org.label-schema.description": "Network Explorer reducer", "org.label-schema.name": "network-explorer/reducer", "org.label-schema.schema-version": "1.0", "org.label-schema.usage": "./README.md" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "da32f06c760e26fb0c9a476711bc83b76c5d12513ab45d38076c7f2359782d38", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": { "7000/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "7000" }, { "HostIp": "::", "HostPort": "7000" } ], "7001/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "7003" }, { "HostIp": "::", "HostPort": "7003" }, { "HostIp": "0.0.0.0", "HostPort": "7002" }, { "HostIp": "::", "HostPort": "7002" }, { "HostIp": "0.0.0.0", "HostPort": "7001" }, { "HostIp": "::", "HostPort": "7001" } ], "7010/tcp": null, "8000/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "8000" }, { "HostIp": "::", "HostPort": "8000" } ] }, "SandboxKey": "/var/run/docker/netns/da32f06c760e", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "29e7a05fc7beb59d78b2262e1290751ab4fe2b21e4a9711a0793e558ab57a43c", "Gateway": "172.17.0.1", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "172.17.0.2", "IPPrefixLen": 16, "IPv6Gateway": "", "MacAddress": "02:42:ac:11:00:02", "Networks": { "bridge": { "IPAMConfig": null, "Links": null, "Aliases": null, "NetworkID": "2fa7167f0c612eda844cb3e9465e92108b5d10a4304033191007c00916eb661e", "EndpointID": "29e7a05fc7beb59d78b2262e1290751ab4fe2b21e4a9711a0793e558ab57a43c", "Gateway": "172.17.0.1", "IPAddress": "172.17.0.2", "IPPrefixLen": 16, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "02:42:ac:11:00:02", "DriverOpts": null } } } } )delim"; class CgroupHandlerTest : public ::testing::Test { protected: void SetUp() override { ASSERT_EQ(0, uv_loop_init(&loop_)); } void TearDown() override { // Clean up loop_ to avoid valgrind and asan complaints about memory leaks. close_uv_loop_cleanly(&loop_); } uv_loop_t loop_; }; TEST_F(CgroupHandlerTest, handle_docker_response) { channel::TestChannel test_channel(std::nullopt, IntakeEncoder::binary); channel::BufferedWriter buffered_writer(test_channel, 1024); ebpf_net::ingest::Writer writer(buffered_writer, monotonic, 0, nullptr); std::unique_ptr curl_engine = CurlEngine::create(&loop_); CgroupHandler::CgroupSettings cgroup_settings; logging::Logger logger(writer); CgroupHandler cgroup_handler(writer, *curl_engine.get(), cgroup_settings, logger); nlohmann::json const dummy_json_response_object = nlohmann::json::parse(dummy_json_response_data); // Populate a map with expected key/value pairs to compare against results from cgroup_handler.handle_docker_response(). std::unordered_map key_value_map; for (auto item : dummy_json_response_object["/Config/Labels"_json_pointer].items()) { key_value_map[item.key()] = item.value().get(); LOG::debug("key_value_map[{}]={}", item.key(), item.value()); } ASSERT_NE(0UL, key_value_map.size()); // Pass the dummy response_data to CgroupHandler::handle_docker_response(). cgroup_handler.handle_docker_response(1, dummy_json_response_object.dump()); // Validate that the writer got the expected values. auto validate_key_value = [&](channel::TestChannel::JsonMessageType const &msg) { LOG::debug("{}", log_waive(msg.dump())); auto key_json_ptr = "/data/key"_json_pointer; auto value_json_ptr = "/data/value"_json_pointer; if (msg.contains(key_json_ptr) && msg.contains(value_json_ptr)) { std::string key = msg[key_json_ptr]; std::string value = msg[value_json_ptr]; LOG::debug("key={}, value={}", key, value); EXPECT_TRUE(key_value_map[key] == value) << "key_value_map[key]=" << key_value_map[key] << " value=" << value; key_value_map.erase(key); } }; test_channel.json_messages_for_each(validate_key_value); EXPECT_EQ(0UL, key_value_map.size()); } ================================================ FILE: collector/kernel/cgroup_prober.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include CgroupProber::CgroupProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, HostInfo const &host_info, std::function periodic_cb, std::function check_cb) : host_info_(host_info), close_dir_error_count_(0) { // END ProbeAlternatives kill_kss_probe_alternatives{ "kill css", { {"on_kill_css", "kill_css"}, // Attaching probe to kill_css fails on some distros and kernel builds, for example Ubuntu Jammy. {"on_kill_css", "css_clear_dir"}, // If the previous two fail try an alternative for kernel versions older than 3.12. {"on_cgroup_destroy_locked", "cgroup_destroy_locked"}, }}; probe_handler.start_probe(skel, kill_kss_probe_alternatives); periodic_cb(); // START ProbeAlternatives css_populate_dir_probe_alternatives{ "css populate dir", { {"on_css_populate_dir", "css_populate_dir"}, {"on_cgroup_populate_dir", "cgroup_populate_dir"}, }}; probe_handler.start_probe(skel, css_populate_dir_probe_alternatives); periodic_cb(); // check both cgroups v1 and v2 because it is possible for active cgroups to exist in both (hybrid mode) // EXISTING cgroups v1 probe_handler.start_probe(skel, "on_cgroup_clone_children_read", "cgroup_clone_children_read"); periodic_cb(); check_cb("cgroup prober startup"); // locate the cgroup v1 mount directory std::string cgroup_v1_mountpoint = find_cgroup_v1_mountpoint(); if (!cgroup_v1_mountpoint.empty()) { // now iterate over cgroups and trigger cgroup_clone_children_read trigger_existing_cgroup_probe(cgroup_v1_mountpoint, "cgroup.clone_children", periodic_cb); check_cb("trigger_cgroup_clone_children_read()"); } /* can remove existing now */ probe_handler.cleanup_probe("cgroup_clone_children_read"); // EXISTING cgroups v2 static const std::string cgroup_v2_first_kernel_version("4.6"); if (host_info_.kernel_version >= cgroup_v2_first_kernel_version) { // Try cgroup_get_from_fd first (only needs kretprobe) int ret = probe_handler.start_kretprobe(skel, "onret_cgroup_get_from_fd", "cgroup_get_from_fd"); bool cgroup_get_from_fd_success = (ret == 0); if (cgroup_get_from_fd_success) { check_cb("cgroup_get_from_fd probe started"); } // If cgroup_get_from_fd failed, fallback to cgroup_control if (!cgroup_get_from_fd_success) { probe_handler.start_probe(skel, "on_cgroup_control", "cgroup_control"); probe_handler.start_kretprobe(skel, "onret_cgroup_control", "cgroup_control"); check_cb("cgroup_control probe started (fallback)"); } // locate the cgroup v2 mount directory std::string cgroup_v2_mountpoint = find_cgroup_v2_mountpoint(); if (!cgroup_v2_mountpoint.empty()) { // now iterate over cgroups and trigger the appropriate probe if (cgroup_get_from_fd_success) { trigger_cgroup_get_from_fd_probe(cgroup_v2_mountpoint, periodic_cb); check_cb("trigger_cgroup_get_from_fd()"); } else { trigger_existing_cgroup_probe(cgroup_v2_mountpoint, "cgroup.controllers", periodic_cb); check_cb("trigger_cgroup_control()"); } } /* cleanup probes */ if (cgroup_get_from_fd_success) { probe_handler.cleanup_kretprobe("cgroup_get_from_fd"); } else { probe_handler.cleanup_kretprobe("cgroup_control"); probe_handler.cleanup_probe("cgroup_control"); } } periodic_cb(); check_cb("cgroup prober cleanup()"); } void CgroupProber::trigger_cgroup_get_from_fd_probe(std::string const &cgroup_dir_name, std::function periodic_cb) { std::stack dirs_stack; dirs_stack.emplace(cgroup_dir_name); while (!dirs_stack.empty()) { periodic_cb(); // get the directory on the top of our stack std::string dir_name(dirs_stack.top()); dirs_stack.pop(); DIR *dir; dir = opendir(dir_name.c_str()); if (!dir) continue; // trigger cgroup_get_from_fd via BPF_PROG_QUERY for this directory int cgroup_fd = open(dir_name.c_str(), O_RDONLY); if (cgroup_fd >= 0) { LOG::debug_in(AgentLogKind::CGROUPS, "cgroup_get_from_fd probe: path={}", dir_name); // Set up BPF_PROG_QUERY attributes union bpf_attr attr = {}; attr.query.target_fd = cgroup_fd; attr.query.attach_type = BPF_CGROUP_INET_INGRESS; // Any valid attach type attr.query.query_flags = 0; attr.query.prog_cnt = 0; // Set to 0 to just get count attr.query.prog_ids = 0; // NULL to just query count // This triggers cgroup_get_from_fd()! int result = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, sizeof(attr)); (void)result; // Suppress unused variable warning close(cgroup_fd); } else { LOG::debug_in(AgentLogKind::CGROUPS, " fail to open path={}", dir_name); } // iterate over the elements of this directory and add any // subdirectories to dirs_stack struct dirent *ent; while ((ent = readdir(dir)) != NULL) { if (ent->d_type == DT_DIR) { // skip over "." and ".." entries in the directory if ((strcmp(ent->d_name, ".") == 0) || (strcmp(ent->d_name, "..") == 0)) continue; dirs_stack.emplace(dir_name + "/" + ent->d_name); } periodic_cb(); } int status = closedir(dir); if (status != 0) { close_dir_error_count_++; } } } void CgroupProber::trigger_existing_cgroup_probe( std::string const &cgroup_dir_name, std::string const &file_name, std::function periodic_cb) { std::stack dirs_stack; dirs_stack.emplace(cgroup_dir_name); while (!dirs_stack.empty()) { periodic_cb(); // get the directory on the top of our stack std::string dir_name(dirs_stack.top()); dirs_stack.pop(); DIR *dir; dir = opendir(dir_name.c_str()); if (!dir) continue; // trigger the cgroup existing probe for this directory std::string path = dir_name + "/" + file_name; LOG::debug_in(AgentLogKind::CGROUPS, "cgroup existing probe: path={}", path); std::ifstream file(path.c_str()); if (file.fail()) { LOG::debug_in(AgentLogKind::CGROUPS, " fail for path={}", path); int status = closedir(dir); if (status != 0) { close_dir_error_count_++; } continue; } else { LOG::debug_in(AgentLogKind::CGROUPS, " success for path={}", path); } std::string line; std::getline(file, line); // iterate over the elements of this directory and add any // subdirectories to dirs_stack struct dirent *ent; while ((ent = readdir(dir)) != NULL) { if (ent->d_type == DT_DIR) { // skip over "." and ".." entries in the directory if ((strcmp(ent->d_name, ".") == 0) || (strcmp(ent->d_name, "..") == 0)) continue; dirs_stack.emplace(dir_name + "/" + ent->d_name); } periodic_cb(); } int status = closedir(dir); if (status != 0) { close_dir_error_count_++; } } } static bool file_exists(std::string file_path) { struct stat sb; if (stat(file_path.c_str(), &sb) == -1) { return false; } return S_ISREG(sb.st_mode); } static bool is_cgroup_v1_mountpoint(std::string dir_path) { static const std::string file_name("/cgroup.clone_children"); return file_exists(dir_path + file_name); } static bool is_cgroup_v2_mountpoint(std::string dir_path) { static const std::string file_name("/cgroup.controllers"); return file_exists(dir_path + file_name); } std::string CgroupProber::find_cgroup_v1_mountpoint() { static const std::vector cgroup_v1_mountpoints = { "/hostfs/sys/fs/cgroup/memory", "/hostfs/cgroup/memory", "/sys/fs/cgroup/memory", "/cgroup/memory"}; for (auto const &mountpoint : cgroup_v1_mountpoints) { if (is_cgroup_v1_mountpoint(mountpoint)) { return mountpoint; } } return std::string(); } std::string CgroupProber::find_cgroup_v2_mountpoint() { static const std::vector cgroup_v2_mountpoints = {"/hostfs/sys/fs/cgroup", "/sys/fs/cgroup"}; for (auto const &mountpoint : cgroup_v2_mountpoints) { if (is_cgroup_v2_mountpoint(mountpoint)) { return mountpoint; } } return std::string(); } ================================================ FILE: collector/kernel/cgroup_prober.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include /* forward declarations */ class ProbeHandler; struct render_bpf_bpf; /** * Adds BPF probes for new and existing cgroups, and iterates through existing * cgroups to obtain an up-to-date view of system cgroups */ class CgroupProber { public: /** * C'tor * * @param probe_handler: a ProbeHandler where new probes can be registered * @param bpf_module: the module from the bpf source code * @param periodic_cb: a callback to be called every once in a while, to * allow user to e.g., flush rings */ CgroupProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, HostInfo const &host_info, std::function periodic_cb, std::function check_cb); int error_count() { return close_dir_error_count_; }; private: /** * Locates the cgroup v1 directory that should be used for probing. */ static std::string find_cgroup_v1_mountpoint(); /** * Locates the cgroup v2 directory that should be used for probing. */ static std::string find_cgroup_v2_mountpoint(); /** * Recursively walks through directory structure and triggers the * corresponding existing croup probe by reading the file_name specified. * * @param cgroup_dir_name: path to directory in which to perform the search * @param file_name: file name to read * @param periodic_cb: callback to call after doing some work. */ void trigger_existing_cgroup_probe( std::string const &cgroup_dir_name, std::string const &file_name, std::function periodic_cb); /** * Recursively walks through directory structure and triggers the * cgroup_get_from_fd probe by calling BPF_PROG_QUERY on each cgroup directory. * * @param cgroup_dir_name: path to directory in which to perform the search * @param periodic_cb: callback to call after doing some work. */ void trigger_cgroup_get_from_fd_probe(std::string const &cgroup_dir_name, std::function periodic_cb); HostInfo const host_info_; int close_dir_error_count_; }; ================================================ FILE: collector/kernel/dns/ares.h ================================================ /* Copyright 1998 by the Massachusetts Institute of Technology. * Copyright (C) 2007-2013 by Daniel Stenberg * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ #ifndef ARES__H #define ARES__H #include "ares_build.h" /* c-ares build definitions */ #include "ares_version.h" /* c-ares version defines */ /* * Define WIN32 when build target is Win32 API */ #if (defined(_WIN32) || defined(__WIN32__)) && !defined(WIN32) && !defined(__SYMBIAN32__) #define WIN32 #endif #include /* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish libc5-based Linux systems. Only include it on system that are known to require it! */ #if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || defined(__minix) || defined(__SYMBIAN32__) || \ defined(__INTEGRITY) || defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || defined(__QNXNTO__) #include #endif #if (defined(NETWARE) && !defined(__NOVELL_LIBC__)) #include #endif #if defined(WATT32) #include #include #include #elif defined(_WIN32_WCE) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include #include #elif defined(WIN32) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include #include #include #else #include #include #endif #if defined(ANDROID) || defined(__ANDROID__) #include #endif typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; #ifdef __cplusplus extern "C" { #endif /* ** c-ares external API function linkage decorations. */ #ifdef CARES_STATICLIB #define CARES_EXTERN #elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) #if defined(CARES_BUILDING_LIBRARY) #define CARES_EXTERN __declspec(dllexport) #else #define CARES_EXTERN __declspec(dllimport) #endif #elif defined(CARES_BUILDING_LIBRARY) && defined(CARES_SYMBOL_HIDING) #define CARES_EXTERN CARES_SYMBOL_SCOPE_EXTERN #else #define CARES_EXTERN #endif #define ARES_SUCCESS 0 /* Server error codes (ARES_ENODATA indicates no relevant answer) */ #define ARES_ENODATA 1 #define ARES_EFORMERR 2 #define ARES_ESERVFAIL 3 #define ARES_ENOTFOUND 4 #define ARES_ENOTIMP 5 #define ARES_EREFUSED 6 /* Locally generated error codes */ #define ARES_EBADQUERY 7 #define ARES_EBADNAME 8 #define ARES_EBADFAMILY 9 #define ARES_EBADRESP 10 #define ARES_ECONNREFUSED 11 #define ARES_ETIMEOUT 12 #define ARES_EOF 13 #define ARES_EFILE 14 #define ARES_ENOMEM 15 #define ARES_EDESTRUCTION 16 #define ARES_EBADSTR 17 /* ares_getnameinfo error codes */ #define ARES_EBADFLAGS 18 /* ares_getaddrinfo error codes */ #define ARES_ENONAME 19 #define ARES_EBADHINTS 20 /* Uninitialized library error code */ #define ARES_ENOTINITIALIZED 21 /* introduced in 1.7.0 */ /* ares_library_init error codes */ #define ARES_ELOADIPHLPAPI 22 /* introduced in 1.7.0 */ #define ARES_EADDRGETNETWORKPARAMS 23 /* introduced in 1.7.0 */ /* More error codes */ #define ARES_ECANCELLED 24 /* introduced in 1.7.0 */ /* Flag values */ #define ARES_FLAG_USEVC (1 << 0) #define ARES_FLAG_PRIMARY (1 << 1) #define ARES_FLAG_IGNTC (1 << 2) #define ARES_FLAG_NORECURSE (1 << 3) #define ARES_FLAG_STAYOPEN (1 << 4) #define ARES_FLAG_NOSEARCH (1 << 5) #define ARES_FLAG_NOALIASES (1 << 6) #define ARES_FLAG_NOCHECKRESP (1 << 7) #define ARES_FLAG_EDNS (1 << 8) /* Option mask values */ #define ARES_OPT_FLAGS (1 << 0) #define ARES_OPT_TIMEOUT (1 << 1) #define ARES_OPT_TRIES (1 << 2) #define ARES_OPT_NDOTS (1 << 3) #define ARES_OPT_UDP_PORT (1 << 4) #define ARES_OPT_TCP_PORT (1 << 5) #define ARES_OPT_SERVERS (1 << 6) #define ARES_OPT_DOMAINS (1 << 7) #define ARES_OPT_LOOKUPS (1 << 8) #define ARES_OPT_SOCK_STATE_CB (1 << 9) #define ARES_OPT_SORTLIST (1 << 10) #define ARES_OPT_SOCK_SNDBUF (1 << 11) #define ARES_OPT_SOCK_RCVBUF (1 << 12) #define ARES_OPT_TIMEOUTMS (1 << 13) #define ARES_OPT_ROTATE (1 << 14) #define ARES_OPT_EDNSPSZ (1 << 15) #define ARES_OPT_NOROTATE (1 << 16) /* Nameinfo flag values */ #define ARES_NI_NOFQDN (1 << 0) #define ARES_NI_NUMERICHOST (1 << 1) #define ARES_NI_NAMEREQD (1 << 2) #define ARES_NI_NUMERICSERV (1 << 3) #define ARES_NI_DGRAM (1 << 4) #define ARES_NI_TCP 0 #define ARES_NI_UDP ARES_NI_DGRAM #define ARES_NI_SCTP (1 << 5) #define ARES_NI_DCCP (1 << 6) #define ARES_NI_NUMERICSCOPE (1 << 7) #define ARES_NI_LOOKUPHOST (1 << 8) #define ARES_NI_LOOKUPSERVICE (1 << 9) /* Reserved for future use */ #define ARES_NI_IDN (1 << 10) #define ARES_NI_IDN_ALLOW_UNASSIGNED (1 << 11) #define ARES_NI_IDN_USE_STD3_ASCII_RULES (1 << 12) /* Addrinfo flag values */ #define ARES_AI_CANONNAME (1 << 0) #define ARES_AI_NUMERICHOST (1 << 1) #define ARES_AI_PASSIVE (1 << 2) #define ARES_AI_NUMERICSERV (1 << 3) #define ARES_AI_V4MAPPED (1 << 4) #define ARES_AI_ALL (1 << 5) #define ARES_AI_ADDRCONFIG (1 << 6) /* Reserved for future use */ #define ARES_AI_IDN (1 << 10) #define ARES_AI_IDN_ALLOW_UNASSIGNED (1 << 11) #define ARES_AI_IDN_USE_STD3_ASCII_RULES (1 << 12) #define ARES_AI_CANONIDN (1 << 13) #define ARES_AI_MASK \ (ARES_AI_CANONNAME | ARES_AI_NUMERICHOST | ARES_AI_PASSIVE | ARES_AI_NUMERICSERV | ARES_AI_V4MAPPED | ARES_AI_ALL | \ ARES_AI_ADDRCONFIG) #define ARES_GETSOCK_MAXNUM \ 16 /* ares_getsock() can return info about this \ many sockets */ #define ARES_GETSOCK_READABLE(bits, num) (bits & (1 << (num))) #define ARES_GETSOCK_WRITABLE(bits, num) (bits & (1 << ((num) + ARES_GETSOCK_MAXNUM))) /* c-ares library initialization flag values */ #define ARES_LIB_INIT_NONE (0) #define ARES_LIB_INIT_WIN32 (1 << 0) #define ARES_LIB_INIT_ALL (ARES_LIB_INIT_WIN32) /* * Typedef our socket type */ #ifndef ares_socket_typedef #ifdef WIN32 typedef SOCKET ares_socket_t; #define ARES_SOCKET_BAD INVALID_SOCKET #else typedef int ares_socket_t; #define ARES_SOCKET_BAD -1 #endif #define ares_socket_typedef #endif /* ares_socket_typedef */ typedef void (*ares_sock_state_cb)(void *data, ares_socket_t socket_fd, int readable, int writable); struct apattern; /* NOTE about the ares_options struct to users and developers. This struct will remain looking like this. It will not be extended nor shrunk in future releases, but all new options will be set by ares_set_*() options instead of with the ares_init_options() function. Eventually (in a galaxy far far away), all options will be settable by ares_set_*() options and the ares_init_options() function will become deprecated. When new options are added to c-ares, they are not added to this struct. And they are not "saved" with the ares_save_options() function but instead we encourage the use of the ares_dup() function. Needless to say, if you add config options to c-ares you need to make sure ares_dup() duplicates this new option. */ struct ares_options { int flags; int timeout; /* in seconds or milliseconds, depending on options */ int tries; int ndots; unsigned short udp_port; unsigned short tcp_port; int socket_send_buffer_size; int socket_receive_buffer_size; struct in_addr *servers; int nservers; char **domains; int ndomains; char *lookups; ares_sock_state_cb sock_state_cb; void *sock_state_cb_data; struct apattern *sortlist; int nsort; int ednspsz; }; struct hostent; struct timeval; struct sockaddr; struct ares_channeldata; typedef struct ares_channeldata *ares_channel; typedef void (*ares_callback)(void *arg, int status, int timeouts, unsigned char *abuf, int alen); typedef void (*ares_host_callback)(void *arg, int status, int timeouts, struct hostent *hostent); typedef void (*ares_nameinfo_callback)(void *arg, int status, int timeouts, char *node, char *service); typedef int (*ares_sock_create_callback)(ares_socket_t socket_fd, int type, void *data); typedef int (*ares_sock_config_callback)(ares_socket_t socket_fd, int type, void *data); CARES_EXTERN int ares_library_init(int flags); CARES_EXTERN int ares_library_init_mem( int flags, void *(*amalloc)(size_t size), void (*afree)(void *ptr), void *(*arealloc)(void *ptr, size_t size)); #if defined(ANDROID) || defined(__ANDROID__) CARES_EXTERN void ares_library_init_jvm(JavaVM *jvm); CARES_EXTERN int ares_library_init_android(jobject connectivity_manager); CARES_EXTERN int ares_library_android_initialized(void); #endif CARES_EXTERN int ares_library_initialized(void); CARES_EXTERN void ares_library_cleanup(void); CARES_EXTERN const char *ares_version(int *version); CARES_EXTERN int ares_init(ares_channel *channelptr); CARES_EXTERN int ares_init_options(ares_channel *channelptr, struct ares_options *options, int optmask); CARES_EXTERN int ares_save_options(ares_channel channel, struct ares_options *options, int *optmask); CARES_EXTERN void ares_destroy_options(struct ares_options *options); CARES_EXTERN int ares_dup(ares_channel *dest, ares_channel src); CARES_EXTERN void ares_destroy(ares_channel channel); CARES_EXTERN void ares_cancel(ares_channel channel); /* These next 3 configure local binding for the out-going socket * connection. Use these to specify source IP and/or network device * on multi-homed systems. */ CARES_EXTERN void ares_set_local_ip4(ares_channel channel, unsigned int local_ip); /* local_ip6 should be 16 bytes in length */ CARES_EXTERN void ares_set_local_ip6(ares_channel channel, const unsigned char *local_ip6); /* local_dev_name should be null terminated. */ CARES_EXTERN void ares_set_local_dev(ares_channel channel, const char *local_dev_name); CARES_EXTERN void ares_set_socket_callback(ares_channel channel, ares_sock_create_callback callback, void *user_data); CARES_EXTERN void ares_set_socket_configure_callback(ares_channel channel, ares_sock_config_callback callback, void *user_data); CARES_EXTERN int ares_set_sortlist(ares_channel channel, const char *sortstr); /* * Virtual function set to have user-managed socket IO. * Note that all functions need to be defined, and when * set, the library will not do any bind nor set any * socket options, assuming the client handles these * through either socket creation or the * ares_sock_config_callback call. */ struct iovec; struct ares_socket_functions { ares_socket_t (*asocket)(int, int, int, void *); int (*aclose)(ares_socket_t, void *); int (*aconnect)(ares_socket_t, const struct sockaddr *, ares_socklen_t, void *); ares_ssize_t (*arecvfrom)(ares_socket_t, void *, size_t, int, struct sockaddr *, ares_socklen_t *, void *); ares_ssize_t (*asendv)(ares_socket_t, const struct iovec *, int, void *); }; CARES_EXTERN void ares_set_socket_functions(ares_channel channel, const struct ares_socket_functions *funcs, void *user_data); CARES_EXTERN void ares_send(ares_channel channel, const unsigned char *qbuf, int qlen, ares_callback callback, void *arg); CARES_EXTERN void ares_query(ares_channel channel, const char *name, int dnsclass, int type, ares_callback callback, void *arg); CARES_EXTERN void ares_search(ares_channel channel, const char *name, int dnsclass, int type, ares_callback callback, void *arg); CARES_EXTERN void ares_gethostbyname(ares_channel channel, const char *name, int family, ares_host_callback callback, void *arg); CARES_EXTERN int ares_gethostbyname_file(ares_channel channel, const char *name, int family, struct hostent **host); CARES_EXTERN void ares_gethostbyaddr(ares_channel channel, const void *addr, int addrlen, int family, ares_host_callback callback, void *arg); CARES_EXTERN void ares_getnameinfo( ares_channel channel, const struct sockaddr *sa, ares_socklen_t salen, int flags, ares_nameinfo_callback callback, void *arg); CARES_EXTERN int ares_fds(ares_channel channel, fd_set *read_fds, fd_set *write_fds); CARES_EXTERN int ares_getsock(ares_channel channel, ares_socket_t *socks, int numsocks); CARES_EXTERN struct timeval *ares_timeout(ares_channel channel, struct timeval *maxtv, struct timeval *tv); CARES_EXTERN void ares_process(ares_channel channel, fd_set *read_fds, fd_set *write_fds); CARES_EXTERN void ares_process_fd(ares_channel channel, ares_socket_t read_fd, ares_socket_t write_fd); CARES_EXTERN int ares_create_query( const char *name, int dnsclass, int type, unsigned short id, int rd, unsigned char **buf, int *buflen, int max_udp_size); CARES_EXTERN int ares_mkquery(const char *name, int dnsclass, int type, unsigned short id, int rd, unsigned char **buf, int *buflen); CARES_EXTERN int ares_expand_name(const unsigned char *encoded, const unsigned char *abuf, int alen, char **s, long *enclen); CARES_EXTERN int ares_expand_string(const unsigned char *encoded, const unsigned char *abuf, int alen, unsigned char **s, long *enclen); /* * NOTE: before c-ares 1.7.0 we would most often use the system in6_addr * struct below when ares itself was built, but many apps would use this * private version since the header checked a HAVE_* define for it. Starting * with 1.7.0 we always declare and use our own to stop relying on the * system's one. */ struct ares_in6_addr { union { unsigned char _S6_u8[16]; } _S6_un; }; struct ares_addrttl { struct in_addr ipaddr; int ttl; }; struct ares_addr6ttl { struct ares_in6_addr ip6addr; int ttl; }; struct ares_srv_reply { struct ares_srv_reply *next; char *host; unsigned short priority; unsigned short weight; unsigned short port; }; struct ares_mx_reply { struct ares_mx_reply *next; char *host; unsigned short priority; }; struct ares_txt_reply { struct ares_txt_reply *next; unsigned char *txt; size_t length; /* length excludes null termination */ }; /* NOTE: This structure is a superset of ares_txt_reply */ struct ares_txt_ext { struct ares_txt_ext *next; unsigned char *txt; size_t length; /* 1 - if start of new record * 0 - if a chunk in the same record */ unsigned char record_start; }; struct ares_naptr_reply { struct ares_naptr_reply *next; unsigned char *flags; unsigned char *service; unsigned char *regexp; char *replacement; unsigned short order; unsigned short preference; }; struct ares_soa_reply { char *nsname; char *hostmaster; unsigned int serial; unsigned int refresh; unsigned int retry; unsigned int expire; unsigned int minttl; }; /* ** Parse the buffer, starting at *abuf and of length alen bytes, previously ** obtained from an ares_search call. Put the results in *host, if nonnull. ** Also, if addrttls is nonnull, put up to *naddrttls IPv4 addresses along with ** their TTLs in that array, and set *naddrttls to the number of addresses ** so written. */ CARES_EXTERN int ares_parse_a_reply(const unsigned char *abuf, int alen, struct hostent **host, struct ares_addrttl *addrttls, int *naddrttls); CARES_EXTERN int ares_parse_aaaa_reply( const unsigned char *abuf, int alen, struct hostent **host, struct ares_addr6ttl *addrttls, int *naddrttls); CARES_EXTERN int ares_parse_ptr_reply(const unsigned char *abuf, int alen, const void *addr, int addrlen, int family, struct hostent **host); CARES_EXTERN int ares_parse_ns_reply(const unsigned char *abuf, int alen, struct hostent **host); CARES_EXTERN int ares_parse_srv_reply(const unsigned char *abuf, int alen, struct ares_srv_reply **srv_out); CARES_EXTERN int ares_parse_mx_reply(const unsigned char *abuf, int alen, struct ares_mx_reply **mx_out); CARES_EXTERN int ares_parse_txt_reply(const unsigned char *abuf, int alen, struct ares_txt_reply **txt_out); CARES_EXTERN int ares_parse_txt_reply_ext(const unsigned char *abuf, int alen, struct ares_txt_ext **txt_out); CARES_EXTERN int ares_parse_naptr_reply(const unsigned char *abuf, int alen, struct ares_naptr_reply **naptr_out); CARES_EXTERN int ares_parse_soa_reply(const unsigned char *abuf, int alen, struct ares_soa_reply **soa_out); CARES_EXTERN void ares_free_string(void *str); CARES_EXTERN void ares_free_hostent(struct hostent *host); CARES_EXTERN void ares_free_data(void *dataptr); CARES_EXTERN const char *ares_strerror(int code); struct ares_addr_node { struct ares_addr_node *next; int family; union { struct in_addr addr4; struct ares_in6_addr addr6; } addr; }; struct ares_addr_port_node { struct ares_addr_port_node *next; int family; union { struct in_addr addr4; struct ares_in6_addr addr6; } addr; int udp_port; int tcp_port; }; CARES_EXTERN int ares_set_servers(ares_channel channel, struct ares_addr_node *servers); CARES_EXTERN int ares_set_servers_ports(ares_channel channel, struct ares_addr_port_node *servers); /* Incomming string format: host[:port][,host[:port]]... */ CARES_EXTERN int ares_set_servers_csv(ares_channel channel, const char *servers); CARES_EXTERN int ares_set_servers_ports_csv(ares_channel channel, const char *servers); CARES_EXTERN int ares_get_servers(ares_channel channel, struct ares_addr_node **servers); CARES_EXTERN int ares_get_servers_ports(ares_channel channel, struct ares_addr_port_node **servers); CARES_EXTERN const char *ares_inet_ntop(int af, const void *src, char *dst, ares_socklen_t size); CARES_EXTERN int ares_inet_pton(int af, const char *src, void *dst); #ifdef __cplusplus } #endif #endif /* ARES__H */ ================================================ FILE: collector/kernel/dns/ares_expand_name.c ================================================ /* Copyright 1998, 2011 by the Massachusetts Institute of Technology. * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ #include #include #include "ares.h" #include /* Maximum number of indirections allowed for a name */ #define MAX_INDIRS 50 /** from ares_nowarn.c **/ /* ** unsigned size_t to signed long */ #define CARES_MASK_SLONG 0x7FFFFFFFL long _aresx_uztosl(size_t uznum) { #ifdef __INTEL_COMPILER #pragma warning(push) #pragma warning(disable : 810) /* conversion may lose significant bits */ #endif return (long)(uznum & (size_t)CARES_MASK_SLONG); #ifdef __INTEL_COMPILER #pragma warning(pop) #endif } /* Expand an RFC1035-encoded domain name given by encoded. The * containing message is given by abuf and alen. The result is written, * NUL-terminated, into s, whose size must be large enough to hold the * domain name, i.e., no smaller than dns_name_length()+1. * *enclen is set to the length of the encoded name (not the length of the * expanded name; the goal is to tell the caller how many bytes to * move forward to get past the encoded name). * * In the simple case, an encoded name is a series of labels, each * composed of a one-byte length (limited to values between 0 and 63 * inclusive) followed by the label contents. The name is terminated * by a zero-length label. * * In the more complicated case, a label may be terminated by an * indirection pointer, specified by two bytes with the high bits of * the first byte (corresponding to INDIR_MASK) set to 11. With the * two high bits of the first byte stripped off, the indirection * pointer gives an offset from the beginning of the containing * message with more labels to decode. Indirection can happen an * arbitrary number of times, so we have to detect loops. * * Since the expanded name uses '.' as a label separator, we use * backslashes to escape periods or backslashes in the expanded name. */ void dns_expand_name(const unsigned char *encoded, const unsigned char *abuf, int alen, char *expanded_name, long *enclen) { int len, indir = 0; const unsigned char *p; char *q = expanded_name; /* No error-checking necessary; it was all done by name_length(). */ p = encoded; while (*p) { if ((*p & INDIR_MASK) == INDIR_MASK) { if (!indir) { *enclen = _aresx_uztosl(p + 2U - encoded); indir = 1; } p = abuf + ((*p & ~INDIR_MASK) << 8 | *(p + 1)); } else { len = *p; p++; while (len--) { if (*p == '.' || *p == '\\') *q++ = '\\'; *q++ = *p; p++; } *q++ = '.'; } } if (!indir) *enclen = _aresx_uztosl(p + 1U - encoded); /* Nuke the trailing period if we wrote one. */ if (q > expanded_name) *(q - 1) = 0; else *q = 0; /* zero terminate the zero-length domain */ } /* Return the length of the expansion of an encoded domain name, or * -1 if the encoding is invalid. */ int dns_name_length(const unsigned char *encoded, const unsigned char *abuf, int alen) { int n = 0, offset, indir = 0, top; /* Allow the caller to pass us abuf + alen and have us check for it. */ if (encoded >= abuf + alen) return -1; while (*encoded) { top = (*encoded & INDIR_MASK); if (top == INDIR_MASK) { /* Check the offset and go there. */ if (encoded + 1 >= abuf + alen) return -1; offset = (*encoded & ~INDIR_MASK) << 8 | *(encoded + 1); if (offset >= alen) return -1; encoded = abuf + offset; /* If we've seen more indirects than the message length, * then there's a loop. */ ++indir; if (indir > alen || indir > MAX_INDIRS) return -1; } else if (top == 0x00) { offset = *encoded; if (encoded + offset + 1 >= abuf + alen) return -1; encoded++; while (offset--) { n += (*encoded == '.' || *encoded == '\\') ? 2 : 1; encoded++; } n++; } else { /* RFC 1035 4.1.4 says other options (01, 10) for top 2 * bits are reserved. */ return -1; } } /* If there were any labels at all, then the number of dots is one * less than the number of labels, so subtract one. */ return (n) ? n - 1 : n; } /** * This version of expand name assumes s is of length DNS_NAME_MAX_LENGTH and * fails if the expanded name exceeds that. If @expanded_len is not NULL, the * expanded length is written there. */ int dns_expand_name_maxlen( const unsigned char *encoded, const unsigned char *abuf, int alen, char *expanded_name, long *enclen, int *expanded_len) { int len = dns_name_length(encoded, abuf, alen); if (len < 0) return ARES_EBADNAME; if (len + 1 > DNS_NAME_MAX_LENGTH) return ARES_ENOMEM; dns_expand_name(encoded, abuf, alen, expanded_name, enclen); if (expanded_len) *expanded_len = len; return ARES_SUCCESS; } ================================================ FILE: collector/kernel/dns/ares_parse_a_aaaa_reply.c ================================================ /* Copyright 1998 by the Massachusetts Institute of Technology. * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ #include #include #include #include "ares.h" #include "ares_dns.h" #include #include /** * The hostname of the reply will be written to @hostname_out, and its length * to @hostname_len. @hostname_out must be at least DNS_NAME_MAX_LENGTH * characters long. */ int dns_parse_a_aaaa_reply( const unsigned char *abuf, int alen, char *hostname_out, int *hostname_len, struct in_addr *in_addrs_out, int *num_in_addrs, struct in6_addr *in6_addrs_out, int *num_in6_addrs) { unsigned int qdcount, ancount; int status, i, rr_type, rr_class, rr_len, rr_ttl, naddrs, n6addrs; int cname_ttl = INT_MAX; /* the TTL imposed by the CNAME chain */ int naliases; long len; const unsigned char *aptr; char *hostname = hostname_out; char rr_name[DNS_NAME_MAX_LENGTH], rr_data[DNS_NAME_MAX_LENGTH]; const int max_in_addrs = (in_addrs_out && num_in_addrs) ? *num_in_addrs : 0; const int max_in6_addrs = (in6_addrs_out && num_in6_addrs) ? *num_in6_addrs : 0; /* Set number of addresses returned to NULL for all failure cases. */ if (num_in_addrs) { *num_in_addrs = 0; } if (num_in6_addrs) { *num_in6_addrs = 0; } /* Give up if abuf doesn't have room for a header. */ if (alen < HFIXEDSZ) { return ARES_EBADRESP; } /* Ensure this is a response */ if (DNS_HEADER_QR(abuf) != 1) { return ARES_EBADRESP; } /* Fetch the question and answer count from the header. */ qdcount = DNS_HEADER_QDCOUNT(abuf); ancount = DNS_HEADER_ANCOUNT(abuf); if (qdcount != 1) { return ARES_EBADRESP; } /* Expand the name from the question, and skip past the question. */ aptr = abuf + HFIXEDSZ; status = dns_expand_name_maxlen(aptr, abuf, alen, hostname, &len, hostname_len); if (status != ARES_SUCCESS) { return status; } if (aptr + len + QFIXEDSZ > abuf + alen) { return ARES_EBADRESP; } aptr += len + QFIXEDSZ; naddrs = 0; n6addrs = 0; naliases = 0; /* Examine each answer resource record (RR) in turn. */ for (i = 0; i < (int)ancount; i++) { /* Decode the RR up to the data field. */ status = dns_expand_name_maxlen(aptr, abuf, alen, rr_name, &len, NULL); if (status != ARES_SUCCESS) { break; } aptr += len; if (aptr + RRFIXEDSZ > abuf + alen) { status = ARES_EBADRESP; break; } rr_type = DNS_RR_TYPE(aptr); rr_class = DNS_RR_CLASS(aptr); rr_len = DNS_RR_LEN(aptr); rr_ttl = DNS_RR_TTL(aptr); aptr += RRFIXEDSZ; if (aptr + rr_len > abuf + alen) { status = ARES_EBADRESP; break; } if (rr_class == C_IN && rr_type == T_A && rr_len == sizeof(struct in_addr) && strcasecmp(rr_name, hostname) == 0) { if (naddrs < max_in_addrs) { struct in_addr *at = &in_addrs_out[naddrs]; if (aptr + sizeof(struct in_addr) > abuf + alen) { /* LCOV_EXCL_START: already checked above */ status = ARES_EBADRESP; break; } /* LCOV_EXCL_STOP */ memcpy(at, aptr, sizeof(struct in_addr)); } naddrs++; status = ARES_SUCCESS; } else if ( rr_class == C_IN && rr_type == T_AAAA && rr_len == sizeof(struct ares_in6_addr) && strcasecmp(rr_name, hostname) == 0) { if (n6addrs < max_in6_addrs) { struct in6_addr *const at = &in6_addrs_out[n6addrs]; if (aptr + sizeof(struct in6_addr) > abuf + alen) { /* LCOV_EXCL_START: already checked above */ status = ARES_EBADRESP; break; } /* LCOV_EXCL_STOP */ memcpy(at, aptr, sizeof(struct in6_addr)); } n6addrs++; status = ARES_SUCCESS; } if (rr_class == C_IN && rr_type == T_CNAME) { /* Record the RR name as an alias. */ naliases++; /* Decode the RR data and replace the hostname with it. */ status = dns_expand_name_maxlen(aptr, abuf, alen, rr_data, &len, NULL); if (status != ARES_SUCCESS) { break; } hostname = rr_data; /* Take the min of the TTLs we see in the CNAME chain. */ if (cname_ttl > rr_ttl) { cname_ttl = rr_ttl; } } aptr += rr_len; if (aptr > abuf + alen) { /* LCOV_EXCL_START: already checked above */ status = ARES_EBADRESP; break; } /* LCOV_EXCL_STOP */ } /* here we are a bit aggressive and write a response even on failure so * caller can choose to use what we got so far */ if (num_in_addrs) { *num_in_addrs = naddrs < max_in_addrs ? naddrs : max_in_addrs; } if (num_in6_addrs) { *num_in6_addrs = n6addrs < max_in6_addrs ? n6addrs : max_in6_addrs; } if (status == ARES_SUCCESS && naddrs == 0 && n6addrs == 0 && naliases == 0) { /* the check for naliases to be zero is to make sure CNAME responses don't get caught here */ status = ARES_ENODATA; } return status; } ================================================ FILE: collector/kernel/dns/ares_parse_query.c ================================================ /* Copyright 1998 by the Massachusetts Institute of Technology. * * Permission to use, copy, modify, and distribute this * software and its documentation for any purpose and without * fee is hereby granted, provided that the above copyright * notice appear in all copies and that both that copyright * notice and this permission notice appear in supporting * documentation, and that the name of M.I.T. not be used in * advertising or publicity pertaining to distribution of the * software without specific, written prior permission. * M.I.T. makes no representations about the suitability of * this software for any purpose. It is provided "as is" * without express or implied warranty. */ #include #include #include #include #include "ares.h" #include "ares_dns.h" #include #include // #define DEBUG_DNS_PARSE_QUERY 1 /** * returns ARES_SUCCESS if this is a valid request or response, * ARES_EBADQUERY otherwise, or on error * * Whether or not this is a request or reply will be returned in *is_response * The question's type will be returned in *type_out * The transaction ID will returned in *qid_out * * The question (hostname) of the request will be written to @question_out, and * its length to @question_len. @hostname_out must be at least * DNS_NAME_MAX_LENGTH characters long. */ int dns_parse_query( const unsigned char *abuf, int alen, int *is_response, uint16_t *type_out, uint16_t *qid_out, char *question_out, int *question_len) { unsigned int qdcount, ancount, nscount; //, arcount; int qr, status, q_type, q_class; long len; uint16_t qid; const unsigned char *aptr; /* Give up if abuf doesn't have room for a header. */ if (alen < HFIXEDSZ) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: alen < HFIXEDSZ\n"); #endif return ARES_EBADQUERY; } /* Fetch the question and answer count from the header. */ qr = DNS_HEADER_QR(abuf); qid = DNS_HEADER_QID(abuf); qdcount = DNS_HEADER_QDCOUNT(abuf); ancount = DNS_HEADER_ANCOUNT(abuf); nscount = DNS_HEADER_NSCOUNT(abuf); // arcount = DNS_HEADER_ARCOUNT(abuf); #if DEBUG_DNS_PARSE_QUERY fprintf( stderr, "qr: %d qid: %d qdcount:%u ancount:%u nscount:%u arcount:%u\n", qr, (int)qid, qdcount, ancount, nscount, arcount); #endif /* Determine if this is a request */ if (qr == 0) { /* if it's a request, only accept requests with a single question and * nothing else */ if (qdcount != 1 || ancount != 0 || nscount != 0) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: request with more than one question or other records\n"); #endif return ARES_EBADQUERY; } if (is_response) *is_response = 0; } else { /* if it's a response, only accept responses to a single question */ if (qdcount != 1) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: response to not single question\n"); #endif return ARES_EBADQUERY; } if (is_response) *is_response = 1; } /* Return the transaction id */ if (qid_out) { *qid_out = qid; } /* Expand the name from the question */ aptr = abuf + HFIXEDSZ; status = dns_expand_name_maxlen(aptr, abuf, alen, question_out, &len, question_len); if (status != ARES_SUCCESS) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: couldn't expand name\n"); #endif return status; } if (aptr + len + QFIXEDSZ > abuf + alen) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: truncated packet\n"); #endif return ARES_EBADQUERY; } aptr += len; q_type = DNS_QUESTION_TYPE(aptr); q_class = DNS_QUESTION_CLASS(aptr); /* Reject anything that isn't an Internet query */ if (q_class != ns_c_in) { #if DEBUG_DNS_PARSE_QUERY fprintf(stderr, "EBADQUERY: non-internet query\n"); #endif return ARES_EBADQUERY; } if (type_out) { *type_out = q_type; } return ARES_SUCCESS; } ================================================ FILE: collector/kernel/dns/c_ares_nameser.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #ifndef ARES_NAMESER_H #define ARES_NAMESER_H /* header file provided by liren@vivisimo.com */ #ifndef HAVE_ARPA_NAMESER_H #define NS_PACKETSZ 512 /* maximum packet size */ #define NS_MAXDNAME 256 /* maximum domain name */ #define NS_MAXCDNAME 255 /* maximum compressed domain name */ #define NS_MAXLABEL 63 #define NS_HFIXEDSZ 12 /* #/bytes of fixed data in header */ #define NS_QFIXEDSZ 4 /* #/bytes of fixed data in query */ #define NS_RRFIXEDSZ 10 /* #/bytes of fixed data in r record */ #define NS_INT16SZ 2 #define NS_INADDRSZ 4 #define NS_IN6ADDRSZ 16 #define NS_CMPRSFLGS 0xc0 /* Flag bits indicating name compression. */ #define NS_DEFAULTPORT 53 /* For both TCP and UDP. */ typedef enum __ns_class { ns_c_invalid = 0, /* Cookie. */ ns_c_in = 1, /* Internet. */ ns_c_2 = 2, /* unallocated/unsupported. */ ns_c_chaos = 3, /* MIT Chaos-net. */ ns_c_hs = 4, /* MIT Hesiod. */ /* Query class values which do not appear in resource records */ ns_c_none = 254, /* for prereq. sections in update requests */ ns_c_any = 255, /* Wildcard match. */ ns_c_max = 65536 } ns_class; typedef enum __ns_type { ns_t_invalid = 0, /* Cookie. */ ns_t_a = 1, /* Host address. */ ns_t_ns = 2, /* Authoritative server. */ ns_t_md = 3, /* Mail destination. */ ns_t_mf = 4, /* Mail forwarder. */ ns_t_cname = 5, /* Canonical name. */ ns_t_soa = 6, /* Start of authority zone. */ ns_t_mb = 7, /* Mailbox domain name. */ ns_t_mg = 8, /* Mail group member. */ ns_t_mr = 9, /* Mail rename name. */ ns_t_null = 10, /* Null resource record. */ ns_t_wks = 11, /* Well known service. */ ns_t_ptr = 12, /* Domain name pointer. */ ns_t_hinfo = 13, /* Host information. */ ns_t_minfo = 14, /* Mailbox information. */ ns_t_mx = 15, /* Mail routing information. */ ns_t_txt = 16, /* Text strings. */ ns_t_rp = 17, /* Responsible person. */ ns_t_afsdb = 18, /* AFS cell database. */ ns_t_x25 = 19, /* X_25 calling address. */ ns_t_isdn = 20, /* ISDN calling address. */ ns_t_rt = 21, /* Router. */ ns_t_nsap = 22, /* NSAP address. */ ns_t_nsap_ptr = 23, /* Reverse NSAP lookup (deprecated). */ ns_t_sig = 24, /* Security signature. */ ns_t_key = 25, /* Security key. */ ns_t_px = 26, /* X.400 mail mapping. */ ns_t_gpos = 27, /* Geographical position (withdrawn). */ ns_t_aaaa = 28, /* Ip6 Address. */ ns_t_loc = 29, /* Location Information. */ ns_t_nxt = 30, /* Next domain (security). */ ns_t_eid = 31, /* Endpoint identifier. */ ns_t_nimloc = 32, /* Nimrod Locator. */ ns_t_srv = 33, /* Server Selection. */ ns_t_atma = 34, /* ATM Address */ ns_t_naptr = 35, /* Naming Authority PoinTeR */ ns_t_kx = 36, /* Key Exchange */ ns_t_cert = 37, /* Certification record */ ns_t_a6 = 38, /* IPv6 address (deprecates AAAA) */ ns_t_dname = 39, /* Non-terminal DNAME (for IPv6) */ ns_t_sink = 40, /* Kitchen sink (experimentatl) */ ns_t_opt = 41, /* EDNS0 option (meta-RR) */ ns_t_apl = 42, /* Address prefix list (RFC3123) */ ns_t_ds = 43, /* Delegation Signer (RFC4034) */ ns_t_sshfp = 44, /* SSH Key Fingerprint (RFC4255) */ ns_t_rrsig = 46, /* Resource Record Signature (RFC4034) */ ns_t_nsec = 47, /* Next Secure (RFC4034) */ ns_t_dnskey = 48, /* DNS Public Key (RFC4034) */ ns_t_tkey = 249, /* Transaction key */ ns_t_tsig = 250, /* Transaction signature. */ ns_t_ixfr = 251, /* Incremental zone transfer. */ ns_t_axfr = 252, /* Transfer zone of authority. */ ns_t_mailb = 253, /* Transfer mailbox records. */ ns_t_maila = 254, /* Transfer mail agent records. */ ns_t_any = 255, /* Wildcard match. */ ns_t_zxfr = 256, /* BIND-specific, nonstandard. */ ns_t_max = 65536 } ns_type; typedef enum __ns_opcode { ns_o_query = 0, /* Standard query. */ ns_o_iquery = 1, /* Inverse query (deprecated/unsupported). */ ns_o_status = 2, /* Name server status query (unsupported). */ /* Opcode 3 is undefined/reserved. */ ns_o_notify = 4, /* Zone change notification. */ ns_o_update = 5, /* Zone update message. */ ns_o_max = 6 } ns_opcode; typedef enum __ns_rcode { ns_r_noerror = 0, /* No error occurred. */ ns_r_formerr = 1, /* Format error. */ ns_r_servfail = 2, /* Server failure. */ ns_r_nxdomain = 3, /* Name error. */ ns_r_notimpl = 4, /* Unimplemented. */ ns_r_refused = 5, /* Operation refused. */ /* these are for BIND_UPDATE */ ns_r_yxdomain = 6, /* Name exists */ ns_r_yxrrset = 7, /* RRset exists */ ns_r_nxrrset = 8, /* RRset does not exist */ ns_r_notauth = 9, /* Not authoritative for zone */ ns_r_notzone = 10, /* Zone of record different from zone section */ ns_r_max = 11, /* The following are TSIG extended errors */ ns_r_badsig = 16, ns_r_badkey = 17, ns_r_badtime = 18 } ns_rcode; #endif /* HAVE_ARPA_NAMESER_H */ #ifndef HAVE_ARPA_NAMESER_COMPAT_H #define PACKETSZ NS_PACKETSZ #define MAXDNAME NS_MAXDNAME #define MAXCDNAME NS_MAXCDNAME #define MAXLABEL NS_MAXLABEL #define HFIXEDSZ NS_HFIXEDSZ #define QFIXEDSZ NS_QFIXEDSZ #define RRFIXEDSZ NS_RRFIXEDSZ #define INDIR_MASK NS_CMPRSFLGS #define NAMESERVER_PORT NS_DEFAULTPORT #define QUERY ns_o_query #define SERVFAIL ns_r_servfail #define NOTIMP ns_r_notimpl #define REFUSED ns_r_refused #undef NOERROR /* it seems this is already defined in winerror.h */ #define NOERROR ns_r_noerror #define FORMERR ns_r_formerr #define NXDOMAIN ns_r_nxdomain #define C_IN ns_c_in #define C_CHAOS ns_c_chaos #define C_HS ns_c_hs #define C_NONE ns_c_none #define C_ANY ns_c_any #define T_A ns_t_a #define T_NS ns_t_ns #define T_MD ns_t_md #define T_MF ns_t_mf #define T_CNAME ns_t_cname #define T_SOA ns_t_soa #define T_MB ns_t_mb #define T_MG ns_t_mg #define T_MR ns_t_mr #define T_NULL ns_t_null #define T_WKS ns_t_wks #define T_PTR ns_t_ptr #define T_HINFO ns_t_hinfo #define T_MINFO ns_t_minfo #define T_MX ns_t_mx #define T_TXT ns_t_txt #define T_RP ns_t_rp #define T_AFSDB ns_t_afsdb #define T_X25 ns_t_x25 #define T_ISDN ns_t_isdn #define T_RT ns_t_rt #define T_NSAP ns_t_nsap #define T_NSAP_PTR ns_t_nsap_ptr #define T_SIG ns_t_sig #define T_KEY ns_t_key #define T_PX ns_t_px #define T_GPOS ns_t_gpos #define T_AAAA ns_t_aaaa #define T_LOC ns_t_loc #define T_NXT ns_t_nxt #define T_EID ns_t_eid #define T_NIMLOC ns_t_nimloc #define T_SRV ns_t_srv #define T_ATMA ns_t_atma #define T_NAPTR ns_t_naptr #define T_KX ns_t_kx #define T_CERT ns_t_cert #define T_A6 ns_t_a6 #define T_DNAME ns_t_dname #define T_SINK ns_t_sink #define T_OPT ns_t_opt #define T_APL ns_t_apl #define T_DS ns_t_ds #define T_SSHFP ns_t_sshfp #define T_RRSIG ns_t_rrsig #define T_NSEC ns_t_nsec #define T_DNSKEY ns_t_dnskey #define T_TKEY ns_t_tkey #define T_TSIG ns_t_tsig #define T_IXFR ns_t_ixfr #define T_AXFR ns_t_axfr #define T_MAILB ns_t_mailb #define T_MAILA ns_t_maila #define T_ANY ns_t_any #endif /* HAVE_ARPA_NAMESER_COMPAT_H */ #endif /* ARES_NAMESER_H */ ================================================ FILE: collector/kernel/dns/dns.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ #define DNS_NAME_MAX_LENGTH 256 #define MAX_ENCODED_IP_ADDRS 16 #define MAX_ENCODED_DNS_MESSAGE \ (/* timestamp */ sizeof(u64) + /* jb message */ jb_ingest__dns_response__data_size + \ /* ip addrs */ (sizeof(u32) * MAX_ENCODED_IP_ADDRS) + /* DNS name */ DNS_NAME_MAX_LENGTH) int dns_name_length(const unsigned char *encoded, const unsigned char *abuf, int alen); void dns_expand_name(const unsigned char *encoded, const unsigned char *abuf, int alen, char *s, long *enclen); int dns_expand_name_maxlen( const unsigned char *encoded, const unsigned char *abuf, int alen, char *s, long *enclen, int *expanded_len); int dns_parse_query( const unsigned char *abuf, int alen, int *is_response, uint16_t *type_out, uint16_t *qid_out, char *question_out, int *question_len); int dns_parse_a_aaaa_reply( const unsigned char *abuf, int alen, char *hostname_out, int *hostname_len, struct in_addr *in_addrs_out, int *num_in_addrs, struct in6_addr *in6_addrs_out, int *num_in6_addrs); #ifdef __cplusplus } #endif /* __cplusplus */ ================================================ FILE: collector/kernel/dns_requests.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "spdlog/common.h" #include #include #include /** * DNS requests key hash function * * @param[in] k The dns request key to hash * @return the hash of the dns request key */ size_t DnsRequests::dns_request_key_hash::operator()(const dns_request_key &k) const noexcept { return std::hash{}(k.qid) ^ std::hash{}(k.type) ^ std::hash{}(k.name); } /** * DNS requests key comparison function * * @param[in] k The first dns request key to compare * @param[in] k2 The second dns request key to compare * @return true if the keys are equal, false otherwise */ bool DnsRequests::dns_request_key_equal_to::operator()(const dns_request_key &k, const dns_request_key &k2) const noexcept { return k.qid == k2.qid && k.type == k2.type && k.name == k2.name; } /** * Add a DNS Request to the data structure * * @param[in] key Key of the DNS request * @param[in] value Value of the DNS request */ void DnsRequests::add(const dns_request_key &key, const dns_request_value &value) { auto iter = dns_requests_.insert(dns_requests_.end(), std::make_pair(key, value)); dns_requests_by_key_.insert(std::make_pair(key, iter)); dns_requests_by_sock_.insert(std::make_pair(value.sk, iter)); } /** * Return a list of DNS Requests that correspond to a particular key * * @param[in] key Key of the DNS Request to look up * @param[out] out The list of matching dns request key/value pairs */ void DnsRequests::lookup(const dns_request_key &key, std::list &out) { auto key_er = dns_requests_by_key_.equal_range(key); for (auto eriter = key_er.first; eriter != key_er.second; eriter++) { out.push_back(eriter->second); } } /** * Return a list of DNS Requests that are older than a particular timestamp * * @param[in] timestamp_ns The timestamp to get requests older than * @param[out] out The list of matching dns request key/value pairs */ void DnsRequests::lookup_older_than(u64 timestamp_ns, std::list &out) { for (auto iter = dns_requests_.begin(); iter != dns_requests_.end(); iter++) { if (iter->second.timestamp_ns >= timestamp_ns) { break; } out.push_back(iter); } } /** * Return a list of DNS Requests that match a particular socket * * @param[in] sk The kernel `struct sock*` pointer of the socket to look up * @param[out] out The list of matching dns request key/value pairs */ void DnsRequests::lookup_socket(u64 sk, std::list &out) { auto sk_er = dns_requests_by_sock_.equal_range(sk); for (auto eriter = sk_er.first; eriter != sk_er.second; eriter++) { out.push_back(eriter->second); } } /** * Removes a specific DNS Request * * @param[in] key The DNS Request to remove, as returned by lookup* functions */ void DnsRequests::remove(const Request &req) { auto key_er = dns_requests_by_key_.equal_range(req->first); for (auto eriter = key_er.first; eriter != key_er.second; eriter++) { if (eriter->second == req) { dns_requests_by_key_.erase(eriter); break; } } auto sk_er = dns_requests_by_sock_.equal_range(req->second.sk); for (auto skiter = sk_er.first; skiter != sk_er.second; skiter++) { if (skiter->second == req) { dns_requests_by_sock_.erase(skiter); break; } } dns_requests_.erase(req); } /** * Removes all matching DNS Requests for a particular key * * @param[in] key Key of the DNS requests to remove */ void DnsRequests::remove_all_with_key(const dns_request_key &key) { auto key_er = dns_requests_by_key_.equal_range(key); for (auto eriter = key_er.first; eriter != key_er.second;) { auto req = eriter->second; auto sk_er = dns_requests_by_sock_.equal_range(req->second.sk); for (auto skiter = sk_er.first; skiter != sk_er.second; skiter++) { if (skiter->second == req) { dns_requests_by_sock_.erase(skiter); break; } } dns_requests_.erase(req); eriter = dns_requests_by_key_.erase(eriter); } } ================================================ FILE: collector/kernel/dns_requests.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include class DnsRequests { // Type declarations public: struct dns_request_key { u16 qid; // transaction id u16 type; // query type std::string name; // query data bool is_rx; // was this request 'sent (client)' or 'received (server)' }; struct dns_request_value { u64 timestamp_ns; // when the query was made according to bpf u64 sk; // socket that sent the dns request }; protected: struct dns_request_key_hash { size_t operator()(const dns_request_key &k) const noexcept; }; struct dns_request_key_equal_to { bool operator()(const dns_request_key &k, const dns_request_key &k2) const noexcept; }; typedef std::list> DnsRequestsList; typedef std::unordered_multimap DnsRequestsByKeyMap; typedef std::unordered_multimap DnsRequestsBySockMap; public: typedef DnsRequestsList::iterator Request; // Public interface public: void add(const dns_request_key &key, const dns_request_value &value); void lookup(const dns_request_key &key, std::list &out); void lookup_older_than(u64 timestamp_ns, std::list &out); void lookup_socket(u64 sk, std::list &out); void remove(const Request &req); void remove_all_with_key(const dns_request_key &key); // Protected member variables protected: DnsRequestsByKeyMap dns_requests_by_key_; DnsRequestsList dns_requests_; DnsRequestsBySockMap dns_requests_by_sock_; }; ================================================ FILE: collector/kernel/entrypoint-kct.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 echo "===================== /etc/os-release =====================" [[ ! -e /etc/os-release ]] || cat /etc/os-release echo "========================= uname -a ========================" uname -a echo "======================= environment =======================" env | sort echo "===========================================================" install_dir=${EBPF_NET_INSTALL_DIR:-/srv} data_dir=${EBPF_NET_DATA_DIR:-/var/run/ebpf_net} dump_dir="${data_dir}/dump" mkdir -p "${data_dir}" "${dump_dir}" if ! mountpoint -q /sys; then mount -t sysfs none /sys || echo "Warning: Could not mount sysfs" fi echo "launching kernel collector test ..." # to run the collector under gdb, set `EBPF_NET_RUN_UNDER_GDB` to the flavor of gdb # you want (e.g.: `cgdb` or `gdb`) - this is intended for development purposes if [[ -n "${EBPF_NET_RUN_UNDER_GDB}" ]]; then if [[ "${#EBPF_NET_GDB_COMMANDS[@]}" -lt 1 ]]; then # default behavior is to run the agent, print a stack trace after it exits # and exit gdb without confirmation EBPF_NET_GDB_COMMANDS=( \ 'set pagination off' 'handle SIGPIPE nostop pass' 'handle SIGUSR1 nostop pass' 'handle SIGUSR2 nostop pass' run bt 'server q' ) fi GDB_ARGS=() for gdb_cmd in "${EBPF_NET_GDB_COMMANDS[@]}"; do GDB_ARGS+=(-ex "${gdb_cmd}") done (set -x; exec "${EBPF_NET_RUN_UNDER_GDB}" -q "${GDB_ARGS[@]}" \ --args "${install_dir}/kernel_collector_test" "$@" \ ) elif [[ -n "${EBPF_NET_RUN_UNDER_VALGRIND}" ]]; then # to run the collector under valgrind, set `EBPF_NET_RUN_UNDER_VALGRIND` to the options to pass to valgrind, # including at minimum the tool you want, for example: # "--tool=memcheck", or # "--tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes", or # "--tool=massif --stacks=yes" # note: to get full symbols from valgrind also build the kernel-collector in debug mode # shellcheck disable=SC2086 (set -x; exec /usr/bin/valgrind ${EBPF_NET_RUN_UNDER_VALGRIND} "${install_dir}/kernel_collector_test" "$@") else # Run the test binary and capture its exit code without inverting it. (set -x; "${install_dir}/kernel_collector_test" "$@") test_exit_code=$? if [[ ${test_exit_code} -ne 0 ]]; then echo "kernel collector test FAILED" cp /srv/core-* /hostfs/data || true if [[ -n "${DELAY_EXIT_ON_FAILURE}" ]]; then echo "DELAY_EXIT_ON_FAILURE is set, doing 'sleep inf'" sleep inf fi fi fi if [ -e /tmp/bpf-dump-file ] then "${install_dir}/bpf_wire_to_json" < /tmp/bpf-dump-file | jq . > /tmp/bpf-dump-file.json fi if [ -e /tmp/intake-dump-file ] then "${install_dir}/intake_wire_to_json" < /tmp/intake-dump-file | jq . > /tmp/intake-dump-file.json fi # Copy converted dumps to host-mounted data directory for artifact collection if [ -d /hostfs/data ]; then if [ -e /tmp/bpf-dump-file.json ]; then cp -f /tmp/bpf-dump-file.json /hostfs/data/ || true fi if [ -e /tmp/intake-dump-file.json ]; then cp -f /tmp/intake-dump-file.json /hostfs/data/ || true # Optional: emit a filtered file with only bpf_log messages from intake JSON jq '.[] | select(.name=="bpf_log")' /tmp/intake-dump-file.json > /hostfs/data/bpf-logs.json 2>/dev/null || true fi fi if [[ -n "${DELAY_EXIT}" ]] then echo "DELAY_EXIT is set, doing 'sleep inf'" sleep inf fi # Propagate the test's exit code so the container exit reflects pass/fail. if [[ -n "${test_exit_code:-}" ]]; then exit "${test_exit_code}" fi ================================================ FILE: collector/kernel/entrypoint.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* PATH_MAX*/ #include #include #include #include #include #include #include #include #include #include static constexpr size_t AGENT_MAX_MEMORY = 300ULL * 1024 * 1024; /* 300 MiB */ /* agent nice(2) value: -20 highest priority, 19 lowest, 0 default */ static constexpr int AGENT_NICE_VALUE = 5; static constexpr auto DISABLE_HTTP_METRICS_VAR = "EBPF_NET_DISABLE_HTTP_METRICS"; static constexpr auto NAMESPACE_OVERRIDE_VAR = "EBPF_NET_AGENT_NAMESPACE"; static constexpr auto CLUSTER_OVERRIDE_VAR = "EBPF_NET_AGENT_CLUSTER"; static constexpr auto SERVICE_OVERRIDE_VAR = "EBPF_NET_AGENT_SERVICE"; static constexpr auto HOST_OVERRIDE_VAR = "EBPF_NET_AGENT_HOST"; static constexpr auto ZONE_OVERRIDE_VAR = "EBPF_NET_AGENT_ZONE"; static void refill_log_rate_limit_cb(uv_timer_t *timer) { LOG::refill_rate_limit_budget(200); } /** * Check whether the binary has sufficient privileges and permissions. * If so, return. * If not, throw an exception with the appropriate errno. * * It can be difficult to directly determine privileges and permissions of a running binary. For example, containers may run * with root as the default user, but unless the container is run with --privileged or equivalent then not all root priviliges * are available. As another example, within a container it may not be possible to determine SELinux status. It may appear to * be disabled when queried in the container, but if enabled on the host it will prevent certain operations even if the * container is running with --privileged or equivalent. * * Instead, attempt to indirectly determine privileges and permissions by running bpf_map_create() as a test operation. Note * that it is intentionally called with invalid parameters so a map isn't actually created. Typical errors from this test * operation are: * EPERM: container is not running with --privileged or equivalent * EACCES: SELinux policy is preventing eBPF operations * EINVAL: privileges and permissions are sufficient - the test operation made it to where the invalid parameters were detected */ void check_permissions() { int fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 0, 0, 0, NULL); if (fd == -1) { switch (errno) { case EPERM: case EACCES: { std::string failstr = fmt::format("Test bpf_map_create() operation failed with errno {}", errno); throw std::system_error(errno, std::generic_category(), failstr); break; } case EINVAL: return; break; default: LOG::warn("Unexpected error checking for sufficient privileges and permissions errno {}: {}", errno, strerror(errno)); return; break; } } } //////////////////////////////////////////////////////////////////////////////// void mount_debugfs_if_required() { struct stat stat_buf = {0}; int ret = stat("/sys/kernel/debug/tracing", &stat_buf); if (ret != 0) { if (errno == ENOENT) { /* directory doesn't exist, try to mount */ ret = mount("debugfs", "/sys/kernel/debug/", "debugfs", 0, ""); if (ret != 0) { throw std::system_error(errno, std::generic_category(), "could not mount debugfs on /sys/kernel/debug"); } } } } //////////////////////////////////////////////////////////////////////////////// /** * Disables stdout and/or stderr output if requested by the user * @param disable_stdout: should we disable stdout * @param disable_stderr: should we disable stderr * * @return 0 on success, non-zero on error (returns the value of errno) */ int control_stdout_stderr(bool disable_stdout, bool disable_stderr) { // if user didn't ask to disable, we're done if ((disable_stdout == false) && (disable_stderr == false)) return 0; // open /dev/null int null_fd = open("/dev/null", O_WRONLY); if (null_fd == -1) return errno; if (disable_stdout) { int res = dup2(null_fd, STDOUT_FILENO); if (res == -1) { int errno_copy = errno; // in case close changes errno close(null_fd); return errno_copy; } } if (disable_stderr) { int res = dup2(null_fd, STDERR_FILENO); if (res == -1) { int errno_copy = errno; // in case close changes errno close(null_fd); return errno_copy; } } // free the file descriptor we dup'd int res = close(null_fd); if (res == -1) return errno; return 0; } //////////////////////////////////////////////////////////////////////////////// void set_resource_limits() { struct rlimit l = {}; l.rlim_max = AGENT_MAX_MEMORY; l.rlim_cur = l.rlim_max; int res = setrlimit(RLIMIT_DATA, &l); if (res != 0) LOG::warn("Could not set agent memory limit"); res = nice(AGENT_NICE_VALUE); if (res == -1) LOG::warn("Could not set agent nice(2) priority"); } //////////////////////////////////////////////////////////////////////////////// void get_uname(struct utsname &unamebuf) { if (uname(&unamebuf)) { throw std::runtime_error("Failed to get system uname"); } LOG::info( "Running on:\n" " sysname: {}\n" " nodename: {}\n" " release: {}\n" " version: {}\n" " machine: {}", unamebuf.sysname, unamebuf.nodename, unamebuf.release, unamebuf.version, unamebuf.machine); } void verify_kernel_blacklist(bool override, struct utsname &unamebuf) { bool kernel_fail = false; struct blacklist_match_line { const char *sysname; const char *nodename; const char *release; const char *version; const char *machine; }; blacklist_match_line failed_line; static blacklist_match_line kernel_blacklist[] = { #include "kernel_blacklist.h" }; int pat_num = 0; for (auto pat : kernel_blacklist) { bool match = true; if (match && pat.sysname && !std::regex_match(unamebuf.sysname, std::regex(pat.sysname))) { match = false; } if (match && pat.nodename && !std::regex_match(unamebuf.nodename, std::regex(pat.nodename))) { match = false; } if (match && pat.release && !std::regex_match(unamebuf.release, std::regex(pat.release))) { match = false; } if (match && pat.version && !std::regex_match(unamebuf.version, std::regex(pat.version))) { match = false; } if (match && pat.machine && !std::regex_match(unamebuf.machine, std::regex(pat.machine))) { match = false; } if (match) { kernel_fail = true; failed_line = pat; break; } pat_num++; } if (kernel_fail) { std::string failstr = fmt::format( "kernel blacklist check: pattern #{}:" " sysname: {}\n" " nodename: {}\n" " release: {}\n" " version: {}\n" " machine: {}", pat_num, failed_line.sysname ? failed_line.sysname : "NULL", failed_line.nodename ? failed_line.nodename : "NULL", failed_line.release ? failed_line.release : "NULL", failed_line.version ? failed_line.version : "NULL", failed_line.machine ? failed_line.machine : "NULL"); if (!override) { LOG::critical("Failed {}", failstr); exit(-1); } else { LOG::info("Overriding {}", failstr); } } } //////////////////////////////////////////////////////////////////////////////// static int kernel_collector_run(int argc, char **argv) { /* Initialize libuv loop */ uv_loop_t loop; int res = uv_loop_init(&loop); if (res != 0) { throw std::runtime_error("uv_loop_init failed"); } // args parsing cli::ArgsParser parser("Kernel Collector."); args::HelpFlag help(*parser, "help", "Display this help menu", {'h', "help"}); args::ValueFlag filter_ns(*parser, "nanoseconds", "Gap between subsequent reports", {"filter-ns"}, 10 * 1000 * 1000ull); args::ValueFlag socket_stats_interval_sec( *parser, "seconds", "Interval between sending socket stats", {"socket-stats-interval-sec"}, 10); args::ValueFlag metadata_timeout_us( *parser, "microseconds", "Microseconds to wait for cloud instance metadata", {"cloud-metadata-timeout-ms"}, std::chrono::microseconds(1s).count()); args::ValueFlag conf_file(*parser, "config_file", "The location of the custom config file", {"config-file"}, ""); args::Flag no_stdout(*parser, "no_stdout", "Disable stdout output", {"no-stdout"}); args::Flag no_stderr(*parser, "no_stderr", "Disable stderr output", {"no-stderr"}); args::Flag override_kernel_blacklist( *parser, "override_kernel_blacklist", "Override kernel blacklist (use at your own risk, can result in kernel panic)", {"override-kernel-blacklist"}); /* crash reporting */ args::Flag disable_log_rate_limit( *parser, "disable_log_rate_limit", "Disable rate limit the logging", {"disable-log-rate-limit"}); args::ValueFlag docker_ns_label( *parser, "label", "Docker label to be used as namespace", {"docker-ns-label"}, ""); auto disable_http_metrics = parser.add_env_flag("disable-http-metrics", "Disable collection of HTTP metrics", DISABLE_HTTP_METRICS_VAR, false); args::Flag enable_userland_tcp_flag( *parser, "userland_tcp", "Enable userland tcp processing (experimental)", {"enable-userland-tcp"}); auto force_docker_metadata = parser.add_flag("force-docker-metadata", "Forces the use of docker metadata"); auto disable_nomad_metadata = parser.add_flag("disable-nomad-metadata", "Disables detection and use of Nomad metadata"); args::ValueFlag docker_metadata_dump_dir( *parser, "docker-metadata-dump-dir", "If set, dump docker metadata to this directory (for debug purposes)", {"docker-metadata-dump-dir"}); args::ValueFlag bpf_dump_file( *parser, "bpf-dump-file", "If set, dumps the stream of eBPF messages to the file given by this flag", {"bpf-dump-file"}); #ifdef CONFIGURABLE_BPF args::ValueFlag bpf_file(*parser, "bpf_file", "File containing bpf code", {"bpf"}, ""); #endif // CONFIGURABLE_BPF #ifndef NDEBUG auto schedule_bpf_lost_samples = parser.add_arg( "schedule-bpf-lost-samples", "internal development - will continuously, at the interval in seconds provided, simulate lost BPF samples (PERF_RECORD_LOST) in BufferedPoller to test KernelCollector restarts via that code path"); #endif parser.new_handler>("agent-log"); parser.new_handler>("channel"); parser.new_handler>("cloud-platform"); parser.new_handler>("utility"); auto &intake_config_handler = parser.new_handler(); SignalManager signal_manager(loop, "kernel-collector"); if (auto result = parser.process(argc, argv); !result.has_value()) { return result.error(); } // Initialize minimal signal handler behavior (ignore SIGPIPE, disable core dumps) signal_manager.handle(); /* * Set docker nameservice label from commandline flags if provided; * fallback to $DOCKER_NS_LABEL environment variable if that exists. */ if (docker_ns_label.Matched()) { CgroupHandler::docker_ns_label_field = docker_ns_label.Get(); } else { char const *docker_ns_label_env = std::getenv("DOCKER_NS_LABEL"); if (docker_ns_label_env != nullptr) { CgroupHandler::docker_ns_label_field = std::string(docker_ns_label_env); } } /* set agent resource limits */ #if USE_ADDRESS_SANITIZER == 0 set_resource_limits(); #endif /* disable stdout/stderr if requested */ { int res = control_stdout_stderr(no_stdout, no_stderr); if (res != 0) { LOG::critical("Could not squelch stdout={} stderr={}: error={}", no_stdout.Get(), no_stderr.Get(), res); sleep(5); // make sure we don't spam too much exit(-1); } } auto agent_id = gen_agent_id(); /* log agent version */ std::string version_text = fmt::format("{}.{}.{}", versions::release.major(), versions::release.minor(), versions::release.patch()); if (auto signature = versions::release.signature(); !signature.empty()) { version_text += fmt::format("-{}", signature); } LOG::info("Starting Kernel Collector version {} ({})", version_text, release_mode_string); LOG::info("Kernel Collector agent ID is {}", agent_id); /* get kernel version */ struct utsname unamebuf; get_uname(unamebuf); /* check kernel blacklist */ verify_kernel_blacklist(override_kernel_blacklist.Matched(), unamebuf); /* * Set http_metrics flag if environment variable specified as well */ bool const enable_http_metrics = !disable_http_metrics; static constexpr char const *enabled_disabled[] = {"Disabled", "Enabled"}; LOG::info("HTTP Metrics: {}", enabled_disabled[enable_http_metrics]); LOG::info("Socket stats interval in seconds: {}", socket_stats_interval_sec.Get()); /* acknowledge userland tcp */ bool const enable_userland_tcp = enable_userland_tcp_flag.Matched(); LOG::info("Userland TCP: {}", enabled_disabled[enable_userland_tcp]); /* Initialize curl */ curlpp::initialize(); std::chrono::microseconds const metadata_timeout(metadata_timeout_us.Get()); /* AWS metadata */ LOG::trace_in(CloudPlatform::aws, "--- resolving AWS metadata ---"); auto const aws_metadata = AwsMetadata::fetch(metadata_timeout); if (aws_metadata) { aws_metadata->print_instance_metadata(); aws_metadata->print_interfaces(); } else { LOG::debug("Unable to fetch AWS metadata: {}", aws_metadata.error().what()); } /* GCP metadata */ LOG::trace_in(CloudPlatform::gcp, "--- resolving GCP metadata ---"); auto const gcp_metadata = GcpInstanceMetadata::fetch(metadata_timeout); if (gcp_metadata) { gcp_metadata->print(); } else { LOG::debug("Unable to fetch GCP metadata: {}", gcp_metadata.error().what()); } /* read label overrides from environment if present */ auto override_agent_namespace = std::string{try_get_env_var(NAMESPACE_OVERRIDE_VAR)}; auto override_agent_cluster = std::string{try_get_env_var(CLUSTER_OVERRIDE_VAR)}; auto override_agent_zone = std::string{try_get_env_var(ZONE_OVERRIDE_VAR)}; auto override_agent_service = std::string{try_get_env_var(SERVICE_OVERRIDE_VAR)}; auto override_agent_host = std::string{try_get_env_var(HOST_OVERRIDE_VAR)}; /* Nomad metadata */ if (!disable_nomad_metadata) { LOG::trace("--- resolving Nomad metadata ---"); if (NomadMetadata const nomad_metadata; nomad_metadata) { nomad_metadata.print(); // respect explicit overrides from environment variables // TODO: what's the equivalent of a namespace in Nomad? if (override_agent_cluster.empty() && !nomad_metadata.ns().empty()) { override_agent_cluster = std::string(nomad_metadata.ns()); } if (override_agent_service.empty() && !nomad_metadata.task_name().empty()) { override_agent_service = std::string(nomad_metadata.task_name()); } } else { LOG::debug("Unable to fetch Nomad metadata - environment variables not found"); } } else { LOG::trace("--- skipping Nomad metadata resolution ---"); } // resolve hostname std::string const hostname = get_host_name(MAX_HOSTNAME_LENGTH).recover([&](auto &error) { LOG::error("Unable to retrieve host information from uname: {}", error); return aws_metadata->id().valid() ? std::string(aws_metadata->id().value()) : "(unknown)"; }); LOG::info("Kernel Collector version {} ({}) started on host {}", version_text, release_mode_string, hostname); config::ConfigFile configuration_data(config::ConfigFile::YamlFormat(), conf_file.Get()); /* use label overrides if present */ if (!override_agent_namespace.empty()) { LOG::debug("overriding agent namespace with '{}'", override_agent_namespace); configuration_data.labels()["namespace"] = override_agent_namespace; } if (!override_agent_cluster.empty()) { LOG::debug("overriding agent cluster with '{}'", override_agent_cluster); configuration_data.labels()["cluster"] = override_agent_cluster; } if (!override_agent_zone.empty()) { LOG::debug("overriding agent zone with '{}'", override_agent_zone); configuration_data.labels()["zone"] = override_agent_zone; } if (!override_agent_service.empty()) { LOG::debug("overriding agent service with '{}'", override_agent_service); configuration_data.labels()["service"] = override_agent_service; } if (!override_agent_host.empty()) { LOG::debug("overriding agent host with '{}'", override_agent_host); configuration_data.labels()["host"] = override_agent_host; } HostInfo host_info{ .os = OperatingSystem::Linux, .os_flavor = integer_value(LinuxDistro::unknown), .os_version = gcp_metadata ? gcp_metadata->image() : "unknown", .kernel_headers_source = KernelHeadersSource::libbpf, .kernel_version = unamebuf.release, .hostname = hostname}; try { check_permissions(); /* mount debugfs if it is not mounted */ mount_debugfs_if_required(); // Create BPF configuration with all required parameters u64 boot_time_adjustment = get_boot_time(); BpfConfiguration bpf_config{ .boot_time_adjustment = boot_time_adjustment, .filter_ns = args::get(filter_ns), .enable_tcp_data_stream = enable_userland_tcp}; uv_timer_t refill_log_rate_limit_timer; if (!disable_log_rate_limit) { LOG::enable_rate_limit(5000); res = uv_timer_init(&loop, &refill_log_rate_limit_timer); if (res != 0) { throw std::runtime_error("Cannot init rate limit timer."); } res = uv_timer_start( &refill_log_rate_limit_timer, refill_log_rate_limit_cb, /*1s*/ 1000, /*1s*/ 1000); if (res != 0) { throw std::runtime_error("Cannot start rate limit timer."); } } /* initialize curl engine */ auto curl_engine = CurlEngine::create(&loop); // Load the intake configuration from the configuration file and from command-line args. config::IntakeConfig intake_config = configuration_data.intake_config(); intake_config_handler.read_config(intake_config); // Initialize our kernel telemetry collector KernelCollector kernel_collector{ bpf_config, intake_config, aws_metadata.try_value(), gcp_metadata.try_value(), configuration_data.labels(), loop, *curl_engine, enable_http_metrics, socket_stats_interval_sec.Get(), CgroupHandler::CgroupSettings{ .force_docker_metadata = *force_docker_metadata, .docker_metadata_dump_dir = docker_metadata_dump_dir ? std::optional(docker_metadata_dump_dir.Get()) : std::nullopt, }, bpf_dump_file.Get(), host_info}; signal_manager.handle_signals({SIGINT, SIGTERM}, std::bind(&KernelCollector::on_close, &kernel_collector)); #ifndef NDEBUG std::unique_ptr debug_bpf_lost_samples_timer; if (schedule_bpf_lost_samples.given() && *schedule_bpf_lost_samples > 0) { std::chrono::seconds const timeout(*schedule_bpf_lost_samples); auto schedule_bpf_lost_samples = [&debug_bpf_lost_samples_timer, timeout]() { if (auto const result = debug_bpf_lost_samples_timer->defer(timeout)) { LOG::info("successfully scheduled inject_bpf_lost_samples() {} from now", timeout); } else { LOG::error("failed to schedule inject_bpf_lost_samples() {} from now: {}", timeout, result.error()); } }; auto inject_bpf_lost_samples = [&kernel_collector, schedule_bpf_lost_samples]() { kernel_collector.debug_bpf_lost_samples(); schedule_bpf_lost_samples(); // reschedule another callback to inject bpf lost samples }; debug_bpf_lost_samples_timer = std::make_unique(loop, inject_bpf_lost_samples); schedule_bpf_lost_samples(); // schedule the first callback to inject bpf lost samples } #endif LOG::debug("starting event loop..."); uv_run(&loop, UV_RUN_DEFAULT); } catch (std::system_error &e) { switch (e.code().value()) { case EPERM: print_troubleshooting_message_and_exit(host_info, TroubleshootItem::operation_not_permitted, e); break; case EACCES: print_troubleshooting_message_and_exit(host_info, TroubleshootItem::permission_denied, e); break; default: print_troubleshooting_message_and_exit(host_info, TroubleshootItem::unexpected_exception, e); break; } return 1; } catch (std::exception &e) { print_troubleshooting_message_and_exit(host_info, TroubleshootItem::unexpected_exception, e); return 1; } return 0; } extern "C" int otn_kernel_collector_main(int argc, const char **argv) { return kernel_collector_run(argc, const_cast(argv)); } ================================================ FILE: collector/kernel/entrypoint.sh ================================================ #!/bin/bash # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 # shellcheck disable=SC1091 [[ ! -e ./debug-info.conf ]] || source ./debug-info.conf if [[ "${EBPF_NET_DEBUG_MODE}" == true ]]; then echo "===================== /etc/os-release =====================" [[ ! -e /etc/os-release ]] || cat /etc/os-release echo "========================= uname -a ========================" uname -a echo "======================= environment =======================" env | sort echo "===========================================================" fi install_dir=${EBPF_NET_INSTALL_DIR:-/srv} data_dir=${EBPF_NET_DATA_DIR:-/var/run/ebpf_net} dump_dir="${data_dir}/dump" mkdir -p "${data_dir}" "${dump_dir}" # With libbpf, kernel headers are no longer needed if ! mountpoint -q /sys; then mount -t sysfs none /sys || echo "Warning: Could not mount sysfs" fi cmd_args=() echo "launching kernel collector..." # on Debug (non-production) images, devs can run in local mode by setting # `EBPF_NET_RUN_LOCAL` to non-empty. if [[ -n "${EBPF_NET_RUN_LOCAL}" ]]; then # shellcheck disable=SC1091 source /srv/local.sh cmd_args+=("${local_cmd_args[@]}") fi # to run the collector under gdb, set `EBPF_NET_RUN_UNDER_GDB` to the flavor of gdb # you want (e.g.: `cgdb` or `gdb`) - this is intended for development purposes if [[ -n "${EBPF_NET_RUN_UNDER_GDB}" ]]; then apt-get update -y apt-get install -y --no-install-recommends "${EBPF_NET_RUN_UNDER_GDB}" if [[ "${#EBPF_NET_GDB_COMMANDS[@]}" -lt 1 ]]; then # default behavior is to run the agent, print a stack trace after it exits # and exit gdb without confirmation EBPF_NET_GDB_COMMANDS=( \ 'set pagination off' 'handle SIGPIPE nostop pass' 'handle SIGUSR1 nostop pass' 'handle SIGUSR2 nostop pass' run bt 'server q' ) fi GDB_ARGS=() for gdb_cmd in "${EBPF_NET_GDB_COMMANDS[@]}"; do GDB_ARGS+=(-ex "${gdb_cmd}") done (set -x; exec "${EBPF_NET_RUN_UNDER_GDB}" -q "${GDB_ARGS[@]}" \ --args "${install_dir}/kernel-collector" "${cmd_args[@]}" "$@" \ ) elif [[ -n "${EBPF_NET_RUN_UNDER_VALGRIND}" ]]; then # to run the collector under valgrind, set `EBPF_NET_RUN_UNDER_VALGRIND` to the options to pass to valgrind, # including at minimum the tool you want, for example: # "--tool=memcheck", or # "--tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes", or # "--tool=massif --stacks=yes" # note: to get full symbols from valgrind also build the kernel-collector in debug mode apt update -y apt install -y valgrind # shellcheck disable=SC2086 (set -x; exec /usr/bin/valgrind ${EBPF_NET_RUN_UNDER_VALGRIND} "${install_dir}/kernel-collector" "${cmd_args[@]}" "$@") else (set -x; exec "${install_dir}/kernel-collector" "${cmd_args[@]}" "$@") fi ================================================ FILE: collector/kernel/fd_reader.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include FDReader::FDReader(int pid) : pid_(pid), task_tid_(-1), fd_dir_(NULL), task_dir_(NULL), task_fd_dir_(NULL), fd_dir_ent_(NULL), task_dir_ent_(NULL) {} FDReader::~FDReader() { // XXX: may want to add checks for the return value of closedir if (task_dir_) closedir(task_dir_); if (fd_dir_) closedir(fd_dir_); } int FDReader::get_inode() { // check if this entry is a fd int fd; if (sscanf(fd_dir_ent_->d_name, "%d", &fd) != 1) return -1; // exit if the current fd_dir_ent_ is either "." or ".." // create the link to this fd char link[64]; if (snprintf(link, sizeof(link), "/proc/%d/fd/%d", pid_, fd) < 0) return -1; // read the link char link_content[20]; int info_len = readlink(link, link_content, sizeof(link_content) - 1); if (info_len == -1) return -1; // exit if there was an error reading the file link_content[info_len] = '\0'; // check whether this was a socket if (strncmp(link_content, "socket:[", strlen("socket:["))) return -1; // strncmp(a,b) == 0 when a == b // get the inode number int ino; if (sscanf(link_content, "socket:[%u]", &ino) != 1) return -1; // sscanf returns the number of items it matched return ino; } int FDReader::next_task() { int tid = -1; while (tid == -1) { task_dir_ent_ = readdir(task_dir_); if (task_dir_ent_ == 0) return -1; // there are no more entries left // check that this dir was not "." or ".." int temp; // XXX: not sure that sscanf won't overwrite a value if (sscanf(task_dir_ent_->d_name, "%d", &temp) != 1) { continue; } tid = temp; } task_tid_ = tid; return 0; } int FDReader::next_fd() { int fd = -1; while (fd == -1) { fd_dir_ent_ = readdir(fd_dir_); if (fd_dir_ent_ == 0) return -1; int temp; // XXX: not sure that sscanf won't overwrite a value if (sscanf(fd_dir_ent_->d_name, "%d", &temp) != 1) { continue; } fd = temp; } return 0; } int FDReader::open_fd_dir() { // create the link to the fd_dir_ char fd_dir_name[64]; if (snprintf(fd_dir_name, sizeof(fd_dir_name), "/proc/%d/fd", pid_) < 0) return -1; // open up the task_dir_ fd_dir_ = opendir(fd_dir_name); if (!fd_dir_) return -1; return 0; } int FDReader::open_task_dir() { // create the link to the task_dir_ char task_dir_name[64]; if (snprintf(task_dir_name, sizeof(task_dir_name), "/proc/%d/task", pid_) < 0) return -1; // open up the task_dir_ task_dir_ = opendir(task_dir_name); if (!task_dir_) return -1; return 0; } int FDReader::open_task_comm() { // create the link to the task comm char link[64]; if (snprintf(link, sizeof(link), "/proc/%d/task/%d/comm", pid_, task_tid_) < 0) return -1; // read the link char link_content[16]; int info_len = readlink(link, link_content, sizeof(link_content) - 1); if (info_len == -1) return -1; // exit if there was an error reading the file link_content[info_len] = '\0'; return 0; } ================================================ FILE: collector/kernel/fd_reader.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include /** * Does two things: * 1. Reads over the task directory of a given pid * i.e. /proc/[pid]/task * 2. Reads over the entries in the fd directory of a given pid * i.e. /proc/[pid]/fd * We report the inode for entries in the fd directory that are sockets */ class FDReader { public: /** * c'tor */ FDReader(int pid); /** * d'tor */ ~FDReader(); /** * Returns the inode number if the fd is for a socket * Returns -1 if the this fd was not a socket */ int get_inode(); /** * Read task_dir_. Returns 0 if a task was found and -1 otherwise. * i.e. skips over non-task entries until we find a task or run out of entries * Sets the task_dir_ent_ and task_tid_ if we do find an entry * */ int next_task(); /** * f_dir_. Returns 0 if an entry was found and -1 otherwise. * i.e. skips over non-task entries until we find a task or run out of entries * Sets the fd_dir_ent_ if we do find an entry */ int next_fd(); /** * Opens the fd directory specified by pid_. * Returns 0 on success, -1 if failed. */ int open_fd_dir(); /** * Opens the task directory specified by pid_. * Returns 0 on success, -1 if failed. */ int open_task_dir(); /** * Opens the comm file specified by pid_ and tid_. * Returns 0 on success, -1 if failed. */ int open_task_comm(); private: int pid_; int task_tid_; DIR *fd_dir_; // /proc/[pid]/fd DIR *task_dir_; // /proc/[pid]/task DIR *task_fd_dir_; // /proc/[pid]/task/[tid]/fd dirent *fd_dir_ent_; dirent *task_dir_ent_; }; ================================================ FILE: collector/kernel/hostport_tuple.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include // Struct represents the 4-tuple of a connection // NOTE: when used as a key, src_port and dst_port are in network byte order // NOTE: hasher relies on the alignment of variables struct hostport_tuple { u32 src_ip; u32 dst_ip; u16 src_port; u16 dst_port; u32 proto; bool operator==(const hostport_tuple &rhs) const { return (this->src_ip == rhs.src_ip) && (this->dst_ip == rhs.dst_ip) && (this->src_port == rhs.src_port) && (this->dst_port == rhs.dst_port) && (this->proto == rhs.proto); } hostport_tuple reversed() const { return {dst_ip, src_ip, dst_port, src_port, proto}; } }; namespace std { template <> struct hash { size_t operator()(const hostport_tuple &t) const noexcept { return (std::size_t)lookup3_hashword((uint32_t *)&t, 4, 0x9e3779b9); } }; } // namespace std ================================================ FILE: collector/kernel/kernel_blacklist.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ // Regexp patterns for kernel blacklist // failures must match all specified uname components // components specified as NULL are not matched // Pattern specification: // { sysname, nodename, release, version, machine } // minikube 4.15.0 {"Linux", "minikube", R"(^4\.15\.0.*)", NULL, NULL}, // Linux 4.19.17 from Ubuntu mainline repo {"Linux", NULL, R"(^4\.19\.57.*)", NULL, NULL}, // Linux 5.1.16 from Ubuntu mainline repo { "Linux", NULL, R"(^5\.1\.16.*)", NULL, NULL } ================================================ FILE: collector/kernel/kernel_collector.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { static constexpr std::chrono::milliseconds TRY_CONNECTING_TIMEOUT = 5s; static constexpr u64 connection_timeout_ms_ = 10000; static constexpr u64 probe_holdoff_timeout_ms_ = 2000; static constexpr u64 polling_timeout_ms_ = 100; static constexpr u64 slow_polling_timeout_ms_ = 1000; /* minimum time allowed between probes */ static constexpr u64 inter_probe_time_ns_ = 120 * 1000 * 1000 * 1000ul; /* max jitter added to time between probes */ static constexpr std::chrono::milliseconds MAX_JITTER_TIME = 10s; } // namespace void __try_connecting_cb(uv_timer_t *timer) { KernelCollector *collector = (KernelCollector *)timer->data; collector->try_connecting(timer); } void __connection_timeout_cb(uv_timer_t *timer) { LOG::trace("KernelCollector: connection timeout"); KernelCollector *collector = (KernelCollector *)timer->data; collector->connection_timeout(timer); } void __probe_holdoff_cb(uv_timer_t *timer) { LOG::trace("KernelCollector: probe holdoff timeout"); KernelCollector *collector = (KernelCollector *)timer->data; collector->probe_holdoff_timeout(timer); } void __polling_steady_state_cb(uv_timer_t *timer) { KernelCollector *collector = (KernelCollector *)timer->data; collector->polling_steady_state(timer); } void __polling_steady_state_slow_cb(uv_timer_t *timer) { KernelCollector *collector = (KernelCollector *)timer->data; collector->polling_steady_state_slow(timer); } void __handle_close_cb(uv_handle_t *handle) { LOG::trace("KernelCollector: closed handle"); } KernelCollector::KernelCollector( const BpfConfiguration &bpf_config, config::IntakeConfig const &intake_config, AwsMetadata const *aws_metadata, GcpInstanceMetadata const *gcp_metadata, std::map config_labels, uv_loop_t &loop, CurlEngine &curl_engine, bool enable_http_metrics, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings cgroup_settings, std::string const &bpf_dump_file, HostInfo host_info) : bpf_config_(bpf_config), intake_config_(std::move(intake_config)), aws_metadata_(aws_metadata), gcp_metadata_(gcp_metadata), config_labels_(config_labels), host_info_(std::move(host_info)), loop_(loop), last_lost_count_(0), encoder_(intake_config_.make_encoder()), callbacks_(*this), primary_channel_(intake_config_.make_channel(loop)), secondary_channel_(intake_config_.create_output_record_file()), upstream_connection_( WRITE_BUFFER_SIZE, intake_config_.allow_compression(), *primary_channel_, secondary_channel_ ? &secondary_channel_ : nullptr), writer_(upstream_connection_.buffered_writer(), monotonic, bpf_config_.boot_time_adjustment, encoder_.get()), last_probe_monotonic_time_ns_(monotonic() - inter_probe_time_ns_), is_connected_(false), curl_engine_(curl_engine), heartbeat_sender_( loop_, [this] { send_heartbeat(); return scheduling::JobFollowUp::ok; }), enable_http_metrics_(enable_http_metrics), socket_stats_interval_sec_(socket_stats_interval_sec), cgroup_settings_(std::move(cgroup_settings)), log_(writer_), kernel_collector_restarter_(*this) { if (!bpf_dump_file.empty()) { auto const error = bpf_dump_file_.create( bpf_dump_file.c_str(), FileDescriptor::Access::write_only, FileDescriptor::Positioning::append, FileDescriptor::Permission::read_write, FileDescriptor::Permission::read); if (error) { LOG::warn("unable to open/create eBPF dump file at '{}': {}", bpf_dump_file, error); } } int res; /* initialize polling timer */ res = uv_timer_init(&loop_, &polling_timer_); if (res != 0) throw std::runtime_error("Could not init polling_timer_"); polling_timer_.data = this; /* initialize slow polling timer */ res = uv_timer_init(&loop_, &slow_timer_); if (res != 0) throw std::runtime_error("Could not init slow_timer_"); slow_timer_.data = this; /* initialize connection timeout timer */ res = uv_timer_init(&loop_, &connection_timeout_); if (res != 0) throw std::runtime_error("Could not init connection_timeout_"); connection_timeout_.data = this; /* initialize probe_holdoff_timeor */ res = uv_timer_init(&loop_, &probe_holdoff_timer_); if (res != 0) throw std::runtime_error("Could not init probe_holdoff_timer_"); probe_holdoff_timer_.data = this; /* initialize try_connecting timer */ res = uv_timer_init(&loop_, &try_connecting_timer_); if (res != 0) throw std::runtime_error("Could not init try_connecting_timer_"); try_connecting_timer_.data = this; /* start in try_connecting state */ enter_try_connecting(); } KernelCollector::~KernelCollector() { on_close(); } void KernelCollector::try_connecting(uv_timer_t *timer) { cleanup_pointers(); try { LOG::info("connecting to {}...", intake_config_); upstream_connection_.connect(callbacks_); } catch (std::exception &e) { LOG::trace("upstream connect threw exception: {}", e.what()); return; } enter_connecting(); } void KernelCollector::connection_timeout(uv_timer_t *timer) { LOG::trace("connection timeout"); upstream_connection_.close(); } void KernelCollector::polling_steady_state(uv_timer_t *timer) { if (disabled_) { return; } /* push data to server */ try { bpf_handler_->start_poll(1, 1); } catch (std::exception &e) { log_.error("BPFHandler::start_poll threw exception '{}'", e.what()); enter_try_connecting(); return; } /* only print when some messages got lost */ if (bpf_handler_->serv_lost_count() > last_lost_count_) { last_lost_count_ = bpf_handler_->serv_lost_count(); LOG::trace("-polling- lost count: {}", bpf_handler_->serv_lost_count()); } } void KernelCollector::polling_steady_state_slow(uv_timer_t *timer) { if (disabled_) { return; } bpf_handler_->slow_poll(); ResourceUsageReporter::report(writer_); upstream_connection_.flush(); } void KernelCollector::on_close() { cleanup_pointers(); upstream_connection_.close(); close_uv_handle_cleanly(reinterpret_cast(&polling_timer_), __handle_close_cb); close_uv_handle_cleanly(reinterpret_cast(&slow_timer_), __handle_close_cb); close_uv_handle_cleanly(reinterpret_cast(&connection_timeout_), __handle_close_cb); close_uv_handle_cleanly(reinterpret_cast(&probe_holdoff_timer_), __handle_close_cb); close_uv_handle_cleanly(reinterpret_cast(&try_connecting_timer_), __handle_close_cb); } void KernelCollector::on_upstream_connected() { LOG::trace("upstream connected"); try { send_connection_metadata(); } catch (std::exception &e) { LOG::error("Exception thrown when sending connection request and agent metadata: {}", e.what()); return; } } void KernelCollector::on_connected() { LOG::trace("Connected, entering probe hold-off"); enter_probe_holdoff(); heartbeat_sender_.start(HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL); } void KernelCollector::probe_holdoff_timeout(uv_timer_t *timer) { auto const upstream_connection_flush_and_close = [this]() { upstream_connection_.flush(); upstream_connection_.close(); }; LOG::trace("Adding probes"); last_probe_monotonic_time_ns_ = monotonic(); auto const handle_exception = [&](TroubleshootItem item, std::exception const &e) { log_.error("Exception during BPFHandler initialization, closing connection: {}", e.what()); print_troubleshooting_message_and_exit(host_info_, item, e, log_, upstream_connection_flush_and_close); }; auto potential_troubleshoot_item = TroubleshootItem::bpf_load_probes_failed; try { bpf_handler_.emplace(loop_, bpf_config_, enable_http_metrics_, bpf_dump_file_, log_, encoder_.get(), host_info_); potential_troubleshoot_item = TroubleshootItem::unexpected_exception; writer_.bpf_compiled(); kernel_collector_restarter_.reset(); bpf_handler_->load_buffered_poller( upstream_connection_.buffered_writer(), bpf_config_.boot_time_adjustment, curl_engine_, socket_stats_interval_sec_, cgroup_settings_, kernel_collector_restarter_); potential_troubleshoot_item = TroubleshootItem::bpf_load_probes_failed; bpf_handler_->load_probes(writer_); /* Start running buf_poller in steady-state */ potential_troubleshoot_item = TroubleshootItem::unexpected_exception; writer_.begin_telemetry(); writer_.collector_health(integer_value(::collector::CollectorStatus::healthy), 0); LOG::info("Agent connected successfully. Telemetry is flowing!"); is_connected_ = true; kernel_collector_restarter_.startup_completed(); enter_polling_state(); } catch (std::system_error &e) { if (e.code().value() == EPERM) { handle_exception(TroubleshootItem::operation_not_permitted, e); return; } handle_exception(potential_troubleshoot_item, e); return; } catch (std::exception &e) { handle_exception(potential_troubleshoot_item, e); return; } } void KernelCollector::send_connection_metadata() { // send a version_info message upstream_connection_.set_compression(false); writer_.version_info(versions::release.major(), versions::release.minor(), versions::release.patch()); upstream_connection_.flush(); upstream_connection_.set_compression(true); // we use strncpy in a way that might truncate the '\0' at the end, so need to // ask GCC (8.0+ not to fail) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstringop-truncation" /* Write config file labels */ #define make_bufs_from_field(struct_name, field1, buf_name1, field2, buf_name2) \ struct struct_name __##struct_name##__##buf_name1; \ char buf_name1[sizeof(__##struct_name##__##buf_name1.field1)] = {}; \ struct struct_name __##struct_name##__##buf_name2; \ char buf_name2[sizeof(__##struct_name##__##buf_name2.field2)] = {}; if (intake_config_.encoder() == IntakeEncoder::binary) { writer_.connect(static_cast(ClientType::kernel), jb_blob{host_info_.hostname}); upstream_connection_.flush(); } writer_.os_info( integer_value(host_info_.os), host_info_.os_flavor, jb_blob{host_info_.os_version}, jb_blob{host_info_.kernel_version}); writer_.report_cpu_cores(std::thread::hardware_concurrency()); writer_.kernel_headers_source(integer_value(host_info_.kernel_headers_source)); for (auto const &label : config_labels_) { writer_.set_config_label(jb_blob{label.first}, jb_blob{label.second}); } /* Kernel version */ constexpr std::string_view kernel_version_label = "__kernel_version"; writer_.set_config_label(jb_blob{kernel_version_label}, jb_blob{host_info_.kernel_version}); upstream_connection_.flush(); #define make_buf_from_field(struct_name, field, buf_name) \ struct struct_name __##struct_name##__##buf_name; \ char buf_name[sizeof(__##struct_name##__##buf_name.field)] = {}; if (aws_metadata_) { writer_.cloud_platform(static_cast(CloudPlatform::aws)); if (auto const &account_id = aws_metadata_->account_id()) { LOG::trace_in(CloudPlatform::aws, "reporting aws account id: {}", account_id.value()); writer_.cloud_platform_account_info(jb_blob{account_id.value()}); } else { LOG::trace_in(CloudPlatform::aws, "no aws account id to report"); } auto id = aws_metadata_->id().value(); if (id.starts_with(std::string_view("i-"))) { id.remove_prefix(2); } writer_.set_node_info( jb_blob{aws_metadata_->az().value()}, jb_blob{aws_metadata_->iam_role().value()}, jb_blob{id}, jb_blob{aws_metadata_->type().value()}); upstream_connection_.flush(); for (auto const &interface : aws_metadata_->network_interfaces()) { for (auto const &ipv4 : interface.private_ipv4s()) { struct sockaddr_in private_sa; int res = inet_pton(AF_INET, ipv4.c_str(), &(private_sa.sin_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__private_ipv4_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.private_ipv4_addr(private_sa.sin_addr.s_addr, (u8 *)vpc_id_buf); } for (auto const &ipv6 : interface.ipv6s()) { struct sockaddr_in6 sa; int res = inet_pton(AF_INET6, ipv6.c_str(), &(sa.sin6_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__ipv6_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.ipv6_addr(sa.sin6_addr.s6_addr, (u8 *)vpc_id_buf); } for (auto const &mapped_ipv4 : interface.mapped_ipv4s()) { struct sockaddr_in public_sa; int res = inet_pton(AF_INET, mapped_ipv4.first.c_str(), &(public_sa.sin_addr)); if (res != 1) { continue; } struct sockaddr_in private_sa; res = inet_pton(AF_INET, mapped_ipv4.second.c_str(), &(private_sa.sin_addr)); if (res != 1) { continue; } make_buf_from_field(jb_ingest__public_to_private_ipv4, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.public_to_private_ipv4(public_sa.sin_addr.s_addr, private_sa.sin_addr.s_addr, (u8 *)vpc_id_buf); } } } else if (gcp_metadata_) { writer_.cloud_platform(static_cast(CloudPlatform::gcp)); // TODO: obtain account_id for GCP and uncomment below // LOG::trace_in(CloudPlatform::gcp), "reporting gcp account id: {}", account_id.value()); // writer_.cloud_platform_account_info(jb_blob{account_id}); writer_.set_node_info( jb_blob{gcp_metadata_->az()}, jb_blob{gcp_metadata_->role()}, jb_blob{gcp_metadata_->hostname()}, jb_blob{gcp_metadata_->type()}); upstream_connection_.flush(); for (auto const &interface : gcp_metadata_->network_interfaces()) { if (auto const ipv4 = interface.ipv4()) { make_buf_from_field(jb_ingest__private_ipv4_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.private_ipv4_addr(ipv4->as_int(), (u8 *)vpc_id_buf); for (auto const &public_ip : interface.public_ips()) { make_buf_from_field(jb_ingest__public_to_private_ipv4, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.public_to_private_ipv4(public_ip.as_int(), ipv4->as_int(), (u8 *)vpc_id_buf); } } else if (auto const ipv6 = interface.ipv6()) { uint8_t ipv6_buffer[16]; ipv6->write_to(ipv6_buffer); make_buf_from_field(jb_ingest__ipv6_addr, vpc_id, vpc_id_buf); strncpy(vpc_id_buf, interface.vpc_id().c_str(), sizeof(vpc_id_buf)); writer_.ipv6_addr(ipv6_buffer, (u8 *)vpc_id_buf); } } } else { writer_.cloud_platform(static_cast(CloudPlatform::unknown)); writer_.set_node_info(jb_blob{/* az */}, jb_blob{/* role */}, jb_blob{host_info_.hostname}, jb_blob{/* instance_type */}); // no network interface data (public/private ip) to send } // pop the "-Wstringop-truncation" warning quelch #pragma GCC diagnostic pop /* Finished sending metadata */ writer_.metadata_complete(0); upstream_connection_.flush(); on_connected(); } void KernelCollector::on_error(int error) { /* we don't want attempts to perform IO on the channel */ heartbeat_sender_.stop(); stop_all_timers(); } void KernelCollector::restart() { /* we don't want attempts to perform IO on the channel */ heartbeat_sender_.stop(); // flush the channel so previously sent messages make it to the reducer upstream_connection_.flush(); // close the channel, it will trigger reconnect via KernelCollector::Callbacks::on_closed() which calls enter_try_connecting() upstream_connection_.close(); } void KernelCollector::cleanup_pointers() { bpf_handler_.reset(); } void KernelCollector::received_data(const u8 *data, int data_len) { std::string_view const response{reinterpret_cast(data), static_cast(data_len)}; LOG::trace("KernelCollector::received_data({}): {}", data_len); switch (intake_config_.encoder()) { case IntakeEncoder::binary: { static constexpr int max_command_length = 8; for (int i = 0; i < data_len; i++) { received_command_ = (received_command_ << 8) | *(data + i); recieved_length_++; if (recieved_length_ == max_command_length) { handle_received_command(received_command_); recieved_length_ = 0; } } } break; } } void KernelCollector::handle_received_command(u64 command) { if (command == static_cast(ServerCommand::DISABLE_SEND)) { LOG::info("Stop sending data, instructed by the server."); disabled_ = true; } } void KernelCollector::send_heartbeat() { writer_.heartbeat(); upstream_connection_.flush(); } KernelCollector::Callbacks::Callbacks(KernelCollector &collector) : collector_(collector) {} u32 KernelCollector::Callbacks::received_data(const u8 *data, int data_len) { collector_.received_data(data, data_len); return data_len; } void KernelCollector::Callbacks::on_error(int err) { LOG::trace("upstream connection error {}", static_cast(-err)); collector_.on_error(err); /* close the channel, it will trigger reconnect */ collector_.upstream_connection_.close(); } void KernelCollector::Callbacks::on_closed() { LOG::trace("closed upstream connection, will reconnect..."); collector_.enter_try_connecting(); } void KernelCollector::Callbacks::on_connect() { LOG::trace("established upstream connection"); collector_.on_upstream_connected(); } void KernelCollector::enter_try_connecting(std::chrono::milliseconds discount) { if (is_connected_) { LOG::info("disconnected from upstream, attempting to reconnect..."); is_connected_ = false; } stop_all_timers(); auto timeout = discount > TRY_CONNECTING_TIMEOUT ? 0ms : TRY_CONNECTING_TIMEOUT - discount; u64 now = monotonic(); if (now - last_probe_monotonic_time_ns_ < inter_probe_time_ns_) { // need to bound using inter_probe_time_ns_ std::chrono::milliseconds const at_least{ (inter_probe_time_ns_ - (now - last_probe_monotonic_time_ns_)) / ((u64)1000 * 1000)}; timeout = std::max(timeout, at_least); } // add jitter { std::random_device rd; std::uniform_int_distribution d(0, integer_time(MAX_JITTER_TIME)); timeout += std::chrono::milliseconds{d(rd)}; } LOG::trace( "KernelCollector: entering TRY_CONNECTING state. Next " "reconnection attempt in {}", timeout); int res = uv_timer_start( &try_connecting_timer_, __try_connecting_cb, integer_time(timeout), integer_time(TRY_CONNECTING_TIMEOUT)); if (res != 0) { throw std::runtime_error("Could not start try_connecting_timer"); } } void KernelCollector::enter_connecting() { stop_all_timers(); LOG::trace("KernelCollector: entering CONNECTING state"); int res = uv_timer_start(&connection_timeout_, __connection_timeout_cb, connection_timeout_ms_, 0); if (res != 0) throw std::runtime_error("Could not start connection_timeout timer"); } void KernelCollector::enter_probe_holdoff() { stop_all_timers(); LOG::trace("KernelCollector: entering PROBE_HOLDOFF state"); int res = uv_timer_start(&probe_holdoff_timer_, __probe_holdoff_cb, probe_holdoff_timeout_ms_, 0); if (res != 0) throw std::runtime_error("Could not start probe_holdoff_timer"); } void KernelCollector::enter_polling_state() { stop_all_timers(); LOG::trace("KernelCollector: entering POLLING state"); int res = uv_timer_start(&polling_timer_, __polling_steady_state_cb, polling_timeout_ms_, polling_timeout_ms_); if (res != 0) throw std::runtime_error("Could not start polling_timer"); res = uv_timer_start(&slow_timer_, __polling_steady_state_slow_cb, slow_polling_timeout_ms_, slow_polling_timeout_ms_); if (res != 0) throw std::runtime_error("Could not start slow_timer"); } void KernelCollector::stop_all_timers() { uv_timer_stop(&try_connecting_timer_); uv_timer_stop(&connection_timeout_); uv_timer_stop(&probe_holdoff_timer_); uv_timer_stop(&polling_timer_); uv_timer_stop(&slow_timer_); } #ifndef NDEBUG void KernelCollector::debug_bpf_lost_samples() { if (bpf_handler_) { bpf_handler_->debug_bpf_lost_samples(); } } #endif ================================================ FILE: collector/kernel/kernel_collector.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include class KernelCollector { friend class KernelCollectorRestarter; friend class KernelCollectorTest; public: /** * c'tor */ KernelCollector( const BpfConfiguration &bpf_config, config::IntakeConfig const &intake_config, AwsMetadata const *aws_metadata, GcpInstanceMetadata const *gcp_metadata, std::map config_labels, uv_loop_t &loop, CurlEngine &curl_engine, bool enable_http_metrics, u64 socket_stats_interval_sec, CgroupHandler::CgroupSettings cgroup_settings, std::string const &bpf_dump_file, HostInfo host_info); /** * d'tor */ virtual ~KernelCollector(); /** * called by the uv_timer callback for establishing server connections */ void try_connecting(uv_timer_t *timer); /** * called by the uv_timer callback for timing out connection attempts */ void connection_timeout(uv_timer_t *timer); /** * called by the uv_timer callback for starting bpf probes */ void probe_holdoff_timeout(uv_timer_t *timer); /** * called by the uv_timer callback for polling processes */ void polling_steady_state(uv_timer_t *timer); /** * called by the uv_timer callback for slow polling processes */ void polling_steady_state_slow(uv_timer_t *timer); /** * called when upstream is connected, to complete connection */ void on_upstream_connected(); /* called from callbacks given to uv_close */ void on_close(); #ifndef NDEBUG /* Debug code for internal development to simulate lost BPF samples (PERF_RECORD_LOST) in BufferedPoller. */ void debug_bpf_lost_samples(); #endif private: class Callbacks : public channel::Callbacks { public: Callbacks(KernelCollector &collector); virtual u32 received_data(const u8 *data, int data_len) override; virtual void on_error(int err) override; virtual void on_closed() override; virtual void on_connect() override; private: KernelCollector &collector_; }; /* sends info from config (if present) and AWS state */ void send_connection_metadata(); /* cleans up shared pointers */ void cleanup_pointers(); // enter try_connecting state - will take `discount` time out of // the hold-off void enter_try_connecting(std::chrono::milliseconds discount = 0ms); /* connecting state, while Tcp is trying to connect and we can time out */ void enter_connecting(); /* probe holdoff. got upstream connection and waiting before adding probes */ void enter_probe_holdoff(); /* enter polling steady state */ void enter_polling_state(); /* stops all timers */ void stop_all_timers(); /* receive data from connection */ void received_data(const u8 *data, int data_len); /* handles command received from the server */ void handle_received_command(u64 command); /* sends a heartbeat message to the server */ void send_heartbeat(); void on_connected(); void on_error(int error); /* called to restart the KernelCollector */ void restart(); private: /* parameters for establishing connections */ BpfConfiguration const bpf_config_; config::IntakeConfig const &intake_config_; AwsMetadata const *aws_metadata_; GcpInstanceMetadata const *gcp_metadata_; const std::map config_labels_; HostInfo const host_info_; // Following 2 variables handle the command received from server. // // |received_command_| holds the paritial command received so far, of // |recieved_length_| bytes. // When |recieved_length_| == max_command_length, handle_recieved_command() // function is invoked. u64 received_command_ = 0; int recieved_length_ = 0; bool disabled_ = false; /* lib_uv objects */ uv_loop_t &loop_; uv_timer_t try_connecting_timer_; uv_timer_t connection_timeout_; uv_timer_t probe_holdoff_timer_; uv_timer_t polling_timer_; uv_timer_t slow_timer_; u64 last_lost_count_; std::unique_ptr<::ebpf_net::ingest::Encoder> encoder_; std::optional bpf_handler_; Callbacks callbacks_; std::unique_ptr primary_channel_; channel::FileChannel secondary_channel_; channel::UpstreamConnection upstream_connection_; ::ebpf_net::ingest::Writer writer_; u64 last_probe_monotonic_time_ns_; // is the agent in healthy steady state -- so we can log when not healthy bool is_connected_; CurlEngine &curl_engine_; scheduling::IntervalScheduler heartbeat_sender_; /* enable/disable features */ bool enable_http_metrics_; u64 socket_stats_interval_sec_; CgroupHandler::CgroupSettings const cgroup_settings_; FileDescriptor bpf_dump_file_; logging::Logger log_; KernelCollectorRestarter kernel_collector_restarter_; }; ================================================ FILE: collector/kernel/kernel_collector_restarter.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include KernelCollectorRestarter::KernelCollectorRestarter(KernelCollector &collector) : collector_(collector) { reset(); } void KernelCollectorRestarter::check_restart() { if (startup_completed_ && restart_requested_) { if (restart_in_progress_) { LOG::debug("KernelCollector restart already in progress"); } else { restart_in_progress_ = true; LOG::debug("restarting KernelCollector"); collector_.restart(); } } } void KernelCollectorRestarter::startup_completed() { LOG::debug("KernelCollectorRestarter: startup completed"); startup_completed_ = true; check_restart(); } void KernelCollectorRestarter::request_restart() { LOG::debug("KernelCollectorRestarter: restart requested"); restart_requested_ = true; check_restart(); } void KernelCollectorRestarter::reset() { startup_completed_ = false; restart_requested_ = false; restart_in_progress_ = false; } ================================================ FILE: collector/kernel/kernel_collector_restarter.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include class KernelCollector; /** * Helper class to facilitate restarting KernelCollector, i.e. when BufferedPoller detects lost BPF samples (PERF_RECORD_LOST). * If KernelCollector startup has completed, then a restart request will be processed immediately. * If KernelCollector startup has not completed, then the processing of a restart request will be deferred until after * KernelCollector startup has completed. * If a restart is already in progress, subsequent restart requests will be ignored. */ class KernelCollectorRestarter { friend class KernelCollector; public: KernelCollectorRestarter(KernelCollector &collector); void startup_completed(); void request_restart(); void reset(); private: void check_restart(); bool startup_completed_; bool restart_requested_; bool restart_in_progress_; KernelCollector &collector_; }; ================================================ FILE: collector/kernel/kernel_collector_test.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define BPF_DUMP_FILE "/tmp/bpf-dump-file" #define INTAKE_DUMP_FILE "/tmp/intake-dump-file" extern "C" { /* bpf source code */ extern char agent_bpf_c[]; extern unsigned int agent_bpf_c_len; } // extern "C" class TestIntakeConfig : public config::IntakeConfig { using config::IntakeConfig::IntakeConfig; bool allow_compression() const { return false; } std::unique_ptr make_channel(uv_loop_t &loop) const override { return std::make_unique(loop, encoder()); } }; // Conditions to be met before stopping test struct StopConditions { std::chrono::seconds timeout_sec; u64 num_sends; std::map names_and_counts; bool wait_for_all_workloads_to_complete; }; class KernelCollectorTest : public CommonTest { protected: void SetUp() override { CommonTest::SetUp(); // Allow relevant HTTP-related logs for this test set_log_whitelist({AgentLogKind::HTTP, AgentLogKind::PROTOCOL, AgentLogKind::BPF, AgentLogKind::PERF}); ASSERT_EQ(0, uv_loop_init(&loop_)); } void TearDown() override { // Clean up loop_ to avoid valgrind and asan complaints about memory leaks. close_uv_loop_cleanly(&loop_); } void start_kernel_collector( IntakeEncoder intake_encoder, StopConditions const &stop_conditions, std::string const &bpf_dump_file = "", std::function const &ingest_msg_cb = {}) { stop_conditions_.emplace(stop_conditions); // This mostly duplicates the KernelCollector setup done in collector/kernel/main.cc. // Create BPF configuration with test parameters u64 boot_time_adjustment = get_boot_time(); test_intake_config_ = TestIntakeConfig("", "", INTAKE_DUMP_FILE, intake_encoder); auto const aws_metadata = AwsMetadata::fetch(1000ms); auto const gcp_metadata = GcpInstanceMetadata::fetch(1000ms); config::ConfigFile configuration_data(config::ConfigFile::YamlFormat(), ""); std::unique_ptr curl_engine = CurlEngine::create(&loop_); bool const enable_http_metrics = true; bool const enable_userland_tcp = false; u64 const socket_stats_interval_sec = 10; BpfConfiguration bpf_config{ .boot_time_adjustment = boot_time_adjustment, .filter_ns = 10 * 1000 * 1000ull, .enable_tcp_data_stream = enable_userland_tcp}; struct utsname unamebuf; if (uname(&unamebuf)) { throw std::runtime_error("Failed to get system uname"); } LOG::info( "Running on:\n" " sysname: {}\n" " nodename: {}\n" " release: {}\n" " version: {}\n" " machine: {}", unamebuf.sysname, unamebuf.nodename, unamebuf.release, unamebuf.version, unamebuf.machine); // resolve hostname std::string const hostname = get_host_name(MAX_HOSTNAME_LENGTH).recover([&](auto &error) { LOG::error("Unable to retrieve host information from uname: {}", error); return aws_metadata->id().valid() ? std::string(aws_metadata->id().value()) : "(unknown)"; }); HostInfo host_info{ .os = OperatingSystem::Linux, .os_flavor = integer_value(LinuxDistro::unknown), .os_version = "unknown", .kernel_headers_source = KernelHeadersSource::libbpf, .kernel_version = unamebuf.release, .hostname = hostname}; kernel_collector_.emplace( bpf_config, *test_intake_config_, aws_metadata.try_value(), gcp_metadata.try_value(), configuration_data.labels(), loop_, *curl_engine, enable_http_metrics, socket_stats_interval_sec, CgroupHandler::CgroupSettings{false, std::nullopt}, bpf_dump_file, host_info); if (ingest_msg_cb) { get_test_channel()->set_sent_msg_cb(ingest_msg_cb); } run_test_stopper(); run_workload_starter(); LOG::info("starting event loop..."); uv_run(&loop_, UV_RUN_DEFAULT); } void stop_kernel_collector() { stop_workloads(); print_json_messages(); if (timeout_exceeded_) { print_stop_conditions(); } print_message_counts(); // NOTE: use EXPECT_s here because ASSERT_s fail fast, returning from the current function, skipping the cleanup below EXPECT_EQ(0ull, get_probe_handler().num_failed_probes_); EXPECT_TRUE(binary_messages_check_counts()); EXPECT_EQ(0ull, get_test_channel()->get_num_failed_sends()); EXPECT_EQ(false, timeout_exceeded_); auto &message_counts = get_test_channel()->get_message_counts(); EXPECT_EQ(0ull, message_counts["bpf_log"]); kernel_collector_->on_close(); uv_stop(&loop_); print_code_timings(); } void run_test_stopper() { auto stop_test_check = [&]() { SCOPED_TIMING(StopTestCheck); auto const &stop_conditions = stop_conditions_->get(); // check for test timeout if (stopwatch_) { timeout_exceeded_ = stopwatch_->elapsed(stop_conditions.timeout_sec); LOG::trace( "stop_test_check() stop_conditions timeout_sec {} exceeded {}", stop_conditions.timeout_sec, timeout_exceeded_); if (timeout_exceeded_) { LOG::error("stop_test_check() test timeout of {} exceeded", stop_conditions.timeout_sec); stop_kernel_collector(); return; } } // wait for all workloads to complete if requested if (stop_conditions.wait_for_all_workloads_to_complete) { LOG::trace("stop_test_check() num_remaining_workloads_ = {}", num_remaining_workloads_); if (num_remaining_workloads_) { stop_test_timer_->defer(std::chrono::seconds(1)); return; } } // check num_sends auto channel = get_test_channel(); auto num_sends = channel->get_num_sends(); LOG::trace( "stop_test_check() channel->get_num_sends() = {} stop_conditions num_sends = {}", num_sends, stop_conditions.num_sends); if (num_sends < stop_conditions.num_sends) { stop_test_timer_->defer(std::chrono::seconds(1)); return; } // check names_and_counts auto &message_counts = channel->get_message_counts(); bool reschedule = false; for (auto const &[name, count] : stop_conditions.names_and_counts) { auto message_count = message_counts[name]; LOG::trace("stop_test_check() message_counts[{}] = {} \tstop count = {}", name, message_count, count); if (message_count < count) { reschedule = true; } } if (reschedule) { stop_test_timer_->defer(std::chrono::seconds(1)); return; } LOG::trace("stop_test_check() stop_conditions have been met - calling stop_kernel_collector()"); stop_kernel_collector(); }; stop_test_timer_ = std::make_unique(loop_, stop_test_check); stop_test_timer_->defer(std::chrono::seconds(1)); } void start_workload(std::function workload_cb) { auto index = workload_index_++; ++num_remaining_workloads_; auto workload_wrapper = [this, workload_cb, index]() { LOG::info("workload {} starting", index); workload_cb(); LOG::info("workload {} complete", index); --num_remaining_workloads_; }; workload_threads_.emplace_back(workload_wrapper); } void add_workload(std::function workload) { workloads_.push_back(std::move(workload)); } void add_workload_processes() { add_workload([]() { system( "exec 1> /tmp/workload-processes.log 2>&1; echo starting workload; set -x; whoami; pwd; ls; cd /tmp; pwd; ls; cd /; pwd; ls; cd ~; pwd; ls; echo workload complete"); }); } void add_workload_curl_otel() { add_workload([]() { system( "exec 1> /tmp/workload-curl-otel.log 2>&1; echo starting workload; for n in $(seq 1 10); do curl https://opentelemetry.io; done; echo workload complete"); }); } void add_workload_curl_localhost() { add_workload([]() { auto pid = fork(); if (pid == 0) { int fd = open("/dev/null", O_WRONLY); dup2(fd, 1); // redirect stdout dup2(fd, 2); // redirect stderr execl("/usr/bin/python3", "python3", "-m", "http.server", "28099", nullptr); exit(1); } system( "exec 1> /tmp/workload-curl-localhost.log 2>&1; echo starting workload; for n in $(seq 1 100); do curl localhost:28099; done; echo workload complete"); kill(pid, SIGTERM); }); } void add_workload_stress_ng_sock() { add_workload([]() { // Emit stress-ng output live to console and also capture to a log file for later inspection. // Use bash for process substitution; fall back gracefully if stdbuf is unavailable. system("bash -lc '" "exec > >(tee -a /tmp/workload-stress-ng-sock.log) 2>&1; " "echo starting workload; " "echo stress-ng version: $(stress-ng --version 2>&1 || true); " "for n in $(seq 1 10); do " " echo [workload-0] iteration $n start $(date -Is); " " if command -v stdbuf >/dev/null 2>&1; then " " stdbuf -oL -eL stress-ng --sock 2 --sock-domain ipv4 --sock-ops 1000 --sock-port 6787 --metrics-brief; " " else " " stress-ng --sock 2 --sock-domain ipv4 --sock-ops 1000 --sock-port 6787 --metrics-brief; " " fi; " " echo [workload-0] iteration $n end $(date -Is); " " sleep .1; " "done; " "echo workload complete'"); }); } void start_workloads() { num_remaining_workloads_ = 0; for (auto workload : workloads_) { start_workload(workload); } }; void run_workload_starter() { auto &message_counts = get_test_channel()->get_message_counts(); auto start_workloads_check = [&]() { LOG::trace("in start_workloads_check()"); if ((message_counts["bpf_compiled"] >= 1) && (message_counts["socket_steady_state"] >= 1) && (message_counts["process_steady_state"] >= 1)) { LOG::trace("start_workloads_check() STARTING"); start_workloads(); // this is where we start timing for purposes of the test timeout stopwatch_.emplace(); } else { start_workloads_timer_->defer(std::chrono::seconds(1)); } }; start_workloads_timer_ = std::make_unique(loop_, start_workloads_check); start_workloads_timer_->defer(std::chrono::seconds(1)); } void stop_workloads() { for (auto &thr : workload_threads_) { if (thr.joinable()) { thr.join(); } } }; void print_stop_conditions() { auto &message_counts = get_test_channel()->get_message_counts(); LOG::debug("stop conditions:"); for (auto const &[name, count] : stop_conditions_->get().names_and_counts) { auto message_count = message_counts[name]; LOG::debug( "stop_conditions[\"{}\"] = {} \t({} received) {}", name, count, message_count, message_count < count ? " FAILED" : ""); } } void print_message_counts() { LOG::debug("message_counts:"); for (auto const &[name, count] : get_test_channel()->get_message_counts()) { LOG::debug("message_counts[\"{}\"] = {}", name, count); } } void print_json_messages() { LOG::trace("json_messages:"); auto print_message = [&](channel::TestChannel::JsonMessageType const &msg) { LOG::trace("{}", log_waive(msg.dump())); }; get_test_channel()->json_messages_for_each(print_message); } // This is an example of using TestChannel::binary_messages_for_each(). It looks at each message, counts the message type, // and compares the counts to TestChannel::message_counts_. bool binary_messages_check_counts() { channel::TestChannel::MessageCountsType check_message_counts; size_t num_binary_messages = 0; auto count_message = [&](channel::TestChannel::BinaryMessageType const &msg) { ++num_binary_messages; std::stringstream ss; json_converter::WireToJsonConverter converter(ss); converter.process(reinterpret_cast(msg.data()), msg.size()); std::string str = "[" + ss.str() + "]"; nlohmann::json const objects = nlohmann::json::parse(str); for (auto const &object : objects) { ++check_message_counts[object["name"]]; } }; get_test_channel()->binary_messages_for_each(count_message); LOG::trace("check_message_counts:"); for (auto const &[name, count] : check_message_counts) { LOG::trace("check_message_counts[\"{}\"] = {}", name, count); } return num_binary_messages ? check_message_counts == get_test_channel()->get_message_counts() : true; } channel::TestChannel *get_test_channel() { return dynamic_cast(kernel_collector_->primary_channel_.get()); } ProbeHandler &get_probe_handler() { if (!kernel_collector_->bpf_handler_) { throw std::runtime_error("std::optional bpf_handler_ does not have a value"); } return kernel_collector_->bpf_handler_->probe_handler_; } uv_loop_t loop_; std::optional test_intake_config_; std::optional kernel_collector_; bool timeout_exceeded_ = false; std::optional> stopwatch_; std::unique_ptr stop_test_timer_; std::unique_ptr start_workloads_timer_; std::vector workload_threads_; size_t workload_index_ = 0; std::atomic num_remaining_workloads_ = std::numeric_limits::max(); std::vector> workloads_; std::optional> stop_conditions_; }; // clang-format off #define NAMES_AND_COUNTS_COMMON \ {"bpf_compiled", 1}, \ {"begin_telemetry", 1}, \ {"close_sock_info", 100}, \ {"cloud_platform", 1}, \ {"dns_response", 10}, \ {"http_response", 10}, \ {"metadata_complete", 1}, \ {"new_sock_info", 100}, \ {"os_info", 1}, \ {"pid_close_info", 5}, \ {"pid_info_create", 5}, \ {"pid_set_comm", 5}, \ {"process_steady_state", 1}, \ {"set_cgroup", 5}, \ {"set_command", 5}, \ {"set_config_label", 1}, \ {"set_node_info", 1}, \ {"set_tgid", 5}, \ {"socket_stats", 100}, \ {"socket_steady_state", 1}, // clang-format on // Test basic kernel-collector functionality, validating that a minimum number of messages are seen as expected from running // process and network workloads. TEST_F(KernelCollectorTest, binary) { StopConditions stop_conditions{ .timeout_sec = std::chrono::seconds(60), .num_sends = 25, .names_and_counts = {NAMES_AND_COUNTS_COMMON}, .wait_for_all_workloads_to_complete = true}; add_workload_processes(); add_workload_curl_otel(); add_workload_curl_localhost(); start_kernel_collector(IntakeEncoder::binary, stop_conditions, BPF_DUMP_FILE); } // This test was originally used to reproduce a race condition in the eBPF code that handles socket close events that would // cause extraneous bpf_log messages by running a socket stress workload. TEST_F(KernelCollectorTest, bpf_log) { StopConditions stop_conditions{ .timeout_sec = std::chrono::minutes(10), .num_sends = 500, .names_and_counts = {}, .wait_for_all_workloads_to_complete = true}; add_workload_stress_ng_sock(); // This will be called for each render message sent from the kernel-collector to 'ingest' (by the reducer in a real system) auto ingest_msg_cb = [&](nlohmann::json const &object) { SCOPED_TIMING(BpfLogTestIngestMsgCb); // Log any bpf_log messages so they are visible in CI output. if (object["name"] == "bpf_log") { LOG::error("bpf_log: {}", log_waive(object.dump())); } }; start_kernel_collector(IntakeEncoder::binary, stop_conditions, BPF_DUMP_FILE, ingest_msg_cb); } ================================================ FILE: collector/kernel/kernel_collector_test_docker/Dockerfile ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 FROM docker.io/bitnami/minideb:trixie@sha256:0766a3b76750541ae2c5f3b806e5201e1b2ca1549a0a036a057726dc9e5dfa4d # ca-certificates are required by libcurl RUN install_packages ca-certificates libcurlpp0 libabsl20240722 libelf1 ENV SSL_CERT_DIR=/etc/ssl/certs ENV EBPF_NET_INSTALL_DIR=/srv ENV EBPF_NET_HOST_DIR=/hostfs ENV EBPF_NET_DATA_DIR=/var/run/ebpf_net ENTRYPOINT [ "/srv/entrypoint-kct.sh" ] RUN install_packages stress-ng RUN install_packages bc cgdb gawk gdb gzip iputils-ping jq netcat-openbsd procps python3 ripgrep vim valgrind curl COPY srv /srv WORKDIR /srv ================================================ FILE: collector/kernel/kernel_symbols.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "kernel_symbols.h" #include #include #include #include namespace { inline std::string_view parse_symbol_name(std::string_view s) { static constexpr std::string_view delimiters = " \t"; // Skip two tokens (address and type). for (size_t i = 0; i < 2; ++i) { auto p = s.find_first_of(delimiters); if (p == std::string_view::npos) { return std::string_view(); } s = views::ltrim_ws(s.substr(p)); } // Return the next token (symbol name). return s.substr(0, s.find_first_of(delimiters)); } } // namespace KernelSymbols read_proc_kallsyms(std::istream &stream) { KernelSymbols symbols; while (stream.good()) { std::string line; std::getline(stream, line); if (line.empty()) { continue; } std::string_view symbol = parse_symbol_name(line); if (symbol.empty()) { throw std::runtime_error("parse error"); } symbols.insert(std::string(symbol)); } return symbols; } KernelSymbols read_proc_kallsyms(const char *path) { std::ifstream file(path); if (!file.is_open()) { throw std::system_error(errno, std::generic_category(), "error opening file"); } return read_proc_kallsyms(file); } ================================================ FILE: collector/kernel/kernel_symbols.h ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #pragma once #include #include #include #include using KernelSymbols = std::unordered_set; // Reads and parses kernel symbol names. // Throws an exception on parsing error. KernelSymbols read_proc_kallsyms(std::istream &stream); // Reads kernel symbol names from a special file in the proc filesystem. // Throws an exception if the file can not be read and its content parsed. KernelSymbols read_proc_kallsyms(const char *path = "/proc/kallsyms"); ================================================ FILE: collector/kernel/kernel_symbols_test.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "kernel_symbols.h" #include #include #include #include static constexpr std::string_view EXAMPLE_KALLSYMS = R"delim( 0000000000000000 T startup_64 0000000000000000 T _stext 0000000000000000 T _text 0000000000000000 T secondary_startup_64 0000000000000000 T secondary_startup_64_no_verify 0000000000000000 t verify_cpu 0000000000000000 T sev_verify_cbit 0000000000000000 T start_cpu0 0000000000000000 T __startup_64 0000000000000000 T startup_64_setup_env 0000000000000000 b ignore_oc [ehci_hcd] 0000000000000000 t iso_stream_find [ehci_hcd] 0000000000000000 r smask_out.94 [ehci_hcd] 0000000000000000 d __UNIQUE_ID_ddebug215.19 [ehci_hcd] 0000000000000000 d __UNIQUE_ID_ddebug121.49 [ehci_hcd] 0000000000000000 t ehci_run.cold [ehci_hcd] ffffffffc09a60cc r __ksymtab_nf_conntrack_alter_reply [nf_conntrack] ffffffffc09aa74e r __kstrtab_nf_conntrack_alter_reply [nf_conntrack] ffffffffc0996bc0 t nf_conntrack_alter_reply [nf_conntrack] ffffffffc09cdbf0 t ctnetlink_dump_tuples_proto [nf_conntrack_netlink] ffffffffc09ce230 t ctnetlink_dump_tuples_ip [nf_conntrack_netlink] )delim"; static constexpr std::string_view UNKNOWN_SYMBOL = "DEFINITELY_NOT_A_KERNEL_SYMBOL"; TEST(KernelSymbolsTest, ReadStream) { std::stringstream stream(EXAMPLE_KALLSYMS.data()); auto ks = read_proc_kallsyms(stream); EXPECT_TRUE(ks.contains("verify_cpu")); EXPECT_TRUE(ks.contains("nf_conntrack_alter_reply")); EXPECT_TRUE(ks.contains("ctnetlink_dump_tuples_ip")); EXPECT_TRUE(ks.contains("iso_stream_find")); EXPECT_FALSE(ks.contains(UNKNOWN_SYMBOL.data())); } TEST(KernelSymbolsTest, ReadProcKallsyms) { auto ks = read_proc_kallsyms(); EXPECT_TRUE(ks.contains("security_sk_free")); EXPECT_TRUE(ks.contains("inet_release")); EXPECT_TRUE(ks.contains("tcp_connect")); EXPECT_TRUE(ks.contains("inet_csk_listen_start")); EXPECT_TRUE(ks.contains("tcp_init_sock")); EXPECT_TRUE(ks.contains("inet_csk_accept")); EXPECT_TRUE(ks.contains("udp_v4_get_port")); EXPECT_TRUE(ks.contains("udp_v6_get_port")); EXPECT_TRUE(ks.contains("tcp4_seq_show")); EXPECT_TRUE(ks.contains("tcp6_seq_show")); EXPECT_TRUE(ks.contains("udp4_seq_show")); EXPECT_TRUE(ks.contains("udp6_seq_show")); EXPECT_TRUE(ks.contains("taskstats_exit")); EXPECT_TRUE(ks.contains("cgroup_exit")); EXPECT_TRUE(ks.contains("cgroup_attach_task")); EXPECT_TRUE(ks.contains("wake_up_new_task")); EXPECT_TRUE(ks.contains("__set_task_comm")); EXPECT_TRUE(ks.contains("get_pid_task")); EXPECT_FALSE(ks.contains(UNKNOWN_SYMBOL.data())); } TEST(KernelSymbolsTest, NoSuchFile) { EXPECT_THROW(read_proc_kallsyms("/NO_SUCH_FILE"), std::system_error); } TEST(KernelSymbolsTest, ParseError) { std::stringstream stream("foo bar"); EXPECT_THROW(read_proc_kallsyms(stream), std::runtime_error); } TEST(KernelSymbolsTest, NoParseError) { std::stringstream stream("foo bar baz"); EXPECT_NO_THROW(read_proc_kallsyms(stream)); } TEST(KernelSymbolsTest, Empty) { std::stringstream stream("\n\n\n"); KernelSymbols ks; EXPECT_NO_THROW(ks = read_proc_kallsyms(stream)); EXPECT_TRUE(ks.empty()); } ================================================ FILE: collector/kernel/main.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 extern "C" int otn_kernel_collector_main(int argc, const char **argv); int main(int argc, char *argv[]) { return otn_kernel_collector_main(argc, const_cast(argv)); } ================================================ FILE: collector/kernel/nat_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include constexpr auto SIZEOF_STRUCT_NF_CONNTRACK_TUPLE_HASH = 56; NatHandler::NatHandler(::ebpf_net::ingest::Writer &writer, logging::Logger &log) : writer_(writer), log_(log) {} /* END */ void NatHandler::handle_nf_nat_cleanup_conntrack(u64 timestamp, struct jb_agent_internal__nf_nat_cleanup_conntrack *msg) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_nf_nat_cleanup_conntrack: ct={}, " "src={}:{}, dst={}:{}, proto={}", msg->ct, IPv4Address::from(msg->src_ip), ntohs(msg->src_port), IPv4Address::from(msg->dst_ip), ntohs(msg->dst_port), msg->proto); } const hostport_tuple ft = { msg->src_ip, msg->dst_ip, msg->src_port, msg->dst_port, msg->proto, }; remove_nat(ft); } /* START */ void NatHandler::handle_nf_conntrack_alter_reply(u64 timestamp, struct jb_agent_internal__nf_conntrack_alter_reply *msg) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_nf_conntrack_alter_reply: ct={}, " "src={}:{}, dst={}:{}, proto={}, " "nat_src={}:{}, nat_dst={}:{}, nat_proto={}", msg->ct, IPv4Address::from(msg->src_ip), ntohs(msg->src_port), IPv4Address::from(msg->dst_ip), ntohs(msg->dst_port), msg->proto, IPv4Address::from(msg->nat_src_ip), ntohs(msg->nat_src_port), IPv4Address::from(msg->nat_dst_ip), ntohs(msg->nat_dst_port), msg->nat_proto); } const hostport_tuple map_from = { .src_ip = msg->src_ip, .dst_ip = msg->dst_ip, .src_port = msg->src_port, .dst_port = msg->dst_port, .proto = msg->proto, }; const hostport_tuple map_to = { .src_ip = msg->nat_src_ip, .dst_ip = msg->nat_dst_ip, .src_port = msg->nat_src_port, .dst_port = msg->nat_dst_port, .proto = msg->proto, }; record_nat(map_from, map_to); // If we've seen an sk for this four-tuple already, we can report to the // server if (auto sk_pos = existing_sk_table_.find(map_from); sk_pos != existing_sk_table_.end()) { u64 sk = sk_pos->second; send_nat_remapping(timestamp, sk, map_to); } else { LOG::trace_in(AgentLogKind::NAT, "sk doesn't exist for this four-tuple yet"); } } /* EXISTING */ void NatHandler::handle_existing_conntrack_tuple(u64 timestamp, struct jb_agent_internal__existing_conntrack_tuple *msg) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_existing_conntrack_tuple: ct={}, dir={}, " "src={}:{}, dst={}:{}, proto={}", msg->ct, msg->dir, IPv4Address::from(msg->src_ip), ntohs(msg->src_port), IPv4Address::from(msg->dst_ip), ntohs(msg->dst_port), msg->proto); } const u64 ct = msg->ct; const u8 dir = msg->dir; const hostport_tuple ft = { msg->src_ip, msg->dst_ip, msg->src_port, msg->dst_port, msg->proto, }; // We have two cases here: either the direction is 0 or 1 // (IP_CT_DIR_ORIGINAL/IP_CT_DIR_REPLY). Because of the way this probe is // triggered, we expect the incoming msgs to be ordered as alternating 0,1 // pairs, where every pair corresponds to a given connection. dir=O is the // first side we see for a connection, so we simply record the four-tuple. // dir=1 should always come after 0, so we check to make sure that the // corresponding dir=0 info is present in our table, and then we record the // connection info if this pair is a NAT. if (dir == 0) { existing_conntrack_table_[ct] = ft; return; } // in the dir==1 case, we reported "ct-sizeof(struct // nf_conntrack_tuple_hash)", which should be the same addr as the // corresponding dir==0 conntrack_tuple. auto search = existing_conntrack_table_.find(ct); if (search == existing_conntrack_table_.end()) { log_.error("existing conntrack not found"); return; } hostport_tuple const &map_from = search->second; hostport_tuple const &map_to = ft; // Check whether this is a NAT-ed connection if (map_from == map_to) { return; } if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "connection is NAT-ed from: src={}:{}, dst={}:{}, proto={}", IPv4Address::from(map_from.src_ip), ntohs(map_from.src_port), IPv4Address::from(map_from.dst_ip), ntohs(map_from.dst_port), map_from.proto); } record_nat(map_from, map_to); // Clean up the other direction for this connection if (auto revct_it = existing_conntrack_table_.find(ct - SIZEOF_STRUCT_NF_CONNTRACK_TUPLE_HASH); revct_it != existing_conntrack_table_.end()) { existing_conntrack_table_.erase(revct_it); } else { LOG::debug_in(AgentLogKind::NAT, "reverse direction not found for ct={}", ct); } } void NatHandler::handle_set_state_ipv4(u64 timestamp, jb_agent_internal__set_state_ipv4 *msg) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_set_state_ipv4: " "sk={}, src={}:{}, dest={}:{}, tx_rx={}", msg->sk, IPv4Address::from(msg->src), msg->sport, IPv4Address::from(msg->dest), msg->dport, msg->tx_rx); } const u64 sk = msg->sk; const hostport_tuple ft = { .src_ip = msg->src, .dst_ip = msg->dest, .src_port = htons(msg->sport), .dst_port = htons(msg->dport), .proto = IPPROTO_TCP, }; record_sk(sk, ft); // We had a NAT table entry before getting the socket info. if (auto mapping = nat_table_.find(ft); mapping != nat_table_.end()) { send_nat_remapping(timestamp, sk, mapping->second); } if (auto rev_mapping = nat_table_rev_.find(ft.reversed()); rev_mapping != nat_table_rev_.end()) { send_nat_remapping(timestamp, sk, rev_mapping->second.reversed()); } } void NatHandler::handle_set_state_ipv6(u64 timestamp, jb_agent_internal__set_state_ipv6 *msg) { // Check if it is an ipv4 address IPv6Address src = IPv6Address::from(msg->src); IPv6Address dst = IPv6Address::from(msg->dest); if (!src.is_ipv4() || !dst.is_ipv4()) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_set_state_ipv6: " "not a v4 address - skipping nat handling sk={}, src={}:{}, dest={}:{}, tx_rx={}", msg->sk, src, msg->sport, dst, msg->dport, msg->tx_rx); } return; } if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::handle_set_state_ipv6: " "sk={}, src={}:{}, dest={}:{}, tx_rx={}", msg->sk, src.to_ipv4().value(), msg->sport, dst.to_ipv4().value(), msg->dport, msg->tx_rx); } const u64 sk = msg->sk; const hostport_tuple ft = { .src_ip = src.to_ipv4().value().as_int(), .dst_ip = dst.to_ipv4().value().as_int(), .src_port = htons(msg->sport), .dst_port = htons(msg->dport), .proto = IPPROTO_TCP, }; record_sk(sk, ft); // We had a NAT table entry before getting the socket info. if (auto mapping = nat_table_.find(ft); mapping != nat_table_.end()) { send_nat_remapping(timestamp, sk, mapping->second); } if (auto rev_mapping = nat_table_rev_.find(ft.reversed()); rev_mapping != nat_table_rev_.end()) { send_nat_remapping(timestamp, sk, rev_mapping->second.reversed()); } } void NatHandler::handle_close_socket(u64 timestamp, jb_agent_internal__close_sock_info *msg) { remove_sk(msg->sk); } void NatHandler::record_sk(u64 sk, hostport_tuple const &ft) { // We were hitting the assert in remove_sk(), which happens if two sk's use // the same four-tuple without a call to remove_sk() in-between. if (auto search = existing_sk_table_.find(ft); search != existing_sk_table_.end()) { const auto &existing_sk = search->second; LOG::debug_in( AgentLogKind::NAT, "NatHandler::record_sk: rewriting existing ft->sk mapping: " "sk={}, existing_sk={}, ft=({}:{},{}:{})", sk, existing_sk, IPv4Address::from(ft.src_ip), ntohs(ft.src_port), IPv4Address::from(ft.dst_ip), ntohs(ft.dst_port)); remove_sk(existing_sk); } // There was also an edge-case where we'd have a memory leak of the same sk // gets used for two-dfferent four-tuples without a call to remove_sk() // in-between. if (auto rev_search = existing_sk_table_rev_.find(sk); rev_search != existing_sk_table_rev_.end()) { const auto &existing_ft = rev_search->second; LOG::debug_in( AgentLogKind::NAT, "NatHandler::record_sk: rewriting existing sk->ft mapping: " "sk={}, existing_ft={{}:{},{}:{}), ft=({}:{},{}:{})", sk, IPv4Address::from(existing_ft.src_ip), ntohs(existing_ft.src_port), IPv4Address::from(existing_ft.dst_ip), ntohs(existing_ft.dst_port), IPv4Address::from(ft.src_ip), ntohs(ft.src_port), IPv4Address::from(ft.dst_ip), ntohs(ft.dst_port)); remove_sk(sk); } existing_sk_table_[ft] = sk; existing_sk_table_rev_[sk] = ft; } void NatHandler::remove_sk(u64 sk) { auto rev_search = existing_sk_table_rev_.find(sk); if (rev_search == existing_sk_table_rev_.end()) { // TODO: this happens pretty frequently. Initially this would happen in the // gap between the end and set_state probes. However, this would also happen // if any socket closes without reaching established as well. return; } auto search = existing_sk_table_.find(rev_search->second); assert(search != existing_sk_table_.end()); // tuple should be in existing_sk_table_ existing_sk_table_rev_.erase(rev_search); existing_sk_table_.erase(search); } void NatHandler::record_nat(hostport_tuple const &map_from, hostport_tuple const &map_to) { // Clean up possible previous records. // if (auto it = nat_table_.find(map_from); it != nat_table_.end()) { // map_from->some_to exist, remove some_to->map_from LOG::debug_in( AgentLogKind::NAT, "NatHandler::record_nat: rewriting existing mapping: " "({}:{},{}:{})->({}:{},{}:{}) with " "({}:{},{}:{})->({}:{},{}:{})", IPv4Address::from(map_from.src_ip), ntohs(map_from.src_port), IPv4Address::from(map_from.dst_ip), ntohs(map_from.dst_port), IPv4Address::from(it->second.src_ip), ntohs(it->second.src_port), IPv4Address::from(it->second.dst_ip), ntohs(it->second.dst_port), IPv4Address::from(map_from.src_ip), ntohs(map_from.src_port), IPv4Address::from(map_from.dst_ip), ntohs(map_from.dst_port), IPv4Address::from(map_to.src_ip), ntohs(map_to.src_port), IPv4Address::from(map_to.dst_ip), ntohs(map_to.dst_port)); nat_table_rev_.erase(it->second); } if (auto rev_it = nat_table_rev_.find(map_to); rev_it != nat_table_rev_.end()) { // map_to->some_from exists, remove some_from->map_to LOG::debug_in( AgentLogKind::NAT, "NatHandler::record_nat: rewriting existing reverse mapping: " "({}:{},{}:{})<-({}:{},{}:{}) with " "({}:{},{}:{})<-({}:{},{}:{})", IPv4Address::from(rev_it->second.src_ip), ntohs(rev_it->second.src_port), IPv4Address::from(rev_it->second.dst_ip), ntohs(rev_it->second.dst_port), IPv4Address::from(map_to.src_ip), ntohs(map_to.src_port), IPv4Address::from(map_to.dst_ip), ntohs(map_to.dst_port), IPv4Address::from(map_from.src_ip), ntohs(map_from.src_port), IPv4Address::from(map_from.dst_ip), ntohs(map_from.dst_port), IPv4Address::from(map_to.src_ip), ntohs(map_to.src_port), IPv4Address::from(map_to.dst_ip), ntohs(map_to.dst_port)); nat_table_.erase(rev_it->second); } nat_table_[map_from] = map_to; nat_table_rev_[map_to] = map_from; } void NatHandler::remove_nat(hostport_tuple const &map_from) { if (auto it = nat_table_.find(map_from); it != nat_table_.end()) { const auto &map_to = it->second; nat_table_rev_.erase(map_to); nat_table_.erase(it); } } hostport_tuple *NatHandler::get_nat_mapping(u32 src, u32 dst, u16 sport, u16 dport, u32 proto) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::get_nat_mapping: src={}:{}, dst={}:{}, proto={}", IPv4Address::from(src), sport, IPv4Address::from(dst), dport, proto); } const hostport_tuple ft = { src, dst, htons(sport), htons(dport), proto, }; if (auto it = nat_table_.find(ft); it != nat_table_.end()) { LOG::trace_in(AgentLogKind::NAT, "mapping found"); return &(it->second); } else { LOG::trace_in(AgentLogKind::NAT, "no mapping found"); return nullptr; } } void NatHandler::send_nat_remapping(u64 timestamp, u64 sk, hostport_tuple const &ft) { if (is_log_whitelisted(AgentLogKind::NAT)) { LOG::trace_in( AgentLogKind::NAT, "NatHandler::send_nat_remapping: sk={}, src={}:{}, dst={}:{}", sk, IPv4Address::from(ft.src_ip), ntohs(ft.src_port), IPv4Address::from(ft.dst_ip), ntohs(ft.dst_port)); } writer_.nat_remapping_tstamp(timestamp, sk, ft.src_ip, ft.dst_ip, ft.src_port, ft.dst_port); } ================================================ FILE: collector/kernel/nat_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include class NatHandler { public: /** * c'tor */ NatHandler(::ebpf_net::ingest::Writer &writer, logging::Logger &log); // end void handle_nf_nat_cleanup_conntrack(u64 timestamp, struct jb_agent_internal__nf_nat_cleanup_conntrack *msg); // start void handle_nf_conntrack_alter_reply(u64 timestamp, struct jb_agent_internal__nf_conntrack_alter_reply *msg); // existing void handle_existing_conntrack_tuple(u64 timestamp, struct jb_agent_internal__existing_conntrack_tuple *msg); void handle_set_state_ipv4(u64 timestamp, jb_agent_internal__set_state_ipv4 *msg); void handle_set_state_ipv6(u64 timestamp, jb_agent_internal__set_state_ipv6 *msg); void handle_close_socket(u64 timestamp, jb_agent_internal__close_sock_info *msg); // Returns a pointer to the corresponding val for a key we lookup // in the nat_table_. If there is no corresponding val, return nullptr. hostport_tuple *get_nat_mapping(u32 src, u32 dst, u16 sport, u16 dport, u32 proto); private: // Maps the IP_CT_DIR_ORIGINAL 4-tuple of a conntrack entry to the // IP_CT_DIR_REPLY 4-tuple of a conntrack entry. Only contains NAT-ed // connections absl::flat_hash_map nat_table_; // Maps replies to originals. absl::flat_hash_map nat_table_rev_; // Maps a struct nf_conntrack_tuple* to the corresponding 4-tuple. // Used when we iterate over existing conntrack entries. absl::flat_hash_map existing_conntrack_table_; // Maps a 4-tuple to its corresponding sk. Used to keep track of // existing sk's so we don't send NAT msgs for sk's we haven't // seen a set_state_ipv4 msg for yet. absl::flat_hash_map existing_sk_table_; // A reverse index of our existing_sk_table_ so that we can do cleanup later. absl::flat_hash_map existing_sk_table_rev_; ::ebpf_net::ingest::Writer &writer_; logging::Logger &log_; // Adds the mapping between a socket and its (local,remote) tuple. void record_sk(u64 sk, hostport_tuple const &ft); // Removes an entry from our existing_sk_table_ void remove_sk(u64 sk); // Adds the specified NAT mapping to internal tables. void record_nat(hostport_tuple const &map_from, hostport_tuple const &map_to); // Removes the specified NAT mapping from internal tables. void remove_nat(hostport_tuple const &map_from); // Sends the nat_remapping message to the specified socket. void send_nat_remapping(u64 timestamp, u64 sk, hostport_tuple const &ft); }; ================================================ FILE: collector/kernel/nat_prober.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #define RCV_BUFFSIZE 8192 // see libnfnetlink/include/libnfnetlink.h for NFNL_BUFFSIZE NatProber::NatProber(ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb) : periodic_cb_(periodic_cb) { // END (cleanup): nf_ct_delete probe_handler.start_probe(skel, "on_nf_ct_delete", "nf_ct_delete"); periodic_cb(); // START (create): __nf_conntrack_confirm entry and return probe_handler.start_probe(skel, "on___nf_conntrack_confirm", "__nf_conntrack_confirm"); periodic_cb(); probe_handler.start_kretprobe(skel, "onret___nf_conntrack_confirm", "__nf_conntrack_confirm"); periodic_cb(); // EXISTING ProbeAlternatives probe_alternatives{ "ctnetlink_existing_conntrack", { // nf_ct_port_tuple_to_nlattr is more widely available and has the same parameters as ctnetlink_dump_tuples {"on_ctnetlink_dump_tuples", "nf_ct_port_tuple_to_nlattr"}, {"on_ctnetlink_dump_tuples", "ctnetlink_dump_tuples"}, // Attaching probe to ctnetlink_dump_tuples fails on some distros and kernel builds, for example Ubuntu Jammy. {"on_ctnetlink_dump_tuples", "ctnetlink_dump_tuples_ip"}, }}; std::string ctnetlink_existing_k_func_name = probe_handler.start_probe(skel, probe_alternatives); periodic_cb(); int res = query_kernel(); if (res != 0) { if (res == EAGAIN || res == EWOULDBLOCK) { LOG::warn( "While probing NAT, netfilter socket finished before seeing " "NLMSG_DONE. {}", std::strerror(res)); } else { LOG::error("NatProber::NatProber() - Error calling query_kernel(): {}", std::strerror(res)); } } periodic_cb(); // Cleanup existing probe_handler.cleanup_probe(ctnetlink_existing_k_func_name); periodic_cb(); } /* Creates a netlink socket and creates a request for the kernel to dump its * conntrack table information. Returns 0 on success and errno on failure. */ int NatProber::query_kernel() { // create a netlink socket // domain: AF_NETLINK, type: SOCK_RAW | SOCK_NONBLOCK, protocol: // NETLINK_NETFILTER int sock_fd = socket(AF_NETLINK, SOCK_RAW | SOCK_NONBLOCK, NETLINK_NETFILTER); if (sock_fd == -1) { LOG::debug("NatProber::query_kernel() - Error opening netlink socket: {}", std::strerror(errno)); return errno; } // Source addr info (where to bind) struct sockaddr_nl src_addr; memset(&src_addr, 0, sizeof(src_addr)); src_addr.nl_family = AF_NETLINK; src_addr.nl_pid = getpid(); // self pid src_addr.nl_groups = 0; // unicast int err = bind(sock_fd, (struct sockaddr *)&src_addr, sizeof(src_addr)); if (err == -1) { int saved_errno = errno; LOG::debug("NatProber::query_kernel() - Error binding netlink socket: {}", std::strerror(saved_errno)); close(sock_fd); return saved_errno; } // Dest addr_info (where to send) struct sockaddr_nl dst_addr; memset(&dst_addr, 0, sizeof(dst_addr)); dst_addr.nl_family = AF_NETLINK; dst_addr.nl_pid = 0; // for the linux kernel dst_addr.nl_groups = 0; // unicast // Setup a netlink/conntrack query message - this is based on the msg created // in the conntrack tool. Specifically the codepath triggered by `sudo // conntrack -L` struct nlct_query_msg { struct nlmsghdr nlh; struct nfgenmsg nfmsg; struct nfattr nfattr1; u32 nfattr_data1; struct nfattr nfattr2; u32 nfattr_data2; }; struct nlct_query_msg msg; memset(&msg, 0, sizeof(msg)); // nlh msg.nlh.nlmsg_len = sizeof(msg); // size of the entire msg msg.nlh.nlmsg_type = (NFNL_SUBSYS_CTNETLINK << 8) | IPCTNL_MSG_CT_GET; msg.nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; msg.nlh.nlmsg_seq = (u32)std::time(nullptr); // current unix time msg.nlh.nlmsg_pid = 0; // for the linux kernel // nfmsg msg.nfmsg.nfgen_family = AF_INET; msg.nfmsg.version = NFNETLINK_V0; msg.nfmsg.res_id = 0; // nfattrs msg.nfattr1.nfa_len = 0x08; // size of nfattr1 + nfattr_data1 msg.nfattr1.nfa_type = CTA_MARK; msg.nfattr_data1 = 0; // empty msg.nfattr2.nfa_len = 0x08; // size of nfattr2 + nfattr_data2 msg.nfattr2.nfa_type = CTA_MARK_MASK; msg.nfattr_data2 = 0; // empty // Send msg err = sendto(sock_fd, (void *)&msg, sizeof(msg), 0, (struct sockaddr *)&dst_addr, sizeof(dst_addr)); if (err == -1) { int saved_errno = errno; LOG::debug("NatProber::query_kernel() - Error sending on netlink socket: {}", std::strerror(saved_errno)); close(sock_fd); return saved_errno; } // Receive responses while (1) { socklen_t addrlen = sizeof(dst_addr); unsigned char buf[RCV_BUFFSIZE]; err = recvfrom(sock_fd, buf, sizeof(buf), 0, (struct sockaddr *)&dst_addr, &addrlen); if (err == -1) { int saved_errno = errno; LOG::debug("NatProber::query_kernel() - Error receiving on netlink socket: {}", std::strerror(saved_errno)); close(sock_fd); return saved_errno; } // If no data was returned then we break - but this is an unexpected case, // so log for debugging purposes. if (err == 0) { LOG::debug("NatProber::query_kernel() - recvfrom = 0"); break; ; } // Break once we're done receiving messages // According to netlink_dump() inside of netlink/af_netlink.c, we expect the // last msg in a dump to consist of only an nlmsghdr with a flag for // NLMSG_DONE. struct nlmsghdr *nlh = (struct nlmsghdr *)buf; if (nlh->nlmsg_type == NLMSG_DONE) { break; } // Ensure we handle perf events in a timely fashion // else we run the risk of overflowing the event buffer periodic_cb_(); } // Cleanup close(sock_fd); return 0; } ================================================ FILE: collector/kernel/nat_prober.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include // Forward declaration for the skeleton struct render_bpf_bpf; /* forward declarations */ class ProbeHandler; /** * Adds BPF probes for nat translations of connections via the kernel's * conntrack tables */ class NatProber { public: /** * C'tor * * @param probe_handler: a ProbeHandler where new probes can be registered * @param bpf_module: the module from the bpf source code * @param periodic_cb: a callback to be called every once in a while, to * allow user to e.g., flush rings */ NatProber(ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb); private: /** * Queries the kernel for conntrack info */ int query_kernel(); std::function periodic_cb_; }; ================================================ FILE: collector/kernel/perf_poller.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include PerfPoller::PerfPoller(PerfContainer &container) : container_(container) {} PerfPoller::~PerfPoller() {} void PerfPoller::start(u64 interval_useconds, u64 n_intervals) { while (n_intervals) { poll(); usleep(interval_useconds); n_intervals--; } } ================================================ FILE: collector/kernel/perf_poller.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include class PerfPoller { public: /** * c'tor */ PerfPoller(PerfContainer &container); /** * d'tor */ virtual ~PerfPoller(); /** * poll method, to be implemented by child class. * * Can use container_->... */ virtual void poll() = 0; /** * Perform @n_intervals pollings, every @interval_useconds microseconds */ void start(u64 interval_useconds, u64 n_intervals); protected: PerfContainer &container_; }; ================================================ FILE: collector/kernel/perf_reader.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include #include #define __STRINGIZE(X) #X #define _STRINGIZE(X) __STRINGIZE(X) PerfContainer::PerfContainer() : n_entries_(0), readers_in_entries_(0) {} void PerfContainer::add_ring(PerfRing &pr) { if (readers_.size() >= BPF_MAX_CPUS) throw std::runtime_error("Only up to " _STRINGIZE(BPF_MAX_CPUS) " cpus are currently supported"); readers_.push_back(pr); } void PerfContainer::add_data_ring(PerfRing &pr) { if (data_readers_.size() >= BPF_MAX_CPUS) throw std::runtime_error("Only up to " _STRINGIZE(BPF_MAX_CPUS) " cpus are currently supported"); data_readers_.push_back(pr); } void PerfContainer::set_callback(uv_loop_t &loop, void *ctx, CALLBACK cb) { for (auto &reader : readers_) { reader.set_callback(loop, ctx, cb); } for (auto &data_reader : data_readers_) { data_reader.set_callback(loop, ctx, cb); } } std::string PerfContainer::inspect(void) { std::string out; size_t num_readers = readers_.size(); out += fmt::format("readers_: size={}\n", num_readers); for (size_t n = 0; n < readers_.size(); n++) { u32 total_bytes; u32 bytes = readers_[n].bytes_remaining(&total_bytes); out += fmt::format(" readers_[{}]: size={} ({}% full)\n", n, bytes, (((double)bytes) / (double)total_bytes) * 100.0); } out += fmt::format("entries_: n_entries_={}. readers_in_entries_={}\n", n_entries_, readers_in_entries_.to_string()); for (u32 n = 0; n < n_entries_; n++) { out += fmt::format(" entries_[{}]={{timestamp {}, reader_index {}}}", n, entries_[n].timestamp, entries_[n].reader_index); n++; } return out; } PerfReader::PerfReader(PerfContainer &container, u64 max_timestamp) : container_(container), max_timestamp_(max_timestamp), active_(true) { for (size_t i = 0; i < container.readers_.size(); i++) { auto &reader = container.readers_[i]; reader.start_read_batch(); auto &data_reader = container.data_readers_[i]; data_reader.start_read_batch(); /* if the reader is already in container.entries_, continue */ if (container.readers_in_entries_.test(i)) continue; /* might need to add to container_.entries_. do so if required */ update_when_not_in_entries(i); } } PerfReader::~PerfReader() { stop(); } bool PerfReader::empty() { return (container_.n_entries_ == 0) || (container_.entries_[0].timestamp > max_timestamp_); } void PerfReader::pop_unpadded_and_copy_to(char *dest) { auto &reader = top(); /* get length */ u32 length = reader.peek_size(); /* copy into buffer */ reader.peek_copy(dest, 0, length); /* release the element */ reader.pop(); update_after_pop(); } void PerfReader::pop_and_copy_to(char *dest) { auto &reader = top(); /* get unpadded length */ u32 unpadded = reader.peek_aligned_u32(sizeof(u32)); /* copy into buffer */ reader.peek_copy(dest, sizeof(u64), unpadded); /* release the element */ reader.pop(); update_after_pop(); } void PerfReader::pop() { auto &reader = top(); reader.pop(); update_after_pop(); } void PerfReader::stop() { if (!active_) return; for (auto &reader : container_.readers_) reader.finish_read_batch(); for (auto &data_reader : container_.data_readers_) data_reader.finish_read_batch(); active_ = false; } void PerfReader::update_after_pop() { /* pop the entry in container_.entries_ */ size_t idx = container_.entries_[0].reader_index; std::pop_heap(&container_.entries_[0], &container_.entries_[container_.n_entries_]); container_.n_entries_--; container_.readers_in_entries_.reset(idx); update_when_not_in_entries(idx); } void PerfReader::update_when_not_in_entries(size_t idx) { PerfRing &reader = container_.readers_[idx]; /* if the reader is empty, no need to add it */ int size = reader.peek_size(); if (size == -ENOENT) return; u64 timestamp = 0ull; /* PERF_RECORD_LOST is inserted as the minimum timestamp so it comes out first */ if (reader.peek_type() == PERF_RECORD_SAMPLE) { /* will have 8 bytes for sample record, 8 bytes timestamp */ assert(size >= 16); timestamp = reader.peek_aligned_u64(sizeof(u64)); } /* add the timestamp */ container_.entries_[container_.n_entries_] = {.timestamp = timestamp, .reader_index = idx}; container_.n_entries_++; std::push_heap(&container_.entries_[0], &container_.entries_[container_.n_entries_]); container_.readers_in_entries_.set(idx); } ================================================ FILE: collector/kernel/perf_reader.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include /** * Container for multiple CPU readers. * * Accessing values is done through PerfReader. PerfReader starts and finishes * read batches, and maintains the entries_ heap. * * If a reader is non-empty, PerfReader will populate an entry in entries_: * * if the next entry is PERF_LOST_RECORD, the timestamp will be ~0ull * * if the next entry is PERF_RECORD_SAMPLE, the timestamp will be the one * encoded in the sample. This code assume that a sample starts with * [ perf_event_header + u32 size + u32 unpadded_size + u64 timestamp ] */ class PerfContainer { public: /** * C'tor * @param cpus: the CPUs whose rings we should map * @param n_pages: the number of data pages for in each ring. */ PerfContainer(); /* disallow copy and assignment */ PerfContainer(const PerfContainer &) = delete; void operator=(const PerfContainer &) = delete; /** * Add the control channel ring to the collection * * Throws if trying to add more than 64 elements */ void add_ring(PerfRing &pr); /** * Add the data channel ring to the collection * * Throws if trying to add more than 64 elements */ void add_data_ring(PerfRing &pr); /** * Set a callback to execute when events show up in the * control channel perf ring * Used if you are not polling for high-speed access * Must be called only after all perf rings are added. */ typedef void CALLBACK(void *ctx); void set_callback(uv_loop_t &loop, void *ctx, CALLBACK cb); /** * Debugging routine to inspect the contents of the perf container */ std::string inspect(void); // returns the number of perf rings in this container std::size_t size() const { return readers_.size(); } // returns a reference to the i-th perf ring PerfRing const &operator[](std::size_t i) const { return readers_[i]; } PerfRing const &data_ring(std::size_t i) const { return data_readers_[i]; } PerfRing &data_ring(std::size_t i) { return data_readers_[i]; } private: friend class PerfReader; /* Readers for each live CPU */ std::vector readers_; std::vector data_readers_; /* (timestamp, reader_index) pairs */ struct PerfEntry { u64 timestamp; size_t reader_index; inline bool operator<(const PerfEntry &other) const { // Reverse the sense of the PerfEntry sort order // this way, our entries_ heap puts the earliest timestamps first, // making this a -min heap- instead of a -max heap- return timestamp > other.timestamp; } }; PerfEntry entries_[BPF_MAX_CPUS]; size_t n_entries_; /* bitmask: which readers from readers_ are already in entries_ */ std::bitset readers_in_entries_; }; /** * Read sorted values from multiple queues. */ class PerfReader { public: /** * C'tor * * Starts a sorted read operation. */ PerfReader(PerfContainer &container, u64 max_timestamp); /* D'tor */ ~PerfReader(); /** * Returns true if there are no more events to read safely while keeping * sort. */ bool empty(); /** * Returns the number of bytes left to read of of the perf ring, and * optionally the total size of the ring */ inline u32 bytes_remaining(u32 *total_bytes); /** * Returns the type of the next value * * Assumes reader is not empty (i.e., !empty()) */ inline u32 peek_type() const { return top().peek_type(); } /** * Returns the total size of the next perf event * * Assumes reader is not empty (i.e., !empty()) */ inline u32 peek_size() { return top().peek_size(); } /** * Returns the length of the payload of the next value * * Assumes reader is not empty (i.e., !empty()) and type==PERF_RECORD_SAMPLE */ inline u16 peek_unpadded_length() { return top().peek_aligned_u32(sizeof(u32)); } /** * Returns the length of the payload of the next value * * Assumes reader is not empty (i.e., !empty()) and type==PERF_RECORD_SAMPLE */ inline u16 peek_rpc_id() { return top().peek_aligned_u16(2 * sizeof(u64)); } /** * Returns the number of lost samples, if type is PERF_RECORD_LOST */ inline u64 peek_n_lost() { return top().peek_aligned_u64(sizeof(u64)); } /** * Returns a view into the sample's contents, without the perf event header. * * The messages are stored in a ring buffer, so if they're located at the end * of the ring and wrap-around to the beginning, then the view will be spread * over two chunks, in the order specified by `first` and `second` members of * the returned pair. Otherwise, the view will have exactly one chunk * represented by `first` in the returned pair and `second` will be empty. * * @assumes: `peek_type()` == `PERF_RECORD_SAMPLE`. */ inline std::pair peek_message() const { auto const &ring = top(); assert(ring.peek_type() == PERF_RECORD_SAMPLE); return ring.peek(); } /** * Returns which cpu index we're reading from next */ inline size_t peek_index() const { size_t idx = container_.entries_[0].reader_index; return idx; } /** * Copies the payload of a sample entry to the specified buffer. * * Assumes reader is not empty (i.e., !empty()) * Assumes type is PERF_RECORD_SAMPLE * Assumes there is enough space in the destination (>= peek_sample_length()) */ void pop_and_copy_to(char *dest); /** * Copies the payload of a sample entry to the specified buffer. * Uses the native size of the payload message instead of a header to * determine the length Assumes there is no padding on the perf_submit side * * Assumes reader is not empty (i.e., !empty()) * Assumes type is PERF_RECORD_SAMPLE * Assumes there is enough space in the destination (>= peek_sample_length()) */ void pop_unpadded_and_copy_to(char *dest); /** * Pops the entry, and updates data structures * * Assumes reader is not empty (i.e., !empty()) */ void pop(); /** * Explicitly finish the batch */ void stop(); private: /** * Returns the reader with the smallest */ inline PerfRing &top() { size_t idx = container_.entries_[0].reader_index; return container_.readers_[idx]; } inline PerfRing const &top() const { size_t idx = container_.entries_[0].reader_index; return container_.readers_[idx]; } /** * Updates container_.entries_ and container_.readers_in_entries_ */ void update_after_pop(); /** * Updates the reader's state given that it is not in container_.entries_ */ void update_when_not_in_entries(size_t idx); /* container to read from */ PerfContainer &container_; /* the maximum timestamp we should accept */ u64 max_timestamp_; /* is the reader active */ bool active_; /* This class acts as a "Guard", so disallow copy and assignment */ PerfReader(const PerfReader &) = delete; void operator=(const PerfReader &) = delete; }; ================================================ FILE: collector/kernel/probe_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include // Include the generated skeleton extern "C" { #include "generated/render_bpf.skel.h" } #include #include #define EVENTS_PERF_RING_N_BYTES (1024 * 4096) #define EVENTS_PERF_RING_N_WATERMARK_BYTES (512 * 4096) #define DATA_CHANNEL_PERF_RING_N_BYTES (256 * 4096) #define DATA_CHANNEL_PERF_RING_N_WATERMARK_BYTES (1) // Helper function to get online CPUs std::vector get_online_cpus() { std::vector cpus; int num_cpus = libbpf_num_possible_cpus(); for (int i = 0; i < num_cpus; i++) { cpus.push_back(i); } return cpus; } ProbeHandler::ProbeHandler(logging::Logger &log) : log_(log), num_failed_probes_(0), stack_trace_count_(0) {}; void ProbeHandler::load_kernel_symbols() { KernelSymbols ks; try { ks = read_proc_kallsyms(); } catch (std::exception &exc) { log_.error("Failed to load kernel symbols: {}", exc.what()); return; } if (!ks.empty()) { LOG::info("Kernel symbols list loaded"); kernel_symbols_ = std::move(ks); } } void ProbeHandler::clear_kernel_symbols() { kernel_symbols_.reset(); } int ProbeHandler::setup_mmap(int cpu, int perf_fd, PerfContainer &perf, bool is_data, u32 n_bytes, u32 n_watermark_bytes) { /* get mmap'd memory */ auto s = std::make_unique(cpu, n_bytes, n_watermark_bytes); int mmap_fd = s->fd(); /* add to perf container */ PerfRing ring(std::move(s)); if (is_data) { perf.add_data_ring(ring); } else { perf.add_ring(ring); } /* set the file descriptor in the kernel */ int res = bpf_map_update_elem(perf_fd, static_cast(&cpu), static_cast(&mmap_fd), 0); if (res < 0) { LOG::error("cannot set perf fd {} for cpu {} to point to ring fd {}", perf_fd, cpu, mmap_fd); return -4; } return 0; } int ProbeHandler::get_bpf_map_fd(struct render_bpf_bpf *skel, const char *map_name) { struct bpf_map *map = get_bpf_map(skel, map_name); if (!map) { LOG::error("Cannot get '{}' map", map_name); return -2; } int map_fd = bpf_map__fd(map); if (map_fd < 0) { LOG::error("'{}' map's fd<0: fd={}", map_name, map_fd); return -3; } return map_fd; } // Callback to route libbpf messages through our logging system static int libbpf_print_messages(enum libbpf_print_level level, const char *format, va_list args) { char buffer[16 * 1024]; int len = vsnprintf(buffer, sizeof(buffer), format, args); // Remove trailing newline if present if (len > 0 && buffer[len - 1] == '\n') { buffer[len - 1] = '\0'; len--; } std::string message(buffer); if (level <= LIBBPF_INFO) LOG::debug("libbpf: {}", message); else LOG::trace("libbpf: {}", message); return len; } struct render_bpf_bpf *ProbeHandler::open_bpf_skeleton() { // Set a custom print callback to route libbpf messages through our logging system libbpf_set_print(libbpf_print_messages); struct render_bpf_bpf *skel = render_bpf_bpf__open(); if (!skel) { LOG::error("Cannot open BPF skeleton"); return nullptr; } return skel; } void ProbeHandler::configure_bpf_skeleton(struct render_bpf_bpf *skel, const BpfConfiguration &config) { if (!skel) { LOG::error("Cannot configure BPF skeleton: null skeleton"); return; } // Configure global variables before loading skel->rodata->boot_time_adjustment = config.boot_time_adjustment; skel->rodata->filter_ns = config.filter_ns; skel->rodata->enable_tcp_data_stream = config.enable_tcp_data_stream ? 1 : 0; LOG::info( "BPF configuration: boot_time_adjustment={}, filter_ns={}, tcp_data_stream={}", config.boot_time_adjustment, config.filter_ns, config.enable_tcp_data_stream ? "enabled" : "disabled"); } void ProbeHandler::destroy_bpf_skeleton(struct render_bpf_bpf *skel) { if (skel) { render_bpf_bpf__destroy(skel); } } int ProbeHandler::load_bpf_skeleton(struct render_bpf_bpf *skel, PerfContainer &perf) { int res = render_bpf_bpf__load(skel); if (res != 0) { LOG::error("Cannot load BPF skeleton, res={}", res); return res; } LOG::info("eBPF program successfully loaded"); /* get events map descriptor */ int events_fd = get_bpf_map_fd(skel, "events"); if (events_fd < 0) { return events_fd; } /* get data_channel map descriptor */ int data_channel_fd = get_bpf_map_fd(skel, "data_channel"); if (data_channel_fd < 0) { return data_channel_fd; } /* get online cpus */ auto online_cpus = get_online_cpus(); /* open mmaps */ for (auto cpu : online_cpus) { res = setup_mmap(cpu, events_fd, perf, false, EVENTS_PERF_RING_N_BYTES, EVENTS_PERF_RING_N_WATERMARK_BYTES); if (res < 0) { return res; } res = setup_mmap(cpu, data_channel_fd, perf, true, DATA_CHANNEL_PERF_RING_N_BYTES, DATA_CHANNEL_PERF_RING_N_WATERMARK_BYTES); if (res < 0) { return res; } } return 0; } struct bpf_map *ProbeHandler::get_bpf_map(struct render_bpf_bpf *skel, const std::string &name) { if (name == "cgroup_exit_active") return skel->maps.cgroup_exit_active; if (name == "tcp_open_sockets") return skel->maps.tcp_open_sockets; if (name == "on_inet_csk_accept_active") return skel->maps.on_inet_csk_accept_active; if (name == "seen_inodes") return skel->maps.seen_inodes; if (name == "udp_get_port_hash") return skel->maps.udp_get_port_hash; if (name == "inet_release_active") return skel->maps.inet_release_active; if (name == "tail_calls") return skel->maps.tail_calls; if (name == "cgroup_control_active") return skel->maps.cgroup_control_active; if (name == "seen_conntracks") return skel->maps.seen_conntracks; if (name == "inet_csk_accept_active") return skel->maps.inet_csk_accept_active; if (name == "tcp_sendmsg_active") return skel->maps.tcp_sendmsg_active; if (name == "tcp_recvmsg_active") return skel->maps.tcp_recvmsg_active; if (name == "events") return skel->maps.events; if (name == "bpf_log_globals_per_cpu") return skel->maps.bpf_log_globals_per_cpu; if (name == "tgid_info_table") return skel->maps.tgid_info_table; if (name == "dead_group_tasks") return skel->maps.dead_group_tasks; if (name == "udp_open_sockets") return skel->maps.udp_open_sockets; if (name == "dns_message_array") return skel->maps.dns_message_array; if (name == "_tcp_connections") return skel->maps._tcp_connections; if (name == "_tcp_control") return skel->maps._tcp_control; if (name == "data_channel") return skel->maps.data_channel; return nullptr; } int ProbeHandler::register_tail_call( struct render_bpf_bpf *skel, const std::string &prog_array_name, int index, const std::string &func_name) { struct bpf_map *prog_array = get_bpf_map(skel, prog_array_name); if (!prog_array) { log_.error("Failed to find prog array map {}", prog_array_name); ++num_failed_probes_; return -1; } // TCP processor programs need to be looked up by function name struct bpf_program *prog = bpf_object__find_program_by_name(skel->obj, func_name.c_str()); if (!prog) { log_.error("Failed to register tail call for {}, could not find program", func_name); ++num_failed_probes_; return -2; } int prog_fd = bpf_program__fd(prog); if (prog_fd < 0) { log_.error("Failed to register tail call for {}, could not get program fd", func_name); ++num_failed_probes_; return -3; } int map_fd = bpf_map__fd(prog_array); int ret = bpf_map_update_elem(map_fd, &index, &prog_fd, 0); if (ret < 0) { log_.error("Failed to update prog array for tail call {}, errno {}", func_name, errno); ++num_failed_probes_; return -4; } tail_calls_.emplace_back(prog_array_name, func_name, prog_fd, index); return 0; } #if DEBUG_ENABLE_STACKTRACE std::string ProbeHandler::get_stack_trace(struct render_bpf_bpf *skel, s32 kernel_stack_id, s32 user_stack_id, u32 tgid) { std::string out; // TODO: Implement stack trace functionality for libbpf // This requires additional implementation for stack trace handling out += "Stack trace functionality not yet implemented for libbpf\n"; stack_trace_count_++; return out; } #endif int ProbeHandler::start_probe_common( struct render_bpf_bpf *skel, bool is_kretprobe, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix) { auto bpf_program = bpf_object__find_program_by_name(skel->obj, func_name.c_str()); if (!bpf_program) { LOG::error("Could not get find program. func_name:{} k_func_name:{}", func_name, k_func_name); return -1; } /* attach the probe */ std::string probe_name = (is_kretprobe ? kretprobe_prefix_ : probe_prefix_) + k_func_name + event_id_suffix; struct bpf_link *link; link = bpf_program__attach_kprobe( bpf_program, is_kretprobe, /* retprobe */ k_func_name.c_str()); if (!link) { LOG::debug_in( AgentLogKind::BPF, "Unable to attach {}. probe_name:{} func_name:{} k_func_name:{} errno:{}", is_kretprobe ? "kretprobe" : "kprobe", probe_name, func_name, k_func_name, errno); return -3; } // Store the link pointer for cleanup probes_.push_back(link); probe_names_.push_back(probe_name); return 0; } int ProbeHandler::start_probe( struct render_bpf_bpf *skel, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix) { auto ret = start_probe_common(skel, false, func_name, k_func_name, event_id_suffix); if (ret != 0) { log_.error("Failed to attach {} kprobe, error {}", k_func_name, ret); ++num_failed_probes_; } return ret; } int ProbeHandler::start_kretprobe( struct render_bpf_bpf *skel, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix) { auto ret = start_probe_common(skel, true, func_name, k_func_name, event_id_suffix); if (ret != 0) { log_.error("Failed to attach {} kretprobe, error {}", k_func_name, ret); ++num_failed_probes_; } return ret; } std::string ProbeHandler::start_probe_common( struct render_bpf_bpf *skel, bool is_kretprobe, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix) { size_t probe_num = 1; size_t num_alternatives = probe_alternatives.func_names.size(); if (num_alternatives == 0) { throw std::runtime_error("ProbeHandler:start_probe_common() no alternatives provided"); } for (const auto &func_and_kfunc : probe_alternatives.func_names) { int ret = start_probe_common(skel, is_kretprobe, func_and_kfunc.func_name, func_and_kfunc.k_func_name, event_id_suffix); if (ret == 0) { LOG::debug_in( AgentLogKind::BPF, "Successfully attached {} {}, alternative {} of {}, func_name={}, k_func_name={}", probe_alternatives.desc, is_kretprobe ? "kretprobe" : "kprobe", probe_num, num_alternatives, func_and_kfunc.func_name, func_and_kfunc.k_func_name); return func_and_kfunc.k_func_name; break; } ++probe_num; } log_.error( "Failed to attach any {} {}, attempted {} alternatives", probe_alternatives.desc, is_kretprobe ? "kretprobe" : "kprobe", num_alternatives); ++num_failed_probes_; return std::string(); } std::string ProbeHandler::start_probe( struct render_bpf_bpf *skel, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix) { return start_probe_common(skel, false, probe_alternatives, event_id_suffix); } std::string ProbeHandler::start_kretprobe( struct render_bpf_bpf *skel, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix) { return start_probe_common(skel, true, probe_alternatives, event_id_suffix); } void ProbeHandler::cleanup_probes() { while (!probes_.empty()) { auto probe = probes_.back(); probes_.pop_back(); std::string probe_name = probe_names_.back(); probe_names_.pop_back(); LOG::debug_in(AgentLogKind::BPF, "cleanup probe for {}", probe_name); // Clean up the bpf_link if (probe) { bpf_link__destroy(probe); } } LOG::debug_in(AgentLogKind::BPF, "Done cleaning up probes"); } void ProbeHandler::cleanup_tail_calls(struct render_bpf_bpf *skel) { while (!tail_calls_.empty()) { const auto &tc = tail_calls_.back(); LOG::debug_in(AgentLogKind::BPF, "cleanup tail call for {} from table {}", tc.func_, tc.table_); struct bpf_map *prog_array = get_bpf_map(skel, tc.table_); if (prog_array) { int map_fd = bpf_map__fd(prog_array); bpf_map_delete_elem(map_fd, &tc.index_); } tail_calls_.pop_back(); } } void ProbeHandler::cleanup_probe_common(const std::string &probe_name) { int i = 0; for (std::vector::iterator it = probe_names_.begin(); it != probe_names_.end(); ++it) { if (!probe_names_[i].compare(probe_name)) { auto probe = probes_[i]; std::string probe_name = probe_names_[i]; probes_.erase(probes_.begin() + i); probe_names_.erase(probe_names_.begin() + i); LOG::debug_in(AgentLogKind::BPF, "cleanup probe for {}", probe_name); // Clean up the bpf_link if (probe) { bpf_link__destroy(probe); } return; } ++i; } log_.error("Error removing probe. {} was not found.", probe_name); } void ProbeHandler::cleanup_probe(const std::string &k_func_name) { if (k_func_name.empty()) return; cleanup_probe_common(probe_prefix_ + k_func_name); } void ProbeHandler::cleanup_kretprobe(const std::string &k_func_name) { if (k_func_name.empty()) return; cleanup_probe_common(kretprobe_prefix_ + k_func_name); } ================================================ FILE: collector/kernel/probe_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include "kernel_symbols.h" #include #include #include #include #include #include #include #include // Forward declaration for the skeleton struct render_bpf_bpf; /** * Configuration structure for BPF global variables * Used to configure the BPF program before loading */ struct BpfConfiguration { u64 boot_time_adjustment = 0; u64 filter_ns = 1000000000; // Default 1 second in nanoseconds bool enable_tcp_data_stream = false; }; /** * ProbeAlternatives encapsulates multiple alternatives to attempt when attaching a probe. Alternatives may be needed due to * differences in kernel versions or builds. */ struct ProbeAlternatives { struct FuncAndKfunc { std::string func_name; std::string k_func_name; }; ProbeAlternatives(std::string desc, std::vector func_names) : desc(std::move(desc)), func_names(std::move(func_names)) {} std::string desc; std::vector func_names; }; /** * Handles the creation of probes and provides info for cleanup to signal * handler */ class ProbeHandler { friend class KernelCollectorTest; public: /** * c'tor */ ProbeHandler(logging::Logger &log); /** * Loads the list of available kernel symbols from /proc/kallsyms. * This list is then used to determine if a kernel function can be instrumented. */ void load_kernel_symbols(); /** * Clears the list of kernel symbols. * Used to free up memory after all probes are started. */ void clear_kernel_symbols(); struct render_bpf_bpf *open_bpf_skeleton(); void configure_bpf_skeleton(struct render_bpf_bpf *skel, const BpfConfiguration &config); int load_bpf_skeleton(struct render_bpf_bpf *skel, PerfContainer &perf); void destroy_bpf_skeleton(struct render_bpf_bpf *skel); /** * BPF table helpers **/ struct bpf_map *get_bpf_map(struct render_bpf_bpf *skel, const std::string &name); /** * Register tail call in table */ int register_tail_call( struct render_bpf_bpf *skel, const std::string &prog_array_name, int index, const std::string &func_name); /** * Starts a kprobe * on failure logs error and increments num_failed_probes_ * @returns 0 on success, negative value on failure */ int start_probe( struct render_bpf_bpf *skel, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix = std::string()); /** * Starts a kretprobe * on failure logs error and increments num_failed_probes_ * @returns 0 on success, negative value on failure */ int start_kretprobe( struct render_bpf_bpf *skel, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix = std::string()); /** * Starts a kprobe from the provided alternatives. * Probes are attempted in order until one succeeds. * If all alternatives fail an error is logged and num_failed_probes_ is incremented. * @returns string containing the k_func_name of the probe that was attached on success, empty string on failure */ std::string start_probe( struct render_bpf_bpf *skel, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix = std::string()); /** * Starts a kretprobe from the provided alternatives. * Probes are attempted in order until one succeeds. * If all alternatives fail an error is logged and num_failed_probes_ is incremented. * @returns string containing the k_func_name of the probe that was attached on success, empty string on failure */ std::string start_kretprobe( struct render_bpf_bpf *skel, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix = std::string()); /** * Handles the cleanup of probes on exit */ void cleanup_probes(); /** * Clean up all the registered tail calls */ void cleanup_tail_calls(struct render_bpf_bpf *skel); /** * Cleans up a single probe */ void cleanup_probe(const std::string &k_func_name); /** * Cleans up a single kretprobe */ void cleanup_kretprobe(const std::string &k_func_name); #if DEBUG_ENABLE_STACKTRACE /** * Gets a stack trace and removes it from the list */ std::string get_stack_trace(struct render_bpf_bpf *skel, s32 kernel_stack_id, s32 user_stack_id, u32 tgid); #endif protected: /** * Common code to start a kprobe or kretprobe * @returns 0 on success, negative value on failure */ int start_probe_common( struct render_bpf_bpf *skel, bool is_kretprobe, const std::string &func_name, const std::string &k_func_name, const std::string &event_id_suffix); /** * Common code to start a kprobe or kretprobe from the provided alternatives * @returns string containing the k_func_name of the probe that was attached on success, empty string on failure */ std::string start_probe_common( struct render_bpf_bpf *skel, bool is_kretprobe, const ProbeAlternatives &probe_alternatives, const std::string &event_id_suffix = std::string()); /** * Common code to clean up a single probe */ void cleanup_probe_common(const std::string &probe_name); /** * Returns the file descriptor for a table declared in bpf */ int get_bpf_map_fd(struct render_bpf_bpf *skel, const char *map_name); /** * Sets up memory mapping for perf rings */ int setup_mmap(int cpu, int events_fd, PerfContainer &perf, bool is_data, u32 n_bytes, u32 n_watermark_bytes); private: static constexpr char probe_prefix_[] = "ebpf_net_p_"; static constexpr char kretprobe_prefix_[] = "ebpf_net_r_"; logging::Logger &log_; struct TailCallTuple { TailCallTuple(const std::string &table, const std::string &func, int fd, int index) : table_(table), func_(func), fd_(fd), index_(index) {} std::string table_; std::string func_; int fd_; int index_; }; std::vector probes_; std::vector tail_calls_; std::vector probe_names_; size_t num_failed_probes_; // number of kprobes, kretprobes, and tail_calls that failed to attach size_t stack_trace_count_; std::optional kernel_symbols_; }; ================================================ FILE: collector/kernel/proc_cmdline.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "proc_cmdline.h" #include #include #include #include // Max number of characters we are willing to read from /proc/PID/cmdline. static constexpr size_t MAX_CMDLINE_READ_SIZE = 256; Expected read_proc_cmdline(u32 pid) { char path[64] = {0}; if (snprintf(path, sizeof(path), "/proc/%d/cmdline", pid) < 0) { return {unexpected, std::error_code()}; } FileDescriptor fd; if (auto error = fd.open(path, FileDescriptor::Access::read_only, FileDescriptor::Positioning::beginning)) { return {unexpected, std::move(error)}; } std::string buffer(MAX_CMDLINE_READ_SIZE, '\0'); if (auto const result = fd.read_all(buffer.data(), buffer.size())) { buffer.resize(*result); return std::move(buffer); } else { return {unexpected, result.error()}; } } Expected try_read_proc_cmdline(u32 pid) { auto r = read_proc_cmdline(pid); if (!r) { if ((r.error().category() == std::generic_category()) && ((r.error().value() == ENOENT) || (r.error().value() == ESRCH))) { // no such process return ""; } } return r; } ================================================ FILE: collector/kernel/proc_cmdline.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include // Reads the process command-line from /proc/PID/cmdline. Expected read_proc_cmdline(u32 pid); // Reads the process command-line from /proc/PID/cmdline. // On ENOENT (no such file or directory) error, returns an empty string // instead of an error code. Expected try_read_proc_cmdline(u32 pid); ================================================ FILE: collector/kernel/proc_net_reader.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include ProcNetReader::ProcNetReader(std::string filename) : tcp_file_(filename), sk_ino_(0), sk_p_(0), sk_state_(0) { // skip the first line of the file std::string line; getline(tcp_file_, line); } ProcNetReader::~ProcNetReader() { tcp_file_.close(); } int ProcNetReader::get_ino() { return sk_ino_; } unsigned long ProcNetReader::get_sk() { return sk_p_; } int ProcNetReader::get_sk_state() { return sk_state_; } int ProcNetReader::next() { std::string line; getline(tcp_file_, line); if (line == "") return 0; std::istringstream issline(line); std::string tk; int tk_id = 0; do { issline >> tk; tk_id++; if (tk_id == 4) { // parse the state sscanf(tk.c_str(), "%x", &sk_state_); } else if (tk_id == 10) { // parse inode_number sscanf(tk.c_str(), "%d", &sk_ino_); } else if (tk_id == 12) { // parse sk pointer sscanf(tk.c_str(), "%lx", &sk_p_); } else if (tk_id > 12) { // done break; } } while (issline); return 1; } ================================================ FILE: collector/kernel/proc_net_reader.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include /** * Reads through "/proc/net/tcp" and "/proc/net/tcp6" */ class ProcNetReader { public: /** * c'tor */ ProcNetReader(std::string filename); /** * d'tor */ ~ProcNetReader(); /** * Accessors for ino_, sk_, and sk_state_; */ int get_ino(); unsigned long get_sk(); int get_sk_state(); // // Read /proc//* for the next entry. // Returns 1 if a process entry was found and 0 otherwise. // int next(); private: std::ifstream tcp_file_; int sk_ino_; unsigned long sk_p_; int sk_state_; }; ================================================ FILE: collector/kernel/proc_reader.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include ProcReader::ProcReader() : pid_(0) { proc_ = opendir("/proc"); if (proc_ == NULL) throw std::runtime_error("ProcReader: Couldn't open proc."); } ProcReader::~ProcReader() { closedir(proc_); } int ProcReader::get_pid() { return pid_; } int ProcReader::is_pid() { if (sscanf(proc_ent_->d_name, "%d", &pid_) != 1) return 0; // exit if the current proc_ent_ isn't a pid directory return 1; } int ProcReader::next() { proc_ent_ = readdir(proc_); if (proc_ent_ == 0) return 0; return 1; } ================================================ FILE: collector/kernel/proc_reader.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include /** * Reads through proc */ class ProcReader { public: /** * c'tor * throws if buff_ can't be malloc-ed */ ProcReader(); /** * d'tor */ ~ProcReader(); /** * Acts as an accessor to pid_ * Assumes we always call is_pid() first. */ int get_pid(); /** * Returns 1 if the current proc_ent_ is a pid directory. Returns 0 otherwise */ int is_pid(); // // Reads /proc/* for the next entry. // Returns 1 if a process entry was found and 0 otherwise. // int next(); private: DIR *proc_; dirent *proc_ent_; int pid_; }; ================================================ FILE: collector/kernel/process_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include namespace { std::string_view comm_to_string(std::uint8_t const (&comm)[16]) { auto length = strnlen(reinterpret_cast(comm), sizeof(comm)); return std::string_view(reinterpret_cast(comm), length); } } // namespace ProcessHandler::ProcessHandler( ::ebpf_net::ingest::Writer &writer, ::ebpf_net::kernel_collector::Index &collector_index, logging::Logger &log) : writer_(writer), collector_index_(collector_index), log_(log), memory_page_bytes_(memory_page_size().try_raise().value()) { LOG::trace_in( AgentLogKind::TRACKED_PROCESS, "ProcessHandler: memory_page_size={} clock_ticks_per_second={}", memory_page_bytes_, clock_ticks_per_second); writer_.system_wide_process_settings(clock_ticks_per_second, memory_page_bytes_); } ProcessHandler::~ProcessHandler() { for (auto &process : processes_) { process.second.handle.put(collector_index_); } processes_.clear(); } void ProcessHandler::on_new_process(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_info const &msg) { const std::string_view comm = comm_to_string(msg.comm); LOG::trace_in( AgentLogKind::PID, "ProcessHandler::{}: timestamp={} pid={} parent={} cgroup=0x{:x} comm='{}'", __func__, timestamp, msg.pid, msg.parent_pid, msg.cgroup, comm); if (processes_.find(msg.pid) != processes_.end()) { ++stats_.duplicate_tgid; #ifdef DEBUG_TGID LOG::debug_in( AgentLogKind::TRACKED_PROCESS, "DUPLICATE TGID pid={} parent={} cgroup=0x{:x} comm='{}'!", msg.pid, msg.parent_pid, msg.cgroup, comm); #endif // DEBUG_TGID } auto weak_handle = collector_index_.tracked_process.by_key({msg.cgroup, msg.pid}); assert(msg.pid == weak_handle.tgid()); assert(msg.cgroup == weak_handle.cgroup()); weak_handle.set_tgid(msg.pid); weak_handle.set_command(to_jb_blob(msg.comm)); weak_handle.set_cgroup(msg.cgroup); processes_[msg.pid] = { .handle = weak_handle.to_handle() #ifdef DEBUG_TGID , .timestamp = timestamp, .cgroup = msg.cgroup, .command = std::string(comm) #endif // DEBUG_TGID }; } void ProcessHandler::on_process_end(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_close const &msg) { const std::string_view comm = comm_to_string(msg.comm); LOG::trace_in(AgentLogKind::PID, "ProcessHandler::{}: timestamp={} pid={} comm='{}'", __func__, timestamp, msg.pid, comm); if (auto i = processes_.find(msg.pid); i != processes_.end()) { i->second.handle.put(collector_index_); processes_.erase(i); } else { ++stats_.missing_tgid; #ifdef DEBUG_TGID LOG::debug_in(AgentLogKind::TRACKED_PROCESS, "MISSING TGID pid={} comm='{}'", msg.pid, comm); #endif // DEBUG_TGID } } void ProcessHandler::on_cgroup_move(std::chrono::nanoseconds timestamp, struct jb_agent_internal__cgroup_attach_task const &msg) { LOG::trace_in( AgentLogKind::PID, "ProcessHandler::{}: timestamp={} pid={} cgroup=0x{:x}", __func__, timestamp, msg.pid, msg.cgroup); if (auto i = processes_.find(msg.pid); i != processes_.end()) { i->second.handle.access(collector_index_).set_cgroup(msg.cgroup); } else { ++stats_.missing_tgid; #ifdef DEBUG_TGID LOG::debug_in(AgentLogKind::TRACKED_PROCESS, "MISSING TGID pid={} cgroup=0x{:x}", msg.pid, msg.cgroup); #endif // DEBUG_TGID } } void ProcessHandler::set_process_command(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_set_comm const &msg) { const std::string_view comm = comm_to_string(msg.comm); LOG::trace_in(AgentLogKind::PID, "ProcessHandler::{}: timestamp={} pid={} comm='{}'", __func__, timestamp, msg.pid, comm); if (auto i = processes_.find(msg.pid); i != processes_.end()) { i->second.handle.access(collector_index_).set_command(to_jb_blob(msg.comm)); } else { ++stats_.missing_tgid; #ifdef DEBUG_TGID LOG::debug_in(AgentLogKind::TRACKED_PROCESS, "MISSING TGID pid={} comm='{}'", msg.pid, comm); #endif // DEBUG_TGID } } void ProcessHandler::pid_exit(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_exit const &msg) { LOG::trace_in( AgentLogKind::PID, "ProcessHandler::{}: timestamp={} tgid={} pid={} exit_code={}", __func__, timestamp, msg.tgid, msg.pid, msg.exit_code); if (auto i = processes_.find(msg.tgid); i != processes_.end()) { i->second.handle.access(collector_index_) .pid_exit_tstamp(integer_time(timestamp), msg.tgid, msg.pid, msg.exit_code); } else { ++stats_.missing_tgid; #ifdef DEBUG_TGID LOG::debug_in(AgentLogKind::TRACKED_PROCESS, "MISSING TGID tgid={} pid={} exit_code={}", msg.tgid, msg.pid, msg.exit_code); #endif // DEBUG_TGID } } #ifdef DEBUG_TGID void ProcessHandler::debug_tgid_dump() { auto const str = [this] { std::ostringstream dump; dump << "-- BEGIN TGID DUMP --\n"; for (auto const &i : processes_) { auto &process = i.second; dump << fmt::format( "tstamp: {}, tgid: {}, cgroup: {}, comm: {}\n", process.timestamp, i.first, process.cgroup, process.command); } dump << "-- END TGID DUMP --\n"; return dump.str(); }(); LOG::debug_in(AgentLogKind::TRACKED_PROCESS, "{}", str); } #endif // DEBUG_TGID ================================================ FILE: collector/kernel/process_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include #include // Turn this on to enable tgid table debugging feature // via: kill -USR1 #ifndef NDEBUG // #define DEBUG_TGID 1 #endif // NDEBUG class ProcessHandler { public: ProcessHandler( ::ebpf_net::ingest::Writer &writer, ::ebpf_net::kernel_collector::Index &collector_index, logging::Logger &log); ~ProcessHandler(); void on_new_process(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_info const &msg); void on_process_end(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_close const &msg); void on_cgroup_move(std::chrono::nanoseconds timestamp, struct jb_agent_internal__cgroup_attach_task const &msg); void set_process_command(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_set_comm const &msg); void pid_exit(std::chrono::nanoseconds timestamp, struct jb_agent_internal__pid_exit const &msg); #ifdef DEBUG_TGID void debug_tgid_dump(); #endif // DEBUG_TGID private: struct ThreadGroupData { ebpf_net::kernel_collector::handles::tracked_process handle; #ifdef DEBUG_TGID std::chrono::nanoseconds timestamp; u64 cgroup; std::string command; #endif // DEBUG_TGID }; // rpc components ::ebpf_net::ingest::Writer &writer_; ::ebpf_net::kernel_collector::Index &collector_index_; // process data absl::flat_hash_map processes_; // logging logging::Logger &log_; std::size_t memory_page_bytes_; struct InternalStats { std::size_t duplicate_tgid = 0; std::size_t missing_tgid = 0; } stats_; }; ================================================ FILE: collector/kernel/process_prober.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include // Forward declaration for skeleton struct render_bpf_bpf; ProcessProber::ProcessProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb, std::function check_cb) { // END probe_handler.start_probe(skel, "on_taskstats_exit", "taskstats_exit"); probe_handler.start_probe(skel, "on_cgroup_exit", "cgroup_exit"); probe_handler.start_kretprobe(skel, "onret_cgroup_exit", "cgroup_exit"); // STATE CHANGE (pid->cgroup) probe_handler.start_probe(skel, "on_cgroup_attach_task", "cgroup_attach_task"); // START /* probe for new process info * Note that other functions we considered were: * sys_execve, sched_fork, __sched_fork, _do_fork, sched_exec */ probe_handler.start_probe(skel, "on_wake_up_new_task", "wake_up_new_task"); probe_handler.start_probe(skel, "on_set_task_comm", "__set_task_comm"); // EXISTING probe_handler.start_kretprobe(skel, "onret_get_pid_task", "get_pid_task"); periodic_cb(); check_cb("process prober startup"); // iterate over /proc/ to trigger on_get_pid_task() trigger_get_pid_task(periodic_cb); check_cb("trigger_get_pid_task()"); // we can remove the probe for existing now probe_handler.cleanup_kretprobe("get_pid_task"); periodic_cb(); check_cb("process prober cleanup()"); } void ProcessProber::trigger_get_pid_task(std::function periodic_cb) { ProcReader proc_reader; while (proc_reader.next()) { periodic_cb(); if (!proc_reader.is_pid()) continue; // skip this entry if this wasn't a pid directory int pid = proc_reader.get_pid(); FDReader fd_reader(pid); int status = fd_reader.open_task_dir(); if (status) continue; // skip this entry because task_dir couldn't be opened // for each thread in this group while (!fd_reader.next_task()) { // read a file from this tid so that our probe can generate a msg. // we don't care about the return value fd_reader.open_task_comm(); periodic_cb(); } } } ================================================ FILE: collector/kernel/process_prober.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include class ProbeHandler; struct render_bpf_bpf; class ProcessProber { public: ProcessProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb, std::function check_cb); private: /** * Iterates over /proc and triggers calls to get_pid_task */ void trigger_get_pid_task(std::function periodic_cb); }; ================================================ FILE: collector/kernel/protocols/protocol_handler_base.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "protocol_handler_base.h" #include "platform/platform.h" #include "spdlog/common.h" #include "spdlog/fmt/bin_to_hex.h" #include "util/log.h" ProtocolHandlerBase::ProtocolHandlerBase(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid) : data_handler_(data_handler), key_(key), pid_(pid) {} ================================================ FILE: collector/kernel/protocols/protocol_handler_base.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include "platform/platform.h" #include "collector/kernel/bpf_src/tcp-processor/tcp_processor.h" class TCPDataHandler; class ProtocolHandlerBase { public: typedef std::shared_ptr ptr_type; ProtocolHandlerBase(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid); virtual ~ProtocolHandlerBase() {} virtual void handle_server_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) = 0; virtual void handle_client_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) = 0; virtual void set_upgrade(ptr_type upgrade) { upgrade_ = upgrade; } virtual ptr_type get_upgrade() { return upgrade_; } inline TCPDataHandler *data_handler() { return data_handler_; } inline tcp_control_key_t control_key() const { return key_; } inline u32 pid() const { return pid_; } private: TCPDataHandler *data_handler_; tcp_control_key_t key_; ptr_type upgrade_; u32 pid_; }; ================================================ FILE: collector/kernel/protocols/protocol_handler_http.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "protocol_handler_http.h" #include "platform/platform.h" #include "protocol_tools.h" #include "spdlog/common.h" #include "spdlog/fmt/bin_to_hex.h" #include "util/log.h" ProtocolHandler_HTTP::ProtocolHandler_HTTP(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid) : ProtocolHandlerBase(data_handler, key, pid), request_timestamp_(0), response_timestamp_(0), client_stream_position_(0), server_stream_position_(0), client_state_(CLIENT_STATE::START), server_state_(SERVER_STATE::START), http_version_major_(0), http_version_minor_(0), http_code_(0) { LOG::debug_in(AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::$ctor(sk={:x})", control_key().sk); } ProtocolHandler_HTTP::~ProtocolHandler_HTTP() { LOG::debug_in(AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::$dtor(sk={:x})", control_key().sk); } void ProtocolHandler_HTTP::transition(CLIENT_STATE new_state) { client_state_ = new_state; } void ProtocolHandler_HTTP::transition(SERVER_STATE new_state) { server_state_ = new_state; } void ProtocolHandler_HTTP::handle_server_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_server_data(sk={:x}): tstamp={}, offset={}, is_recv={}, data_len={}, data:\n{}", tstamp, control_key().sk, offset, stream_type_to_string(stream_type), data_len, std::string_view((const char *)data, data_len)); // HTTP server-side state machine // TODO: Add buffering if state processing can't proceed without more bytes if (offset > server_stream_position_) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_server_data: server stream skipped {} bytes from {} to {}", offset - server_stream_position_, server_stream_position_, offset); } else if (offset > server_stream_position_) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_server_data: server stream went backward {} bytes from {} to {}", server_stream_position_ - offset, server_stream_position_, offset); } while (server_state_ != SERVER_STATE::END) { switch (server_state_) { // Beginning of stream case SERVER_STATE::START: response_timestamp_ = tstamp; transition(SERVER_STATE::VERSION_AND_CODE); break; // Wait for HTTP version and code case SERVER_STATE::VERSION_AND_CODE: { if (data_len < 12) { // TODO: Eventually, buffer and just return transition(SERVER_STATE::STOP); break; } // Pattern match against HTTP/x.y if (!prefix_check(data, 12, "HTTP/") || data[6] != '.' || data[8] != ' ') { transition(SERVER_STATE::STOP); break; } http_version_major_ = char_to_number(data[5]); http_version_minor_ = char_to_number(data[7]); // Check for invalid HTTP version if (http_version_major_ == -1 || http_version_minor_ == -1) { transition(SERVER_STATE::STOP); break; } // Convert HTTP response code to integer int http_code_2 = char_to_number(data[9]); int http_code_1 = char_to_number(data[10]); int http_code_0 = char_to_number(data[11]); // Ensure each digit of HTTP response code is valid if (http_code_2 == -1 || http_code_1 == -1 || http_code_0 == -1) { transition(SERVER_STATE::STOP); break; } http_code_ = (u16)(http_code_2 * 100 + http_code_1 * 10 + http_code_0); // Submit the http response code, and latency u64 latency = response_timestamp_ - request_timestamp_; enum CLIENT_SERVER_TYPE client_server = (stream_type == ST_SEND) ? SC_SERVER : SC_CLIENT; LOG::debug_in( AgentLogKind::HTTP, "handle_http_response: tstamp={}, sk={:x}, pid={}, code={}, " "latency_ns={}, client_server={}", response_timestamp_, control_key().sk, pid(), http_code_, latency, client_server_type_to_string(client_server)); data_handler()->writer().http_response_tstamp( response_timestamp_, control_key().sk, pid(), http_code_, latency, (u8)client_server); transition(SERVER_STATE::STOP); } break; // Stop server side processing case SERVER_STATE::STOP: data_handler()->enable_stream(control_key(), stream_type, false); transition(SERVER_STATE::END); break; // End of server side processing case SERVER_STATE::END: default: LOG::error("ProtocolHandler_HTTP::handle_server_data: invalid server state"); return; } } server_stream_position_ += data_len; } void ProtocolHandler_HTTP::handle_client_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_client_data(sk={:x}): tstamp={}, offset={}, is_recv={}, data_len={}, data:\n{}", tstamp, control_key().sk, offset, stream_type_to_string(stream_type), data_len, std::string_view((const char *)data, data_len)); // HTTP client-side state machine // TODO: Add buffering if state processing can't proceed without more bytes if (offset > client_stream_position_) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_client_data: client stream skipped {} bytes from {} to {}", offset - client_stream_position_, client_stream_position_, offset); } else if (offset > client_stream_position_) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_HTTP::handle_client_data: client stream went backward {} bytes from {} to {}", client_stream_position_ - offset, client_stream_position_, offset); } while (client_state_ != CLIENT_STATE::END) { switch (client_state_) { // Beginning of stream case CLIENT_STATE::START: request_timestamp_ = tstamp; transition(CLIENT_STATE::STOP); break; // Stop client side processing case CLIENT_STATE::STOP: data_handler()->enable_stream(control_key(), stream_type, false); transition(CLIENT_STATE::END); break; // End of client side processing case CLIENT_STATE::END: default: LOG::error("ProtocolHandler_HTTP::handle_client_data: invalid client state"); return; } } client_stream_position_ += data_len; } ================================================ FILE: collector/kernel/protocols/protocol_handler_http.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include "../tcp_data_handler.h" #include "protocol_handler_base.h" class ProtocolHandler_HTTP : public ProtocolHandlerBase { private: u64 request_timestamp_; u64 response_timestamp_; u64 client_stream_position_; u64 server_stream_position_; enum class CLIENT_STATE { START, STOP, END } client_state_; enum class SERVER_STATE { START, VERSION_AND_CODE, STOP, END } server_state_; int http_version_major_; int http_version_minor_; int http_code_; void transition(CLIENT_STATE new_state); void transition(SERVER_STATE new_state); public: ProtocolHandler_HTTP(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid); virtual ~ProtocolHandler_HTTP(); virtual void handle_server_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) override; virtual void handle_client_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) override; }; ================================================ FILE: collector/kernel/protocols/protocol_handler_unknown.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include "protocol_handler_unknown.h" #include "absl/base/internal/endian.h" #include "platform/platform.h" #include "protocol_tools.h" #include "spdlog/common.h" #include "spdlog/fmt/bin_to_hex.h" #include "util/log.h" #include ProtocolHandler_UNKNOWN::ProtocolHandler_UNKNOWN(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid) : ProtocolHandlerBase(data_handler, key, pid) { LOG::debug_in(AgentLogKind::PROTOCOL, "ProtocolHandler_UNKNOWN::$ctor(sk={:x})", control_key().sk); available_protocols_ = TCP_PROTOCOL_MASK; } ProtocolHandler_UNKNOWN::~ProtocolHandler_UNKNOWN() { LOG::debug_in(AgentLogKind::PROTOCOL, "ProtocolHandler_UNKNOWN::$dtor(sk={:x})", control_key().sk); } void ProtocolHandler_UNKNOWN::handle_server_data( u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_UNKNOWN::handle_server_data(sk={:x}): tstamp={}, offset={}, is_recv={}, data_len={}, data={}", tstamp, control_key().sk, offset, stream_type_to_string(stream_type), data_len, std::string_view((const char *)data, data_len)); // Currently protocol detection only happens with client data // XXX: when we implement other protocols, we may need to validate a server response before upgrading, potentially available_protocols_ = 0; data_handler()->enable_stream(control_key(), false); } TCP_PROTOCOL_DETECT_RESULT ProtocolHandler_UNKNOWN::detect_http(u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) { TCP_PROTOCOL_DETECT_RESULT res = TPD_UNKNOWN; // XXX: This is not complete. Eventually we should implement buffering if data_len is less than 4, since // XXX: in theory 3 bytes could show up, followed by the rest of the data, and since TCP is a stream protocol. if (data_len < 4) return res; switch (U32IDX(data, 0)) { case U32CC("GET "): case U32CC("PUT "): return TPD_SUCCESS; case U32CC("HEAD"): case U32CC("POST"): if (data_len >= 5) { res = data[4] == ' ' ? TPD_SUCCESS : TPD_FAILED; } break; case U32CC("DELE"): if (data_len >= 7) { res = (data[4] == 'T' && data[5] == 'E' && data[6] == ' ') ? TPD_SUCCESS : TPD_FAILED; } break; case U32CC("CONN"): if (data_len >= 8) { res = U32IDX(data, 1) == U32CC("ECT ") ? TPD_SUCCESS : TPD_FAILED; } break; case U32CC("OPTI"): if (data_len >= 8) { res = U32IDX(data, 1) == U32CC("ONS ") ? TPD_SUCCESS : TPD_FAILED; } break; case U32CC("TRAC"): if (data_len >= 6) { res = (data[4] == 'E' && data[5] == ' ') ? TPD_SUCCESS : TPD_FAILED; } break; case U32CC("PATC"): if (data_len >= 6) { res = (data[4] == 'H' && data[5] == ' ') ? TPD_SUCCESS : TPD_FAILED; } break; default: res = TPD_FAILED; break; } return res; } void ProtocolHandler_UNKNOWN::handle_client_data( u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) { LOG::debug_in( AgentLogKind::PROTOCOL, "ProtocolHandler_UNKNOWN::handle_client_data(sk={:x}): tstamp={}, offset={}, is_recv={}, data_len={}, data:\n{}", tstamp, control_key().sk, offset, stream_type_to_string(stream_type), data_len, std::string_view((const char *)data, data_len)); // All the client-side protocol detection routines we want to run std::list> client_protocols = { {TCPPROTO_HTTP, &ProtocolHandler_UNKNOWN::detect_http}, //{TCPPROTO_MYSQL, &ProtocolHandler_UNKNOWN::detect_mysql}, }; // Run the available detection routines for (auto it : client_protocols) { int tcpproto = it.first; PROTOCOL_DETECT_FUNC detect = it.second; if (available_protocols_ & TCP_PROTOCOL_BIT(tcpproto)) { // Run the detection if it's still qualified auto res = (this->*detect)(offset, stream_type, data, data_len); if (res == TPD_SUCCESS) { // If we -definitely- detected a particular protocol, upgrade to its handler ProtocolHandlerBase::ptr_type upgrade = data_handler()->create_protocol_handler(tcpproto, control_key(), pid()); set_upgrade(upgrade); return; } else if (res == TPD_FAILED) { // Remove any that are disqualified available_protocols_ &= ~TCP_PROTOCOL_BIT(tcpproto); } // If res==TPD_UNKNOWN, then continue checking with the next protocol } } // When all protocols are done just stop listening completely if (available_protocols_ == 0) { data_handler()->enable_stream(control_key(), false); } } ================================================ FILE: collector/kernel/protocols/protocol_handler_unknown.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include "../tcp_data_handler.h" #include "protocol_handler_base.h" class ProtocolHandler_UNKNOWN : public ProtocolHandlerBase { public: ProtocolHandler_UNKNOWN(TCPDataHandler *data_handler, const tcp_control_key_t &key, u32 pid); virtual ~ProtocolHandler_UNKNOWN(); virtual void handle_server_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) override; virtual void handle_client_data(u64 tstamp, u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len) override; protected: typedef TCP_PROTOCOL_DETECT_RESULT (ProtocolHandler_UNKNOWN::*PROTOCOL_DETECT_FUNC)( u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len); TCP_PROTOCOL_DETECT_RESULT detect_http(u64 offset, STREAM_TYPE stream_type, const u8 *data, size_t data_len); private: u32 available_protocols_; }; ================================================ FILE: collector/kernel/protocols/protocol_tools.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once inline bool prefix_check(const void *haystack, const size_t haystack_len, const char *needle) { size_t needle_len = strlen(needle); return (haystack_len >= needle_len && memcmp(haystack, needle, needle_len) == 0); } inline int char_to_number(char x) { if (x < '0' || x > '9') return -1; return (int)(x - '0'); } #define U32IDX(X, N) (absl::little_endian::FromHost32(((u32 *)(X))[(N)])) #define _U32CH(X, N) (((u32)((X)[(N)])) << (8 * (N))) #define U32CC(X) (_U32CH(X, 0) | _U32CH(X, 1) | _U32CH(X, 2) | _U32CH(X, 3)) ================================================ FILE: collector/kernel/socket_prober.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include #include static constexpr u32 periodic_cb_mask = 0x3f; SocketProber::SocketProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb, std::function check_cb, logging::Logger &log) : log_(log) { // END // NOTE: Covers all protocols probe_handler.start_probe(skel, "on_security_sk_free", "security_sk_free"); // inet END probe_handler.start_probe(skel, "on_inet_release", "inet_release"); probe_handler.start_kretprobe(skel, "onret_inet_release", "inet_release"); // CHANGE OF STATE probe_handler.start_probe(skel, "on_tcp_connect", "tcp_connect"); probe_handler.start_probe(skel, "on_inet_csk_listen_start", "inet_csk_listen_start"); // START probe_handler.start_probe(skel, "on_tcp_init_sock", "tcp_init_sock"); // NOTE: these probes also sends out state information probe_handler.start_kretprobe(skel, "onret_inet_csk_accept", "inet_csk_accept"); probe_handler.start_probe(skel, "on_inet_csk_accept", "inet_csk_accept"); // UDP START probe_handler.start_kretprobe(skel, "onret_udp_v4_get_port", "udp_v4_get_port"); probe_handler.start_kretprobe(skel, "onret_udp_v6_get_port", "udp_v6_get_port"); probe_handler.start_probe(skel, "on_udp_v4_get_port", "udp_v4_get_port"); probe_handler.start_probe(skel, "on_udp_v6_get_port", "udp_v6_get_port"); // EXISTING probe_handler.start_probe(skel, "on_tcp4_seq_show", "tcp4_seq_show"); probe_handler.start_probe(skel, "on_tcp6_seq_show", "tcp6_seq_show"); probe_handler.start_probe(skel, "on_udp4_seq_show", "udp4_seq_show"); probe_handler.start_probe(skel, "on_udp6_seq_show", "udp6_seq_show"); periodic_cb(); check_cb("socket prober startup"); /* First step: fill up the "seen_inodes" bpf hashmap: inode -> pid */ struct bpf_map *seen_inodes_map = probe_handler.get_bpf_map(skel, "seen_inodes"); int map_fd = bpf_map__fd(seen_inodes_map); // Clear the map u32 key, next_key; while (bpf_map_get_next_key(map_fd, &key, &next_key) == 0) { bpf_map_delete_elem(map_fd, &next_key); key = next_key; } periodic_cb(); check_cb("clear inode table"); fill_inode_to_pid_map(map_fd, periodic_cb); check_cb("fill_inode_to_pid_map()"); // now iterate over processes again but look through network namespaces // for new ns, read tcp and tcp6. this will trigger tcp46_seq_show trigger_seq_show(periodic_cb); check_cb("trigger_seq_show()"); /* can remove existing now */ probe_handler.cleanup_probe("tcp4_seq_show"); periodic_cb(); check_cb("socket prober cleanup (1)"); probe_handler.cleanup_probe("tcp6_seq_show"); periodic_cb(); check_cb("socket prober cleanup (2)"); probe_handler.cleanup_probe("udp4_seq_show"); periodic_cb(); check_cb("socket prober cleanup (3)"); probe_handler.cleanup_probe("udp6_seq_show"); periodic_cb(); check_cb("socket prober cleanup (4)"); } void SocketProber::fill_inode_to_pid_map(int map_fd, std::function periodic_cb) { // iterate over /proc/ ProcReader proc_reader; u32 proc_count = 0; u32 n_update_failures = 0; while (proc_reader.next()) { // every few procs, call periodic_cb, in case a lot of them are skipped if (((++proc_count) & periodic_cb_mask) == 0) periodic_cb(); if (!proc_reader.is_pid()) continue; // skip this entry if this wasn't a pid directory int pid = proc_reader.get_pid(); FDReader fd_reader(pid); int status = fd_reader.open_task_dir(); if (status) { LOG::trace_in(AgentLogKind::SOCKET, "skipping entry because task_dir couldn't be opened pid={}", pid); continue; // skip this entry because task_dir couldn't be opened } // for each fd of this pid. status = fd_reader.open_fd_dir(); if (status) { LOG::trace_in(AgentLogKind::SOCKET, "skipping entry because fd_dir couldn't be opened pid={}", pid); continue; // skip this pid if fd_dir couldn't be opened } u32 inode_count = 0; while (!fd_reader.next_fd()) { int ino = fd_reader.get_inode(); if (ino > 0) { u32 key = (u32)ino; u32 lookup_pid = 0; int lookup_result = bpf_map_lookup_elem(map_fd, &key, &lookup_pid); if (lookup_result == 0) { LOG::trace_in( AgentLogKind::SOCKET, "Duplicate file descriptor for pid={}, ino={} (lookup_pid={})", pid, ino, lookup_pid); continue; } int update_result = bpf_map_update_elem(map_fd, &key, &pid, 0); if (update_result != 0) { // log at most 10 times if (++n_update_failures < 10) { LOG::debug("Error updating hash_map: {} - {}", update_result, strerror(errno)); } continue; } LOG::trace_in(AgentLogKind::SOCKET, "Added inode to hash_map: pid={}, ino={}", pid, ino); } // every few inodes, call periodic_cb if (((++inode_count) & periodic_cb_mask) == 0) periodic_cb(); } // call periodic_cb for every pid periodic_cb(); } if (n_update_failures != 0) { log_.warn("Recovering existing socket inodes got {} total update failures", n_update_failures); } } void SocketProber::trigger_seq_show(std::function periodic_cb) { ProcReader proc_reader; std::set done_network_namespaces; // iterate over /proc/ while (proc_reader.next()) { periodic_cb(); if (!proc_reader.is_pid()) { continue; // skip this entry if this wasn't a pid directory } int pid = proc_reader.get_pid(); int network_namespace = get_network_namespace(pid); if (network_namespace == -1) continue; // something went wrong on this pid so skip to the next one /* if we've seen this namespace, don't re-process */ auto ns_it = done_network_namespaces.find(network_namespace); if (ns_it != done_network_namespaces.end()) continue; /* new network namespace -- process it */ done_network_namespaces.insert(network_namespace); read_proc_net_tcp("/proc/" + std::to_string(pid) + "/net/tcp", periodic_cb); read_proc_net_tcp("/proc/" + std::to_string(pid) + "/net/tcp6", periodic_cb); read_proc_net_udp("/proc/" + std::to_string(pid) + "/net/udp", periodic_cb); read_proc_net_udp("/proc/" + std::to_string(pid) + "/net/udp6", periodic_cb); } periodic_cb(); } void SocketProber::read_proc_net_tcp(const std::string &filename, std::function periodic_cb) { ProcNetReader proc_net_reader(filename); u32 sk_count = 0; while (proc_net_reader.next()) { // every few sk's, call periodic_cb if (((++sk_count) & periodic_cb_mask) == 0) periodic_cb(); // u64 sk_p = proc_net_reader.get_sk(); // int sk_state = proc_net_reader.get_sk_state(); // if ((sk_state != 1) && (sk_state != 10)) { // LOG::trace_in(AgentLogKind::SOCKET, "sk {:x} state not listen/established: {}", sk_p, sk_state); // continue; // } // int sk_ino = proc_net_reader.get_ino(); // if (sk_ino == 0) { // LOG::debug("sk {:x} sk_ino was 0: {}", sk_p, sk_ino); // continue; // skip if ino=0 (sk will already be closed) //} /* TODO: log? */ // std::cout << "successful sk: " << sk_p << "\t sk_ino: " << sk_ino << // std::endl; } } void SocketProber::read_proc_net_udp(const std::string &filename, std::function periodic_cb) { ProcNetReader proc_net_reader(filename); u32 sk_count = 0; while (proc_net_reader.next()) { /* just iterate to get udp sockets in udp_seq_show */ // every few sk's, call periodic_cb if (((++sk_count) & periodic_cb_mask) == 0) periodic_cb(); } } int SocketProber::get_network_namespace(int pid) { char link[64]; int network_namespace; snprintf(link, sizeof(link), "/proc/%d/ns/net", pid); char link_content[32]; int info_len = readlink(link, link_content, sizeof(link_content) - 1); if (info_len == -1) // TODO: add logging return -1; link_content[info_len] = '\0'; if (strncmp(link_content, "net:[", strlen("net:["))) // TODO: add logging return -1; // throw std::runtime_error("get_network_namespace: readlink should // start with net:["); sscanf(link_content, "net:[%u]", &network_namespace); return network_namespace; } ================================================ FILE: collector/kernel/socket_prober.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include /* forward declarations */ struct render_bpf_bpf; class ProbeHandler; /** * Adds BPF probes for new and existing sockets, and iterates through existing * sockets, to obtain an up-to-date view of system sockets */ class SocketProber { public: /** * C'tor * * @param probe_handler: a ProbeHandler where new probes can be registered * @param bpf_module: the module from the bpf source code * @param periodic_cb: a callback to be called every once in a while, to * allow user to e.g., flush rings */ SocketProber( ProbeHandler &probe_handler, struct render_bpf_bpf *skel, std::function periodic_cb, std::function check_cb, logging::Logger &log); private: /** * Fills the given map with a mapping of inode->pid of existing sockets * * @param map: the inode->pid map to fill * @param periodic_cb: callback to call after doing some work. */ void fill_inode_to_pid_map(int map_fd, std::function periodic_cb); /** * Iterates through proc, and triggers the corresponding seq_show functions * for all supported types of existing sockets by reading proc namespaces * * @param periodic_cb: callback to call after doing some work. */ void trigger_seq_show(std::function periodic_cb); /** * Reads a file in /proc//net/{tcp,tcp6} * * @param filename: the file to read * @param periodic_cb: callback to call after doing some work. */ void read_proc_net_tcp(const std::string &filename, std::function periodic_cb); /** * Reads a file in /proc//net/{udp,udp6} * * @param filename: the file to read * @param periodic_cb: callback to call after doing some work. */ void read_proc_net_udp(const std::string &filename, std::function periodic_cb); /** * Returns the network namespace the pid lives in, by reading /proc * * @param pid: the pid to check * @returns namespace ID on success, -1 on failure. */ int get_network_namespace(int pid); private: logging::Logger &log_; }; ================================================ FILE: collector/kernel/socket_table.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include /** * Struct used to keep connection statistics to be sent periodically */ struct tcp_statistics { u64 diff_bytes_acked = 0; u32 diff_delivered = 0; u32 diff_retrans = 0; u32 max_srtt = 0; u32 diff_rcv_holes = 0; u64 diff_bytes_received = 0; u32 diff_rcv_delivered = 0; u32 max_rcv_rtt = 0; /* is this statistic valid? since we cannot dequeue ourselves, this enables * code to invalidate this entry so it will be ignored */ bool valid = false; }; struct tcp_socket_entry { /* last state observed */ u64 bytes_acked = 0; u32 packets_delivered = 0; u32 packets_retrans = 0; u64 bytes_received = 0; u32 rcv_holes = 0; u32 rcv_delivered = 0; }; struct udp_statistics { bool valid = false; u32 packets = 0; u64 bytes = 0; u32 drops = 0; }; struct udp_remote_endpoint { std::array addr = {0, 0, 0, 0}; u16 port = 0; /** Address Family if address changed since last report, 0 otherwise */ u8 changed_af = 0; }; struct udp_socket_entry { /* local address info */ std::array laddr = {0, 0, 0, 0}; u16 lport = 0; bool reported = false; /* did we send to backend */ u32 pid = 0; u64 sk = 0; struct udp_remote_endpoint addrs[2] = {{}}; /* 0 for TX, 1 for RX */ }; ================================================ FILE: collector/kernel/tcp_data_handler.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include "spdlog/common.h" #include "spdlog/fmt/bin_to_hex.h" #include #include #include #include #include #include #include #include #include #include #include #include "protocols/protocol_handler_base.h" #include "protocols/protocol_handler_http.h" #include "protocols/protocol_handler_unknown.h" bool TCPDataHandler::tcp_control_key_t_comparator::operator()(const tcp_control_key_t &a, const tcp_control_key_t &b) const { return a.sk < b.sk; } TCPDataHandler::TCPDataHandler( uv_loop_t &loop, ProbeHandler &probe_handler, struct render_bpf_bpf *skel, ::ebpf_net::ingest::Writer &writer, PerfContainer &container, logging::Logger &log) : loop_(loop), skel_(skel), writer_(writer), container_(container), log_(log) { // Get tcp control hash table map file descriptor struct bpf_map *tcp_control_map = probe_handler.get_bpf_map(skel_, "_tcp_control"); tcp_control_map_fd_ = bpf_map__fd(tcp_control_map); } TCPDataHandler::~TCPDataHandler() {} std::shared_ptr TCPDataHandler::create_protocol_handler(int protocol, const tcp_control_key_t &key, u32 pid) { switch (protocol) { case TCPPROTO_HTTP: return std::make_shared(this, key, pid); case TCPPROTO_UNKNOWN: return std::make_shared(this, key, pid); default: throw std::runtime_error("invalid protocol"); } return nullptr; } void TCPDataHandler::upgrade_protocol_handler( std::shared_ptr new_handler, std::shared_ptr original_handler) { tcp_control_key_t key = original_handler->control_key(); #ifndef NDEBUG tcp_control_key_t newkey = new_handler->control_key(); assert(key.sk == newkey.sk); #endif auto it = protocol_handlers_.find(key); if (it == protocol_handlers_.end()) { throw std::runtime_error("update from invalid protocol"); } it->second = new_handler; } // toggle enabling one side of a stream void TCPDataHandler::enable_stream(const tcp_control_key_t &key, STREAM_TYPE stream_type, bool enable) { // Get the file descriptor for the tcp control map int fd = tcp_control_map_fd_; // Get the key pointer in a form that bpf can accept void *keyptr = const_cast(static_cast(&key)); // Get the previous value tcp_control_value_t value; int err; if ((err = bpf_map_lookup_elem(fd, keyptr, &value)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, lookup failed, sk={:x}, err={}", key.sk, err); return; } // Change the desired values value.streams[stream_type].enable = enable; // Update the value if it is still in the table if ((err = bpf_map_update_elem(fd, keyptr, &value, BPF_EXIST)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, update failed, sk={:x}, err={}", key.sk, err); return; } LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream: sk={}, stream_type={}, enable={}", key.sk, stream_type_to_string(stream_type), enable ? "true" : "false"); } // toggle enabling both sides of a stream void TCPDataHandler::enable_stream(const tcp_control_key_t &key, bool enable) { // Get the file descriptor for the tcp control map int fd = tcp_control_map_fd_; // Get the key pointer in a form that bpf can accept void *keyptr = const_cast(static_cast(&key)); // Get the previous value tcp_control_value_t value; int err; if ((err = bpf_map_lookup_elem(fd, keyptr, &value)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, lookup failed, sk={:x}, err={}", key.sk, err); return; } // Change the desired values value.streams[ST_SEND].enable = enable; value.streams[ST_RECV].enable = enable; // Update the value if it is still in the table if ((err = bpf_map_update_elem(fd, keyptr, &value, BPF_EXIST)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, update failed, sk={:x}, err={}", key.sk, err); return; } LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream: sk={:x}, stream_type=BOTH, enable={}", key.sk, enable ? "true" : "false"); } void TCPDataHandler::update_stream_start(const tcp_control_key_t &key, STREAM_TYPE stream_type, u64 start) { // Get the file descriptor for the tcp control map int fd = tcp_control_map_fd_; // Get the key pointer in a form that bpf can accept void *keyptr = const_cast(static_cast(&key)); // Get the previous value tcp_control_value_t value; int err; if ((err = bpf_map_lookup_elem(fd, keyptr, &value)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, lookup failed, sk={:x}, err={}", key.sk, err); return; } // Change the desired values value.streams[stream_type].start = start; // Update the value if it is still in the table if ((err = bpf_map_update_elem(fd, keyptr, &value, BPF_EXIST)) < 0) { // It's okay for this to fail, it means that BPF deleted the tcp connection record // before this got called, which means we should do nothing in this case LOG::debug_in( AgentLogKind::PROTOCOL, "enable_stream called on non-existing key, update failed, sk={:x}, err={}", key.sk, err); return; } LOG::debug_in( AgentLogKind::PROTOCOL, "update_stream_start: sk={:x}, stream_type={}, start={}", key.sk, stream_type_to_string(stream_type), start); } void TCPDataHandler::process( size_t idx, u64 tstamp, u64 sk, u32 pid, u32 length, u64 offset, STREAM_TYPE stream_type, CLIENT_SERVER_TYPE client_server) { // Send to the appropriate protocol handler tcp_control_key_t key{.sk = sk}; // find the appropriate protocol handler auto phiter = protocol_handlers_.find(key); if (phiter == protocol_handlers_.end()) { // data for new socket, so create a protocol handler for it auto ret = protocol_handlers_.insert(std::make_pair(key, create_protocol_handler(TCPPROTO_UNKNOWN, key, pid))); phiter = ret.first; } // Get the data ring for the same cpu as the control ring // we're on to ensure we get the right data PerfRing &ring = container_.data_ring(idx); u32 length_remaining = length; u64 current_offset = offset; while (length_remaining > 0) { auto ring_size = ring.peek_size(); if (ring_size == -ENOENT) { LOG::debug_in(AgentLogKind::PROTOCOL, "TCPDataHandler::process: ring buffer is empty"); break; } // If we have an sample, process it // peek_type assumes ring is non-empty int type = ring.peek_type(); if (type == PERF_RECORD_SAMPLE) { u32 padded_chunk_length = ring_size; // PERF_SAMPLE_RAW adds 32 bits of length per documentation of perf_event_open const unsigned int min_length = sizeof(u32); // round up max_length to a multiple of 8 bytes for padding const unsigned int max_length = ((sizeof(u32) + DATA_CHANNEL_CHUNK_MAX) + 7) & ~7; // Ensure we don't overflow if (padded_chunk_length < min_length) { log_.error("got message < sizeof header: padded_chunk_length({}) < min_length({})", padded_chunk_length, min_length); return; } if (padded_chunk_length > max_length) { log_.error("got message > sizeof message: padded_chunk_length({}) > max_length({})", padded_chunk_length, max_length); return; } // process tcp data LOG::debug_in(AgentLogKind::PROTOCOL, "tcp_data_handler: processing data (padded_chunk_length={})", padded_chunk_length); char buf[max_length]; /* copy into buffer */ ring.peek_copy(buf, 0, padded_chunk_length); /* release the element */ ring.pop(); // get pointer to data and length of data data_channel_header_t *header = (data_channel_header_t *)(buf + sizeof(u32)); const u8 *data = (const u8 *)(buf + sizeof(u32) + sizeof(data_channel_header_t)); u32 data_len = header->length; // make sure we don't read past end const unsigned int chunk_length = data_len + sizeof(u32) + sizeof(data_channel_header_t); if (chunk_length > padded_chunk_length) { log_.error( "got chunk_length > padded_chunk_length: chunk_length({}) > padded_chunk_length({})", chunk_length, padded_chunk_length); return; } // make sure our read won't take us past the length remaining on this data if (data_len > length_remaining) { // there is more data in the buffer than we need // just process the remaining length data_len = length_remaining; } // process the data with the protocol handler // possibly upgrading the handler ProtocolHandlerBase::ptr_type phb = phiter->second; bool do_upgrade; do { do_upgrade = false; if (client_server == SC_SERVER) { phb->handle_server_data(tstamp, current_offset, stream_type, data, data_len); } else { phb->handle_client_data(tstamp, current_offset, stream_type, data, data_len); } ProtocolHandlerBase::ptr_type upgrade = phb->get_upgrade(); if (upgrade) { upgrade_protocol_handler(upgrade, phb); phb = upgrade; do_upgrade = true; } } while (do_upgrade); // offset for the next chunk current_offset += data_len; length_remaining -= data_len; } else if (type == PERF_RECORD_LOST) { u64 n_lost = ring.peek_aligned_u64(sizeof(u64)); ring.pop(); lost_record_total_count_ = lost_record_total_count_ + n_lost; log_.warn("tcp_data_handler: lost {} records ({} total)", n_lost, lost_record_total_count_); } else { log_.error("tcp_data_handler: unexpected record type {}", type); // skip this one ring.pop(); } } } void TCPDataHandler::handle_close_socket(u64 sk) { // Send to the appropriate protocol handler tcp_control_key_t key{ .sk = sk, }; // end-of-socket auto phiter = protocol_handlers_.find(key); if (phiter == protocol_handlers_.end()) { // This is okay since sockets that send no data won't have a protocol handler // LOG::debug_in(AgentLogKind::PROTOCOL, "missing protocol handler for key(sk={:x})", key.sk); } else { // remove protocol handler at end of socket protocol_handlers_.erase(phiter); } } ================================================ FILE: collector/kernel/tcp_data_handler.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include "collector/agent_log.h" #include "collector/kernel/bpf_src/tcp-processor/tcp_processor.h" #include "collector/kernel/perf_reader.h" #include "protocols/protocol_handler_base.h" /* forward declarations */ struct render_bpf_bpf; class ProbeHandler; class TCPDataHandler { public: /** * c'tor */ TCPDataHandler( uv_loop_t &loop, ProbeHandler &probe_handler, struct render_bpf_bpf *skel, ::ebpf_net::ingest::Writer &writer, PerfContainer &container, logging::Logger &log); /** * d'tor */ virtual ~TCPDataHandler(); // Data processing entrypoint void process( size_t idx, u64 tstamp, u64 sk, u32 pid, u32 length, u64 offset, STREAM_TYPE stream_type, CLIENT_SERVER_TYPE client_server); void handle_close_socket(u64 sk); // Output inline ::ebpf_net::ingest::Writer &writer() { return writer_; } // tcp kernel->userland throttling control backchannel void enable_stream(const tcp_control_key_t &key, STREAM_TYPE stream_type, bool enable); void enable_stream(const tcp_control_key_t &key, bool enable); void update_stream_start(const tcp_control_key_t &key, STREAM_TYPE stream_type, u64 start); // create a new protocol handler, possibly for upgrade ProtocolHandlerBase::ptr_type create_protocol_handler(int protocol, const tcp_control_key_t &key, u32 pid); protected: void upgrade_protocol_handler(ProtocolHandlerBase::ptr_type new_handler, ProtocolHandlerBase::ptr_type original_handler); struct tcp_control_key_t_comparator { bool operator()(const tcp_control_key_t &a, const tcp_control_key_t &b) const; }; protected: uv_loop_t &loop_; struct render_bpf_bpf *skel_; ::ebpf_net::ingest::Writer &writer_; PerfContainer &container_; u64 lost_record_total_count_ = 0; std::map, tcp_control_key_t_comparator> protocol_handlers_; int tcp_control_map_fd_; logging::Logger &log_; }; ================================================ FILE: collector/kernel/troubleshoot_item.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME TroubleshootItem #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(none, 0, "") \ X(bpf_load_probes_failed, 1, "") \ X(operation_not_permitted, 2, "") \ X(permission_denied, 3, "") \ X(unexpected_exception, 4, "") #define ENUM_DEFAULT none #include ================================================ FILE: collector/kernel/troubleshooting.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include #include #include #include static constexpr auto EXIT_SLEEP_GRACE_PERIOD_DEFAULT = 60s; static constexpr auto EXIT_SLEEP_FOREVER = std::chrono::seconds::max(); void close_agent(int exit_code, std::function flush_and_close, std::chrono::seconds exit_sleep_sec) { if (flush_and_close) { flush_and_close(); } #if NDEBUG // delay exit for release builds but not for debug builds to prevent test failures from blocking std::this_thread::sleep_for(exit_sleep_sec); #endif exit(exit_code); } void print_troubleshooting_message_and_exit( HostInfo const &info, TroubleshootItem item, std::exception const &e, std::optional> log, std::function flush_and_close) { if (item == TroubleshootItem::none) { return; } auto const item_name = to_string(item); auto const os_name = to_string(info.os); auto const distro_name = to_string(static_cast(info.os_flavor)); auto const headers_source = to_string(info.kernel_headers_source); auto const exception_message = std::string(e.what()); auto const item_fmt = fmt::format( "troubleshoot item {} (os={},flavor={},headers_src={},kernel={}): {}", item_name, os_name, distro_name, headers_source, info.kernel_version, exception_message); if (log) { log->get().error(item_fmt); } else { LOG::error(item_fmt); } auto exit_sleep_sec = EXIT_SLEEP_GRACE_PERIOD_DEFAULT; switch (item) { case TroubleshootItem::bpf_load_probes_failed: { std::cout << fmt::format( R"TROUBLESHOOTING( Failed to load eBPF probes for the Linux distro '{}' running kernel version {}. {} )TROUBLESHOOTING", distro_name, info.kernel_version, item_fmt); break; } case TroubleshootItem::operation_not_permitted: { exit_sleep_sec = EXIT_SLEEP_FOREVER; std::cout << fmt::format( R"TROUBLESHOOTING( Insufficient permissions to perform a priviliged operation. Priviliged operations include mounting debugfs, loading eBPF code and probes, etc. Make sure to run as privileged user, and/or with --privileged flag or equivalant. {} Blocking to avoid failure retry loop. )TROUBLESHOOTING", item_fmt); break; } case TroubleshootItem::permission_denied: { exit_sleep_sec = EXIT_SLEEP_FOREVER; std::cout << fmt::format( R"TROUBLESHOOTING( Operation failed with permission denied. This can occur due to SELinux policy enforcement. If SELinux is enabled, make sure to first run the provided script to apply an SELinux policy allowing eBPF operations. {} Blocking to avoid failure retry loop. )TROUBLESHOOTING", item_fmt); break; } case TroubleshootItem::unexpected_exception: { std::cout << fmt::format( R"TROUBLESHOOTING( Unexpected exception. {} )TROUBLESHOOTING", item_fmt); break; } default: { std::cout << R"TROUBLESHOOTING( Unknown error happened in the Kernel Collector. )TROUBLESHOOTING"; break; } } std::cout << std::endl; std::cout.flush(); close_agent(-1, flush_and_close, exit_sleep_sec); } ================================================ FILE: collector/kernel/troubleshooting.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include /** * Prints a troubleshooting message for the given item. * * In case the item is unrecoverable, calls `exit()` and doesn't return. */ void print_troubleshooting_message_and_exit( HostInfo const &info, TroubleshootItem item, std::exception const &e, std::optional> log = std::nullopt, std::function flush_and_close = nullptr); ================================================ FILE: collector/server_command.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include enum class ServerCommand : u64 { NONE, DISABLE_SEND = 0xe5c94272c6a3028ful, }; ================================================ FILE: common/client_server_type.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once // This enum is used in BPF, Agent, and Server. Must be a normal enum because BPF/BCC is C, not C++. // Server/client type enum enum CLIENT_SERVER_TYPE { SC_CLIENT = 0, SC_SERVER = 1, }; #ifndef _PROCESSING_BPF inline const char *client_server_type_to_string(enum CLIENT_SERVER_TYPE client_server) { switch (client_server) { case SC_CLIENT: return "CLIENT"; case SC_SERVER: return "SERVER"; default: break; } throw std::runtime_error("invalid CLIENT_SERVER_TYPE value"); } #endif ================================================ FILE: common/client_type.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME ClientType #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(kernel, 1, "") \ X(cloud, 2, "") \ X(k8s, 3, "") \ X(ingest, 4, "") \ X(matching, 5, "") \ X(aggregation, 6, "") \ X(liveness_probe, 7, "") \ X(readiness_probe, 8, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/cloud_platform.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME CloudPlatform #define ENUM_TYPE std::uint16_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(aws, 1, "") \ X(gcp, 2, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/collected_blob_type.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME CollectedBlobType #define ENUM_TYPE std::uint16_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(proc_pid_stat, 1, "") \ X(proc_pid_io, 2, "") \ X(proc_pid_status, 3, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/collector_status.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME CollectorStatus #define ENUM_NAMESPACE collector #define ENUM_TYPE std::uint16_t #define ENUM_ELEMENTS(X) \ X(healthy, 0, "") \ X(unknown, 1, "") \ X(aws_describe_regions_error, 2, "") \ X(aws_describe_network_interfaces_error, 3, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/component.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAMESPACE common #define ENUM_NAME Component #define ENUM_TYPE std::uint16_t #define ENUM_ELEMENTS(X) \ X(none, 0, "") \ X(debug_data, 1, "") #define ENUM_DEFAULT none #include ================================================ FILE: common/constants.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include namespace kernel { constexpr u16 MIN_CGROUP_CPU_SHARES = 2; constexpr u16 MAX_CGROUP_CPU_SHARES = 1024; constexpr u32 DEFAULT_CGROUP_QUOTA = 100'000; } // namespace kernel namespace versions { extern const VersionInfo release; } // namespace versions ================================================ FILE: common/host_info.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include struct HostInfo { OperatingSystem const os; std::uint8_t const os_flavor; std::string const os_version; KernelHeadersSource const kernel_headers_source; std::string const kernel_version; std::string const hostname; }; ================================================ FILE: common/http_status_code.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #define ENUM_NAME HttpStatusCode #define ENUM_TYPE std::uint16_t #define ENUM_ELEMENTS(X) \ \ /* informational */ \ X(Continue, 100, "") \ X(SwitchingProtocols, 101, "") \ X(Processing, 102, "") \ X(EarlyHints, 103, "") \ \ /* success */ \ X(OK, 200, "") \ X(Created, 201, "") \ X(Accepted, 202, "") \ X(NonAuthoritativeInformation, 203, "") \ X(NoContent, 204, "") \ X(ResetContent, 205, "") \ X(PartialContent, 206, "") \ X(MultiStatus, 207, "") \ X(AlreadyReported, 208, "") \ X(InstanceManipulationUsed, 226, "") \ \ /* redirection */ \ X(MultipleChoices, 300, "") \ X(MovedPermanently, 301, "") \ X(Found, 302, "") \ X(SeeOther, 303, "") \ X(NotModified, 304, "") \ X(UseProxy, 305, "") \ X(SwitchProxy, 306, "") \ X(TemporaryRedirect, 307, "") \ X(PermanentRedirect, 308, "") \ \ /* client error */ \ X(BadRequest, 400, "") \ X(Unauthorized, 401, "") \ X(PaymentRequired, 402, "") \ X(Forbidden, 403, "") \ X(NotFound, 404, "") \ X(MethodNotAllowed, 405, "") \ X(NotAcceptable, 406, "") \ X(ProxyAuthenticationRequired, 407, "") \ X(RequestTimeout, 408, "") \ X(Conflict, 409, "") \ X(Gone, 410, "") \ X(LengthRequired, 411, "") \ X(PreconditionFailed, 412, "") \ X(PayloadTooLarge, 413, "") \ X(URITooLong, 414, "") \ X(UnsupportedMediaType, 415, "") \ X(RangeNotSatisfiable, 416, "") \ X(ExpectationFailed, 417, "") \ X(ImATeapot, 418, "") \ X(MisdirectedRequest, 421, "") \ X(UnprocessableEntity, 422, "") \ X(Locked, 423, "") \ X(FailedDependency, 424, "") \ X(TooEarly, 425, "") \ X(UpgradeRequired, 426, "") \ X(PreconditionRequired, 428, "") \ X(TooManyRequests, 429, "") \ X(RequestHeaderFieldsTooLarge, 431, "") \ X(UnavailableForLegalReasons, 451, "") \ \ /* server error */ \ X(InternalServerError, 500, "") \ X(NotImplemented, 501, "") \ X(BadGateway, 502, "") \ X(ServiceUnavailable, 503, "") \ X(GatewayTimeout, 504, "") \ X(HTTPVersionNotSupported, 505, "") \ X(VariantAlsoNegotiates, 506, "") \ X(InsufficientStorage, 507, "") \ X(LoopDetected, 508, "") \ X(NotExtended, 510, "") \ X(NetworkAuthenticationRequired, 511, "") #include constexpr bool is_informational_class(HttpStatusCode code) { auto const value = static_cast>(code); return value >= 100 && value < 200; } constexpr bool is_success_class(HttpStatusCode code) { auto const value = static_cast>(code); return value >= 200 && value < 300; } constexpr bool is_redirection_class(HttpStatusCode code) { auto const value = static_cast>(code); return value >= 300 && value < 400; } constexpr bool is_client_error_class(HttpStatusCode code) { auto const value = static_cast>(code); return value >= 400 && value < 500; } constexpr bool is_server_error_class(HttpStatusCode code) { auto const value = static_cast>(code); return value >= 500 && value < 600; } ================================================ FILE: common/intake_encoder.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME IntakeEncoder #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) X(binary, 0, "") #define ENUM_DEFAULT binary #include inline std::string_view format_as(IntakeEncoder v) { return to_string(v); } ================================================ FILE: common/kernel_headers_source.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME KernelHeadersSource #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(pre_installed, 1, "") \ X(pre_fetched, 2, "") \ X(dont_fetch, 3, "") \ X(fetched, 4, "") \ X(libbpf, 5, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/linux_distro.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME LinuxDistro #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(debian, 1, "") \ X(ubuntu, 2, "") \ X(rhel, 3, "") \ X(centos, 4, "") \ X(amazon, 5, "") \ X(gcp_cos, 6, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/operating_system.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME OperatingSystem #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(Linux, 1, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: common/port_protocol.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #define ENUM_NAME PortProtocol #define ENUM_TYPE std::uint8_t #define ENUM_ELEMENTS(X) \ X(unknown, 0, "") \ X(TCP, 1, "") \ X(UDP, 2, "") \ X(HTTP, 3, "") \ X(PROXY, 4, "") \ X(SCTP, 5, "") #define ENUM_DEFAULT unknown #include ================================================ FILE: config/CMakeLists.txt ================================================ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 add_library( config_file STATIC config_file.cc ) target_link_libraries( config_file intake_config yamlcpp spdlog ) add_library( intake_config STATIC intake_config.cc ) target_link_libraries( intake_config render_ebpf_net_ingest_writer tcp_channel libuv-interface args_parser file_ops environment_variables ) ================================================ FILE: config/config_file.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include namespace config { ConfigFile::ConfigFile(YamlFormat, std::string const &path, FailMode fail) : intake_config_(IntakeConfig::DEFAULT_CONFIG) { if (path.empty()) { return; } try { YAML::Node yaml = YAML::LoadFile(path); if (auto labels = yaml["labels"]) { if (!labels.IsMap() && !labels.IsNull()) { LOG::warn("Ignoring 'labels' in config file: 'labels' should be a map."); } else { for (auto const &i : labels) { auto key = i.first.as(); auto value = i.second.as(); if ((key.length() > 20) || (value.length() > 40)) { LOG::warn( "Ignoring label '{}': '{}'. " "key and value lengths must be max 20, 40 chars.", key, value); continue; } LOG::info("Node label has been set in config: '{}':'{}'", key, value); labels_[std::move(key)] = std::move(value); } } } else { LOG::info("No \"labels\" were specified."); } if (auto intake = yaml["intake"]) { if (!intake.IsMap() && !intake.IsNull()) { LOG::warn("Ignoring 'intake' in config file: 'intake' should be a map."); } else { auto host = intake["host"].as(); auto port = intake["port"].as(); intake_config_.host(host); intake_config_.port(port); } } } catch (std::exception const &e) { LOG::error("Config file at {} could not be loaded.", path); if (fail == FailMode::exception) { throw e; } } } } // namespace config ================================================ FILE: config/config_file.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include namespace config { class ConfigFile { public: struct YamlFormat {}; enum class FailMode { silent, exception }; ConfigFile(YamlFormat, std::string const &path, FailMode fail = FailMode::exception); using LabelsMap = std::map; LabelsMap const &labels() const { return labels_; } LabelsMap &labels() { return labels_; } IntakeConfig const &intake_config() const { return intake_config_; } private: LabelsMap labels_; IntakeConfig intake_config_; }; } // namespace config ================================================ FILE: config/intake_config.cc ================================================ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 #include #include #include #include #include #include namespace config { const IntakeConfig IntakeConfig::DEFAULT_CONFIG = IntakeConfig( "127.0.0.1", // host "8000", // port "", // record_output_path IntakeEncoder::binary // encoder ); FileDescriptor IntakeConfig::create_output_record_file() const { FileDescriptor fd; LOG::debug("intake record file: `{}`", record_path_); if (!record_path_.empty()) { if (auto const error = fd.create(record_path_.c_str(), FileDescriptor::Access::write_only)) { LOG::error("failed to create intake record file at `{}`", record_path_); } else { LOG::debug("created intake record file at `{}`", record_path_); } } return fd; } std::unique_ptr IntakeConfig::make_channel(uv_loop_t &loop) const { if (host_.empty()) { throw std::invalid_argument("missing intake host value"); } if (port_.empty()) { throw std::invalid_argument("missing intake port value"); } return std::make_unique(loop, host_, port_); } void IntakeConfig::read_from_env(IntakeConfig &config) { if (std::string_view value = try_get_env_var(INTAKE_HOST_VAR); !value.empty()) { config.host_ = value; } if (std::string_view value = try_get_env_var(INTAKE_PORT_VAR); !value.empty()) { config.port_ = value; } if (std::string_view value = try_get_env_var(INTAKE_RECORD_OUTPUT_PATH_VAR); !value.empty()) { config.record_path_ = value; } if (std::string_view value = try_get_env_var(INTAKE_INTAKE_ENCODER_VAR); !value.empty()) { config.encoder_ = try_enum_from_string(value, IntakeEncoder::binary); } } IntakeConfig::ArgsHandler::ArgsHandler(cli::ArgsParser &parser) : host_(parser.add_arg( "intake-host", "IP address or host name of the reducer to which telemetry is to be sent")), port_(parser.add_arg( "intake-port", "TCP port number on which the reducer is listening for collector connections")), encoder_(parser.add_arg( "intake-encoder", "Chooses the intake encoder to use" " - this relates to the sink used to dump collected telemetry to")) {} void IntakeConfig::ArgsHandler::read_config(IntakeConfig &config) { IntakeConfig::read_from_env(config); if (host_) { config.host(*host_); } if (port_) { config.port(*port_); } if (encoder_) { config.encoder(*encoder_); } } } // namespace config ================================================ FILE: config/intake_config.h ================================================ /* * Copyright The OpenTelemetry Authors * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include #include #include #include #include #include #include #include #include #include #include namespace config { class IntakeConfig { // environment variable names used by `read_from_env()` static constexpr auto INTAKE_HOST_VAR = "EBPF_NET_INTAKE_HOST"; static constexpr auto INTAKE_PORT_VAR = "EBPF_NET_INTAKE_PORT"; static constexpr auto INTAKE_INTAKE_ENCODER_VAR = "EBPF_NET_INTAKE_ENCODER"; static constexpr auto INTAKE_RECORD_OUTPUT_PATH_VAR = "EBPF_NET_RECORD_INTAKE_OUTPUT_PATH"; public: static const IntakeConfig DEFAULT_CONFIG; IntakeConfig() {} /** * Constructs an intake config object. * * host: the host to connect to for intake * port: the port to connect to for intake * secondary_output: when given, path to a file into which to record all traffic sent upstream */ IntakeConfig( std::string host, std::string port, std::string record_output_path = {}, IntakeEncoder encoder = IntakeEncoder::binary) : host_(std::move(host)), port_(std::move(port)), record_path_(std::move(record_output_path)), encoder_(encoder) {} std::string const &host() const { return host_; } std::string const &port() const { return port_; } void host(std::string const &host) { host_ = host; } void port(std::string const &port) { port_ = port; } /** * If a secondary output has been set, opens or creates the output file and * returns its file descriptor. * * Otherwise, returns an invalid file descriptor. */ FileDescriptor create_output_record_file() const; void encoder(IntakeEncoder encoder) { encoder_ = encoder; } IntakeEncoder encoder() const { return encoder_; } virtual bool allow_compression() const { return encoder_ == IntakeEncoder::binary; } virtual std::unique_ptr make_channel(uv_loop_t &loop) const; std::unique_ptr<::ebpf_net::ingest::Encoder> make_encoder() const { switch (encoder_) { default: // Use default encoder. return nullptr; } } /** * Reads intake configuration from existing environment variables. * * NOTE: this function reads environment variables so it's advisable to call it * before any thread is created, given that reading/writing to environment * variables is not thread safe and we can't control 3rd party libraries. */ static void read_from_env(IntakeConfig &config); template friend Out &&operator<<(Out &&out, IntakeConfig const &config) { out << config.host_ << ':' << config.port_ << " (" << config.encoder_ << ')'; return std::forward(out); } struct ArgsHandler; private: std::string host_; std::string port_; std::string record_path_; IntakeEncoder encoder_ = IntakeEncoder::binary; }; struct IntakeConfig::ArgsHandler : cli::ArgsParser::Handler { ArgsHandler(cli::ArgsParser &parser); void read_config(IntakeConfig &config); private: cli::ArgsParser::ArgProxy host_; cli::ArgsParser::ArgProxy port_; cli::ArgsParser::ArgProxy encoder_; }; } // namespace config namespace fmt { template <> struct formatter : fmt::ostream_formatter {}; } // namespace fmt ================================================ FILE: config.h.cmake_in ================================================ #cmakedefine01 CONFIGURABLE_BPF #cmakedefine01 DEBUG_LOG #cmakedefine01 ENABLE_CODE_TIMING #cmakedefine01 USE_ADDRESS_SANITIZER ================================================ FILE: crates/build/otn_link_build.rs ================================================ pub fn run() { use std::env; // Rebuild if any of the link env vars change println!("cargo:rerun-if-env-changed=OTN_LINK_SEARCH"); println!("cargo:rerun-if-env-changed=OTN_LINK_LIBS"); println!("cargo:rerun-if-env-changed=OTN_LINK_ARGS"); if let Ok(search) = env::var("OTN_LINK_SEARCH") { for p in search.split(':').filter(|s| !s.is_empty()) { println!("cargo:rustc-link-search=native={}", p); } } if let Ok(libs) = env::var("OTN_LINK_LIBS") { for spec in libs.split(';').filter(|s| !s.is_empty()) { // spec format: kind=name (e.g., static=agentlib or dylib=stdc++) if let Some((kind, name)) = spec.split_once('=') { println!("cargo:rustc-link-lib={}={}", kind, name); } else { // default to "dylib" if kind omitted (not expected) println!("cargo:rustc-link-lib={}", spec); } } } if let Ok(args) = env::var("OTN_LINK_ARGS") { for arg in args.split(';').filter(|s| !s.is_empty()) { println!("cargo:rustc-link-arg={}", arg); } } } ================================================ FILE: crates/cloud-collector-bin/Cargo.toml ================================================ [package] name = "cloud-collector-bin" version = "0.1.0" edition = "2021" [dependencies] cloud-collector-sys = { path = "../cloud-collector-sys" } [[bin]] name = "cloud-collector" path = "src/main.rs" ================================================ FILE: crates/cloud-collector-bin/src/main.rs ================================================ fn main() { let code = cloud_collector_sys::run_with_args(std::env::args_os()); std::process::exit(code); } ================================================ FILE: crates/cloud-collector-sys/Cargo.toml ================================================ [package] name = "cloud-collector-sys" version = "0.1.0" edition = "2021" [lib] name = "cloud_collector_sys" path = "src/lib.rs" [dependencies] encoder_ebpf_net_ingest = { path = "../render/ebpf_net/ingest" } encoder_ebpf_net_cloud_collector = { path = "../render/ebpf_net/cloud_collector" } ================================================ FILE: crates/cloud-collector-sys/build.rs ================================================ include!("../build/otn_link_build.rs"); fn main() { run(); } ================================================ FILE: crates/cloud-collector-sys/src/lib.rs ================================================ use std::os::raw::{c_char, c_int}; // Ensure encoder crates are linked so C++ static libs can resolve their // extern "C" symbols via our dependency graph. #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_cloud_collector; #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_ingest; extern "C" { pub fn otn_cloud_collector_main(argc: c_int, argv: *const *const c_char) -> c_int; } pub fn run_with_args(args: I) -> i32 where I: IntoIterator, S: Into, { use std::ffi::CString; let mut c_strings: Vec = Vec::new(); for arg in args { let s: std::ffi::OsString = arg.into(); let bytes = std::os::unix::ffi::OsStringExt::into_vec(s); c_strings.push(CString::new(bytes).expect("argv contains NUL byte")); } let ptrs: Vec<*const c_char> = c_strings.iter().map(|s| s.as_ptr()).collect(); unsafe { otn_cloud_collector_main(ptrs.len() as c_int, ptrs.as_ptr()) } } ================================================ FILE: crates/element-queue/Cargo.toml ================================================ [package] name = "element-queue" version = "0.1.0" edition = "2021" license = "Apache-2.0" [lib] name = "element_queue" path = "src/lib.rs" ================================================ FILE: crates/element-queue/src/layout.rs ================================================ use core::ptr; use core::ptr::{addr_of, addr_of_mut}; /// C-compatible shared indices for ElementQueue contiguous layout. #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct ElementQueueShared { pub elem_head: u32, pub buf_head: u32, pub elem_tail: u32, pub buf_tail: u32, } impl ElementQueueShared { /// Initialize shared indices to zero using volatile writes to match C semantics. pub unsafe fn init(shared: *mut ElementQueueShared) { shared.init_zero() } } /// Trait providing volatile read/write accessors on raw pointers to the /// shared header. Enables pointer-style calls like `self.shared.set_buf_head(v)`. pub(crate) trait ElementQueueSharedOps { fn init_zero(self); // Volatile getters (READ_ONCE semantics) fn get_elem_head(self) -> u32; fn get_buf_head(self) -> u32; fn get_elem_tail(self) -> u32; fn get_buf_tail(self) -> u32; // Volatile setters (WRITE_ONCE semantics) fn set_elem_head(self, v: u32); fn set_buf_head(self, v: u32); fn set_elem_tail(self, v: u32); fn set_buf_tail(self, v: u32); } impl ElementQueueSharedOps for *mut ElementQueueShared { #[inline] fn init_zero(self) { unsafe { ptr::write_volatile(addr_of_mut!((*self).elem_head), 0); ptr::write_volatile(addr_of_mut!((*self).buf_head), 0); ptr::write_volatile(addr_of_mut!((*self).elem_tail), 0); ptr::write_volatile(addr_of_mut!((*self).buf_tail), 0); } } #[inline] fn get_elem_head(self) -> u32 { unsafe { ptr::read_volatile(addr_of!((*self).elem_head)) } } #[inline] fn get_buf_head(self) -> u32 { unsafe { ptr::read_volatile(addr_of!((*self).buf_head)) } } #[inline] fn get_elem_tail(self) -> u32 { unsafe { ptr::read_volatile(addr_of!((*self).elem_tail)) } } #[inline] fn get_buf_tail(self) -> u32 { unsafe { ptr::read_volatile(addr_of!((*self).buf_tail)) } } #[inline] fn set_elem_head(self, v: u32) { unsafe { ptr::write_volatile(addr_of_mut!((*self).elem_head), v) } } #[inline] fn set_buf_head(self, v: u32) { unsafe { ptr::write_volatile(addr_of_mut!((*self).buf_head), v) } } #[inline] fn set_elem_tail(self, v: u32) { unsafe { ptr::write_volatile(addr_of_mut!((*self).elem_tail), v) } } #[inline] fn set_buf_tail(self, v: u32) { unsafe { ptr::write_volatile(addr_of_mut!((*self).buf_tail), v) } } } /// Compute the size of a contiguous element queue buffer in bytes. pub fn contig_size(n_elems: u32, buf_len: u32) -> usize { core::mem::size_of::() + (n_elems as usize) * core::mem::size_of::() + (buf_len as usize) } ================================================ FILE: crates/element-queue/src/lib.rs ================================================ #![forbid(unsafe_op_in_unsafe_fn)] /// Linux-style negative error codes preserved for parity with C implementation. use crate::layout::ElementQueueSharedOps; pub mod errno { pub const EINVAL: i32 = -22; pub const ENOSPC: i32 = -28; pub const ENOENT: i32 = -2; pub const EAGAIN: i32 = -11; pub const ENOSYS: i32 = -38; } pub mod layout; pub mod raw; // Re-export for backwards-compatibility with the previous single-module layout. pub use layout::{contig_size, ElementQueueShared}; pub use raw::{ElementQueue, EqError, ReadBatch, WriteBatch}; /// Owned contiguous storage for an element queue, similar to MemElementQueueStorage. pub struct MemElementQueueStorage { buf: Vec, n_elems: u32, buf_len: u32, } impl MemElementQueueStorage { pub fn new(n_elems: u32, buf_len: u32) -> Self { let size = contig_size(n_elems, buf_len); // Allocate with u64 alignment and round up size to u64 words let words = (size + 7) / 8; let mut buf = vec![0u64; words]; // Initialize shared header let ptr = buf.as_mut_ptr() as *mut u8; (ptr as *mut ElementQueueShared).init_zero(); Self { buf, n_elems, buf_len, } } pub fn n_elems(&self) -> u32 { self.n_elems } pub fn buf_len(&self) -> u32 { self.buf_len } pub fn data_ptr(&self) -> *mut u8 { self.buf.as_ptr() as *mut u8 } pub fn make_queue(&self) -> Result { // Safety: `self.buf` is contiguous and has the expected layout unsafe { ElementQueue::new_from_contiguous(self.n_elems, self.buf_len, self.data_ptr()) } } /// Construct a queue over an external contiguous buffer. /// Safety: `data` must have the expected layout and size. pub unsafe fn queue_from_ptr( data: *mut u8, n_elems: u32, buf_len: u32, ) -> Result { unsafe { ElementQueue::new_from_contiguous(n_elems, buf_len, data) } } } #[cfg(test)] mod tests { use super::*; #[test] fn layout_size() { let n_elems = 8u32; let buf_len = 64u32; let size = contig_size(n_elems, buf_len); assert_eq!( size, core::mem::size_of::() + (n_elems as usize) * 4 + buf_len as usize ); } #[test] fn write_read_basic() { let storage = MemElementQueueStorage::new(8, 128); let mut q = storage.make_queue().unwrap(); // Writer via guard let mut wb = q.start_write(); wb.write(5).unwrap().copy_from_slice(b"hello"); wb.write(8).unwrap().copy_from_slice(b"world!!!"); let _ = wb.finish(); // Reader via guard let rb = q.start_read(); // peek first assert_eq!(rb.peek_len().unwrap(), 5); let v1 = rb.read().unwrap(); assert_eq!(v1, b"hello"); // second let v2 = rb.read().unwrap(); assert_eq!(v2, b"world!!!"); let _ = rb.finish(); } #[test] fn wrap_around_alignment() { let storage = MemElementQueueStorage::new(8, 64); let mut q = storage.make_queue().unwrap(); // Fill close to end to force wrap let mut wb = q.start_write(); wb.write(30).unwrap().copy_from_slice(&vec![0u8; 30]); wb.write(5).unwrap().copy_from_slice(&vec![1u8; 5]); // 5 -> aligned to 8 let _ = wb.finish(); let rb = q.start_read(); let _ = rb.read().unwrap(); let second = rb.read().unwrap(); assert_eq!(second.len(), 5); assert!(second.iter().all(|&b| b == 1)); let _ = rb.finish(); } #[test] fn peek_value_u64() { let storage = MemElementQueueStorage::new(8, 128); let mut q = storage.make_queue().unwrap(); let ts: u64 = 0x1122334455667788; let mut wb = q.start_write(); // write timestamp as 8 bytes wb.write(8).unwrap().copy_from_slice(&ts.to_le_bytes()); let _ = wb.finish(); let rb = q.start_read(); let v: u64 = rb.peek_value::().unwrap(); assert_eq!(v, ts.to_le()); let got = rb.read().unwrap(); assert_eq!(got.len(), 8); let _ = rb.finish(); } } ================================================ FILE: crates/element-queue/src/raw.rs ================================================ use core::cell::Cell; use core::ptr::{self, NonNull}; use core::sync::atomic::{fence, Ordering}; use crate::errno; use crate::layout::{ElementQueueShared, ElementQueueSharedOps}; #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum EqError { InvalidArg, NoSpace, NoEntry, TryAgain, Unexpected, } impl EqError { pub fn code(self) -> i32 { use EqError::*; match self { InvalidArg => errno::EINVAL, NoSpace => errno::ENOSPC, NoEntry => errno::ENOENT, TryAgain => errno::EAGAIN, Unexpected => errno::ENOSYS, } } } /// Unsafe, low-level queue core. Mirrors the C implementation closely. /// /// Safety: Methods that dereference raw pointers encapsulate the same /// assumptions as the original C code. Public, higher-level wrappers should /// be preferred when possible. pub struct ElementQueue { // Masks (size - 1), must be power-of-two sizes. elem_mask: u32, buf_mask: u32, // Local cached indices, following batching protocol. elem_head: u32, buf_head: u32, elem_tail: u32, buf_tail: u32, // Raw pointers into contiguous storage. shared: *mut ElementQueueShared, elems: NonNull<[u32]>, data: NonNull<[u8]>, } impl core::fmt::Debug for ElementQueue { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("ElementQueue") .field("elem_mask", &self.elem_mask) .field("buf_mask", &self.buf_mask) .field("elem_head", &self.elem_head) .field("buf_head", &self.buf_head) .field("elem_tail", &self.elem_tail) .field("buf_tail", &self.buf_tail) .finish() } } impl ElementQueue { /// Create a queue over contiguous memory ([shared][elems][data]). /// /// Safety: `data` must be a valid, writable pointer to at least /// `contig_size(n_elems, buf_len)` bytes with the expected layout. pub unsafe fn new_from_contiguous( n_elems: u32, buf_len: u32, data: *mut u8, ) -> Result { if data.is_null() { return Err(EqError::InvalidArg); } if n_elems == 0 || n_elems & (n_elems - 1) != 0 { return Err(EqError::InvalidArg); } if buf_len == 0 || buf_len & (buf_len - 1) != 0 { return Err(EqError::InvalidArg); } // Layout: shared, then elem ring, then data buffer. let shared = data as *mut ElementQueueShared; // SAFETY: pointer arithmetic within the contiguous buffer layout let elems = unsafe { shared.add(1) } as *mut u32; let data_ptr = unsafe { elems.add(n_elems as usize) } as *mut u8; // Create raw slice pointers for elems and data (unsafe boundary here only). let elems_slice: *mut [u32] = ptr::slice_from_raw_parts_mut(elems, n_elems as usize); let data_slice: *mut [u8] = ptr::slice_from_raw_parts_mut(data_ptr, buf_len as usize); // Load current shared indices using volatile reads (per-field like C). let elem_head = shared.get_elem_head(); let buf_head = shared.get_buf_head(); let elem_tail = shared.get_elem_tail(); let buf_tail = shared.get_buf_tail(); Ok(ElementQueue { elem_mask: n_elems - 1, buf_mask: buf_len - 1, elem_head, buf_head, elem_tail, buf_tail, shared, elems: unsafe { NonNull::new_unchecked(elems_slice) }, data: unsafe { NonNull::new_unchecked(data_slice) }, }) } } /// Write batch guard with discard-on-drop semantics. /// /// - Holds local copies of tails which are advanced during the batch. /// - `finish(self)` commits local tails to the queue and publishes to shared. /// - Dropping without `finish()` discards local progress (no commit/publish). pub struct WriteBatch<'q> { q: &'q mut ElementQueue, elem_tail: Cell, buf_tail: Cell, } impl ElementQueue { /// Start a write batch (producer) and return a guard. /// Loads consumer-published heads to check space. pub fn start_write<'q>(&'q mut self) -> WriteBatch<'q> { // Read the heads published by consumer (volatile per-field to mirror ACCESS_ONCE). self.elem_head = self.shared.get_elem_head(); self.buf_head = self.shared.get_buf_head(); debug_assert!((self.buf_tail as i64 - self.buf_head as i64) >= 0); debug_assert!((self.elem_tail as i64 - self.elem_head as i64) >= 0); WriteBatch { elem_tail: Cell::new(self.elem_tail), buf_tail: Cell::new(self.buf_tail), q: self, } } } impl<'q> WriteBatch<'q> { pub fn write<'a>(&'a mut self, len: u32) -> Result<&'a mut [u8], EqError> { let aligned_len = (len + 7) & !7; if aligned_len > self.q.buf_mask { return Err(EqError::InvalidArg); } // Element ring full? if self.elem_tail.get().wrapping_sub(self.q.elem_head) >= (self.q.elem_mask + 1) { return Err(EqError::NoSpace); } let buf_mask = self.q.buf_mask; let buf_tail = ElementQueue::__next_offset_by_len(self.buf_tail.get(), buf_mask, aligned_len); // Enough space in data buffer? Use wrapping arithmetic to mirror C semantics. let used = buf_tail .wrapping_add(aligned_len) .wrapping_sub(self.q.buf_head); if used > buf_mask + 1 { return Err(EqError::NoSpace); } // Reserve locally: update tails self.buf_tail.set(buf_tail.wrapping_add(aligned_len)); let idx = (self.elem_tail.get() & self.q.elem_mask) as usize; // record element length in ring (visible to reader after publish) self.q.elems_mut()[idx] = len; self.elem_tail.set(self.elem_tail.get().wrapping_add(1)); let start = (buf_tail & buf_mask) as usize; let end = start + len as usize; Ok(&mut self.q.data_mut()[start..end]) } pub fn finish(self) -> &'q mut ElementQueue { // Commit local tails and publish to shared with release ordering self.q.elem_tail = self.elem_tail.get(); self.q.buf_tail = self.buf_tail.get(); fence(Ordering::Release); self.q.shared.set_buf_tail(self.q.buf_tail); self.q.shared.set_elem_tail(self.q.elem_tail); self.q } /// Move one element from `reader` to this writer within their active batches. pub fn move_from(&mut self, reader: &ReadBatch<'_>) -> Result { let src_len = reader.peek_len()? as usize; let dst = self.write(src_len as u32)?; let src = reader.read()?; debug_assert_eq!(src.len(), src_len); dst.copy_from_slice(src); Ok(src_len) } } /// Read batch guard with discard-on-drop semantics. /// /// - Holds local copies of heads which are advanced during the batch. /// - `finish(self)` commits local heads to the queue and publishes to shared. /// - Dropping without `finish()` discards local progress (no commit/publish). pub struct ReadBatch<'q> { q: &'q mut ElementQueue, elem_head: Cell, buf_head: Cell, } impl ElementQueue { /// Start a read batch (consumer) and return a guard. /// Loads writer-published tail and establishes acquire ordering. pub fn start_read<'q>(&'q mut self) -> ReadBatch<'q> { // We trust sizes in elem ring; only need elem_tail here. self.elem_tail = self.shared.get_elem_tail(); // Acquire barrier: subsequent reads of elems/data happen-after. fence(Ordering::Acquire); ReadBatch { elem_head: Cell::new(self.elem_head), buf_head: Cell::new(self.buf_head), q: self, } } } impl<'q> ReadBatch<'q> { pub fn peek_len(&self) -> Result { if self.q.elem_tail == self.elem_head.get() { return Err(EqError::NoEntry); } let idx = (self.elem_head.get() & self.q.elem_mask) as usize; Ok(self.q.elems_ref()[idx]) } pub fn peek(&self) -> Result<&[u8], EqError> { let len = self.peek_len()?; let aligned_len = (len + 7) & !7; let offset = ElementQueue::__next_offset_by_len(self.buf_head.get(), self.q.buf_mask, aligned_len) & self.q.buf_mask; let start = offset as usize; let end = start + len as usize; Ok(&self.q.data_ref()[start..end]) } /// Peek a typed value from the next element without advancing the queue. /// Returns Err(NoEntry) if empty or Err(InvalidArg) if the element is too small. pub fn peek_value(&self) -> Result { let bytes = self.peek()?; let need = core::mem::size_of::(); if bytes.len() < need { return Err(EqError::InvalidArg); } // Copy bytes into an uninitialized T and assume init. let mut tmp = core::mem::MaybeUninit::::uninit(); unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), tmp.as_mut_ptr() as *mut u8, need); Ok(tmp.assume_init()) } } /// Read next element, returning a slice into the data buffer. pub fn read(&self) -> Result<&[u8], EqError> { let len = self.peek_len()?; let aligned_len = (len + 7) & !7; let offset = ElementQueue::__next_offset_by_len(self.buf_head.get(), self.q.buf_mask, aligned_len); let start = (offset & self.q.buf_mask) as usize; let end = start + len as usize; // advance local heads self.elem_head.set(self.elem_head.get().wrapping_add(1)); self.buf_head.set(offset.wrapping_add(aligned_len)); Ok(&self.q.data_ref()[start..end]) } pub fn finish(self) -> &'q mut ElementQueue { // Commit local heads and publish to shared with release ordering self.q.elem_head = self.elem_head.get(); self.q.buf_head = self.buf_head.get(); fence(Ordering::Release); self.q.shared.set_buf_head(self.q.buf_head); self.q.shared.set_elem_head(self.q.elem_head); self.q } } impl ElementQueue { #[inline] pub fn elem_count(&self) -> u32 { self.elem_tail.wrapping_sub(self.elem_head) } #[inline] pub fn elem_capacity(&self) -> u32 { self.elem_mask + 1 } #[inline] pub fn buf_used(&self) -> u32 { self.buf_tail.wrapping_sub(self.buf_head) } #[inline] pub fn buf_capacity(&self) -> u32 { self.buf_mask + 1 } #[inline] fn __next_offset_by_len(buf_offset: u32, buf_mask: u32, len: u32) -> u32 { // If this write would cross the end of buffer page, wrap to the start. let end = buf_offset.wrapping_add(len).wrapping_sub(1); let page_end = end & !buf_mask; let page_start = buf_offset & !buf_mask; if ((page_end.wrapping_sub(page_start)) as i32) > 0 { page_end } else { buf_offset } } #[inline] fn elems_ref(&self) -> &[u32] { // SAFETY: elems points to valid initialized slice for the life of self unsafe { self.elems.as_ref() } } #[inline] fn elems_mut(&mut self) -> &mut [u32] { // SAFETY: unique &mut self borrow guarantees exclusive access unsafe { self.elems.as_mut() } } #[inline] fn data_ref(&self) -> &[u8] { // SAFETY: data points to valid initialized slice for the life of self unsafe { self.data.as_ref() } } #[inline] fn data_mut(&mut self) -> &mut [u8] { // SAFETY: unique &mut self borrow guarantees exclusive access unsafe { self.data.as_mut() } } } ================================================ FILE: crates/k8s-collector/Cargo.toml ================================================ [package] name = "k8s-collector" version = "0.1.0" edition = "2021" [dependencies] clap = { version = "4", features = ["derive"] } tokio = { version = "1", features = ["sync", "macros", "rt-multi-thread", "time", "net", "io-util"], default-features = false } futures-util = { version = "0.3" } thiserror = "1" log = "0.4" # K8s watcher stack kube = { version = "2.0.1", features = ["runtime", "client", "rustls-tls"] } k8s-openapi = { version = "0.26" } serde = { version = "1", features = ["derive"] } serde_json = "1" encoder_ebpf_net_ingest = { path = "../render/ebpf_net/ingest" } lz4_flex = { version = "0.11", features = ["frame"] } tokio-util = { version = "0.7", features = ["io-util"] } hostname = "0.4" [dev-dependencies] k8s-openapi = { version = "0.26", features = ["v1_30"] } proptest = "1" anyhow = "1" env_logger = "0.11" serde_json = "1" ================================================ FILE: crates/k8s-collector/src/collector.rs ================================================ //! Orchestrates the layered pipeline: kube watchers → tombstone adapters → //! matcher → encoder → writer. //! //! The function [`run`] connects to the Kubernetes API, wires the streams, and //! drives the end-to-end loop including reducer connection management. use std::pin::Pin; use futures_util::{future, Stream, StreamExt}; use log::{error, info, warn}; use tokio::time::{interval, sleep, Duration, Instant}; use crate::config::Config; use crate::convert_to_meta::{job_to_owner, pod_to_meta, rs_to_owner}; use crate::encode; use crate::matcher::Matcher; use crate::output::RenderEvent; use crate::tombstone_adapter::TombstoneAdapter; use crate::types::{OwnerMeta, PodMeta}; use crate::writer::Writer; use kube::runtime::watcher::Event; use kube::{ runtime::{ reflector::{self}, watcher::{self, Event as KEvent}, }, Api, Client, }; use k8s_openapi::api::apps::v1::ReplicaSet; use k8s_openapi::api::batch::v1::Job; use k8s_openapi::api::core::v1::Pod; type PodStream = Pin>, kube::runtime::watcher::Error>> + Send>>; type OwnerStream = Pin< Box>, kube::runtime::watcher::Error>> + Send>, >; /// High-level collector pipeline used by both the binary and tests. /// /// Owns kube watcher streams, Stores, tombstone adapters, and the Matcher, /// exposing a `next_messages` API that yields batches of `RenderEvent`s. pub struct Collector { pod_stream: PodStream, rs_owner_stream: OwnerStream, job_owner_stream: OwnerStream, rs_owners_writer: reflector::store::Writer, job_owners_writer: reflector::store::Writer, pods_writer: reflector::store::Writer, matcher: Matcher, rs_owner_tomb: TombstoneAdapter, job_owner_tomb: TombstoneAdapter, pod_tomb: TombstoneAdapter, } impl Collector { /// Construct a new collector pipeline using the default Kubernetes client. pub async fn new(cfg: Config) -> Result { let client = Client::try_default() .await .map_err(|e| crate::Error::KubeInit(e.to_string()))?; Ok(Self::with_client(cfg, client)) } /// Construct a new collector pipeline from an existing Kubernetes client. pub fn with_client(cfg: Config, client: Client) -> Self { let pods_api: Api = Api::all(client.clone()); let rs_api: Api = Api::all(client.clone()); let job_api: Api = Api::all(client); let wc = watcher::Config::default(); // Watcher streams mapped to our Event protocol let pod_stream = watcher::watcher(pods_api, wc.clone()).map(|e| map_kube_event(e, pod_to_meta)); let rs_stream = watcher::watcher(rs_api, wc.clone()).map(|e| map_kube_event(e, rs_to_owner)); let job_stream = watcher::watcher(job_api, wc).map(|e| map_kube_event(e, job_to_owner)); let (rs_owners_store, rs_owners_writer) = reflector::store::store::(); let (job_owners_store, job_owners_writer) = reflector::store::store::(); let (pods_store, pods_writer) = reflector::store::store::(); let matcher = Matcher::new(rs_owners_store, job_owners_store, pods_store); let rs_owner_tomb = TombstoneAdapter::new(cfg.delete_ttl, cfg.delete_capacity); let job_owner_tomb = TombstoneAdapter::new(cfg.delete_ttl, cfg.delete_capacity); let pod_tomb = TombstoneAdapter::new(cfg.delete_ttl, cfg.delete_capacity); Self { pod_stream: Box::pin(pod_stream), rs_owner_stream: Box::pin(rs_stream), job_owner_stream: Box::pin(job_stream), rs_owners_writer, job_owners_writer, pods_writer, matcher, rs_owner_tomb, job_owner_tomb, pod_tomb, } } /// Fetch the next batch of render events from either the pod or owner /// watcher stream. /// /// Returns: /// - `Ok(events)` with zero or more `RenderEvent`s; the caller can skip /// empty batches and continue calling to wait for more data. /// - `Err` on watcher failure. pub async fn next_messages(&mut self) -> Result, crate::Error> { loop { let mut out = Vec::new(); tokio::select! { ev = self.pod_stream.next() => { match ev { Some(Ok(events)) => { for e in events { for fwd in self.pod_tomb.handle(e) { self.pods_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_pod(fwd)); } } } Some(Err(err)) => { warn!("pod watcher error: {err:?}; will retry on next tick"); continue; } None => { error!("pod watcher stream terminated unexpectedly"); return Err(crate::Error::Stopped); } } } ev = self.rs_owner_stream.next() => { match ev { Some(Ok(events)) => { for e in events { for fwd in self.rs_owner_tomb.handle(e) { self.rs_owners_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_owner(fwd)); } } } Some(Err(err)) => { warn!("owner watcher error: {err:?}; will retry on next tick"); continue; } None => { error!("owner watcher stream terminated unexpectedly"); return Err(crate::Error::Stopped); } } } ev = self.job_owner_stream.next() => { match ev { Some(Ok(events)) => { for e in events { for fwd in self.job_owner_tomb.handle(e) { self.job_owners_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_owner(fwd)); } } } Some(Err(err)) => { warn!("owner watcher error: {err:?}; will retry on next tick"); continue; } None => { error!("owner watcher stream terminated unexpectedly"); return Err(crate::Error::Stopped); } } } } if !out.is_empty() { return Ok(out); } } } /// Start a new resync epoch using the current store snapshots. pub fn start_new_epoch(&mut self) -> Vec { self.matcher.start_new_epoch() } /// Produce a human-readable snapshot of the internal matcher/stores. /// /// This is primarily intended for tests and debugging to inspect the /// Stores and live pod bookkeeping when assertions fail. pub fn debug_snapshot(&self) -> String { self.matcher.debug_snapshot() } } /// Run the collector to completion. /// /// This function: /// - Starts kube watchers for Pods, ReplicaSets, and Jobs /// - Applies tombstone adapters to both owner and pod streams /// - Matches and encodes events, writing them to the reducer /// - Reconnects aggressively on socket errors and starts a new epoch per connect pub async fn run(cfg: Config) -> Result<(), crate::Error> { use tokio::net::TcpStream; let (major, minor, patch) = collector_version(); info!( "Starting k8s-collector version {}.{}.{}", major, minor, patch ); let mut pipeline = Collector::new(cfg.clone()).await?; let hostname = collector_hostname(); info!("Hostname: {}", hostname); let addr = std::net::SocketAddr::new( cfg.intake_host .parse() .unwrap_or(std::net::Ipv4Addr::LOCALHOST.into()), cfg.intake_port, ); 'reconnect: loop { // Connect info!("k8s-collector: connecting to reducer at {}", addr); let stream = match TcpStream::connect(addr).await { Ok(s) => { info!("k8s-collector: connected to reducer at {}", addr); s } Err(err) => { warn!( "k8s-collector: failed to connect to reducer at {}: {err}; retrying in 1s", addr ); sleep(Duration::from_secs(1)).await; continue; } }; let mut writer = Writer::new(stream); if let Err(err) = perform_handshake(&mut writer, &hostname).await { warn!("k8s-collector: handshake with reducer failed: {err}; reconnecting"); sleep(Duration::from_secs(1)).await; continue 'reconnect; } // New epoch on connect for ev in pipeline.start_new_epoch() { let buf = encode::encode(&ev, timestamp()); if let Err(err) = writer.send(&buf).await { // Connection failed while sending epoch; restart outer loop warn!("k8s-collector: failed to send epoch event to reducer: {err}; reconnecting"); sleep(Duration::from_secs(1)).await; continue 'reconnect; } } if let Err(err) = writer.flush().await { warn!("k8s-collector: flush after epoch send failed: {err}; reconnecting"); sleep(Duration::from_secs(1)).await; continue 'reconnect; } let mut heartbeat = interval(Duration::from_secs(5)); let mut flush_deadline: Option = None; // Inner loop: forward events until connection drops or the pipeline fails 'connection: loop { let flush_deadline_opt = flush_deadline; tokio::select! { res = pipeline.next_messages() => { match res { Ok(events) => { let had_events = !events.is_empty(); for ev in events { let buf = encode::encode(&ev, timestamp()); if let Err(err) = writer.send(&buf).await { // Connection dropped; break to reconnect. warn!( "k8s-collector: send to reducer failed during event forwarding: {err}; reconnecting" ); break 'connection; } } if had_events && flush_deadline.is_none() { flush_deadline = Some(Instant::now() + Duration::from_millis(100)); } } Err(e) => { // Fatal pipeline error (e.g., watcher stream terminated): propagate up. error!("collector pipeline failed: {e}"); return Err(e); } } } _ = heartbeat.tick() => { let buf = encode::encode_heartbeat(timestamp()); if let Err(err) = writer.send(&buf).await { warn!( "k8s-collector: send to reducer failed during heartbeat: {err}; reconnecting" ); break 'connection; } if let Err(err) = writer.flush().await { warn!( "k8s-collector: flush after heartbeat failed: {err}; reconnecting" ); break 'connection; } flush_deadline = None; } _ = async { if let Some(deadline) = flush_deadline_opt { tokio::time::sleep_until(deadline).await; } else { future::pending::<()>().await; } } => { if let Err(err) = writer.flush().await { warn!( "k8s-collector: flush after flush-deadline failed: {err}; reconnecting" ); break 'connection; } flush_deadline = None; } } } // Drop connection, wait and reconnect sleep(Duration::from_secs(1)).await; } } /// Map a kube watcher event to one or more internal events by applying a /// converter function. fn map_kube_event( ev: Result, kube::runtime::watcher::Error>, f: fn(K) -> T, ) -> Result>, kube::runtime::watcher::Error> { Ok(match ev? { KEvent::Apply(obj) => vec![Event::Apply(f(obj))], KEvent::Delete(obj) => vec![Event::Delete(f(obj))], KEvent::Init => vec![Event::Init], KEvent::InitApply(obj) => vec![Event::InitApply(f(obj))], KEvent::InitDone => vec![Event::InitDone], }) } fn collector_version() -> (u32, u32, u32) { fn parse(name: &str) -> Option { std::env::var(name).ok()?.parse().ok() } let major = parse("EBPF_NET_MAJOR_VERSION").unwrap_or(0); let minor = parse("EBPF_NET_MINOR_VERSION").unwrap_or(11); let patch = parse("EBPF_NET_PATCH_VERSION").unwrap_or(0); (major, minor, patch) } fn collector_hostname() -> String { if let Ok(h) = std::env::var("K8S_COLLECTOR_HOSTNAME") { return h; } match hostname::get() { Ok(name) => name.to_string_lossy().into_owned(), Err(_) => "k8s-collector".to_string(), } } async fn perform_handshake(writer: &mut Writer, hostname: &str) -> std::io::Result<()> where W: tokio::io::AsyncWrite + Unpin + Send + 'static, { // First message: uncompressed version_info to negotiate compression and // pass the minimum accepted version check on the reducer. let (major, minor, patch) = collector_version(); let ver_buf = encode::encode_version_info(major, minor, patch, timestamp()); writer.send(&ver_buf).await?; writer.flush().await?; // Second message: connect as a k8s collector with hostname. This // authenticates the connection and enables handlers for k8s pod messages. // // ClientType::k8s = 3 (see common/client_type.h). let connect_buf = encode::encode_connect(3, hostname, timestamp()); writer.send(&connect_buf).await?; writer.flush().await?; Ok(()) } /// Current UNIX time in nanoseconds. fn timestamp() -> u64 { use std::time::{SystemTime, UNIX_EPOCH}; SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_nanos() as u64) .unwrap_or(0) } ================================================ FILE: crates/k8s-collector/src/config.rs ================================================ //! Configuration for the Kubernetes metadata collector. //! //! Settings cover endpoint selection and deletion tombstone policy. use std::time::Duration; #[derive(Clone, Debug)] pub struct Config { /// Reducer host to connect to (ingest TCP server) pub intake_host: String, /// Reducer port to connect to pub intake_port: u16, /// Max time to retain delete tombstones to satisfy straggling updates. pub delete_ttl: Duration, /// Max number of delete tombstones to retain (older ones evicted). pub delete_capacity: usize, } impl Default for Config { /// Reasonable defaults: /// - connect to `127.0.0.1:8000` /// - retain tombstones for 60s up to 10k entries fn default() -> Self { Self { intake_host: "127.0.0.1".into(), intake_port: 8000, delete_ttl: Duration::from_secs(60), delete_capacity: 10_000, } } } ================================================ FILE: crates/k8s-collector/src/convert_to_meta.rs ================================================ //! Converters from Kubernetes API objects to compact metadata. //! //! These functions strip extraneous fields and normalize key properties used by //! the matcher and output layers. use k8s_openapi::api::apps::v1::ReplicaSet; use k8s_openapi::api::batch::v1::Job; use k8s_openapi::api::core::v1::Pod; use kube::ResourceExt; use crate::types::{ContainerMeta, OwnerKind, OwnerMeta, OwnerRef, PodMeta}; /// Convert a Kubernetes `Pod` into a [`PodMeta`]. /// /// - Picks the first controller ownerReference as the pod owner (if any) /// - Collects container runtime IDs, names, and images from status.container_statuses /// - Computes a deterministic version string from container images pub fn pod_to_meta(pod: Pod) -> PodMeta { let uid = pod.metadata.uid.clone().unwrap_or_default(); let name = pod.name_any(); let namespace = pod.namespace().unwrap_or_else(|| "default".to_string()); let host_network = pod .spec .as_ref() .and_then(|s| s.host_network) .unwrap_or(false); let ip = pod .status .as_ref() .and_then(|s| s.pod_ip.clone()) .unwrap_or_default(); let containers = pod .status .as_ref() .and_then(|s| s.container_statuses.as_ref()) .map(|v| { v.iter() .map(|cs| ContainerMeta { id: cs.container_id.clone().unwrap_or_default(), name: cs.name.clone(), image: cs.image.clone(), }) .collect::>() }) .unwrap_or_default(); let images = pod .status .as_ref() .and_then(|s| s.container_statuses.as_ref()) .map(|v| v.iter().map(|cs| cs.image.clone()).collect::>()) .unwrap_or_default(); let version = build_version_string(images); let owner = pod .metadata .owner_references .unwrap_or_default() .into_iter() .find(|o| o.controller.unwrap_or(false)) .map(|o| OwnerRef { kind: o.kind.parse().unwrap_or(OwnerKind::Other), uid: o.uid, name: o.name, }); PodMeta { uid, ip, name, namespace, owner, host_network, version, containers, } } /// Convert a Kubernetes `ReplicaSet` into an [`OwnerMeta`]. /// /// If the ReplicaSet has a controller owner (e.g., Deployment), it is stored in /// `controller` for escalation by the matcher. pub fn rs_to_owner(rs: ReplicaSet) -> OwnerMeta { let uid = rs.metadata.uid.unwrap_or_default(); let controller = rs .metadata .owner_references .unwrap_or_default() .into_iter() .find(|o| o.controller.unwrap_or(false)) .map(|o| OwnerRef { kind: o.kind.parse().unwrap_or(OwnerKind::Other), uid: o.uid, name: o.name, }); OwnerMeta { uid, controller } } /// Convert a Kubernetes `Job` into an [`OwnerMeta`]. /// /// If the Job has a controller owner (e.g., CronJob), it is stored in /// `controller` for escalation by the matcher. pub fn job_to_owner(job: Job) -> OwnerMeta { let uid = job.metadata.uid.unwrap_or_default(); let controller = job .metadata .owner_references .unwrap_or_default() .into_iter() .find(|o| o.controller.unwrap_or(false)) .map(|o| OwnerRef { kind: o.kind.parse().unwrap_or(OwnerKind::Other), uid: o.uid, name: o.name, }); OwnerMeta { uid, controller } } /// Build a stable version string from a set of image references. /// /// Images are quoted, sorted, and joined by commas to maintain consistency /// across updates and ordering changes. fn build_version_string(images: Vec) -> String { let mut imgs = images .into_iter() .filter(|s| !s.is_empty()) .map(|i| format!("'{}'", i)) .collect::>(); imgs.sort(); imgs.join(",") } ================================================ FILE: crates/k8s-collector/src/encode.rs ================================================ //! Encoder for render events into ingestion wire buffers. //! //! Uses the generated encoder functions from `encoder_ebpf_net_ingest` to //! produce exact-sized buffers expected by the reducer. The `encode` function //! maps a [`RenderEvent`](crate::output::RenderEvent) into a ready-to-send //! buffer. use core::ffi::c_char; use crate::output::RenderEvent; use crate::types::OwnerKind; // Ensure encoder crate is linked so its #[no_mangle] symbols are available #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_ingest; #[repr(C)] struct JbBlob { buf: *const c_char, len: u16, } extern "C" { fn ebpf_net_ingest_encode_pod_new_with_name( dest: *mut u8, dest_len: u32, tstamp: u64, uid: JbBlob, ip: u32, owner_name: JbBlob, pod_name: JbBlob, owner_kind: u8, owner_uid: JbBlob, is_host_network: u8, ns: JbBlob, version: JbBlob, ); fn ebpf_net_ingest_encode_pod_container( dest: *mut u8, dest_len: u32, tstamp: u64, uid: JbBlob, container_id: JbBlob, container_name: JbBlob, container_image: JbBlob, ); fn ebpf_net_ingest_encode_pod_delete(dest: *mut u8, dest_len: u32, tstamp: u64, uid: JbBlob); fn ebpf_net_ingest_encode_pod_resync( dest: *mut u8, dest_len: u32, tstamp: u64, resync_count: u64, ); } extern "C" { fn ebpf_net_ingest_encode_version_info( dest: *mut u8, dest_len: u32, tstamp: u64, major: u32, minor: u32, patch: u32, ); } extern "C" { fn ebpf_net_ingest_encode_connect( dest: *mut u8, dest_len: u32, tstamp: u64, collector_type: u8, hostname: JbBlob, ); } extern "C" { fn ebpf_net_ingest_encode_heartbeat(dest: *mut u8, dest_len: u32, tstamp: u64); } /// Construct a `JbBlob` view from a Rust `&str`. fn as_blob(s: &str) -> JbBlob { JbBlob { buf: s.as_ptr() as *const c_char, len: s.len() as u16, } } /// Map [`OwnerKind`] into the reducer's numeric enum codes. fn owner_kind_code(k: OwnerKind) -> u8 { match k { OwnerKind::ReplicaSet => 0, OwnerKind::Deployment => 1, OwnerKind::Job => 2, OwnerKind::CronJob => 3, OwnerKind::NoOwner => 254, OwnerKind::Other => 255, } } /// Parse an IPv4 string as a host-order `u32`. fn ipv4_to_u32(ip: &str) -> u32 { ip.parse::().map(u32::from).unwrap_or(0) } /// Encode a [`RenderEvent`] into a ready-to-send wire buffer. pub fn encode(event: &RenderEvent, tstamp: u64) -> Vec { match event { RenderEvent::PodResync { epoch } => { let len = 8 + 16; // timestamp + struct let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_pod_resync( buf.as_mut_ptr(), buf.len() as u32, tstamp, *epoch, ); } buf } RenderEvent::PodDelete { uid } => { let consumed = 4 + uid.len(); let len = 8 + consumed; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_pod_delete( buf.as_mut_ptr(), buf.len() as u32, tstamp, as_blob(uid), ); } buf } RenderEvent::PodContainer { uid, container } => { let consumed = 10 + uid.len() + container.id.len() + container.name.len() + container.image.len(); let len = 8 + consumed; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_pod_container( buf.as_mut_ptr(), buf.len() as u32, tstamp, as_blob(uid), as_blob(&container.id), as_blob(&container.name), as_blob(&container.image), ); } buf } RenderEvent::PodNew { uid, ip, pod_name, ns, version, owner_kind, owner_uid, owner_name, is_host_network, } => { let consumed = 20 + uid.len() + owner_name.len() + pod_name.len() + owner_uid.len() + ns.len() + version.len(); let len = 8 + consumed; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_pod_new_with_name( buf.as_mut_ptr(), buf.len() as u32, tstamp, as_blob(uid), ipv4_to_u32(ip), as_blob(owner_name), as_blob(pod_name), owner_kind_code(*owner_kind), as_blob(owner_uid), if *is_host_network { 1 } else { 0 }, as_blob(ns), as_blob(version), ); } buf } } } /// Encode a `version_info` handshake message. pub fn encode_version_info(major: u32, minor: u32, patch: u32, tstamp: u64) -> Vec { let len = 8 + 16; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_version_info( buf.as_mut_ptr(), buf.len() as u32, tstamp, major, minor, patch, ); } buf } /// Encode a `connect` message identifying this collector. /// /// `collector_type` is the numeric `ClientType` enum value (e.g., 3 for k8s /// collectors; see `common/client_type.h`). pub fn encode_connect(collector_type: u8, hostname: &str, tstamp: u64) -> Vec { let consumed = 5u32 + hostname.len() as u32; let len = 8 + consumed as usize; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_connect( buf.as_mut_ptr(), buf.len() as u32, tstamp, collector_type, as_blob(hostname), ); } buf } /// Encode a `heartbeat` message. pub fn encode_heartbeat(tstamp: u64) -> Vec { let len = 8 + 2; let mut buf = vec![0u8; len]; unsafe { ebpf_net_ingest_encode_heartbeat(buf.as_mut_ptr(), buf.len() as u32, tstamp); } buf } ================================================ FILE: crates/k8s-collector/src/lib.rs ================================================ //! Kubernetes metadata collector (Rust). //! //! This crate implements a layered pipeline to watch Kubernetes resources, //! correlate Pods with their effective owners, and stream normalized messages //! to the reducer. The layers are: //! - convert_to_meta: map Kubernetes API objects to compact metadata structs //! - tombstone_adapter: retain recent Delete events to bridge race windows //! - matcher: match Pods to owners, handle escalation (RS→Deployment, Job→CronJob) //! - output/encode: translate matched events into encoded render buffers //! - writer: async TCP client with optional LZ4 streaming compression //! - collector: orchestrates watchers, adapters, matcher, and writer //! //! The public entry point is [`run_with_config`]. pub mod collector; pub mod config; pub mod convert_to_meta; pub mod encode; pub mod matcher; pub mod output; pub mod tombstone_adapter; pub mod types; pub mod writer; use crate::config::Config; #[derive(thiserror::Error, Debug)] pub enum Error { #[error("collector stopped")] Stopped, #[error("failed to initialize Kubernetes client: {0}")] KubeInit(String), #[error("failed to initialize Tokio runtime: {0}")] RuntimeInit(String), } /// Convenience runner used by the binary: runs the layered pipeline with kube watchers. /// /// Creates a Tokio runtime and executes [`collector::run`]. /// /// Errors indicate runtime initialization failure or fatal errors in the /// orchestrated tasks. pub fn run_with_config(cfg: Config) -> Result<(), Error> { let rt = tokio::runtime::Runtime::new().map_err(|e| Error::RuntimeInit(e.to_string()))?; rt.block_on(async move { collector::run(cfg).await }) } ================================================ FILE: crates/k8s-collector/src/matcher.rs ================================================ //! Pod↔Owner matching engine. //! //! Maintains external kube reflectors Stores for owners (ReplicaSets and Jobs) //! and pods, correlates pods with their //! effective owners, and produces normalized render events: //! - Emit PodNew(+containers) when a pod is resolvable and has an IP //! - Emit PodContainer updates for subsequent Apply on live pods //! - Emit PodDelete when a live pod is deleted //! - Resync epochs on InitDone (double-buffer snapshot becomes active). //! //! Design notes: //! - Stores are provided externally to `Matcher` and are fed by kube reflectors. //! ReplicaSet and Job owners use separate Stores so that independent //! re-initialization (`Init`/`InitApply`/`InitDone`) of their watcher streams //! cannot clobber each other's snapshot state. //! On `InitDone`, `Store::state()` for each owner Store exposes its new //! snapshot; `Matcher` uses the combined view in `start_new_epoch()` to //! re-emit resolvable pods. //! - `Lookup` is implemented for `PodMeta` and `OwnerMeta` so `Store` keys by //! UID as a cluster-scoped identifier. We construct `ObjectRef` keys via //! small helpers (`pod_ref`, `owner_ref`) and use `Store::get` for O(1) //! lookups. //! //! Owner escalation: RS→Deployment, Job→CronJob (when controller owner exists). use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use crate::output::RenderEvent; use crate::types::{OwnerKind, OwnerMeta, PodMeta}; use kube::runtime::reflector::{Lookup, ObjectRef, Store}; use kube::runtime::watcher::Event; // Implement kube-runtime's Lookup trait for our compact meta types so that // they can be used with reflector Stores. We key by UID as a cluster-scoped // identifier to avoid namespace/name collisions and reduce state surface. impl Lookup for PodMeta { type DynamicType = (); fn kind(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("PodMeta") } fn group(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("opentelemetry.network") } fn version(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("v1") } fn plural(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("podmeta") } fn name(&self) -> Option> { Some(Cow::Borrowed(self.uid.as_str())) } fn namespace(&self) -> Option> { None } fn resource_version(&self) -> Option> { None } fn uid(&self) -> Option> { Some(Cow::Borrowed(self.uid.as_str())) } } impl Lookup for OwnerMeta { type DynamicType = (); fn kind(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("OwnerMeta") } fn group(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("opentelemetry.network") } fn version(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("v1") } fn plural(_: &Self::DynamicType) -> Cow<'_, str> { Cow::Borrowed("ownermeta") } fn name(&self) -> Option> { Some(Cow::Borrowed(self.uid.as_str())) } fn namespace(&self) -> Option> { None } fn resource_version(&self) -> Option> { None } fn uid(&self) -> Option> { Some(Cow::Borrowed(self.uid.as_str())) } } pub struct Matcher { epoch: u64, rs_owners: Store, job_owners: Store, waiting_by_owner: HashMap>, // owner uid -> pod uids pods: Store, live_pods: HashSet, } impl Matcher { /// Create a new matcher with the provided Stores. /// /// `rs_owners` and `job_owners` are kept separate to avoid cross-talk /// between independently re-initialized ReplicaSet and Job watcher streams. pub fn new( rs_owners: Store, job_owners: Store, pods: Store, ) -> Self { Self { epoch: 0, rs_owners, job_owners, waiting_by_owner: HashMap::new(), pods, live_pods: HashSet::new(), } } /// Helper: build an ObjectRef for a Pod by UID (cluster-scoped key) fn pod_ref(uid: &str) -> ObjectRef { ObjectRef::::new(uid) } /// Helper: build an ObjectRef for an Owner by UID (cluster-scoped key) fn owner_ref(uid: &str) -> ObjectRef { ObjectRef::::new(uid) } /// Helper: look up an owner meta by UID across ReplicaSet and Job Stores. fn get_owner_meta(&self, uid: &str) -> Option { self.rs_owners .get(&Self::owner_ref(uid)) .or_else(|| self.job_owners.get(&Self::owner_ref(uid))) .as_deref() .cloned() } /// Helper: check whether an owner UID exists in any owner Store. fn owner_exists(&self, uid: &str) -> bool { self.rs_owners.get(&Self::owner_ref(uid)).is_some() || self.job_owners.get(&Self::owner_ref(uid)).is_some() } /// Handle a single Owner event and return any resulting render events. pub fn handle_owner(&mut self, ev: Event) -> Vec { match ev { Event::Init => Vec::new(), Event::InitApply(_m) => Vec::new(), Event::Apply(m) => { let uid = m.uid.clone(); // Drain waiters for this owner if any let mut out = Vec::new(); if let Some(pods) = self.waiting_by_owner.remove(&uid) { for puid in pods { if let Some(p) = self.pods.get(&Self::pod_ref(&puid)).as_deref().cloned() { out.extend(self.try_emit_pod(p)); } } } out } Event::Delete(_m) => Vec::new(), Event::InitDone => { // Scan waiting pods against the now-initialized owner store let mut out = Vec::new(); let keys: Vec = self.waiting_by_owner.keys().cloned().collect(); for owner_uid in keys { if self.owner_exists(&owner_uid) { if let Some(pods) = self.waiting_by_owner.remove(&owner_uid) { for puid in pods { if let Some(p) = self.pods.get(&Self::pod_ref(&puid)).as_deref().cloned() { out.extend(self.try_emit_pod(p)); } } } } } out } } } /// Handle a single Pod event and return any resulting render events. pub fn handle_pod(&mut self, ev: Event) -> Vec { match ev { Event::Init => Vec::new(), Event::InitApply(_p) => Vec::new(), Event::Apply(p) => { if self.live_pods.contains(&p.uid) { // Live pod: containers only return p .containers .into_iter() .map(|c| RenderEvent::PodContainer { uid: p.uid.clone(), container: c, }) .collect(); } self.try_emit_pod(p) } Event::Delete(p) => { let mut out = Vec::new(); if self.live_pods.remove(&p.uid) { out.push(RenderEvent::PodDelete { uid: p.uid.clone() }); } for waiters in self.waiting_by_owner.values_mut() { waiters.retain(|w| w != &p.uid); } out } Event::InitDone => self.start_new_epoch(), } } /// Start a new resync epoch, clearing waiting queues and emitting a /// `PodResync` event followed by any pods now resolvable from the current /// store contents. pub fn start_new_epoch(&mut self) -> Vec { self.epoch = self.epoch.wrapping_add(1); self.waiting_by_owner.clear(); let mut out = vec![RenderEvent::PodResync { epoch: self.epoch }]; // Iterate pod store snapshot and emit resolvable ones for p in self.pods.state().into_iter().map(|a| (*a).clone()) { out.extend(self.try_emit_pod(p)); } out } /// Produce a human-readable snapshot of the current matcher state. /// /// Intended for use in tests and debugging to help understand mismatches /// between the pod/owner Stores and emitted events during resync. pub fn debug_snapshot(&self) -> String { use std::fmt::Write; let mut out = String::new(); let _ = writeln!(out, "Matcher debug snapshot:"); let _ = writeln!(out, " epoch={}", self.epoch); let _ = writeln!(out, " live_pods={:?}", self.live_pods); let _ = writeln!( out, " waiting_by_owner keys={:?}", self.waiting_by_owner.keys() ); let _ = writeln!(out, " rs_owners_store:"); for owner in self.rs_owners.state().into_iter().map(|a| (*a).clone()) { let _ = writeln!(out, " {:?}", owner); } let _ = writeln!(out, " job_owners_store:"); for owner in self.job_owners.state().into_iter().map(|a| (*a).clone()) { let _ = writeln!(out, " {:?}", owner); } let _ = writeln!(out, " pods_store:"); for pod in self.pods.state().into_iter().map(|a| (*a).clone()) { let _ = writeln!(out, " {:?}", pod); } out } /// Attempt to emit `PodNew` (+containers) for the given pod when resolvable. /// fn try_emit_pod(&mut self, p: PodMeta) -> Vec { if !p.has_ip() { return Vec::new(); } // No owner let owner = match &p.owner { None => { self.live_pods.insert(p.uid.clone()); return crate::output::RenderEvent::from_pod_new(&p, None); } Some(o) => o.clone(), }; if !owner.kind.is_rs_or_job() { self.live_pods.insert(p.uid.clone()); return crate::output::RenderEvent::from_pod_new(&p, Some(&owner)); } // Owner is RS/Job: check if owner info is known and possibly escalate if let Some(om) = self.get_owner_meta(&owner.uid) { let escalated = match &om.controller { Some(ctrl) if matches!(ctrl.kind, OwnerKind::Deployment | OwnerKind::CronJob) => { ctrl.clone() } _ => owner, }; self.live_pods.insert(p.uid.clone()); return crate::output::RenderEvent::from_pod_new(&p, Some(&escalated)); } // Not known yet: queue let w = self.waiting_by_owner.entry(owner.uid.clone()).or_default(); if !w.iter().any(|u| u == &p.uid) { w.push(p.uid.clone()); } Vec::new() } } #[cfg(test)] mod tests { use super::*; use crate::types::{ContainerMeta, OwnerRef}; use kube::runtime::reflector::store; struct Harness { m: Matcher, rs_owners_w: kube::runtime::reflector::store::Writer, job_owners_w: kube::runtime::reflector::store::Writer, pods_w: kube::runtime::reflector::store::Writer, } impl Harness { fn new() -> Self { let (rs_owners_store, rs_owners_w) = store::store::(); let (job_owners_store, job_owners_w) = store::store::(); let (pods_store, pods_w) = store::store::(); let m = Matcher::new(rs_owners_store, job_owners_store, pods_store); Self { m, rs_owners_w, job_owners_w, pods_w, } } fn handle_rs_owner(&mut self, ev: Event) -> Vec { self.rs_owners_w.apply_watcher_event(&ev); self.m.handle_owner(ev) } fn handle_job_owner(&mut self, ev: Event) -> Vec { self.job_owners_w.apply_watcher_event(&ev); self.m.handle_owner(ev) } fn handle_pod(&mut self, ev: Event) -> Vec { self.pods_w.apply_watcher_event(&ev); self.m.handle_pod(ev) } } fn pod(uid: &str, ip: &str, owner: Option) -> PodMeta { PodMeta { uid: uid.into(), ip: ip.into(), name: format!("pod-{}", uid), namespace: "default".into(), owner, host_network: false, version: "'img:1'".into(), containers: vec![ContainerMeta { id: format!("c-{}", uid), name: "c".into(), image: "img:1".into(), }], } } fn rs_owner(uid: &str, controller: Option) -> OwnerMeta { OwnerMeta { uid: uid.into(), controller, } } fn ref_kind(kind: OwnerKind, uid: &str, name: &str) -> OwnerRef { OwnerRef { kind, uid: uid.into(), name: name.into(), } } #[test] // Pod arrives first, then owner Apply arrives later. // The pod should be emitted once owner information is available. fn pod_first_then_owner_emits_on_owner_apply() { let mut h = Harness::new(); let events = h.handle_pod(Event::Apply(pod( "p1", "10.0.0.1", Some(ref_kind(OwnerKind::ReplicaSet, "r1", "rs")), ))); assert!(events.is_empty()); let out = h.handle_rs_owner(Event::Apply(rs_owner( "r1", Some(ref_kind(OwnerKind::Deployment, "d1", "dep")), ))); assert_eq!(out.len(), 2); match &out[0] { RenderEvent::PodNew { owner_kind, owner_uid, .. } => { assert_eq!(*owner_kind, OwnerKind::Deployment); assert_eq!(owner_uid, "d1"); } _ => panic!("expected PodNew"), } match &out[1] { RenderEvent::PodContainer { uid, .. } => assert_eq!(uid, "p1"), _ => panic!("expected PodContainer"), } } #[test] // Owner arrives first, then pod Apply. // The pod should be emitted immediately using escalated owner when applicable. fn owner_first_then_pod_emits_immediately() { let mut h = Harness::new(); let _ = h.handle_rs_owner(Event::Apply(rs_owner( "r2", Some(ref_kind(OwnerKind::Deployment, "d2", "dep")), ))); let out = h.handle_pod(Event::Apply(pod( "p2", "10.0.0.2", Some(ref_kind(OwnerKind::ReplicaSet, "r2", "rs")), ))); assert_eq!(out.len(), 2); match &out[0] { RenderEvent::PodNew { owner_kind, owner_uid, .. } => { assert_eq!(*owner_kind, OwnerKind::Deployment); assert_eq!(owner_uid, "d2"); } _ => panic!("expected PodNew"), } match &out[1] { RenderEvent::PodContainer { uid, .. } => assert_eq!(uid, "p2"), _ => panic!("expected PodContainer"), } } #[test] fn initdone_starts_epoch_and_emits_resolvable() { let mut h = Harness::new(); let _ = h.handle_rs_owner(Event::InitApply(rs_owner( "r3", Some(ref_kind(OwnerKind::Deployment, "d3", "dep")), ))); // Complete owner init to swap snapshot into the owner Store let _ = h.handle_rs_owner(Event::InitDone); let _ = h.handle_pod(Event::InitApply(pod( "p3", "10.0.0.3", Some(ref_kind(OwnerKind::ReplicaSet, "r3", "rs")), ))); // Complete pod init to swap snapshot into the pod Store and start new epoch let out = h.handle_pod(Event::InitDone); assert!(matches!(out.first(), Some(RenderEvent::PodResync { .. }))); assert!(out.iter().any(|e| matches!(e, RenderEvent::PodNew { .. }))); } #[test] // When a pod has no owner, emit PodNew with NoOwner and containers. fn ownerless_pod_emits_with_no_owner() { let mut h = Harness::new(); let out = h.handle_pod(Event::Apply(pod("p0", "10.0.0.10", None))); assert_eq!(out.len(), 2); match &out[0] { RenderEvent::PodNew { owner_kind, owner_uid, .. } => { assert_eq!(*owner_kind, OwnerKind::NoOwner); assert!(owner_uid.is_empty()); } _ => panic!("expected PodNew"), } match &out[1] { RenderEvent::PodContainer { uid, .. } => assert_eq!(uid, "p0"), _ => panic!("expected PodContainer"), } } #[test] // If the pod is owned by a non-RS/Job owner, emit directly with that owner. fn non_rs_job_owner_emits_directly() { let mut h = Harness::new(); let owner = ref_kind(OwnerKind::Other, "x1", "owner"); let out = h.handle_pod(Event::Apply(pod("p9", "10.0.0.9", Some(owner.clone())))); assert_eq!(out.len(), 2); match &out[0] { RenderEvent::PodNew { owner_kind, owner_uid, owner_name, .. } => { assert_eq!(*owner_kind, OwnerKind::Other); assert_eq!(owner_uid, "x1"); assert_eq!(owner_name, "owner"); } _ => panic!("expected PodNew"), } match &out[1] { RenderEvent::PodContainer { uid, .. } => assert_eq!(uid, "p9"), _ => panic!("expected PodContainer"), } } #[test] // Job→CronJob escalation: if the Job's controller is a CronJob, // emit the pod with the CronJob as the effective owner. fn job_cronjob_escalation() { let mut h = Harness::new(); // pod owned by Job j1 but owner meta has controller CronJob c1 let _ = h.handle_job_owner(Event::Apply(rs_owner( "j1", Some(ref_kind(OwnerKind::CronJob, "c1", "cj")), ))); // Actually RS owner meta function used for RS/Job both - here using rs_owner to avoid boilerplate as both use OwnerMeta let out = h.handle_pod(Event::Apply(pod( "pj", "10.0.0.20", Some(ref_kind(OwnerKind::Job, "j1", "job")), ))); assert_eq!(out.len(), 2); match &out[0] { RenderEvent::PodNew { owner_kind, owner_uid, owner_name, .. } => { assert_eq!(*owner_kind, OwnerKind::CronJob); assert_eq!(owner_uid, "c1"); assert_eq!(owner_name, "cj"); } _ => panic!("expected PodNew"), } } #[test] // Deleting a live pod should produce a PodDelete event and clear its state. fn delete_emits_pod_delete_when_live() { let mut h = Harness::new(); let _ = h.handle_rs_owner(Event::Apply(rs_owner( "r4", Some(ref_kind(OwnerKind::Deployment, "d4", "dep")), ))); let _ = h.handle_pod(Event::Apply(pod( "pd", "10.0.0.44", Some(ref_kind(OwnerKind::ReplicaSet, "r4", "rs")), ))); let out = h.handle_pod(Event::Delete(pod( "pd", "10.0.0.44", Some(ref_kind(OwnerKind::ReplicaSet, "r4", "rs")), ))); assert_eq!(out.len(), 1); match &out[0] { RenderEvent::PodDelete { uid } => assert_eq!(uid, "pd"), _ => panic!("expected PodDelete"), } } #[test] // Owner flaps Apply+Delete: the delete is retained in the tombstone adapter. // While retained, the store still reflects the owner Apply, so a pod Apply // that depends on this owner should still emit. fn owner_flaps_apply_delete_then_pod_apply_emits() { let mut h = Harness::new(); let mut owner_tomb = crate::tombstone_adapter::TombstoneAdapter::new( std::time::Duration::from_secs(60), 100, ); // Owner Apply forwarded for e in owner_tomb.handle(Event::Apply(rs_owner( "ro", Some(ref_kind(OwnerKind::Deployment, "dep", "dep")), ))) { let _ = h.handle_rs_owner(e); } // Owner Delete retained (no events forwarded) assert!(owner_tomb .handle(Event::Delete(rs_owner( "ro", Some(ref_kind(OwnerKind::Deployment, "dep", "dep")) ))) .next() .is_none()); // Pod Apply should emit now because owner is still present in store let out = h.handle_pod(Event::Apply(pod( "pp", "10.0.0.60", Some(ref_kind(OwnerKind::ReplicaSet, "ro", "rs")), ))); assert!(out.iter().any(|e| matches!(e, RenderEvent::PodNew { .. }))); } #[test] // Pod flaps Apply+Delete: the delete is retained in the pod tombstone adapter. // The pod Apply is forwarded (no emit because owner missing); on a later owner Apply, // the pod should still emit despite the retained delete. fn pod_flaps_apply_delete_then_owner_apply_emits() { let mut h = Harness::new(); let mut pod_tomb = crate::tombstone_adapter::TombstoneAdapter::new( std::time::Duration::from_secs(60), 100, ); // Pod Apply forwarded to matcher (no owner yet -> no output) for e in pod_tomb.handle(Event::Apply(pod( "pp2", "10.0.0.61", Some(ref_kind(OwnerKind::ReplicaSet, "ro2", "rs")), ))) { let out = h.handle_pod(e); assert!(out.is_empty()); } // Pod Delete retained by tombstone adapter assert!(pod_tomb .handle(Event::Delete(pod( "pp2", "10.0.0.61", Some(ref_kind(OwnerKind::ReplicaSet, "ro2", "rs")) ))) .next() .is_none()); // Owner Apply arrives -> should emit pod despite pending delete let out = h.handle_rs_owner(Event::Apply(rs_owner( "ro2", Some(ref_kind(OwnerKind::Deployment, "dep2", "dep")), ))); assert!(out.iter().any(|e| matches!(e, RenderEvent::PodNew { .. }))); } } ================================================ FILE: crates/k8s-collector/src/output.rs ================================================ //! Normalized events produced by the matcher and consumed by the encoder. //! //! Events mirror the messages the reducer expects. `PodNew` is accompanied by //! one or more `PodContainer` events. `PodResync` indicates a new epoch. use crate::types::{ContainerMeta, OwnerKind, OwnerRef, PodMeta}; #[derive(Clone, Debug)] pub enum RenderEvent { /// Begin a new resync epoch with a monotonically increasing counter. PodResync { epoch: u64 }, PodNew { /// Pod UID uid: String, /// Pod IPv4 string ip: String, /// Pod name pod_name: String, /// Namespace ns: String, /// Deterministic version from images version: String, /// Effective owner kind owner_kind: OwnerKind, /// Effective owner UID (empty for NoOwner) owner_uid: String, /// Effective owner name (pod name for NoOwner) owner_name: String, /// Host network flag is_host_network: bool, }, PodContainer { /// Pod UID for which the container is reported uid: String, /// Container metadata container: ContainerMeta, }, /// Delete a previously live pod PodDelete { uid: String }, } impl RenderEvent { /// Expand a `PodNew` event into the full set of render events; includes /// `PodNew` itself and one `PodContainer` per container. pub fn from_pod_new(pod: &PodMeta, owner: Option<&OwnerRef>) -> Vec { let mut out = Vec::new(); let (owner_kind, owner_uid, owner_name) = match owner { Some(o) => (o.kind, o.uid.clone(), o.name.clone()), None => (OwnerKind::NoOwner, String::new(), pod.name.clone()), }; out.push(RenderEvent::PodNew { uid: pod.uid.clone(), ip: pod.ip.clone(), pod_name: pod.name.clone(), ns: pod.namespace.clone(), version: pod.version.clone(), owner_kind, owner_uid, owner_name, is_host_network: pod.host_network, }); for c in &pod.containers { out.push(RenderEvent::PodContainer { uid: pod.uid.clone(), container: c.clone(), }); } out } } ================================================ FILE: crates/k8s-collector/src/tombstone_adapter.rs ================================================ use std::collections::VecDeque; use std::time::{Duration, Instant}; use kube::runtime::watcher::Event; /// Stream-level adapter that retains Delete events temporarily. /// /// - For Apply/InitApply/Init events: emit expired deletes first, then forward the event unchanged. /// - For Delete events: retain internally; if capacity exceeded or entries have expired, emit those deletes. /// - For InitDone: clear the retained deletes (drop them) and forward InitDone. pub struct TombstoneAdapter { ttl: Duration, cap: usize, q: VecDeque<(Instant, T)>, } impl TombstoneAdapter { /// Create a new adapter with the given time-to-live and capacity. pub fn new(ttl: Duration, cap: usize) -> Self { Self { ttl, cap, q: VecDeque::new(), } } /// Handle a new event and return an iterator over forwarded events: /// - Delete: retained; may emit evicted/expired deletes /// - Apply/Init/InitApply: emit expired deletes first, then the event /// - InitDone: clear retention and forward InitDone pub fn handle(&mut self, ev: Event) -> impl Iterator> { let mut out: Vec> = Vec::new(); let now = Instant::now(); let mut drain_expired = |out: &mut Vec>| { // Only inspect the timestamp by reference; clone `T` only when expired loop { match self.q.front() { Some((ts, _)) if now.duration_since(*ts) >= self.ttl => { if let Some((_, t)) = self.q.pop_front() { out.push(Event::Delete(t)); } } _ => break, } } }; match ev { Event::Init => { // Forward init after flushing expired deletes drain_expired(&mut out); out.push(Event::Init); } Event::InitApply(t) => { drain_expired(&mut out); out.push(Event::InitApply(t)); } Event::Apply(t) => { drain_expired(&mut out); out.push(Event::Apply(t)); } Event::Delete(t) => { // retain; drop expired first drain_expired(&mut out); self.q.push_back((now, t)); while self.q.len() > self.cap { if let Some((_, t)) = self.q.pop_front() { out.push(Event::Delete(t)); } } } Event::InitDone => { // Drop retained deletes on init-done; forward the signal self.q.clear(); out.push(Event::InitDone); } } out.into_iter() } } #[cfg(test)] mod tests { use super::*; #[test] // Capacity eviction: when capacity is 1, inserting a second Delete // should evict (and thus emit) the oldest retained Delete event. fn capacity_evicts_oldest_delete() { let mut t = TombstoneAdapter::new(Duration::from_secs(60), 1); // First delete retained, nothing emitted assert!(t.handle(Event::Delete(1)).next().is_none()); // Second delete should evict the first let mut evs = t.handle(Event::Delete(2)); match evs.next() { Some(Event::Delete(x)) => assert_eq!(x, 1), _ => panic!("expected Delete(1)"), } assert!(evs.next().is_none()); } #[test] // InitDone clears the retention queue, so no previously retained // deletes should be surfaced after it. The InitDone marker itself // is forwarded. fn initdone_clears_queue() { let mut t = TombstoneAdapter::new(Duration::from_secs(60), 2); // Retain two deletes assert!(t.handle(Event::Delete(10)).next().is_none()); assert!(t.handle(Event::Delete(20)).next().is_none()); // InitDone should drop queue and forward InitDone let mut evs = t.handle(Event::InitDone); matches!(evs.next(), Some(Event::InitDone)); assert!(evs.next().is_none()); // Another delete should not surface the old deletes anymore, they are dropped let mut evs2 = t.handle(Event::Delete(30)); if evs2.next().is_some() { panic!("expected event - queue should be empty"); } assert!(evs2.next().is_none()); } #[test] // TTL expiration: with ttl=0, a retained Delete is expired immediately // on the next handle() call and emitted before forwarding the new event. fn ttl_expires_before_apply() { // ttl=0 forces immediate expiration on next handle let mut t = TombstoneAdapter::new(Duration::from_secs(0), 10); assert!(t.handle(Event::Delete(42)).next().is_none()); // Next non-delete should surface the expired delete first let mut it = t.handle(Event::Apply(7)); match it.next() { Some(Event::Delete(x)) => assert_eq!(x, 42), _ => panic!("expected Delete(42)"), } match it.next() { Some(Event::Apply(x)) => assert_eq!(x, 7), _ => panic!("expected Apply(7)"), } assert!(it.next().is_none()); } } ================================================ FILE: crates/k8s-collector/src/types.rs ================================================ //! Compact metadata types used by the collector. //! //! These types are derived from Kubernetes objects and used internally by the //! matcher and output layers. use std::fmt; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OwnerKind { /// apps/v1 ReplicaSet ReplicaSet, /// apps/v1 Deployment Deployment, /// batch/v1 Job Job, /// batch/v1 CronJob CronJob, /// Explicitly indicates no owner NoOwner, /// Any other kind; treated as opaque Other, } impl OwnerKind { pub fn is_rs_or_job(self) -> bool { matches!(self, OwnerKind::ReplicaSet | OwnerKind::Job) } } impl std::str::FromStr for OwnerKind { type Err = (); /// Parse a human-readable Kubernetes kind string into an [`OwnerKind`]. fn from_str(s: &str) -> Result { Ok(match s { "ReplicaSet" => OwnerKind::ReplicaSet, "Deployment" => OwnerKind::Deployment, "Job" => OwnerKind::Job, "CronJob" => OwnerKind::CronJob, "NoOwner" => OwnerKind::NoOwner, _ => OwnerKind::Other, }) } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct OwnerRef { /// Kubernetes owner kind pub kind: OwnerKind, /// Owner UID pub uid: String, /// Owner name pub name: String, } impl fmt::Display for OwnerRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "{}:{}:{}", match self.kind { OwnerKind::ReplicaSet => "ReplicaSet", OwnerKind::Deployment => "Deployment", OwnerKind::Job => "Job", OwnerKind::CronJob => "CronJob", OwnerKind::NoOwner => "NoOwner", OwnerKind::Other => "Other", }, self.name, self.uid ) } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct ContainerMeta { /// Container runtime ID (e.g., docker://... or containerd://...) pub id: String, /// Container name pub name: String, /// Container image reference pub image: String, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct PodMeta { /// Pod UID pub uid: String, /// Pod IP address (empty if not yet assigned) pub ip: String, /// Pod name pub name: String, /// Namespace name pub namespace: String, /// Controller owner (if any) pub owner: Option, /// Whether the Pod uses host networking pub host_network: bool, /// Deterministic version built from container images pub version: String, /// Containers observed for this Pod pub containers: Vec, } impl PodMeta { pub fn has_ip(&self) -> bool { !self.ip.is_empty() } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct OwnerMeta { /// UID of the owner (ReplicaSet or Job) pub uid: String, /// Its owner reference, if any (e.g. Deployment for RS, CronJob for Job) pub controller: Option, } ================================================ FILE: crates/k8s-collector/src/writer.rs ================================================ //! Async TCP writer to the reducer with LZ4 streaming compression. //! //! Semantics: //! - On connect, the first message is sent uncompressed to negotiate //! decompression on the reducer side. //! - Subsequent messages are appended to a single persistent LZ4 frame. //! - `flush()` calls the encoder's `flush()` and drains any newly produced bytes //! to the socket, then flushes the socket. //! //! The writer only writes the newly produced compressed tail per call; the //! underlying frame buffer is retained between calls and only reset on //! reconnect. use std::io; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio_util::io::SyncIoBridge; /// Internal sink state: uncompressed passthrough initially, then a persistent /// LZ4 frame once the first message has been sent. enum Sink { Uncompressed(W), Compressed { encoder: Box>>, }, } pub struct Writer { // Invariant: always Some; None is a temporary state during transitions sink: Option>, } impl Writer { /// Create a new writer. The first message is sent uncompressed; subsequent /// messages are appended to a persistent LZ4 frame. pub fn new(sink: W) -> Self { Self { sink: Some(Sink::Uncompressed(sink)), } } /// Send a single message, applying the protocol described above. pub async fn send(&mut self, buf: &[u8]) -> io::Result<()> { // Take ownership of the sink enum to allow moving `W` between variants let sink = self.sink.take().expect("unreachable: sink must be set"); match sink { Sink::Uncompressed(mut w) => { // First message: send uncompressed, then switch to compressed mode w.write_all(buf).await?; // Switch to compressed mode where the encoder writes directly to the inner sink let bridge = SyncIoBridge::new(w); let encoder = lz4_flex::frame::FrameEncoder::new(bridge); self.sink = Some(Sink::Compressed { encoder: Box::new(encoder), }); Ok(()) } Sink::Compressed { mut encoder } => { use std::io::Write; // Perform synchronous compression in a blocking task; the bridge // will block on the async writer without panicking. let data = buf.to_vec(); let res = tokio::task::spawn_blocking(move || { encoder.write_all(&data)?; Ok::<_, io::Error>(encoder) }) .await .map_err(|e| io::Error::other(format!("join error: {}", e)))??; self.sink = Some(Sink::Compressed { encoder: res }); Ok(()) } } } /// Flush any pending compressed bytes and the underlying socket. pub async fn flush(&mut self) -> io::Result<()> { // We need ownership of the encoder to run flush inside spawn_blocking, // so we temporarily take the enum and put it back after flushing. let sink = self.sink.take().expect("unreachable: sink must be set"); match sink { Sink::Uncompressed(mut w) => { w.flush().await?; self.sink = Some(Sink::Uncompressed(w)); Ok(()) } Sink::Compressed { encoder } => { // Flush the encoder which flushes the underlying sink via the bridge use std::io::Write; let res = tokio::task::spawn_blocking(move || { let mut enc = encoder; enc.flush()?; Ok::<_, io::Error>(enc) }) .await .map_err(|e| io::Error::other(format!("join error: {}", e)))??; self.sink = Some(Sink::Compressed { encoder: res }); Ok(()) } } } } #[cfg(test)] mod tests { use lz4_flex::frame::FrameDecoder; use std::io::Read; use std::pin::Pin; use std::sync::{Arc, Mutex}; use tokio::io::AsyncWrite; /// Simple in-memory AsyncWrite sink used for tests. /// Bytes written are appended to the shared buffer. struct TestSink(pub Arc>>); impl AsyncWrite for TestSink { fn poll_write( self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &[u8], ) -> std::task::Poll> { let mut g = self.0.lock().unwrap(); g.extend_from_slice(buf); std::task::Poll::Ready(Ok(buf.len())) } fn poll_flush( self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } fn poll_shutdown( self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } } #[test] // Property-based test: with compression enabled, the first flush exposes the // uncompressed handshake bytes; subsequent flushes append to the persistent // LZ4 frame. Decoding the combined compressed bytes after both flushes must // equal the concatenation of the payloads. fn flush_visibility_compressed_prop() { // Property-based: first flush exposes the uncompressed handshake (a), // subsequent flushes append to the persistent LZ4 frame (b then b+c). proptest::proptest!(|(a in "[ -~]{0,64}", b in "[ -~]{0,64}", c in "[ -~]{0,64}")| { let buf = Arc::new(Mutex::new(Vec::new())); let sink = TestSink(buf.clone()); let rt = tokio::runtime::Runtime::new().unwrap(); let a2 = a.clone(); let b2 = b.clone(); let c2 = c.clone(); rt.block_on(async move { let mut w = super::Writer::new(sink); // First write is uncompressed handshake w.send(a2.as_bytes()).await.unwrap(); w.flush().await.unwrap(); // Verify handshake visible assert_eq!(&buf.lock().unwrap()[..], a2.as_bytes()); // Second write goes into persistent frame w.send(b2.as_bytes()).await.unwrap(); w.flush().await.unwrap(); let snap = buf.lock().unwrap().clone(); let (first, rest) = snap.split_at(a2.len()); assert_eq!(first, a2.as_bytes()); let mut dec = FrameDecoder::new(rest); let mut out = Vec::new(); dec.read_to_end(&mut out).unwrap(); assert_eq!(out, b2.as_bytes()); // Third write appends to same frame w.send(c2.as_bytes()).await.unwrap(); w.flush().await.unwrap(); let snap2 = buf.lock().unwrap().clone(); let (_first2, rest2) = snap2.split_at(a2.len()); let mut dec2 = FrameDecoder::new(rest2); let mut out2 = Vec::new(); dec2.read_to_end(&mut out2).unwrap(); let mut expected = Vec::new(); expected.extend_from_slice(b2.as_bytes()); expected.extend_from_slice(c2.as_bytes()); assert_eq!(out2, expected); }); }); } } ================================================ FILE: crates/k8s-collector/tests/collector_prop.rs ================================================ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Duration; use k8s_collector::matcher::Matcher; use k8s_collector::output::RenderEvent; use k8s_collector::tombstone_adapter::TombstoneAdapter; use k8s_collector::types::{ContainerMeta, OwnerKind, OwnerMeta, OwnerRef, PodMeta}; use kube::runtime::reflector::store; use kube::runtime::watcher::Event; use proptest::prelude::*; /// Minimal, synchronous harness that mirrors the production Collector pipeline: /// - tombstone adapters in front of /// - reflector writers feeding Stores used by /// - Matcher, which produces RenderEvents. struct PipelineHarness { matcher: Matcher, rs_owners_writer: store::Writer, job_owners_writer: store::Writer, pods_writer: store::Writer, rs_owner_tomb: TombstoneAdapter, job_owner_tomb: TombstoneAdapter, pod_tomb: TombstoneAdapter, } impl PipelineHarness { fn new() -> Self { let (rs_owners_store, rs_owners_writer) = store::store::(); let (job_owners_store, job_owners_writer) = store::store::(); let (pods_store, pods_writer) = store::store::(); let matcher = Matcher::new(rs_owners_store, job_owners_store, pods_store); let rs_owner_tomb = TombstoneAdapter::new(Duration::from_secs(0), 1024); let job_owner_tomb = TombstoneAdapter::new(Duration::from_secs(0), 1024); let pod_tomb = TombstoneAdapter::new(Duration::from_secs(0), 1024); Self { matcher, rs_owners_writer, job_owners_writer, pods_writer, rs_owner_tomb, job_owner_tomb, pod_tomb, } } fn apply_rs_owner(&mut self, ev: Event) -> Vec { let mut out = Vec::new(); for fwd in self.rs_owner_tomb.handle(ev) { self.rs_owners_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_owner(fwd)); } out } fn apply_job_owner(&mut self, ev: Event) -> Vec { let mut out = Vec::new(); for fwd in self.job_owner_tomb.handle(ev) { self.job_owners_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_owner(fwd)); } out } fn apply_pod(&mut self, ev: Event) -> Vec { let mut out = Vec::new(); for fwd in self.pod_tomb.handle(ev) { self.pods_writer.apply_watcher_event(&fwd); out.extend(self.matcher.handle_pod(fwd)); } out } } /// How a pod is owned in Kubernetes, restricted to the shapes /// the collector actually supports (RS/Job with optional controller). #[derive(Clone, Debug)] enum OwnerScenario { NoOwner, ReplicaSetOnly, ReplicaSetWithDeployment, JobOnly, JobWithCronJob, } impl OwnerScenario { fn from_tag(tag: u8) -> Self { match tag % 5 { 0 => OwnerScenario::NoOwner, 1 => OwnerScenario::ReplicaSetOnly, 2 => OwnerScenario::ReplicaSetWithDeployment, 3 => OwnerScenario::JobOnly, _ => OwnerScenario::JobWithCronJob, } } /// Build the k8s meta objects for this scenario: /// - OwnerRef embedded into PodMeta (if any) /// - OwnerMeta objects for ReplicaSet/Job owners /// - The effective owner after escalation (Deployment/CronJob) for the model. fn build_for_pod(&self, pod_id: u8) -> (Option, Vec, Option) { let rs_uid = format!("rs-{pod_id}"); let rs_name = format!("rs-{pod_id}"); let dep_uid = format!("dep-{pod_id}"); let dep_name = format!("dep-{pod_id}"); let job_uid = format!("job-{pod_id}"); let job_name = format!("job-{pod_id}"); let cron_uid = format!("cron-{pod_id}"); let cron_name = format!("cron-{pod_id}"); match self { OwnerScenario::NoOwner => (None, Vec::new(), None), OwnerScenario::ReplicaSetOnly => { let pod_owner = OwnerRef { kind: OwnerKind::ReplicaSet, uid: rs_uid.clone(), name: rs_name.clone(), }; let owner_meta = OwnerMeta { uid: rs_uid, controller: None, }; (Some(pod_owner.clone()), vec![owner_meta], Some(pod_owner)) } OwnerScenario::ReplicaSetWithDeployment => { let pod_owner = OwnerRef { kind: OwnerKind::ReplicaSet, uid: rs_uid.clone(), name: rs_name.clone(), }; let controller = OwnerRef { kind: OwnerKind::Deployment, uid: dep_uid.clone(), name: dep_name.clone(), }; let owner_meta = OwnerMeta { uid: rs_uid, controller: Some(controller.clone()), }; (Some(pod_owner), vec![owner_meta], Some(controller)) } OwnerScenario::JobOnly => { let pod_owner = OwnerRef { kind: OwnerKind::Job, uid: job_uid.clone(), name: job_name.clone(), }; let owner_meta = OwnerMeta { uid: job_uid, controller: None, }; (Some(pod_owner.clone()), vec![owner_meta], Some(pod_owner)) } OwnerScenario::JobWithCronJob => { let pod_owner = OwnerRef { kind: OwnerKind::Job, uid: job_uid.clone(), name: job_name.clone(), }; let controller = OwnerRef { kind: OwnerKind::CronJob, uid: cron_uid.clone(), name: cron_name.clone(), }; let owner_meta = OwnerMeta { uid: job_uid, controller: Some(controller.clone()), }; (Some(pod_owner), vec![owner_meta], Some(controller)) } } } } #[derive(Clone, Debug)] struct ModelPod { pod_id: u8, uid: String, scenario: OwnerScenario, present: bool, has_ip: bool, effective_owner: Option, } #[derive(Clone, Debug, Default)] struct ModelState { pods: BTreeMap, epoch: u64, } impl ModelState { fn is_live(&self, pod_id: u8) -> bool { self.pods .get(&pod_id) .map(|p| p.present && p.has_ip) .unwrap_or(false) } fn live_uids(&self) -> BTreeSet { self.pods .values() .filter(|p| p.present && p.has_ip) .map(|p| p.uid.clone()) .collect() } } /// Helper to construct a simple PodMeta (single container, fixed image) /// with a stable UID/IP derived from pod_id. fn build_pod_meta(pod_id: u8, owner: Option) -> PodMeta { let uid = format!("pod-{pod_id}"); PodMeta { uid, ip: format!("10.0.0.{pod_id}"), name: format!("pod-{pod_id}"), namespace: "default".into(), owner, host_network: false, version: "'img:1'".into(), containers: vec![ContainerMeta { id: format!("c-{pod_id}"), name: "c".into(), image: "img:1".into(), }], } } #[derive(Clone, Debug)] enum Op { AddPod { pod_id: u8, scenario: OwnerScenario }, DeletePod { pod_id: u8 }, Reinit, } #[test] fn prop_collector_matches_pod_model() { let config = ProptestConfig { cases: 64, ..ProptestConfig::default() }; // Each step carries: // - tag: selects op kind and also flips arrival order in AddPod // - pod_id: small identifier mapped to a UID // - owner_tag: mapped into an OwnerScenario proptest!(config, |(ops in proptest::collection::vec((any::(), any::(), any::()), 0..128))| { let mut harness = PipelineHarness::new(); let mut model = ModelState::default(); let mut first_seen: HashMap = HashMap::new(); let mut last_epoch: u64 = 0; for (tag, pod_id, owner_tag) in ops { let scenario = OwnerScenario::from_tag(owner_tag); let op = match tag % 3 { 0 => Op::AddPod { pod_id, scenario }, 1 => Op::DeletePod { pod_id }, _ => Op::Reinit, }; let mut step_events: Vec = Vec::new(); match op { Op::AddPod { pod_id, scenario } => { // Add or update a pod with a particular ownership scenario. // We randomize whether owners or pod arrive first and assert: // - exactly one PodNew when the pod becomes live // - PodNew comes from the second arrival // - PodNew owner fields match the scenario. let was_live = model.is_live(pod_id); let (pod_owner_ref, owners_meta, effective_owner) = scenario.build_for_pod(pod_id); let uid = format!("pod-{pod_id}"); let pod_meta = build_pod_meta(pod_id, pod_owner_ref.clone()); let owners_first = (tag & 0x80) != 0; let mut owner_events = Vec::new(); let mut pod_events = Vec::new(); if owners_first { for owner in owners_meta.iter().cloned() { match scenario { OwnerScenario::ReplicaSetOnly | OwnerScenario::ReplicaSetWithDeployment => { owner_events.extend(harness.apply_rs_owner(Event::Apply(owner))); } OwnerScenario::JobOnly | OwnerScenario::JobWithCronJob => { owner_events.extend(harness.apply_job_owner(Event::Apply(owner))); } OwnerScenario::NoOwner => {} } } pod_events.extend(harness.apply_pod(Event::Apply(pod_meta))); } else { pod_events.extend(harness.apply_pod(Event::Apply(pod_meta))); for owner in owners_meta.iter().cloned() { match scenario { OwnerScenario::ReplicaSetOnly | OwnerScenario::ReplicaSetWithDeployment => { owner_events.extend(harness.apply_rs_owner(Event::Apply(owner))); } OwnerScenario::JobOnly | OwnerScenario::JobWithCronJob => { owner_events.extend(harness.apply_job_owner(Event::Apply(owner))); } OwnerScenario::NoOwner => {} } } } model.pods.insert( pod_id, ModelPod { pod_id, uid: uid.clone(), scenario, present: true, has_ip: true, effective_owner, }, ); let now_live = model.is_live(pod_id); if !was_live && now_live { let num_podnew_owner = owner_events .iter() .filter(|ev| matches!(ev, RenderEvent::PodNew { uid: ev_uid, .. } if ev_uid == &uid)) .count(); let num_podnew_pod = pod_events .iter() .filter(|ev| matches!(ev, RenderEvent::PodNew { uid: ev_uid, .. } if ev_uid == &uid)) .count(); let total_podnew = num_podnew_owner + num_podnew_pod; prop_assert_eq!(total_podnew, 1, "expected exactly one PodNew for uid {}", uid); let mp = model.pods.get(&pod_id).expect("model should contain pod after AddPod"); match &mp.scenario { OwnerScenario::NoOwner => { prop_assert_eq!(num_podnew_pod, 1, "NoOwner PodNew should come from pod_events"); prop_assert_eq!(num_podnew_owner, 0, "NoOwner PodNew should not come from owner_events"); } _ => { if owners_first { prop_assert_eq!(num_podnew_pod, 1, "owners_first: PodNew should come from pod_events"); prop_assert_eq!(num_podnew_owner, 0, "owners_first: PodNew should not come from owner_events"); } else { prop_assert_eq!(num_podnew_owner, 1, "pod_first: PodNew should come from owner_events"); prop_assert_eq!(num_podnew_pod, 0, "pod_first: PodNew should not come from pod_events"); } } } let podnew_ev = owner_events .iter() .chain(pod_events.iter()) .find(|ev| matches!(ev, RenderEvent::PodNew { uid: ev_uid, .. } if ev_uid == &uid)) .expect("PodNew must be present for live transition"); if let RenderEvent::PodNew { owner_kind, owner_uid, owner_name, pod_name, .. } = podnew_ev { match &mp.effective_owner { None => { prop_assert_eq!( *owner_kind, OwnerKind::NoOwner, "AddPod: expected NoOwner for uid {}", uid ); prop_assert!( owner_uid.is_empty(), "AddPod: owner_uid should be empty for NoOwner uid {}", uid ); prop_assert_eq!( owner_name, pod_name, "AddPod: owner_name should equal pod_name for NoOwner uid {}", uid ); } Some(o) => { prop_assert_eq!( *owner_kind, o.kind, "AddPod: owner_kind mismatch for uid {}", uid ); prop_assert_eq!( owner_uid, &o.uid, "AddPod: owner_uid mismatch for uid {}", uid ); prop_assert_eq!( owner_name, &o.name, "AddPod: owner_name mismatch for uid {}", uid ); } } } } step_events.extend(owner_events); step_events.extend(pod_events); } Op::DeletePod { pod_id } => { // Delete a pod if present; we only assert indirectly via // subsequent resync snapshots (PodNew should no longer appear). if let Some(mp) = model.pods.get_mut(&pod_id) { mp.present = false; let (owner_ref, _owners, _effective_owner) = mp.scenario.build_for_pod(pod_id); let pod_meta = build_pod_meta(pod_id, owner_ref); let events = harness.apply_pod(Event::Delete(pod_meta)); step_events.extend(events); } } Op::Reinit => { // Simulate a watcher re-init: Init/InitApply/InitDone for // owners then pods. Matcher should emit: // - a new PodResync{epoch} // - PodNew for exactly the live pods in the model. model.epoch = model.epoch.wrapping_add(1); let mut rs_owners_snapshot: Vec = Vec::new(); let mut job_owners_snapshot: Vec = Vec::new(); for mp in model.pods.values() { if !mp.present { continue; } let (_owner_ref, owners_meta, _effective_owner) = mp.scenario.build_for_pod(mp.pod_id); match mp.scenario { OwnerScenario::ReplicaSetOnly | OwnerScenario::ReplicaSetWithDeployment => { rs_owners_snapshot.extend(owners_meta); } OwnerScenario::JobOnly | OwnerScenario::JobWithCronJob => { job_owners_snapshot.extend(owners_meta); } OwnerScenario::NoOwner => {} } } let mut pods_snapshot: Vec = Vec::new(); for mp in model.pods.values() { if !mp.present { continue; } let (owner_ref, _owners_meta, _effective_owner) = mp.scenario.build_for_pod(mp.pod_id); let pod_meta = build_pod_meta(mp.pod_id, owner_ref); pods_snapshot.push(pod_meta); } let mut epoch_events: Vec = Vec::new(); // RS owners re-init epoch_events.extend(harness.apply_rs_owner(Event::Init)); for owner in rs_owners_snapshot.into_iter() { epoch_events.extend(harness.apply_rs_owner(Event::InitApply(owner))); } epoch_events.extend(harness.apply_rs_owner(Event::InitDone)); // Job owners re-init epoch_events.extend(harness.apply_job_owner(Event::Init)); for owner in job_owners_snapshot.into_iter() { epoch_events.extend(harness.apply_job_owner(Event::InitApply(owner))); } epoch_events.extend(harness.apply_job_owner(Event::InitDone)); epoch_events.extend(harness.apply_pod(Event::Init)); for pod_meta in pods_snapshot.into_iter() { epoch_events.extend(harness.apply_pod(Event::InitApply(pod_meta))); } epoch_events.extend(harness.apply_pod(Event::InitDone)); let mut resync_epoch: Option = None; let mut seen_resync = false; let mut snapshot_uids: BTreeSet = BTreeSet::new(); for ev in &epoch_events { match ev { RenderEvent::PodResync { epoch } => { if !seen_resync { resync_epoch = Some(*epoch); seen_resync = true; } } RenderEvent::PodNew { uid, .. } => { if seen_resync { snapshot_uids.insert(uid.clone()); } } _ => {} } } prop_assert!(seen_resync, "expected PodResync event on Reinit"); let epoch = resync_epoch.unwrap(); prop_assert!(epoch > last_epoch, "epoch should increase monotonically"); last_epoch = epoch; let live = model.live_uids(); prop_assert_eq!(snapshot_uids, live, "snapshot PodNew uids must equal live set"); step_events.extend(epoch_events); } } for ev in &step_events { match ev { RenderEvent::PodNew { uid, owner_kind, owner_uid, owner_name, pod_name, .. } => { // First event for a uid must be PodNew. if !first_seen.contains_key(uid) { first_seen.insert(uid.clone(), "PodNew"); } let mp_opt = model .pods .values() .find(|p| &p.uid == uid); prop_assert!(mp_opt.is_some(), "PodNew for unknown uid {}", uid); let mp = mp_opt.unwrap(); match &mp.effective_owner { None => { prop_assert_eq!(*owner_kind, OwnerKind::NoOwner, "expected NoOwner for uid {}", uid); prop_assert!(owner_uid.is_empty(), "owner_uid should be empty for NoOwner uid {}", uid); prop_assert_eq!(owner_name, pod_name, "owner_name should equal pod_name for NoOwner uid {}", uid); } Some(o) => { prop_assert_eq!(*owner_kind, o.kind, "owner_kind mismatch for uid {}", uid); prop_assert_eq!(owner_uid, &o.uid, "owner_uid mismatch for uid {}", uid); prop_assert_eq!(owner_name, &o.name, "owner_name mismatch for uid {}", uid); } } } RenderEvent::PodContainer { uid, .. } => { // PodContainer should only follow PodNew for a uid. if !first_seen.contains_key(uid) { prop_assert!(false, "first event for uid {} was PodContainer, not PodNew", uid); } } RenderEvent::PodDelete { uid } => { // PodDelete should only follow PodNew for a uid. if !first_seen.contains_key(uid) { prop_assert!(false, "first event for uid {} was PodDelete, not PodNew", uid); } } _ => {} } } } }); } ================================================ FILE: crates/k8s-collector/tests/integration_test.rs ================================================ use std::path::PathBuf; use std::time::Duration; use anyhow::{bail, Context}; use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec, ReplicaSet, ReplicaSetSpec}; use k8s_openapi::api::batch::v1::{CronJob, CronJobSpec, Job, JobSpec}; use k8s_openapi::api::core::v1::{Container, Namespace, Pod, PodSpec, PodStatus, PodTemplateSpec}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, ObjectMeta, OwnerReference}; use kube::{ api::{Api, DeleteParams, ListParams, PostParams, ResourceExt}, Client, }; use serde_json::json; use tokio::time::Instant; use k8s_collector::collector::Collector; use k8s_collector::config::Config; use k8s_collector::output::RenderEvent; use k8s_collector::types::OwnerKind; use log::info; fn init_test_logger() { if std::env::var_os("RUST_BACKTRACE").is_none() { std::env::set_var("RUST_BACKTRACE", "1"); } if std::env::var_os("RUST_LIB_BACKTRACE").is_none() { std::env::set_var("RUST_LIB_BACKTRACE", "1"); } let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) .is_test(true) .try_init(); } async fn ensure_namespace(client: &Client, name: &str) -> anyhow::Result<()> { let namespaces: Api = Api::all(client.clone()); match namespaces.get(name).await { Ok(_) => Ok(()), Err(kube::Error::Api(ae)) if ae.code == 404 => { let ns: Namespace = serde_json::from_value(json!({ "apiVersion": "v1", "kind": "Namespace", "metadata": { "name": name }, }))?; namespaces.create(&PostParams::default(), &ns).await?; Ok(()) } Err(e) => Err(e.into()), } } async fn delete_if_exists(api: &Api, name: &str) -> anyhow::Result<()> where T: k8s_openapi::Metadata + Clone + serde::de::DeserializeOwned + serde::Serialize + std::fmt::Debug, { let dp = DeleteParams { grace_period_seconds: Some(0), ..DeleteParams::default() }; match api.delete(name, &dp).await { Ok(_o) => Ok(()), Err(kube::Error::Api(ae)) if ae.code == 404 => Ok(()), Err(e) => Err(e.into()), } } async fn wait_for_pod_running_by_label( pods: &Api, label_selector: &str, timeout: Duration, ) -> anyhow::Result { let deadline = Instant::now() + timeout; loop { let lp = ListParams::default().labels(label_selector); let pod_list = pods.list(&lp).await?; for pod in pod_list { if let Some(status) = &pod.status { if status.phase.as_deref() == Some("Running") && containers_ready(status) { return Ok(pod); } } } if Instant::now() >= deadline { bail!( "timed out waiting for a Running pod with label selector {}", label_selector ); } tokio::time::sleep(Duration::from_secs(1)).await; } } fn containers_ready(status: &PodStatus) -> bool { status .container_statuses .as_ref() .map(|statuses| statuses.iter().all(|cs| cs.ready)) .unwrap_or(false) } async fn create_sleep_deployment( client: &Client, namespace: &str, name: &str, ) -> anyhow::Result { let deployments: Api = Api::namespaced(client.clone(), namespace); let labels: std::collections::BTreeMap = [("k8s-collector-e2e".to_string(), name.to_string())] .into_iter() .collect(); let deployment = Deployment { metadata: ObjectMeta { name: Some(name.to_string()), namespace: Some(namespace.to_string()), ..ObjectMeta::default() }, spec: Some(DeploymentSpec { replicas: Some(1), selector: LabelSelector { match_labels: Some(labels.clone()), ..LabelSelector::default() }, template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..ObjectMeta::default() }), spec: Some(PodSpec { containers: vec![Container { name: "main".into(), image: Some("busybox".into()), command: Some(vec!["/bin/sh".into(), "-c".into(), "sleep 3600".into()]), image_pull_policy: Some("IfNotPresent".into()), ..Container::default() }], ..PodSpec::default() }), }, ..DeploymentSpec::default() }), status: None, }; Ok(deployments .create(&PostParams::default(), &deployment) .await?) } async fn create_replicaset( client: &Client, namespace: &str, name: &str, ) -> anyhow::Result { let replicasets: Api = Api::namespaced(client.clone(), namespace); let labels: std::collections::BTreeMap = [("k8s-collector-e2e".to_string(), name.to_string())] .into_iter() .collect(); let rs = ReplicaSet { metadata: ObjectMeta { name: Some(name.to_string()), namespace: Some(namespace.to_string()), ..ObjectMeta::default() }, spec: Some(ReplicaSetSpec { replicas: Some(1), selector: LabelSelector { match_labels: Some(labels.clone()), ..LabelSelector::default() }, template: Some(PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..ObjectMeta::default() }), spec: Some(PodSpec { containers: vec![Container { name: "main".into(), image: Some("busybox".into()), command: Some(vec!["/bin/sh".into(), "-c".into(), "sleep 3600".into()]), image_pull_policy: Some("IfNotPresent".into()), ..Container::default() }], ..PodSpec::default() }), }), ..ReplicaSetSpec::default() }), status: None, }; Ok(replicasets.create(&PostParams::default(), &rs).await?) } async fn create_cronjob(client: &Client, namespace: &str, name: &str) -> anyhow::Result { let cronjobs: Api = Api::namespaced(client.clone(), namespace); let labels: std::collections::BTreeMap = [("k8s-collector-e2e".to_string(), name.to_string())] .into_iter() .collect(); let cj = CronJob { metadata: ObjectMeta { name: Some(name.to_string()), namespace: Some(namespace.to_string()), ..ObjectMeta::default() }, spec: Some(CronJobSpec { schedule: "* * * * *".into(), job_template: k8s_openapi::api::batch::v1::JobTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels.clone()), ..ObjectMeta::default() }), spec: Some(JobSpec { template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..ObjectMeta::default() }), spec: Some(PodSpec { containers: vec![Container { name: "main".into(), image: Some("busybox".into()), command: Some(vec![ "/bin/sh".into(), "-c".into(), "sleep 3600".into(), ]), image_pull_policy: Some("IfNotPresent".into()), ..Container::default() }], restart_policy: Some("Never".into()), ..PodSpec::default() }), }, ..JobSpec::default() }), }, ..CronJobSpec::default() }), status: None, }; Ok(cronjobs.create(&PostParams::default(), &cj).await?) } async fn create_job_with_cron_owner( client: &Client, namespace: &str, cron: &CronJob, job_name: &str, ) -> anyhow::Result { let jobs: Api = Api::namespaced(client.clone(), namespace); let cron_meta = cron.metadata.clone(); let cron_name = cron_meta .name .clone() .context("CronJob missing metadata.name")?; let cron_uid = cron_meta .uid .clone() .context("CronJob missing metadata.uid")?; let labels: std::collections::BTreeMap = [("k8s-collector-e2e".to_string(), job_name.to_string())] .into_iter() .collect(); let job = Job { metadata: ObjectMeta { name: Some(job_name.to_string()), namespace: Some(namespace.to_string()), owner_references: Some(vec![OwnerReference { api_version: "batch/v1".into(), kind: "CronJob".into(), name: cron_name, uid: cron_uid, controller: Some(true), ..OwnerReference::default() }]), ..ObjectMeta::default() }, spec: Some(JobSpec { template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..ObjectMeta::default() }), spec: Some(PodSpec { containers: vec![Container { name: "main".into(), image: Some("busybox".into()), command: Some(vec!["/bin/sh".into(), "-c".into(), "sleep 3600".into()]), image_pull_policy: Some("IfNotPresent".into()), ..Container::default() }], restart_policy: Some("Never".into()), ..PodSpec::default() }), }, ..JobSpec::default() }), status: None, }; Ok(jobs.create(&PostParams::default(), &job).await?) } struct ObservedPod { uid: String, name: String, expect_owner_kind: OwnerKind, expect_owner_name: String, saw_pod_new: bool, saw_container: bool, } fn update_observed_from_event(obs: &mut ObservedPod, ev: &RenderEvent) { match ev { RenderEvent::PodNew { uid, owner_kind, owner_name, .. } if uid == &obs.uid => { obs.saw_pod_new = true; assert_eq!( *owner_kind, obs.expect_owner_kind, "unexpected owner kind for pod {}", uid ); assert_eq!( owner_name, &obs.expect_owner_name, "unexpected owner name for pod {}", uid ); info!( "k8s-collector e2e: verified owner for pod uid={} name={} -> kind={:?} name={}", uid, obs.name, owner_kind, owner_name ); } RenderEvent::PodContainer { uid, .. } if uid == &obs.uid => { obs.saw_container = true; } _ => {} } } fn observed_done(obs: &ObservedPod) -> bool { obs.saw_pod_new && obs.saw_container } async fn pump_until(collector: &mut Collector, timeout: Duration, mut f: F) -> anyhow::Result<()> where F: FnMut(&RenderEvent) -> bool, { let deadline = Instant::now() + timeout; loop { if Instant::now() >= deadline { info!( "k8s-collector e2e: pump_until outer deadline {:?} elapsed; collector snapshot:\n{}", timeout, collector.debug_snapshot() ); bail!("timed out waiting for expected collector events"); } let remaining = deadline .checked_duration_since(Instant::now()) .unwrap_or_else(|| Duration::from_secs(1)); let batch = match tokio::time::timeout(remaining, collector.next_messages()).await { Ok(Ok(batch)) => batch, Ok(Err(e)) => { info!( "k8s-collector e2e: pump_until collector.next_messages() returned error: {:?}; snapshot:\n{}", e, collector.debug_snapshot() ); return Err(e.into()); } Err(elapsed) => { info!( "k8s-collector e2e: pump_until tokio timeout after {:?} while waiting for collector.next_messages(); snapshot:\n{}", remaining, collector.debug_snapshot() ); bail!( "tokio timeout waiting for collector.next_messages(): {}", elapsed ); } }; if batch.is_empty() { continue; } for ev in &batch { info!("k8s-collector e2e: pump_until observed event: {:?}", ev); if f(ev) { return Ok(()); } } } } #[tokio::test] #[cfg(target_os = "linux")] #[ignore] async fn test_k8s_collector_end_to_end_e2e() -> anyhow::Result<()> { init_test_logger(); // Use a per-run suffix to ensure that resource names are unique across test // invocations, so we never accidentally match pods from a previous run. let run_suffix = { use std::time::{SystemTime, UNIX_EPOCH}; let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_nanos(); let hex = format!("{:x}", nanos); // Take the last 6 hex chars to keep names short but unique-enough. hex.chars() .rev() .take(6) .collect::() .chars() .rev() .collect::() }; info!("k8s-collector e2e: using run suffix {}", run_suffix); // Skip the test if no kubeconfig is available; this environment // does not guarantee a Kubernetes cluster. let kubeconfig_env = std::env::var_os("KUBECONFIG"); let default_kubeconfig_missing = std::env::var_os("HOME") .map(|home| { let mut p = PathBuf::from(home); p.push(".kube/config"); !p.exists() }) .unwrap_or(true); if kubeconfig_env.is_none() && default_kubeconfig_missing { eprintln!( "KUBECONFIG not set and default kubeconfig missing; skipping k8s-collector e2e test" ); return Ok(()); } let client = Client::try_default().await?; let namespace = "k8s-collector-e2e"; info!( "k8s-collector e2e: starting test in namespace {}", namespace ); ensure_namespace(&client, namespace).await?; info!("k8s-collector e2e: namespace {} is ready", namespace); let pods: Api = Api::namespaced(client.clone(), namespace); let deployments: Api = Api::namespaced(client.clone(), namespace); let jobs: Api = Api::namespaced(client.clone(), namespace); let cronjobs: Api = Api::namespaced(client.clone(), namespace); let replicasets: Api = Api::namespaced(client.clone(), namespace); // Per-run resource names with a unique suffix to avoid collisions // between test invocations. let pre_deploy_name = format!("kc-e2e-pre-deploy-{}", run_suffix); let pre_cron_name = format!("kc-e2e-pre-cron-{}", run_suffix); let pre_job_name = format!("kc-e2e-pre-job-{}", run_suffix); let pre_rs_name = format!("kc-e2e-pre-rs-{}", run_suffix); let new_deploy_name = format!("kc-e2e-new-deploy-{}", run_suffix); let new_cron_name = format!("kc-e2e-new-cron-{}", run_suffix); let new_job_name = format!("kc-e2e-new-job-{}", run_suffix); // Best-effort cleanup from previous runs with the same naming pattern. let _ = delete_if_exists(&deployments, &pre_deploy_name).await; let _ = delete_if_exists(&deployments, &new_deploy_name).await; let _ = delete_if_exists(&cronjobs, &pre_cron_name).await; let _ = delete_if_exists(&cronjobs, &new_cron_name).await; let _ = delete_if_exists(&jobs, &pre_job_name).await; let _ = delete_if_exists(&jobs, &new_job_name).await; let _ = delete_if_exists(&replicasets, &pre_rs_name).await; info!("k8s-collector e2e: completed pre-test cleanup of prior resources"); // Pre-existing Deployment and CronJob+Job before the collector starts. let pre_deploy = create_sleep_deployment(&client, namespace, &pre_deploy_name).await?; let pre_cron = create_cronjob(&client, namespace, &pre_cron_name).await?; let _pre_job = create_job_with_cron_owner(&client, namespace, &pre_cron, &pre_job_name).await?; let pre_rs = create_replicaset(&client, namespace, &pre_rs_name).await?; // Wait for their pods to be running and capture UIDs. let pre_deploy_pod = wait_for_pod_running_by_label( &pods, &format!("k8s-collector-e2e={}", pre_deploy_name), Duration::from_secs(120), ) .await?; let pre_deploy_uid = pre_deploy_pod .metadata .uid .clone() .context("pre-deploy pod missing uid")?; let pre_deploy_pod_name = pre_deploy_pod.name_any(); let pre_cron_pod = wait_for_pod_running_by_label( &pods, &format!("k8s-collector-e2e={}", pre_job_name), Duration::from_secs(120), ) .await?; let pre_cron_pod_uid = pre_cron_pod .metadata .uid .clone() .context("pre-cron pod missing uid")?; let pre_cron_pod_name = pre_cron_pod.name_any(); let pre_rs_pod = wait_for_pod_running_by_label( &pods, &format!("k8s-collector-e2e={}", pre_rs_name), Duration::from_secs(120), ) .await?; let pre_rs_uid = pre_rs_pod .metadata .uid .clone() .context("pre-rs pod missing uid")?; let pre_rs_pod_name = pre_rs_pod.name_any(); info!( "k8s-collector e2e: pre-existing pods ready: deploy name={} uid={}, cronpod name={} uid={}, rs name={} uid={}", pre_deploy_pod_name, pre_deploy_uid, pre_cron_pod_name, pre_cron_pod_uid, pre_rs_pod_name, pre_rs_uid ); let cfg = Config::default(); // delete_ttl/delete_capacity already have defaults; we use them as-is. let mut collector = Collector::with_client(cfg, client.clone()); info!("k8s-collector e2e: collector pipeline initialized"); let mut pre_deploy_obs = ObservedPod { uid: pre_deploy_uid.clone(), name: pre_deploy_pod_name.clone(), expect_owner_kind: OwnerKind::Deployment, expect_owner_name: pre_deploy.name_any(), saw_pod_new: false, saw_container: false, }; let mut pre_cron_obs = ObservedPod { uid: pre_cron_pod_uid.clone(), name: pre_cron_pod_name.clone(), expect_owner_kind: OwnerKind::CronJob, expect_owner_name: pre_cron.name_any(), saw_pod_new: false, saw_container: false, }; let mut pre_rs_obs = ObservedPod { uid: pre_rs_uid.clone(), name: pre_rs_pod_name.clone(), expect_owner_kind: OwnerKind::ReplicaSet, expect_owner_name: pre_rs.name_any(), saw_pod_new: false, saw_container: false, }; let mut last_epoch = 0u64; // Pump until we have seen a PodResync and both pre-existing pods fully emitted. pump_until( &mut collector, Duration::from_secs(60), |ev: &RenderEvent| { if let RenderEvent::PodResync { epoch } = ev { last_epoch = *epoch; } update_observed_from_event(&mut pre_deploy_obs, ev); update_observed_from_event(&mut pre_cron_obs, ev); update_observed_from_event(&mut pre_rs_obs, ev); last_epoch > 0 && observed_done(&pre_deploy_obs) && observed_done(&pre_cron_obs) && observed_done(&pre_rs_obs) }, ) .await?; assert!(last_epoch > 0, "expected at least one resync epoch"); info!( "k8s-collector e2e: observed initial resync epoch {} and full emission for pre-existing pods", last_epoch ); // Create new Deployment and CronJob+Job while collector is running. info!("k8s-collector e2e: creating new Deployment and CronJob/Job resources"); let new_deploy = create_sleep_deployment(&client, namespace, &new_deploy_name).await?; let new_cron = create_cronjob(&client, namespace, &new_cron_name).await?; let _new_job = create_job_with_cron_owner(&client, namespace, &new_cron, &new_job_name).await?; let new_deploy_pod = wait_for_pod_running_by_label( &pods, &format!("k8s-collector-e2e={}", new_deploy_name), Duration::from_secs(120), ) .await?; let new_deploy_uid = new_deploy_pod .metadata .uid .clone() .context("new-deploy pod missing uid")?; let new_deploy_pod_name = new_deploy_pod.name_any(); let new_cron_pod = wait_for_pod_running_by_label( &pods, &format!("k8s-collector-e2e={}", new_job_name), Duration::from_secs(120), ) .await?; let new_cron_pod_uid = new_cron_pod .metadata .uid .clone() .context("new-cron pod missing uid")?; let new_cron_pod_name = new_cron_pod.name_any(); info!( "k8s-collector e2e: new pods ready: deploy name={} uid={}, cronpod name={} uid={}", new_deploy_pod_name, new_deploy_uid, new_cron_pod_name, new_cron_pod_uid ); let mut new_deploy_obs = ObservedPod { uid: new_deploy_uid.clone(), name: new_deploy_pod_name.clone(), expect_owner_kind: OwnerKind::Deployment, expect_owner_name: new_deploy.name_any(), saw_pod_new: false, saw_container: false, }; let mut new_cron_obs = ObservedPod { uid: new_cron_pod_uid.clone(), name: new_cron_pod_name.clone(), expect_owner_kind: OwnerKind::CronJob, expect_owner_name: new_cron.name_any(), saw_pod_new: false, saw_container: false, }; pump_until( &mut collector, Duration::from_secs(60), |ev: &RenderEvent| { update_observed_from_event(&mut new_deploy_obs, ev); update_observed_from_event(&mut new_cron_obs, ev); observed_done(&new_deploy_obs) && observed_done(&new_cron_obs) }, ) .await?; info!("k8s-collector e2e: observed PodNew+PodContainer for new deploy and cronjob pods"); // Force an explicit resync epoch and verify it includes all pods. info!( "k8s-collector e2e: forcing explicit resync epoch after last_epoch={}", last_epoch ); let epoch_events = collector.start_new_epoch(); let mut resync_epoch = None; let mut seen_uids = Vec::new(); for ev in &epoch_events { match ev { RenderEvent::PodResync { epoch } => { resync_epoch = Some(*epoch); } RenderEvent::PodNew { uid, .. } => { seen_uids.push(uid.clone()); } _ => {} } } let resync_epoch = resync_epoch.context("expected PodResync event from start_new_epoch")?; assert!( resync_epoch > last_epoch, "expected resync epoch to be greater than previous epoch" ); info!( "k8s-collector e2e: resync epoch {} emitted PodNew snapshot for {} pods", resync_epoch, seen_uids.len() ); // For each observed pod, if it still exists in the cluster under the same // name+UID, require that it appears in the resync snapshot. Pods that have // been deleted or replaced are skipped. When a UID is missing, emit // detailed debugging about the snapshot contents. for obs in [ &pre_deploy_obs, &pre_cron_obs, &pre_rs_obs, &new_deploy_obs, &new_cron_obs, ] { match pods.get(&obs.name).await { Ok(pod) => { let current_uid = pod.metadata.uid.as_deref().unwrap_or_default(); if current_uid == obs.uid { if !seen_uids.contains(&obs.uid) { info!( "k8s-collector e2e: missing expected uid {} in resync snapshot; dumping PodNew events", obs.uid ); for ev in &epoch_events { if let RenderEvent::PodNew { uid, pod_name, ns, .. } = ev { info!( "k8s-collector e2e: snapshot PodNew pod_name={} ns={} uid={}", pod_name, ns, uid ); } } info!("k8s-collector e2e: snapshot seen_uids={:?}", seen_uids); info!( "k8s-collector e2e: collector matcher state snapshot:\n{}", collector.debug_snapshot() ); panic!("expected uid {} in resync snapshot", obs.uid); } info!( "k8s-collector e2e: snapshot contains live pod name={} uid={}", obs.name, obs.uid ); } else { info!( "k8s-collector e2e: pod name={} originally uid={} now uid={}; treating as replaced", obs.name, obs.uid, current_uid ); } } Err(kube::Error::Api(ae)) if ae.code == 404 => { info!( "k8s-collector e2e: pod name={} uid={} no longer exists; skipping snapshot assertion", obs.name, obs.uid ); } Err(e) => return Err(e.into()), } } // Best-effort cleanup. let _ = delete_if_exists(&deployments, &pre_deploy_name).await; let _ = delete_if_exists(&deployments, &new_deploy_name).await; let _ = delete_if_exists(&cronjobs, &pre_cron_name).await; let _ = delete_if_exists(&cronjobs, &new_cron_name).await; let _ = delete_if_exists(&jobs, &pre_job_name).await; let _ = delete_if_exists(&jobs, &new_job_name).await; Ok(()) } ================================================ FILE: crates/k8s-collector-bin/Cargo.toml ================================================ [package] name = "k8s-collector-bin" version = "0.1.0" edition = "2021" [[bin]] name = "k8s-collector" path = "src/main.rs" [dependencies] clap = { version = "4", features = ["derive"] } k8s-collector = { path = "../k8s-collector" } env_logger = "0.11" k8s-openapi = { version = "0.26", features = ["v1_30"] } ================================================ FILE: crates/k8s-collector-bin/src/main.rs ================================================ //! CLI entrypoint for the Kubernetes metadata collector. //! //! Parses configuration from flags and environment variables, then executes //! the collector pipeline. use clap::Parser; use k8s_collector::config::Config; #[derive(Parser, Debug)] #[command( name = "k8s-collector", about = "OpenTelemetry eBPF Kubernetes Metadata Collector (Rust)" )] struct Cli { /// Print configuration values and exit #[arg(long = "print-config")] print_config: bool, /// Deletion tombstone TTL in seconds #[arg(long = "delete-ttl-secs", default_value_t = 60)] delete_ttl_secs: u64, /// Deletion tombstone capacity #[arg(long = "delete-capacity", default_value_t = 10_000)] delete_capacity: usize, /// Reducer intake host #[arg(long = "intake-host")] intake_host: Option, /// Reducer intake port #[arg(long = "intake-port")] intake_port: Option, } /// Build the final [`Config`] from CLI flags and environment variables. fn build_config(cli: &Cli) -> Result { // Allow overriding the TTL via env if present let ttl_secs: u64 = std::env::var("K8S_COLLECTOR_DELETE_TTL_SECS") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(cli.delete_ttl_secs); let ttl = std::time::Duration::from_secs(ttl_secs); let cap: usize = std::env::var("K8S_COLLECTOR_DELETE_CAPACITY") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(cli.delete_capacity); let mut cfg = Config::default(); // Intake host/port via env or args if let Some(h) = std::env::var("EBPF_NET_INTAKE_HOST") .ok() .or(cli.intake_host.clone()) { cfg.intake_host = h; } if let Some(p) = std::env::var("EBPF_NET_INTAKE_PORT") .ok() .and_then(|v| v.parse().ok()) .or(cli.intake_port) { cfg.intake_port = p; } cfg.delete_ttl = ttl; cfg.delete_capacity = cap; Ok(cfg) } /// Print the effective configuration in a human-readable form. fn print_config(cfg: &Config) { println!("intake_host: {}", cfg.intake_host); println!("intake_port: {}", cfg.intake_port); println!("delete_ttl_secs: {}", cfg.delete_ttl.as_secs()); println!("delete_capacity: {}", cfg.delete_capacity); } fn main() { // Initialize logging for the collector binary. let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) .try_init(); let cli = Cli::parse(); let cfg = match build_config(&cli) { Ok(c) => c, Err(e) => { eprintln!("{}", e); std::process::exit(2); } }; if cli.print_config { print_config(&cfg); return; } // In this minimal integration, run validates config and returns. if let Err(e) = k8s_collector::run_with_config(cfg) { eprintln!("collector error: {}", e); std::process::exit(1); } } ================================================ FILE: crates/k8s-relay-bin/Cargo.toml ================================================ [package] name = "k8s-relay-bin" version = "0.1.0" edition = "2021" [dependencies] k8s-relay-sys = { path = "../k8s-relay-sys" } [[bin]] name = "k8s-relay" path = "src/main.rs" ================================================ FILE: crates/k8s-relay-bin/src/main.rs ================================================ fn main() { let code = k8s_relay_sys::run_with_args(std::env::args_os()); std::process::exit(code); } ================================================ FILE: crates/k8s-relay-sys/Cargo.toml ================================================ [package] name = "k8s-relay-sys" version = "0.1.0" edition = "2021" [lib] name = "k8s_relay_sys" path = "src/lib.rs" [dependencies] encoder_ebpf_net_ingest = { path = "../render/ebpf_net/ingest" } ================================================ FILE: crates/k8s-relay-sys/build.rs ================================================ include!("../build/otn_link_build.rs"); fn main() { run(); } ================================================ FILE: crates/k8s-relay-sys/src/lib.rs ================================================ use std::os::raw::{c_char, c_int}; // Ensure encoder crates are linked so C++ static libs can resolve their // extern "C" symbols via our dependency graph. #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_ingest; extern "C" { pub fn otn_k8s_relay_main(argc: c_int, argv: *const *const c_char) -> c_int; } pub fn run_with_args(args: I) -> i32 where I: IntoIterator, S: Into, { use std::ffi::CString; let mut c_strings: Vec = Vec::new(); for arg in args { let s: std::ffi::OsString = arg.into(); let bytes = std::os::unix::ffi::OsStringExt::into_vec(s); c_strings.push(CString::new(bytes).expect("argv contains NUL byte")); } let ptrs: Vec<*const c_char> = c_strings.iter().map(|s| s.as_ptr()).collect(); unsafe { otn_k8s_relay_main(ptrs.len() as c_int, ptrs.as_ptr()) } } ================================================ FILE: crates/kernel-collector-bin/Cargo.toml ================================================ [package] name = "kernel-collector-bin" version = "0.1.0" edition = "2021" [dependencies] kernel-collector-sys = { path = "../kernel-collector-sys" } [[bin]] name = "kernel-collector" path = "src/main.rs" ================================================ FILE: crates/kernel-collector-bin/src/main.rs ================================================ fn main() { // Capture the process args and forward to the C++ entrypoint. let code = kernel_collector_sys::run_with_args(std::env::args_os()); // Propagate the exit code to the OS. std::process::exit(code); } ================================================ FILE: crates/kernel-collector-sys/Cargo.toml ================================================ [package] name = "kernel-collector-sys" version = "0.1.0" edition = "2021" [lib] name = "kernel_collector_sys" path = "src/lib.rs" [dependencies] encoder_ebpf_net_ingest = { path = "../render/ebpf_net/ingest" } encoder_ebpf_net_kernel_collector = { path = "../render/ebpf_net/kernel_collector" } ================================================ FILE: crates/kernel-collector-sys/build.rs ================================================ include!("../build/otn_link_build.rs"); fn main() { run(); } ================================================ FILE: crates/kernel-collector-sys/src/lib.rs ================================================ use std::os::raw::{c_char, c_int}; // Ensure encoder crates are linked so C++ static libs can resolve their // extern "C" symbols via our dependency graph. #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_ingest; #[allow(unused_extern_crates)] extern crate encoder_ebpf_net_kernel_collector; extern "C" { pub fn otn_kernel_collector_main(argc: c_int, argv: *const *const c_char) -> c_int; } pub fn run_with_args(args: I) -> i32 where I: IntoIterator, S: Into, { use std::ffi::CString; let mut c_strings: Vec = Vec::new(); for arg in args { let s: std::ffi::OsString = arg.into(); let bytes = std::os::unix::ffi::OsStringExt::into_vec(s); c_strings.push(CString::new(bytes).expect("argv contains NUL byte")); } let ptrs: Vec<*const c_char> = c_strings.iter().map(|s| s.as_ptr()).collect(); // Ensure null-terminated argv if desired by C libraries (not strictly required for main) // ptrs.push(std::ptr::null()); unsafe { otn_kernel_collector_main(ptrs.len() as c_int, ptrs.as_ptr()) } } ================================================ FILE: crates/otlp_export/Cargo.toml ================================================ [package] name = "otlp_export" version = "0.1.0" edition = "2021" [lib] name = "otlp_export" crate-type = ["rlib", "staticlib"] [dependencies] cxx = { version = "1.0" } tokio = { version = "1", features = ["rt-multi-thread"] } opentelemetry-proto = { version = "0.31", default-features = false, features = ["gen-tonic", "metrics"] } tonic = { version = "0.14", features = ["transport"] } [build-dependencies] cxx-build = { version = "1.0" } ================================================ FILE: crates/otlp_export/src/lib.rs ================================================ #![allow(clippy::new_without_default)] #[cxx::bridge] pub mod ffi { /// Lightweight key/value label representation. #[derive(Debug, Clone)] pub struct Label { pub key: String, pub value: String, } /// Statistics mirroring the counters tracked by the C++ OTLP client. #[derive(Debug)] pub struct PublisherStats { pub bytes_sent: u64, pub bytes_failed: u64, pub data_points_sent: u64, pub data_points_failed: u64, pub requests_sent: u64, pub requests_failed: u64, pub unknown_response_tags: u64, } /// Metric kind to encode. #[repr(i32)] #[derive(Debug)] pub enum MetricKind { Sum = 0, Gauge = 1, } extern "Rust" { type Publisher; /// Create a new OTLP publisher. `endpoint` is host:port or full endpoint string. fn otlp_publisher_new(endpoint: &str) -> Box; /// Publish a u64 metric point. fn publish_metric_u64( self: &mut Publisher, name: &str, unit: &str, description: &str, kind: MetricKind, labels: &Vec