Repository: fluent/fluentd Branch: master Commit: 357c753e92b2 Files: 578 Total size: 3.6 MB Directory structure: gitextract_m1i_qpi2/ ├── .deepsource.toml ├── .github/ │ ├── DISCUSSION_TEMPLATE/ │ │ ├── q-a-japanese.yml │ │ └── q-a.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ └── feature_request.yml │ ├── ISSUE_TEMPLATE.md │ ├── PULL_REQUEST_TEMPLATE.md │ ├── dependabot.yml │ └── workflows/ │ ├── backport.yml │ ├── benchmark.yml │ ├── rubocop.yml │ ├── scorecards.yml │ ├── stale-actions.yml │ ├── test-ruby-head.yml │ └── test.yml ├── .gitignore ├── .rubocop.yml ├── ADOPTERS.md ├── AUTHORS ├── CHANGELOG.md ├── CONTRIBUTING.md ├── GOVERNANCE.md ├── Gemfile ├── GithubWorkflow.md ├── LICENSE ├── MAINTAINERS.md ├── README.md ├── Rakefile ├── SECURITY.md ├── bin/ │ ├── fluent-binlog-reader │ ├── fluent-ca-generate │ ├── fluent-cap-ctl │ ├── fluent-ctl │ ├── fluent-debug │ ├── fluent-plugin-config-format │ └── fluent-plugin-generate ├── code-of-conduct.md ├── example/ │ ├── copy_roundrobin.conf │ ├── counter.conf │ ├── filter_stdout.conf │ ├── in_forward.conf │ ├── in_forward_client.conf │ ├── in_forward_shared_key.conf │ ├── in_forward_tls.conf │ ├── in_forward_users.conf │ ├── in_forward_workers.conf │ ├── in_http.conf │ ├── in_out_forward.conf │ ├── in_sample_blocks.conf │ ├── in_sample_with_compression.conf │ ├── in_syslog.conf │ ├── in_tail.conf │ ├── in_tcp.conf │ ├── in_udp.conf │ ├── logevents.conf │ ├── multi_filters.conf │ ├── out_copy.conf │ ├── out_exec_filter.conf │ ├── out_file.conf │ ├── out_forward.conf │ ├── out_forward_buf_file.conf │ ├── out_forward_client.conf │ ├── out_forward_heartbeat_none.conf │ ├── out_forward_sd.conf │ ├── out_forward_shared_key.conf │ ├── out_forward_tls.conf │ ├── out_forward_users.conf │ ├── out_null.conf │ ├── sd.yaml │ ├── secondary_file.conf │ ├── suppress_config_dump.conf │ ├── v0_12_filter.conf │ ├── v1_literal_example.conf │ └── worker_section.conf ├── fluent.conf ├── fluentd.gemspec ├── lib/ │ └── fluent/ │ ├── agent.rb │ ├── capability.rb │ ├── clock.rb │ ├── command/ │ │ ├── binlog_reader.rb │ │ ├── bundler_injection.rb │ │ ├── ca_generate.rb │ │ ├── cap_ctl.rb │ │ ├── cat.rb │ │ ├── ctl.rb │ │ ├── debug.rb │ │ ├── fluentd.rb │ │ ├── plugin_config_formatter.rb │ │ └── plugin_generator.rb │ ├── compat/ │ │ ├── call_super_mixin.rb │ │ ├── detach_process_mixin.rb │ │ ├── exec_util.rb │ │ ├── file_util.rb │ │ ├── filter.rb │ │ ├── formatter.rb │ │ ├── formatter_utils.rb │ │ ├── handle_tag_and_time_mixin.rb │ │ ├── handle_tag_name_mixin.rb │ │ ├── input.rb │ │ ├── output.rb │ │ ├── output_chain.rb │ │ ├── parser.rb │ │ ├── parser_utils.rb │ │ ├── propagate_default.rb │ │ ├── record_filter_mixin.rb │ │ ├── set_tag_key_mixin.rb │ │ ├── set_time_key_mixin.rb │ │ ├── socket_util.rb │ │ ├── string_util.rb │ │ ├── structured_format_mixin.rb │ │ └── type_converter.rb │ ├── config/ │ │ ├── basic_parser.rb │ │ ├── configure_proxy.rb │ │ ├── dsl.rb │ │ ├── element.rb │ │ ├── error.rb │ │ ├── literal_parser.rb │ │ ├── parser.rb │ │ ├── section.rb │ │ ├── types.rb │ │ ├── v1_parser.rb │ │ ├── yaml_parser/ │ │ │ ├── fluent_value.rb │ │ │ ├── loader.rb │ │ │ ├── parser.rb │ │ │ └── section_builder.rb │ │ └── yaml_parser.rb │ ├── config.rb │ ├── configurable.rb │ ├── counter/ │ │ ├── base_socket.rb │ │ ├── client.rb │ │ ├── error.rb │ │ ├── mutex_hash.rb │ │ ├── server.rb │ │ ├── store.rb │ │ └── validator.rb │ ├── counter.rb │ ├── daemon.rb │ ├── daemonizer.rb │ ├── engine.rb │ ├── env.rb │ ├── error.rb │ ├── event.rb │ ├── event_router.rb │ ├── ext_monitor_require.rb │ ├── file_wrapper.rb │ ├── filter.rb │ ├── fluent_log_event_router.rb │ ├── formatter.rb │ ├── input.rb │ ├── label.rb │ ├── load.rb │ ├── log/ │ │ └── console_adapter.rb │ ├── log.rb │ ├── match.rb │ ├── mixin.rb │ ├── msgpack_factory.rb │ ├── oj_options.rb │ ├── output.rb │ ├── output_chain.rb │ ├── parser.rb │ ├── plugin/ │ │ ├── bare_output.rb │ │ ├── base.rb │ │ ├── buf_file.rb │ │ ├── buf_file_single.rb │ │ ├── buf_memory.rb │ │ ├── buffer/ │ │ │ ├── chunk.rb │ │ │ ├── file_chunk.rb │ │ │ ├── file_single_chunk.rb │ │ │ └── memory_chunk.rb │ │ ├── buffer.rb │ │ ├── compressable.rb │ │ ├── exec_util.rb │ │ ├── file_util.rb │ │ ├── filter.rb │ │ ├── filter_grep.rb │ │ ├── filter_parser.rb │ │ ├── filter_record_transformer.rb │ │ ├── filter_stdout.rb │ │ ├── formatter.rb │ │ ├── formatter_csv.rb │ │ ├── formatter_hash.rb │ │ ├── formatter_json.rb │ │ ├── formatter_ltsv.rb │ │ ├── formatter_msgpack.rb │ │ ├── formatter_out_file.rb │ │ ├── formatter_single_value.rb │ │ ├── formatter_stdout.rb │ │ ├── formatter_tsv.rb │ │ ├── in_debug_agent.rb │ │ ├── in_dummy.rb │ │ ├── in_exec.rb │ │ ├── in_forward.rb │ │ ├── in_gc_stat.rb │ │ ├── in_http.rb │ │ ├── in_monitor_agent.rb │ │ ├── in_object_space.rb │ │ ├── in_sample.rb │ │ ├── in_syslog.rb │ │ ├── in_tail/ │ │ │ ├── group_watch.rb │ │ │ └── position_file.rb │ │ ├── in_tail.rb │ │ ├── in_tcp.rb │ │ ├── in_udp.rb │ │ ├── in_unix.rb │ │ ├── input.rb │ │ ├── metrics.rb │ │ ├── metrics_local.rb │ │ ├── multi_output.rb │ │ ├── out_buffer.rb │ │ ├── out_copy.rb │ │ ├── out_exec.rb │ │ ├── out_exec_filter.rb │ │ ├── out_file.rb │ │ ├── out_forward/ │ │ │ ├── ack_handler.rb │ │ │ ├── connection_manager.rb │ │ │ ├── error.rb │ │ │ ├── failure_detector.rb │ │ │ ├── handshake_protocol.rb │ │ │ ├── load_balancer.rb │ │ │ └── socket_cache.rb │ │ ├── out_forward.rb │ │ ├── out_http.rb │ │ ├── out_null.rb │ │ ├── out_relabel.rb │ │ ├── out_roundrobin.rb │ │ ├── out_secondary_file.rb │ │ ├── out_stdout.rb │ │ ├── out_stream.rb │ │ ├── output.rb │ │ ├── owned_by_mixin.rb │ │ ├── parser.rb │ │ ├── parser_apache.rb │ │ ├── parser_apache2.rb │ │ ├── parser_apache_error.rb │ │ ├── parser_csv.rb │ │ ├── parser_json.rb │ │ ├── parser_ltsv.rb │ │ ├── parser_msgpack.rb │ │ ├── parser_multiline.rb │ │ ├── parser_nginx.rb │ │ ├── parser_none.rb │ │ ├── parser_regexp.rb │ │ ├── parser_syslog.rb │ │ ├── parser_tsv.rb │ │ ├── sd_file.rb │ │ ├── sd_srv.rb │ │ ├── sd_static.rb │ │ ├── service_discovery.rb │ │ ├── socket_util.rb │ │ ├── storage.rb │ │ ├── storage_local.rb │ │ └── string_util.rb │ ├── plugin.rb │ ├── plugin_helper/ │ │ ├── cert_option.rb │ │ ├── child_process.rb │ │ ├── compat_parameters.rb │ │ ├── counter.rb │ │ ├── event_emitter.rb │ │ ├── event_loop.rb │ │ ├── extract.rb │ │ ├── formatter.rb │ │ ├── http_server/ │ │ │ ├── app.rb │ │ │ ├── methods.rb │ │ │ ├── request.rb │ │ │ ├── router.rb │ │ │ ├── server.rb │ │ │ └── ssl_context_builder.rb │ │ ├── http_server.rb │ │ ├── inject.rb │ │ ├── metrics.rb │ │ ├── parser.rb │ │ ├── record_accessor.rb │ │ ├── retry_state.rb │ │ ├── server.rb │ │ ├── service_discovery/ │ │ │ ├── manager.rb │ │ │ └── round_robin_balancer.rb │ │ ├── service_discovery.rb │ │ ├── socket.rb │ │ ├── socket_option.rb │ │ ├── storage.rb │ │ ├── thread.rb │ │ └── timer.rb │ ├── plugin_helper.rb │ ├── plugin_id.rb │ ├── process.rb │ ├── registry.rb │ ├── root_agent.rb │ ├── rpc.rb │ ├── source_only_buffer_agent.rb │ ├── static_config_analysis.rb │ ├── supervisor.rb │ ├── system_config.rb │ ├── test/ │ │ ├── base.rb │ │ ├── driver/ │ │ │ ├── base.rb │ │ │ ├── base_owned.rb │ │ │ ├── base_owner.rb │ │ │ ├── event_feeder.rb │ │ │ ├── filter.rb │ │ │ ├── formatter.rb │ │ │ ├── input.rb │ │ │ ├── multi_output.rb │ │ │ ├── output.rb │ │ │ ├── parser.rb │ │ │ ├── storage.rb │ │ │ └── test_event_router.rb │ │ ├── filter_test.rb │ │ ├── formatter_test.rb │ │ ├── helpers.rb │ │ ├── input_test.rb │ │ ├── log.rb │ │ ├── output_test.rb │ │ ├── parser_test.rb │ │ └── startup_shutdown.rb │ ├── test.rb │ ├── time.rb │ ├── timezone.rb │ ├── tls.rb │ ├── unique_id.rb │ ├── variable_store.rb │ ├── version.rb │ ├── win32api.rb │ └── winsvc.rb ├── tasks/ │ ├── backport/ │ │ └── backporter.rb │ ├── backport.rb │ ├── benchmark/ │ │ ├── conf/ │ │ │ └── in_tail.conf │ │ └── patch_in_tail.rb │ └── benchmark.rb ├── templates/ │ ├── new_gem/ │ │ ├── Gemfile │ │ ├── README.md.erb │ │ ├── Rakefile │ │ ├── fluent-plugin.gemspec.erb │ │ ├── lib/ │ │ │ └── fluent/ │ │ │ └── plugin/ │ │ │ ├── filter.rb.erb │ │ │ ├── formatter.rb.erb │ │ │ ├── input.rb.erb │ │ │ ├── output.rb.erb │ │ │ ├── parser.rb.erb │ │ │ └── storage.rb.erb │ │ └── test/ │ │ ├── helper.rb.erb │ │ └── plugin/ │ │ ├── test_filter.rb.erb │ │ ├── test_formatter.rb.erb │ │ ├── test_input.rb.erb │ │ ├── test_output.rb.erb │ │ ├── test_parser.rb.erb │ │ └── test_storage.rb.erb │ └── plugin_config_formatter/ │ ├── param.md-compact.erb │ ├── param.md-table.erb │ ├── param.md.erb │ └── section.md.erb └── test/ ├── command/ │ ├── test_binlog_reader.rb │ ├── test_ca_generate.rb │ ├── test_cap_ctl.rb │ ├── test_cat.rb │ ├── test_ctl.rb │ ├── test_fluentd.rb │ ├── test_plugin_config_formatter.rb │ └── test_plugin_generator.rb ├── compat/ │ ├── test_calls_super.rb │ └── test_parser.rb ├── config/ │ ├── assertions.rb │ ├── test_config_parser.rb │ ├── test_configurable.rb │ ├── test_configure_proxy.rb │ ├── test_dsl.rb │ ├── test_element.rb │ ├── test_literal_parser.rb │ ├── test_plugin_configuration.rb │ ├── test_section.rb │ ├── test_system_config.rb │ ├── test_types.rb │ └── test_yaml_parser.rb ├── counter/ │ ├── test_client.rb │ ├── test_error.rb │ ├── test_mutex_hash.rb │ ├── test_server.rb │ ├── test_store.rb │ └── test_validator.rb ├── helper.rb ├── helpers/ │ ├── fuzzy_assert.rb │ └── process_extenstion.rb ├── log/ │ └── test_console_adapter.rb ├── plugin/ │ ├── data/ │ │ ├── 2010/ │ │ │ └── 01/ │ │ │ ├── 20100102-030405.log │ │ │ ├── 20100102-030406.log │ │ │ └── 20100102.log │ │ ├── log/ │ │ │ ├── bar │ │ │ ├── foo/ │ │ │ │ ├── bar.log │ │ │ │ └── bar2 │ │ │ └── test.log │ │ ├── log_numeric/ │ │ │ ├── 01.log │ │ │ ├── 02.log │ │ │ ├── 12.log │ │ │ └── 14.log │ │ └── sd_file/ │ │ ├── config │ │ ├── config.json │ │ ├── config.yaml │ │ ├── config.yml │ │ └── invalid_config.yml │ ├── in_tail/ │ │ ├── test_fifo.rb │ │ ├── test_io_handler.rb │ │ └── test_position_file.rb │ ├── out_forward/ │ │ ├── test_ack_handler.rb │ │ ├── test_connection_manager.rb │ │ ├── test_handshake_protocol.rb │ │ ├── test_load_balancer.rb │ │ └── test_socket_cache.rb │ ├── test_bare_output.rb │ ├── test_base.rb │ ├── test_buf_file.rb │ ├── test_buf_file_single.rb │ ├── test_buf_memory.rb │ ├── test_buffer.rb │ ├── test_buffer_chunk.rb │ ├── test_buffer_file_chunk.rb │ ├── test_buffer_file_single_chunk.rb │ ├── test_buffer_memory_chunk.rb │ ├── test_compressable.rb │ ├── test_file_util.rb │ ├── test_filter.rb │ ├── test_filter_grep.rb │ ├── test_filter_parser.rb │ ├── test_filter_record_transformer.rb │ ├── test_filter_stdout.rb │ ├── test_formatter_csv.rb │ ├── test_formatter_hash.rb │ ├── test_formatter_json.rb │ ├── test_formatter_ltsv.rb │ ├── test_formatter_msgpack.rb │ ├── test_formatter_out_file.rb │ ├── test_formatter_single_value.rb │ ├── test_formatter_tsv.rb │ ├── test_in_debug_agent.rb │ ├── test_in_exec.rb │ ├── test_in_forward.rb │ ├── test_in_gc_stat.rb │ ├── test_in_http.rb │ ├── test_in_monitor_agent.rb │ ├── test_in_object_space.rb │ ├── test_in_sample.rb │ ├── test_in_syslog.rb │ ├── test_in_tail.rb │ ├── test_in_tcp.rb │ ├── test_in_udp.rb │ ├── test_in_unix.rb │ ├── test_input.rb │ ├── test_metadata.rb │ ├── test_metrics.rb │ ├── test_metrics_local.rb │ ├── test_multi_output.rb │ ├── test_out_buffer.rb │ ├── test_out_copy.rb │ ├── test_out_exec.rb │ ├── test_out_exec_filter.rb │ ├── test_out_file.rb │ ├── test_out_forward.rb │ ├── test_out_http.rb │ ├── test_out_null.rb │ ├── test_out_relabel.rb │ ├── test_out_roundrobin.rb │ ├── test_out_secondary_file.rb │ ├── test_out_stdout.rb │ ├── test_out_stream.rb │ ├── test_output.rb │ ├── test_output_as_buffered.rb │ ├── test_output_as_buffered_backup.rb │ ├── test_output_as_buffered_compress.rb │ ├── test_output_as_buffered_overflow.rb │ ├── test_output_as_buffered_retries.rb │ ├── test_output_as_buffered_secondary.rb │ ├── test_output_as_standard.rb │ ├── test_owned_by.rb │ ├── test_parser.rb │ ├── test_parser_apache.rb │ ├── test_parser_apache2.rb │ ├── test_parser_apache_error.rb │ ├── test_parser_csv.rb │ ├── test_parser_json.rb │ ├── test_parser_labeled_tsv.rb │ ├── test_parser_msgpack.rb │ ├── test_parser_multiline.rb │ ├── test_parser_nginx.rb │ ├── test_parser_none.rb │ ├── test_parser_regexp.rb │ ├── test_parser_syslog.rb │ ├── test_parser_tsv.rb │ ├── test_sd_file.rb │ ├── test_sd_srv.rb │ ├── test_storage.rb │ ├── test_storage_local.rb │ └── test_string_util.rb ├── plugin_helper/ │ ├── data/ │ │ └── cert/ │ │ ├── cert-key.pem │ │ ├── cert-with-CRLF.pem │ │ ├── cert-with-no-newline.pem │ │ ├── cert.pem │ │ ├── cert_chains/ │ │ │ ├── ca-cert-key.pem │ │ │ ├── ca-cert.pem │ │ │ ├── cert-key.pem │ │ │ └── cert.pem │ │ ├── empty.pem │ │ ├── generate_cert.rb │ │ ├── with_ca/ │ │ │ ├── ca-cert-key-pass.pem │ │ │ ├── ca-cert-key.pem │ │ │ ├── ca-cert-pass.pem │ │ │ ├── ca-cert.pem │ │ │ ├── cert-key-pass.pem │ │ │ ├── cert-key.pem │ │ │ ├── cert-pass.pem │ │ │ └── cert.pem │ │ └── without_ca/ │ │ ├── cert-key-pass.pem │ │ ├── cert-key.pem │ │ ├── cert-pass.pem │ │ └── cert.pem │ ├── http_server/ │ │ ├── test_app.rb │ │ ├── test_request.rb │ │ └── test_route.rb │ ├── service_discovery/ │ │ ├── test_manager.rb │ │ └── test_round_robin_balancer.rb │ ├── test_cert_option.rb │ ├── test_child_process.rb │ ├── test_compat_parameters.rb │ ├── test_event_emitter.rb │ ├── test_event_loop.rb │ ├── test_extract.rb │ ├── test_formatter.rb │ ├── test_http_server_helper.rb │ ├── test_inject.rb │ ├── test_metrics.rb │ ├── test_parser.rb │ ├── test_record_accessor.rb │ ├── test_retry_state.rb │ ├── test_server.rb │ ├── test_service_discovery.rb │ ├── test_socket.rb │ ├── test_storage.rb │ ├── test_thread.rb │ └── test_timer.rb ├── scripts/ │ ├── exec_script.rb │ ├── fluent/ │ │ └── plugin/ │ │ ├── formatter1/ │ │ │ └── formatter_test1.rb │ │ ├── formatter2/ │ │ │ └── formatter_test2.rb │ │ ├── formatter_known.rb │ │ ├── out_test.rb │ │ ├── out_test2.rb │ │ └── parser_known.rb │ └── windows_service_test.ps1 ├── test_capability.rb ├── test_clock.rb ├── test_config.rb ├── test_configdsl.rb ├── test_daemonizer.rb ├── test_engine.rb ├── test_event.rb ├── test_event_router.rb ├── test_event_time.rb ├── test_file_wrapper.rb ├── test_filter.rb ├── test_fluent_log_event_router.rb ├── test_formatter.rb ├── test_input.rb ├── test_log.rb ├── test_match.rb ├── test_mixin.rb ├── test_msgpack_factory.rb ├── test_oj_options.rb ├── test_output.rb ├── test_plugin.rb ├── test_plugin_classes.rb ├── test_plugin_helper.rb ├── test_plugin_id.rb ├── test_process.rb ├── test_root_agent.rb ├── test_source_only_buffer_agent.rb ├── test_static_config_analysis.rb ├── test_supervisor.rb ├── test_test_drivers.rb ├── test_time_formatter.rb ├── test_time_parser.rb ├── test_tls.rb ├── test_unique_id.rb └── test_variable_store.rb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .deepsource.toml ================================================ version = 1 test_patterns = ["test/**/test_*.rb"] exclude_patterns = [ "bin/**", "docs/**", "example/**" ] [[analyzers]] name = "ruby" enabled = true ================================================ FILE: .github/DISCUSSION_TEMPLATE/q-a-japanese.yml ================================================ title: "[QA (Japanese)]" labels: ["Q&A (Japanese)"] body: - type: markdown attributes: value: | 日本語で気軽に質問するためのカテゴリです。もし他の人が困っているのを見つけたらぜひ回答してあげてください。 - type: textarea id: question attributes: label: やりたいこと description: | 何について困っているのかを書いてください。試したことや実際の結果を示してください。 期待する挙動と実際の結果の違いがあればそれも書くのをおすすめします。 validations: required: true - type: textarea id: configuration attributes: label: 設定した内容 description: | どのような設定をして期待する挙動を実現しようとしたのかを書いてください。(例: fluentd.confの内容を貼り付ける) render: apache - type: textarea id: logs attributes: label: ログの内容 description: | Fluentdのログを提示してください。エラーログがあると回答の助けになります。(例: fluentd.logの内容を貼り付ける) render: shell - type: textarea id: environment attributes: label: 環境について description: | - Fluentd or td-agent version: `fluentd --version` or `td-agent --version` - Operating system: `cat /etc/os-release` - Kernel version: `uname -r` どんな環境で困っているかの情報がないと、再現できないため誰も回答できないことがあります。 必要な情報を記入することをおすすめします。 value: | - Fluentd version: - TD Agent version: - Fluent Package version: - Docker image (tag): - Operating system: - Kernel version: render: markdown ================================================ FILE: .github/DISCUSSION_TEMPLATE/q-a.yml ================================================ title: "[Q&A]" labels: ["Q&A"] body: - type: markdown attributes: value: | It is recommended to support each other. - type: textarea id: question attributes: label: What is a problem? description: | A clear and concise description of what you want to happen. What exactly did you do (or not do) that was effective (or ineffective)? validations: required: true - type: textarea id: configuration attributes: label: Describe the configuration of Fluentd description: | If there is the actual configuration of Fluentd, it will help. - type: textarea id: logs attributes: label: Describe the logs of Fluentd description: | If there are error logs of Fluentd, it will help. - type: textarea id: environment attributes: label: Environment description: | - Fluentd or td-agent version: `fluentd --version` or `td-agent --version` - Operating system: `cat /etc/os-release` - Kernel version: `uname -r` Please describe your environment information. If will help to support. value: | - Fluentd version: - TD Agent version: - Fluent Package version: - Docker image (tag): - Operating system: - Kernel version: render: markdown ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: Bug Report description: Create a report with a procedure for reproducing the bug labels: "waiting-for-triage" body: - type: markdown attributes: value: | Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem. - type: textarea id: description attributes: label: Describe the bug description: A clear and concise description of what the bug is validations: required: true - type: textarea id: reproduce attributes: label: To Reproduce description: Steps to reproduce the behavior validations: required: true - type: textarea id: expected attributes: label: Expected behavior description: A clear and concise description of what you expected to happen validations: required: true - type: textarea id: environment attributes: label: Your Environment description: | - Fluentd or its package version: `fluentd --version` (Fluentd, fluent-package) or `td-agent --version` (td-agent) - Operating system: `cat /etc/os-release` - Kernel version: `uname -r` Tip: If you hit the problem with older fluentd version, try latest version first. value: | - Fluentd version: - Package version: - Operating system: - Kernel version: render: markdown validations: required: true - type: textarea id: configuration attributes: label: Your Configuration description: | Write your configuration here. Minimum reproducible fluentd.conf is recommended. render: apache validations: required: true - type: textarea id: logs attributes: label: Your Error Log description: Write your ALL error log here render: shell validations: required: true - type: textarea id: addtional-context attributes: label: Additional context description: Add any other context about the problem here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Ask a Question url: https://github.com/fluent/fluentd/discussions about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions - name: Feedback a Fluentd Use-Case/Testimonial url: https://github.com/fluent/fluentd-website/issues/new?template=testimonials.yml about: Feedback your Fluentd use-case/testimonial, How do you use Fluentd in your service? ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ name: Feature request description: Suggest an idea for this project labels: "waiting-for-triage" body: - type: markdown attributes: value: | Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem. - type: textarea id: description attributes: label: Is your feature request related to a problem? Please describe. description: | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] validations: required: true - type: textarea id: solution attributes: label: Describe the solution you'd like description: A clear and concise description of what you want to happen. validations: required: true - type: textarea id: alternative attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. validations: required: true - type: textarea id: addtional-context attributes: label: Additional context description: Add any other context or screenshots about the feature request here. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem. **Got a question or problem?** RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you. If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd). Don't use Github issue for asking questions. Here are examples: - I installed xxx plugin but it doesn't work. Why? - Fluentd starts but logs are not sent to xxx. Am I wrong? - I want to do xxx. How to realize it with plugins? We may close such questions to keep clear repository for developers and users. Github issue is mainly for submitting a bug report or feature request. See below. If you can't judge your case is a bug or not, use mailing list or slack first. ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ **Which issue(s) this PR fixes**: Fixes # **What this PR does / why we need it**: **Docs Changes**: **Release Note**: ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: 'github-actions' directory: '/' schedule: interval: 'weekly' ================================================ FILE: .github/workflows/backport.yml ================================================ name: Backport Pull Requests on: schedule: # Sun 10:00 (JST) - cron: '0 1 * * 0' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: test: permissions: contents: write pull-requests: write runs-on: ubuntu-latest continue-on-error: false strategy: fail-fast: false matrix: ruby-version: ['3.4'] task: ['backport:v1_19'] name: Backport PR ( ${{ matrix.task }} ) steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: # fetch all history for all branches to execute cherry-pick fetch-depth: 0 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true - name: Set up git identity run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - name: Run backport task ( ${{ matrix.task }} ) shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | bundle exec rake ${{ matrix.task }} ================================================ FILE: .github/workflows/benchmark.yml ================================================ name: Benchmark on: push: branches: [master] pull_request: branches: [master] workflow_dispatch: permissions: read-all concurrency: group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: test: runs-on: ${{ matrix.os }} continue-on-error: false strategy: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] ruby-version: ['4.0'] name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} - name: Install dependencies run: bundle install - name: Run Benchmark shell: bash # Ensure to use bash shell on all platforms run: | bundle exec rake benchmark:run:in_tail | tee -a $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/rubocop.yml ================================================ name: RucoCop Security & Performance Check on: push: paths-ignore: - '*.md' - 'lib/fluent/version.rb' pull_request: paths-ignore: - '*.md' - 'lib/fluent/version.rb' workflow_dispatch: concurrency: group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true permissions: read-all jobs: rubocop: runs-on: ubuntu-latest continue-on-error: false strategy: fail-fast: false matrix: ruby-version: ['4.0'] name: Ruby ${{ matrix.ruby-version }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} - name: Install dependencies run: | bundle install gem install rubocop-performance - name: Run RuboCop run: rubocop ================================================ FILE: .github/workflows/scorecards.yml ================================================ name: Scorecard supply-chain security on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '38 12 * * 2' push: branches: [ "master" ] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecard analysis runs-on: ubuntu-latest permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. # contents: read # actions: read steps: - name: "Checkout code" uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. # repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers # - Allows the repository to include the Scorecard badge. # - See https://github.com/ossf/scorecard-action#publishing-results. # For private repositories: # - `publish_results` will always be set to `false`, regardless # of the value entered here. publish_results: true # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@b1bff81932f5cdfc8695c7752dcee935dcd061c8 # v4.33.0 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/stale-actions.yml ================================================ name: "Mark or close stale issues and PRs" on: schedule: - cron: "00 10 * * *" permissions: read-all jobs: stale: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 days-before-close: 7 stale-issue-message: "This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days" stale-pr-message: "This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days" close-issue-message: "This issue was automatically closed because of stale in 7 days" close-pr-message: "This PR was automatically closed because of stale in 7 days" stale-pr-label: "stale" stale-issue-label: "stale" exempt-issue-labels: "waiting-for-triage,bug,enhancement,feature request,pending,work-in-progress,v1,v2" exempt-pr-labels: "waiting-for-triage,bug,enhancement,feature request,pending,work-in-progress,v1,v2" exempt-all-assignees: true exempt-all-milestones: true ================================================ FILE: .github/workflows/test-ruby-head.yml ================================================ name: Test with Ruby head on: schedule: - cron: '11 14 * * 0' workflow_dispatch: permissions: read-all jobs: test: runs-on: ${{ matrix.os }} continue-on-error: false strategy: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] ruby-version: ['head'] env: RUBYOPT: "--disable-frozen_string_literal" name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} - name: Install addons if: ${{ matrix.os == 'ubuntu-latest' }} run: sudo apt-get install libgmp3-dev libcap-ng-dev - name: Install dependencies run: bundle install - name: Run tests run: bundle exec rake test TESTOPTS="-v --report-slow-tests --no-show-detail-immediately" ================================================ FILE: .github/workflows/test.yml ================================================ name: Test on: push: branches: [master] paths-ignore: - '*.md' - 'lib/fluent/version.rb' pull_request: branches: [master] paths-ignore: - '*.md' - 'lib/fluent/version.rb' workflow_dispatch: concurrency: group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true permissions: read-all jobs: test: runs-on: ${{ matrix.os }} continue-on-error: false strategy: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] ruby-version: ['4.0', '3.4', '3.3', '3.2'] env: RUBYOPT: "--disable-frozen_string_literal" name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} - name: Install addons if: ${{ matrix.os == 'ubuntu-latest' }} run: sudo apt-get install libgmp3-dev libcap-ng-dev - name: Install dependencies run: bundle install - name: Run tests run: bundle exec rake test TESTOPTS="-v --report-slow-tests --no-show-detail-immediately" test-windows-service: runs-on: windows-latest strategy: fail-fast: false matrix: ruby-version: ['4.0', '3.4', '3.3', '3.2'] name: Windows service (Ruby ${{ matrix.ruby-version }}) steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Ruby uses: ruby/setup-ruby@dffb23f65a78bba8db45d387d5ea1bbd6be3ef18 # v1.293.0 with: ruby-version: ${{ matrix.ruby-version }} - name: Install dependencies run: | bundle install rake install - name: Run tests run: test\scripts\windows_service_test.ps1 ================================================ FILE: .gitignore ================================================ Gemfile.lock INSTALL NEWS Makefile Makefile.in README ac aclocal.m4 autom4te.cache confdefs.h config.log config.status configure deps/ fluent-cat fluent-gem fluentd pkg/* tmp/* test/tmp/* test/config/tmp/* make_dist.sh Gemfile.local .ruby-version *.swp coverage/* .vagrant/ cov-int/ cov-fluentd.tar.gz .vscode .idea/ .bundle/ /vendor/ ================================================ FILE: .rubocop.yml ================================================ plugins: - rubocop-performance AllCops: Exclude: - 'vendor/**/*' NewCops: enable SuggestExtensions: false TargetRubyVersion: 3.4 # # Policy: Check Security & Performance in primary use-cases # (Disable most of cosmetic rules) # Lint: Enabled: false Style: Enabled: false Gemspec: Enabled: false Naming: Enabled: false Layout: Enabled: false Metrics: Enabled: false Security: Enabled: true Performance: Enabled: true # # False positive or exceptional cases # # False positive because it's intentional Security/Open: Exclude: - lib/fluent/plugin/buffer/chunk.rb Enabled: true # False positive because it's intentional Security/Eval: Exclude: - lib/fluent/config/dsl.rb - lib/fluent/plugin.rb - lib/fluent/plugin/in_debug_agent.rb Enabled: true # False positive because send method must accept literals. Performance/StringIdentifierArgument: Exclude: - test/plugin/test_in_tcp.rb - test/plugin/test_in_udp.rb - test/counter/test_server.rb - test/plugin/test_out_forward.rb - lib/fluent/plugin/out_forward.rb Enabled: true Performance/StringInclude: Exclude: - 'test/**/*' # It was not improved with String#include? - lib/fluent/command/plugin_config_formatter.rb Enabled: true # False positive for include? against constant ranges. # Almost same between include? and cover?. # See https://github.com/rubocop/rubocop-jp/issues/20 Performance/RangeInclude: Exclude: - lib/fluent/plugin/parser_multiline.rb Enabled: true # Allow using &method(:func) Performance/MethodObjectAsBlock: Exclude: - 'test/**/*' Enabled: false # Allow block.call Performance/RedundantBlockCall: Exclude: - 'test/**/*' - 'lib/fluent/plugin_helper/*.rb' - 'lib/fluent/plugin/*.rb' - 'lib/fluent/compat/*.rb' - 'lib/fluent/config/*.rb' - 'lib/fluent/*.rb' Enabled: true # # TODO: low priority to be fixed # Performance/ConstantRegexp: Exclude: - 'test/**/*' Enabled: true Performance/Sum: Exclude: - 'test/**/*' Enabled: true Performance/CollectionLiteralInLoop: Exclude: - 'test/**/*' Enabled: true ================================================ FILE: ADOPTERS.md ================================================ # Fluentd Adopters Fluentd is widely used by hundred of companies, please refer to the testimonial section of our project website to learn more about it: https://www.fluentd.org/testimonials ================================================ FILE: AUTHORS ================================================ FURUHASHI Sadayuki NAKAGAWA Masahiro ================================================ FILE: CHANGELOG.md ================================================ # v1.20 ## Release v1.20.0 - TBD ### Bug fix * http_server helper: Fix IPv6 bind address support in URI construction * Fixed `URI::InvalidURIError` when binding to IPv6 addresses (e.g., `::`, `::1`) * IPv6 addresses are now properly bracketed in URIs per RFC 3986 (e.g., `http://[::]:24231`) * Handles pre-bracketed addresses correctly to avoid double-bracketing * Affects all plugins using http_server helper with IPv6 bind addresses # v1.19 ## Release v1.19.2 - 2026/02/13 ### Bug Fix * out_forward: add timeout to establish_connection to prevent infinite loop https://github.com/fluent/fluentd/pull/5138 * gem: use latest net-http to solve IPv6 addr error https://github.com/fluent/fluentd/pull/5192 * in_tail: fix error when files without read permission are included in glob patterns https://github.com/fluent/fluentd/pull/5222 * command/fluentd: load win32/registry when edit registry for Ruby 4.0 https://github.com/fluent/fluentd/pull/5221 * plugin_helper/http_server: Ensure request body is closed to prevent socket leaks in POST requests https://github.com/fluent/fluentd/pull/5234 * config: fix duplicate config file loading in config_include_dir https://github.com/fluent/fluentd/pull/5235 * gem: add ostruct gem as dependency for Ruby 4.0 https://github.com/fluent/fluentd/pull/5251 ### Misc * config: warn when backed-up config file will be included https://github.com/fluent/fluentd/pull/5252 * filter_record_transformer: use cgi/escape to avoid Ruby 4.0 deprecation https://github.com/fluent/fluentd/pull/5256 * CI fixes * https://github.com/fluent/fluentd/pull/5257 * https://github.com/fluent/fluentd/pull/5229 * https://github.com/fluent/fluentd/pull/5225 * https://github.com/fluent/fluentd/pull/5186 * https://github.com/fluent/fluentd/pull/5184 * https://github.com/fluent/fluentd/pull/5176 ## Release v1.19.1 - 2025/11/06 ### Bug Fix * YAML config: Supports parsing array format https://github.com/fluent/fluentd/pull/5139 ### Misc * gem: fix uri gem version to keep IPv6 tests https://github.com/fluent/fluentd/pull/5144 * CI fixes * https://github.com/fluent/fluentd/pull/5055 * https://github.com/fluent/fluentd/pull/5057 * https://github.com/fluent/fluentd/pull/5063 * https://github.com/fluent/fluentd/pull/5064 * https://github.com/fluent/fluentd/pull/5077 * https://github.com/fluent/fluentd/pull/5078 * https://github.com/fluent/fluentd/pull/5136 * https://github.com/fluent/fluentd/pull/5140 ## Release v1.19.0 - 2025/07/30 ### Enhancement New features: * Add zstd compression support https://github.com/fluent/fluentd/pull/4657 * Buffer: add `zstd` to `compress` option. * out_file: add `zstd` to `compress` option. * out_forward: add `zstd` to `compress` option. (Experimental) * in_forward: support `zstd` format. * buffer: add feature to evacuate chunk files when retry limit https://github.com/fluent/fluentd/pull/4986 * out_http: TLS1.3 support https://github.com/fluent/fluentd/pull/4859 * out_stdout: support output to STDOUT independently of Fluentd logger by setting `use_logger` to `false` https://github.com/fluent/fluentd/pull/5036 * out_file: add symlink_path_use_relative option to use relative path instead of absolute path in symlink_path https://github.com/fluent/fluentd/pull/4904 * System configuration: Add forced_stacktrace_level to force the log level of stacktraces. https://github.com/fluent/fluentd/pull/5008 * System configuration: support built-in config files https://github.com/fluent/fluentd/pull/4893 Metrics: * metrics: enable input metrics by default https://github.com/fluent/fluentd/pull/4966 * in_tail: add "tracked_file_count" metrics to see how many log files are being tracked https://github.com/fluent/fluentd/pull/4980 * output: add metrics for number of writing events in secondary https://github.com/fluent/fluentd/pull/4971 * output: add metrics for dropped oldest chunk count https://github.com/fluent/fluentd/pull/4981 Others: * in_forward: enable skip_invalid_event by default not to process broken data https://github.com/fluent/fluentd/pull/5003 * buf_file: reinforce buffer file corruption check https://github.com/fluent/fluentd/pull/4998 * in_http: allow empty Origin header requests to pass CORS checks https://github.com/fluent/fluentd/pull/4866 * in_tail: add warning for directory permission https://github.com/fluent/fluentd/pull/4865 * Add logging for errors about loading dependencies on startup https://github.com/fluent/fluentd/pull/4858 * Performance improvements * https://github.com/fluent/fluentd/pull/4759 https://github.com/fluent/fluentd/pull/4760 https://github.com/fluent/fluentd/pull/4763 https://github.com/fluent/fluentd/pull/4764 https://github.com/fluent/fluentd/pull/4769 https://github.com/fluent/fluentd/pull/4813 https://github.com/fluent/fluentd/pull/4817 https://github.com/fluent/fluentd/pull/4835 https://github.com/fluent/fluentd/pull/4845 https://github.com/fluent/fluentd/pull/4881 https://github.com/fluent/fluentd/pull/4884 https://github.com/fluent/fluentd/pull/4886 https://github.com/fluent/fluentd/pull/4930 https://github.com/fluent/fluentd/pull/4931 https://github.com/fluent/fluentd/pull/4932 https://github.com/fluent/fluentd/pull/4933 https://github.com/fluent/fluentd/pull/4934 https://github.com/fluent/fluentd/pull/4995 ### Bug Fix * in_tail: fixed where specifying only encoding parameter might cause data corruption (affects since v0.14.12). https://github.com/fluent/fluentd/pull/5010 * formatter_csv: fix memory leak https://github.com/fluent/fluentd/pull/4864 * server plugin helper: ensure to close all connections at shutdown https://github.com/fluent/fluentd/pull/5026 * Fixed a bug where the default `umask` was not set to `0` when using `--daemon` (td-agent, fluent-package) since v1.14.6. https://github.com/fluent/fluentd/pull/4836 * `--umask` command line option: Fixed so that it is applied when Fluentd runs with `--daemon` (fluent-package) as well as when Fluentd runs with `--no-supervisor`. https://github.com/fluent/fluentd/pull/4836 * Windows: Stop the service when the supervisor is dead https://github.com/fluent/fluentd/pull/4909 * Windows: Fixed an issue where stopping the service immediately after startup could leave the processes. https://github.com/fluent/fluentd/pull/4782 * Windows: Fixed an issue where stopping service sometimes can not be completed forever. https://github.com/fluent/fluentd/pull/4782 ### Misc * in_monitor_agent: stop using CGI.parse due to support Ruby 3.5 https://github.com/fluent/fluentd/pull/4962 * HTTP server plugin helper: stop using CGI.parse due to support Ruby 3.5 https://github.com/fluent/fluentd/pull/4962 * config: change inspect format https://github.com/fluent/fluentd/pull/4914 * console_adapter: support console gem v1.30 https://github.com/fluent/fluentd/pull/4857 * gemspec: fix io-event and io-stream version to avoid unstable behavior on Windows https://github.com/fluent/fluentd/pull/5042 * in_http: replace WEBrick::HTTPUtils.parse_query with URI Note that at least, this makes it unable to use ; delimiter. https://github.com/fluent/fluentd/pull/4900 * http_server: stop fallback to WEBrick https://github.com/fluent/fluentd/pull/4899 * metrics: add getter method automatically https://github.com/fluent/fluentd/pull/4978 * http_server helper: add `header` method for `Request` https://github.com/fluent/fluentd/pull/4903 * multi_output: fix metrics name https://github.com/fluent/fluentd/pull/4979 * plugin_id: fix typo https://github.com/fluent/fluentd/pull/4964 * CI fixes * https://github.com/fluent/fluentd/pull/4728 https://github.com/fluent/fluentd/pull/4730 https://github.com/fluent/fluentd/pull/4746 https://github.com/fluent/fluentd/pull/4747 https://github.com/fluent/fluentd/pull/4748 https://github.com/fluent/fluentd/pull/4750 https://github.com/fluent/fluentd/pull/4755 https://github.com/fluent/fluentd/pull/4820 https://github.com/fluent/fluentd/pull/4874 https://github.com/fluent/fluentd/pull/4877 * Fixes RuboCop's remarks https://github.com/fluent/fluentd/pull/4928 * CI: Add benchmark scripts https://github.com/fluent/fluentd/pull/4989 # v1.18 ## Release v1.18.0 - 2024/11/29 ### Enhancement * Add zero-downtime-restart feature for non-Windows https://github.com/fluent/fluentd/pull/4624 * Add with-source-only feature https://github.com/fluent/fluentd/pull/4661 * `fluentd` command: Add `--with-source-only` option * System configuration: Add `with_source_only` option * Embedded plugin: Add `out_buffer` plugin, which can be used for buffering and relabeling events https://github.com/fluent/fluentd/pull/4661 * Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays https://github.com/fluent/fluentd/pull/4580 * Example: `key {"foo":"#{1 + 1}"} => key {"foo":"2"}` * Please note that this is not backward compatible, although we assume that this will never affect to actual existing configs. * In case the behavior changes unintentionally, you can disable this feature by surrounding the entire value with single quotes. * `key '{"foo":"#{1 + 1}"}' => key {"foo":"#{1 + 1}"}` * transport tls: Use SSL_VERIFY_NONE by default https://github.com/fluent/fluentd/pull/4718 * transport tls: Add ensure_fips option to ensure FIPS compliant mode https://github.com/fluent/fluentd/pull/4720 * plugin_helper/server: Add receive_buffer_size parameter in transport section https://github.com/fluent/fluentd/pull/4649 * filter_parser: Now able to handle multiple parsed results https://github.com/fluent/fluentd/pull/4620 * in_http: add `add_tag_prefix` option https://github.com/fluent/fluentd/pull/4655 * System configuration: add `path` option in `log` section https://github.com/fluent/fluentd/pull/4604 ### Bug Fix * command: fix NoMethodError of --daemon under Windows https://github.com/fluent/fluentd/pull/4716 * `fluentd` command: fix `--plugin` (`-p`) option not to overwrite default value https://github.com/fluent/fluentd/pull/4605 ### Misc * http_server: Ready to support Async 2.0 gem https://github.com/fluent/fluentd/pull/4619 * Minor code refactoring * https://github.com/fluent/fluentd/pull/4641 * CI fixes * https://github.com/fluent/fluentd/pull/4638 * https://github.com/fluent/fluentd/pull/4644 * https://github.com/fluent/fluentd/pull/4675 * https://github.com/fluent/fluentd/pull/4676 * https://github.com/fluent/fluentd/pull/4677 * https://github.com/fluent/fluentd/pull/4686 # v1.17 ## Release v1.17.1 - 2024/08/19 ### Enhancement * out_http: Add `compress gzip` option https://github.com/fluent/fluentd/pull/4528 * in_exec: Add `encoding` option to handle non-ascii characters https://github.com/fluent/fluentd/pull/4533 * in_tail: Add throttling metrics https://github.com/fluent/fluentd/pull/4578 * compat: Improve method call performance https://github.com/fluent/fluentd/pull/4588 * in_sample: Add `reuse_record` parameter to reuse the sample data https://github.com/fluent/fluentd/pull/4586 * `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins. * This increases the load when generating large amounts of sample data. * You can use this new parameter to have the same performance as before. ### Bug Fix * logger: Fix LoadError with console gem v1.25 https://github.com/fluent/fluentd/pull/4492 * parser_json: Fix wrong LoadError warning https://github.com/fluent/fluentd/pull/4522 * in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set https://github.com/fluent/fluentd/pull/4530 * yaml_parser: Support $log_level element https://github.com/fluent/fluentd/pull/4482 ### Misc * Comment out inappropriate default configuration about out_forward https://github.com/fluent/fluentd/pull/4523 * gemspec: Remove unnecessary files from released gem https://github.com/fluent/fluentd/pull/4534 * plugin-generator: Update gemspec to remove unnecessary files https://github.com/fluent/fluentd/pull/4535 * Suppress non-parenthesis warnings https://github.com/fluent/fluentd/pull/4594 * Fix FrozenError in http_server plugin helper https://github.com/fluent/fluentd/pull/4598 * Add logger gem dependency for Ruby 3.5 https://github.com/fluent/fluentd/pull/4589 * out_file: Add warn message for symlink_path setting https://github.com/fluent/fluentd/pull/4502 ## Release v1.17.0 - 2024/04/30 ### Enhancement * in_http: Recognize CSP reports as JSON data https://github.com/fluent/fluentd/pull/4282 * out_http: Add option to reuse connections https://github.com/fluent/fluentd/pull/4330 * in_tail: Expand glob capability for square brackets and one character matcher https://github.com/fluent/fluentd/pull/4401 * out_http: Support AWS Signature Version 4 authentication https://github.com/fluent/fluentd/pull/4459 ### Bug Fix * Make sure `parser_json` and `parser_msgpack` return `Hash`. Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`. https://github.com/fluent/fluentd/pull/4474 * filter_parser: Add error event for multiple parsed results https://github.com/fluent/fluentd/pull/4478 ### Misc * Raise minimum required ruby version https://github.com/fluent/fluentd/pull/4288 * Require missing dependent gems as of Ruby 3.4-dev https://github.com/fluent/fluentd/pull/4411 * Minor code refactoring https://github.com/fluent/fluentd/pull/4294 https://github.com/fluent/fluentd/pull/4299 https://github.com/fluent/fluentd/pull/4302 https://github.com/fluent/fluentd/pull/4320 * CI fixes https://github.com/fluent/fluentd/pull/4369 https://github.com/fluent/fluentd/pull/4433 https://github.com/fluent/fluentd/pull/4452 https://github.com/fluent/fluentd/pull/4477 * github: unify YAML file extension to .yml https://github.com/fluent/fluentd/pull/4429 # v1.16 ## Release v1.16.10 - 2025/09/12 ### Bug Fix * server plugin helper: ensure to close all connections at shutdown https://github.com/fluent/fluentd/pull/5088 ### Misc * CI improvemnts https://github.com/fluent/fluentd/pull/5083 https://github.com/fluent/fluentd/pull/5085 https://github.com/fluent/fluentd/pull/5086 https://github.com/fluent/fluentd/pull/5091 https://github.com/fluent/fluentd/pull/5092 ## Release v1.16.9 - 2025/05/14 ### Bug Fix * winsvc: Fix bug where service accidentally stops after starting. The previous version (v1.16.8) should not be used for Windows Service. https://github.com/fluent/fluentd/pull/4955 ### Misc * CI improvements https://github.com/fluent/fluentd/pull/4956 ## Release v1.16.8 - 2025/05/01 **This version has a critical bug about Windows Service. Do not use this version.** (https://github.com/fluent/fluentd/pull/4955) ### Bug Fix * winsvc: Stop the service when the supervisor is dead https://github.com/fluent/fluentd/pull/4942 * formatter_csv: Fix memory leak https://github.com/fluent/fluentd/pull/4920 ### Misc * Add fiddle as dependency gem for Ruby 3.5 on Windows https://github.com/fluent/fluentd/pull/4919 * Refactoring code https://github.com/fluent/fluentd/pull/4921 https://github.com/fluent/fluentd/pull/4922 https://github.com/fluent/fluentd/pull/4926 https://github.com/fluent/fluentd/pull/4943 * CI improvements https://github.com/fluent/fluentd/pull/4821 https://github.com/fluent/fluentd/pull/4850 https://github.com/fluent/fluentd/pull/4851 https://github.com/fluent/fluentd/pull/4862 https://github.com/fluent/fluentd/pull/4915 https://github.com/fluent/fluentd/pull/4923 https://github.com/fluent/fluentd/pull/4925 https://github.com/fluent/fluentd/pull/4927 ## Release v1.16.7 - 2025/01/29 ### Bug Fix * Windows: Fix `NoMethodError` of `--daemon` option https://github.com/fluent/fluentd/pull/4796 * Windows: Fixed an issue where stopping the service immediately after startup could leave the processes https://github.com/fluent/fluentd/pull/4782 * Windows: Fixed an issue where stopping service sometimes can not be completed forever https://github.com/fluent/fluentd/pull/4782 ### Misc * Windows: Add workaround for unexpected exception https://github.com/fluent/fluentd/pull/4747 * README: remove deprecated google analytics beacon https://github.com/fluent/fluentd/pull/4797 * CI improvements https://github.com/fluent/fluentd/pull/4723 https://github.com/fluent/fluentd/pull/4788 https://github.com/fluent/fluentd/pull/4789 https://github.com/fluent/fluentd/pull/4790 https://github.com/fluent/fluentd/pull/4791 https://github.com/fluent/fluentd/pull/4793 https://github.com/fluent/fluentd/pull/4794 https://github.com/fluent/fluentd/pull/4795 https://github.com/fluent/fluentd/pull/4798 https://github.com/fluent/fluentd/pull/4799 https://github.com/fluent/fluentd/pull/4800 https://github.com/fluent/fluentd/pull/4801 https://github.com/fluent/fluentd/pull/4803 ## Release v1.16.6 - 2024/08/16 ### Bug Fix * YAML config syntax: Fix issue where `$log_level` element was not supported correctly https://github.com/fluent/fluentd/pull/4486 * parser_json: Fix wrong LoadError warning https://github.com/fluent/fluentd/pull/4592 * `fluentd` command: Fix `--plugin` (`-p`) option not to overwrite default value https://github.com/fluent/fluentd/pull/4605 ### Misc * out_file: Add warn message for symlink_path setting https://github.com/fluent/fluentd/pull/4512 * Keep console gem v1.23 to avoid LoadError https://github.com/fluent/fluentd/pull/4510 ## Release v1.16.5 - 2024/03/27 ### Bug Fix * Buffer: Fix emit error of v1.16.4 sometimes failing to process large data exceeding chunk size limit https://github.com/fluent/fluentd/pull/4447 ## Release v1.16.4 - 2024/03/14 ### Bug Fix * Fix to avoid processing discarded chunks in write_step_by_step. It fixes not to raise pile of IOError when many `chunk bytes limit exceeds` errors are occurred. https://github.com/fluent/fluentd/pull/4342 * in_tail: Fix tail watchers in `rotate_wait` state not being managed. https://github.com/fluent/fluentd/pull/4334 ### Misc * buffer: Avoid unnecessary log processing. It will improve performance. https://github.com/fluent/fluentd/pull/4331 ## Release v1.16.3 - 2023/11/14 ### Bug Fix * in_tail: Fix a stall bug on !follow_inode case https://github.com/fluent/fluentd/pull/4327 * in_tail: add warning for silent stop on !follow_inodes case https://github.com/fluent/fluentd/pull/4339 * Buffer: Fix NoMethodError with empty unstaged chunk arrays https://github.com/fluent/fluentd/pull/4303 * Fix for rotate_age where Fluentd passes as Symbol https://github.com/fluent/fluentd/pull/4311 ## Release v1.16.2 - 2023/07/14 ### Bug Fix * in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`, which causes stopping tailing the file https://github.com/fluent/fluentd/pull/4208 * in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log duplication https://github.com/fluent/fluentd/pull/4237 * in_tail: Fix warning log about overwriting entry when `follow_inodes` https://github.com/fluent/fluentd/pull/4214 * in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes` https://github.com/fluent/fluentd/pull/4239 * MessagePackFactory: Make sure to reset local unpacker to prevent received broken data from affecting other receiving data https://github.com/fluent/fluentd/pull/4178 * Fix failure to launch Fluentd on Windows when the log path isn't specified in the command line https://github.com/fluent/fluentd/pull/4188 * logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly https://github.com/fluent/fluentd/pull/4229 * Update sigdump to 0.2.5 to fix wrong value of object counts https://github.com/fluent/fluentd/pull/4225 ### Misc * in_tail: Check detaching inode when `follow_inodes` https://github.com/fluent/fluentd/pull/4191 * in_tail: Add debug log for pos file compaction https://github.com/fluent/fluentd/pull/4228 * Code improvements detected by RuboCop Performance https://github.com/fluent/fluentd/pull/4201 https://github.com/fluent/fluentd/pull/4210 * Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each` https://github.com/fluent/fluentd/pull/4159 ## Release v1.16.1 - 2023/04/17 ### Enhancement * in_tcp: Add `message_length_limit` to drop large incoming data https://github.com/fluent/fluentd/pull/4137 ### Bug Fix * Fix NameError of `SecondaryFileOutput` when setting secondary other than `out_secondary_file` https://github.com/fluent/fluentd/pull/4124 * Server helper: Suppress error of `UDPServer` over `max_bytes` on Windows https://github.com/fluent/fluentd/pull/4131 * Buffer: Fix that `compress` setting causes unexpected error when receiving already compressed MessagePack https://github.com/fluent/fluentd/pull/4147 ### Misc * Update MAINTAINERS.md https://github.com/fluent/fluentd/pull/4119 * Update security policy https://github.com/fluent/fluentd/pull/4123 * Plugin template: Remove unnecessary code https://github.com/fluent/fluentd/pull/4128 * Revive issue auto closer https://github.com/fluent/fluentd/pull/4116 * Fix a link for the repository of td-agent https://github.com/fluent/fluentd/pull/4145 * in_udp: add test of message_length_limit https://github.com/fluent/fluentd/pull/4117 * Fix a typo of an argument of `Fluent::EventStream#each` https://github.com/fluent/fluentd/pull/4148 * Test in_tcp: Fix undesirable way to assert logs https://github.com/fluent/fluentd/pull/4138 ## Release v1.16.0 - 2023/03/29 ### Enhancement * in_tcp: Add `send_keepalive_packet` option https://github.com/fluent/fluentd/pull/3961 * buffer: backup broken file chunk https://github.com/fluent/fluentd/pull/4025 * Add warning messages for restoring buffer with `flush_at_shutdown true` https://github.com/fluent/fluentd/pull/4027 * Add logs for time period of restored buffer possibly broken https://github.com/fluent/fluentd/pull/4028 ### Bug Fix * http_server_helper: Fix format of log messages originating from Async gem https://github.com/fluent/fluentd/pull/3987 * Change to not generate a sigdump file after receiving a `SIGTERM` signal on non-Windows https://github.com/fluent/fluentd/pull/4034 https://github.com/fluent/fluentd/pull/4043 * out_forward: fix error of ack handling conflict on stopping with `require_ack_response` enabled https://github.com/fluent/fluentd/pull/4030 * Fix problem that some `system` configs are not reflected https://github.com/fluent/fluentd/pull/4064 https://github.com/fluent/fluentd/pull/4065 https://github.com/fluent/fluentd/pull/4086 https://github.com/fluent/fluentd/pull/4090 https://github.com/fluent/fluentd/pull/4096 * Fix bug that the logger outputs some initial log messages without applying some settings such as `format` https://github.com/fluent/fluentd/pull/4091 * Windows: Fix a bug that the wrong log file is reopened with log rotate setting when flushing or graceful reloading https://github.com/fluent/fluentd/pull/4054 * Fix race condition of out_secondary_file https://github.com/fluent/fluentd/pull/4081 * Suppress warning using different secondary for out_secondary_file https://github.com/fluent/fluentd/pull/4087 * Fix value of `system_config.workers` at `run_configure`. Change argument type of `Fluent::Plugin::Base::configure()` to `Fluent::Config::Element` only. https://github.com/fluent/fluentd/pull/4066 * Fix bug that Fluentd sometimes tries to use an unavailable port and fails to start on Windows https://github.com/fluent/fluentd/pull/4092 ### Misc * Add method for testing `filtered_with_time` https://github.com/fluent/fluentd/pull/3899 * Replace `$$` with `Process.pid` https://github.com/fluent/fluentd/pull/4040 * Relax required webric gem version https://github.com/fluent/fluentd/pull/4061 * CI fixes to support Ruby 3.2 https://github.com/fluent/fluentd/pull/3968 https://github.com/fluent/fluentd/pull/3996 https://github.com/fluent/fluentd/pull/3997 * Other CI fixes https://github.com/fluent/fluentd/pull/3969 https://github.com/fluent/fluentd/pull/3990 https://github.com/fluent/fluentd/pull/4013 https://github.com/fluent/fluentd/pull/4033 https://github.com/fluent/fluentd/pull/4044 https://github.com/fluent/fluentd/pull/4050 https://github.com/fluent/fluentd/pull/4062 https://github.com/fluent/fluentd/pull/4074 https://github.com/fluent/fluentd/pull/4082 https://github.com/fluent/fluentd/pull/4085 * Update MAINTAINERS.md https://github.com/fluent/fluentd/pull/4026 https://github.com/fluent/fluentd/pull/4069 # v1.15 ## Release v1.15.3 - 2022/11/02 ### Bug Fix * Support glob for `!include` directive in YAML config format https://github.com/fluent/fluentd/pull/3917 * Remove meaningless oj options https://github.com/fluent/fluentd/pull/3929 * Fix log initializer to correctly create per-process files on Windows https://github.com/fluent/fluentd/pull/3939 * out_file: Fix the multi-worker check with `` directive https://github.com/fluent/fluentd/pull/3942 ### Misc * Fix broken tests on Ruby 3.2 https://github.com/fluent/fluentd/pull/3883 https://github.com/fluent/fluentd/pull/3922 ## Release v1.15.2 - 2022/08/22 ### Enhancement * Add a new system configuration `enable_jit` https://github.com/fluent/fluentd/pull/3857 ### Bug Fix * out_file: Fix append mode with `--daemon` flag https://github.com/fluent/fluentd/pull/3864 * child_process: Plug file descriptor leak https://github.com/fluent/fluentd/pull/3844 ### Misc * Drop win32-api gem to support Ruby 3.2 https://github.com/fluent/fluentd/pull/3849 https://github.com/fluent/fluentd/pull/3866 ## Release v1.15.1 - 2022/07/27 ### Bug Fix * Add support for concurrent append in out_file https://github.com/fluent/fluentd/pull/3808 ### Misc * in_tail: Show more information on skipping update_watcher https://github.com/fluent/fluentd/pull/3829 ## Release v1.15.0 - 2022/06/29 ### Enhancement * in_tail: Add log throttling in files based on group rules https://github.com/fluent/fluentd/pull/3535 https://github.com/fluent/fluentd/pull/3771 * Add `dump` command to fluent-ctl https://github.com/fluent/fluentd/pull/3680 * Handle YAML configuration format on configuration file https://github.com/fluent/fluentd/pull/3712 * Add `restart_worker_interval` parameter in `` directive to set interval to restart workers that has stopped for some reason. https://github.com/fluent/fluentd/pull/3768 ### Bug fixes * out_forward: Fix to update timeout of cached sockets https://github.com/fluent/fluentd/pull/3711 * in_tail: Fix a possible crash on file rotation when `follow_inodes true` https://github.com/fluent/fluentd/pull/3754 * output: Fix a possible crash of flush thread https://github.com/fluent/fluentd/pull/3755 * in_tail: Fix crash bugs on Ruby 3.1 on Windows https://github.com/fluent/fluentd/pull/3766 * in_tail: Fix a bug that in_tail cannot open non-ascii path on Windows https://github.com/fluent/fluentd/pull/3774 * Fix a bug that fluentd doesn't release its own log file even after rotated by external tools https://github.com/fluent/fluentd/pull/3782 ### Misc * in_tail: Simplify TargetInfo related code https://github.com/fluent/fluentd/pull/3489 * Fix a wrong issue number in CHANGELOG https://github.com/fluent/fluentd/pull/3700 * server helper: Add comments to linger_timeout behavior about Windows https://github.com/fluent/fluentd/pull/3701 * service_discovery: Fix typo https://github.com/fluent/fluentd/pull/3724 * test: Fix unstable tests and warnings https://github.com/fluent/fluentd/pull/3745 https://github.com/fluent/fluentd/pull/3753 https://github.com/fluent/fluentd/pull/3767 https://github.com/fluent/fluentd/pull/3783 https://github.com/fluent/fluentd/pull/3784 https://github.com/fluent/fluentd/pull/3785 https://github.com/fluent/fluentd/pull/3787 # v1.14 ## Release v1.14.6 - 2022/03/31 ### Enhancement * Enable server plugins to specify socket-option `SO_LINGER` https://github.com/fluent/fluentd/pull/3644 * Add `--umask` command line parameter https://github.com/fluent/fluentd/pull/3671 https://github.com/fluent/fluentd/pull/3679 ### Bug fixes * Fix metric name typo https://github.com/fluent/fluentd/pull/3630 https://github.com/fluent/fluentd/pull/3673 * Apply modifications in pipeline to the records being passed to `@ERROR` label https://github.com/fluent/fluentd/pull/3631 * Fix wrong calculation of retry interval https://github.com/fluent/fluentd/pull/3640 https://github.com/fluent/fluentd/pull/3649 https://github.com/fluent/fluentd/pull/3685 https://github.com/fluent/fluentd/pull/3686 * Support IPv6 address for `rpc_endpoint` in `system` config https://github.com/fluent/fluentd/pull/3641 ### Misc * CI: Support Ruby 3.1 except Windows https://github.com/fluent/fluentd/pull/3619 * Switch to GitHub Discussions https://github.com/fluent/fluentd/pull/3654 * Fix CHANGELOG.md heading styles https://github.com/fluent/fluentd/pull/3648 * Declare `null_value_pattern` as `regexp` https://github.com/fluent/fluentd/pull/3650 ## Release v1.14.5 - 2022/02/09 ### Enhancement * Add support for "application/x-ndjson" to `in_http` https://github.com/fluent/fluentd/pull/3616 * Add support for ucrt binary for Windows https://github.com/fluent/fluentd/pull/3613 ### Bug fixes * Don't retry when `retry_max_times == 0` https://github.com/fluent/fluentd/pull/3608 * Fix hang-up issue during TLS handshake in `out_forward` https://github.com/fluent/fluentd/pull/3601 * Bump up required ServerEngine to v2.2.5 https://github.com/fluent/fluentd/pull/3599 * Fix "invalid byte sequence is replaced" warning on Kubernetes https://github.com/fluent/fluentd/pull/3596 * Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1 https://github.com/fluent/fluentd/pull/3592 ## Release v1.14.4 - 2022/01/06 ### Enhancement * `in_tail`: Add option to skip long lines (`max_line_size`) https://github.com/fluent/fluentd/pull/3565 ### Bug fix * Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size` https://github.com/fluent/fluentd/pull/3560 * On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true. https://github.com/fluent/fluentd/pull/3579 * test: Fix unstable test cases https://github.com/fluent/fluentd/pull/3574 https://github.com/fluent/fluentd/pull/3577 ## Release v1.14.3 - 2021/11/26 ### Enhancement * Changed to accept `http_parser.rb` 0.8.0. `http_parser.rb` 0.8.0 is ready for Ractor. https://github.com/fluent/fluentd/pull/3544 ### Bug fix * in_tail: Fixed a bug that no new logs are read when `enable_stat_watcher true` and `enable_watch_timer false` is set. https://github.com/fluent/fluentd/pull/3541 * in_tail: Fixed a bug that the beginning and initial lines are lost after startup when `read_from_head false` and path includes wildcard '*'. https://github.com/fluent/fluentd/pull/3542 * Fixed a bug that processing messages were lost when BufferChunkOverflowError was thrown even though only a specific message size exceeds chunk_limit_size. https://github.com/fluent/fluentd/pull/3553 https://github.com/fluent/fluentd/pull/3562 ### Misc * Bump up required version of `win32-service` gem. newer version is required to implement additional `fluent-ctl` commands. https://github.com/fluent/fluentd/pull/3556 ## Release v1.14.2 - 2021/10/29 IMPORTANT: This release contain the fix for CVE-2021-41186 - ReDoS vulnerability in `parser_apache2`. This vulnerability is affected from Fluentd v0.14.14 to v1.14.1. We recommend to upgrade Fluentd to v1.14.2 or use patched version of `parser_apache2` plugin. ### Enhancement * fluent-cat: Add `--event-time` option to send specified event time for testing. https://github.com/fluent/fluentd/pull/3528 ### Bug fix * Fixed to generate correct epoch timestamp even after switching Daylight Saving Time https://github.com/fluent/fluentd/pull/3524 * Fixed ReDoS vulnerability in parser_apache2. This vulnerability is caused by a certain pattern of a broken apache log. ## Release v1.14.1 - 2021/09/29 ### Enhancement * in_tail: Added file related metrics. These metrics should be collected same as fluent-bit's in_tail. https://github.com/fluent/fluentd/pull/3504 * out_forward: Changed to use metrics mechanism for node statistics https://github.com/fluent/fluentd/pull/3506 ### Bug fix * in_tail: Fixed a crash bug that it raise undefined method of eof? error. This error may happen only when `read_bytes_limit_per_second` was specified. https://github.com/fluent/fluentd/pull/3500 * out_forward: Fixed a bug that node statistics information is not included correctly. https://github.com/fluent/fluentd/pull/3503 https://github.com/fluent/fluentd/pull/3507 * Fixed a error when using `@include` directive It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3. https://github.com/fluent/fluentd/pull/3517 * out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success` It didn't work even if a user set it. https://github.com/fluent/fluentd/pull/3515 * Fixed not to output nanoseconds field of next retry time in warning log Then, inappropriate labels in log are also fixed. (retry_time -> retry_times, next_retry_seconds -> next_retry_time) https://github.com/fluent/fluentd/pull/3518 ## Release v1.14.0 - 2021/08/30 ### Enhancement * Added `enable_input_metrics`, `enable_size_metrics` system configuration parameter This feature might need to pay higher CPU cost, so input event metrics features are disabled by default. These features are also enabled by `--enable-input-metrics`,`--enable-size-metrics` command line option. https://github.com/fluent/fluentd/pull/3440 * Added reserved word `@ROOT` for getting root router. This is incompatible change. Do not use `@ROOT` for label name. https://github.com/fluent/fluentd/pull/3358 * in_syslog: Added `send_keepalive_packet` option https://github.com/fluent/fluentd/pull/3474 * in_http: Added `cors_allow_credentials` option. This option tells browsers whether to expose the response to frontend when the credentials mode is "include". https://github.com/fluent/fluentd/pull/3481 https://github.com/fluent/fluentd/pull/3491 ### Bug fix * in_tail: Fixed a bug that deleted paths are not removed from pos file by file compaction at start up https://github.com/fluent/fluentd/pull/3467 * in_tail: Revived a warning message of retrying unaccessible file https://github.com/fluent/fluentd/pull/3478 * TLSServer: Fixed a crash bug on logging peer host name errors https://github.com/fluent/fluentd/pull/3483 ### Misc * Added metrics plugin mechanism The implementations is changed to use metrics plugin. In the future, 3rd party plugin will be able to handle these metrics. https://github.com/fluent/fluentd/pull/3471 https://github.com/fluent/fluentd/pull/3473 https://github.com/fluent/fluentd/pull/3479 https://github.com/fluent/fluentd/pull/3484 # v1.13 ## Release v1.13.3 - 2021/07/27 ### Bug fix * in_tail: Care DeletePending state on Windows https://github.com/fluent/fluentd/pull/3457 https://github.com/fluent/fluentd/pull/3460 * in_tail: Fix some pos_file bugs. Avoid deleting pos_file entries unexpectedly when both `pos_file_compaction_interval` and `follow_inode` are enabled. Use `bytesize` instead of `size` for path length. https://github.com/fluent/fluentd/pull/3459 * in_tail: Fix detecting rotation twice on `follow_inode`. https://github.com/fluent/fluentd/pull/3466 ### Misc * Remove needless spaces in a sample config file https://github.com/fluent/fluentd/pull/3456 ## Release v1.13.2 - 2021/07/12 ### Enhancement * fluent-plugin-generate: Storage plugin was supported. https://github.com/fluent/fluentd/pull/3426 * parser_json: Added support to customize configuration of oj options. Use `FLUENT_OJ_OPTION_BIGDECIMAL_LOAD`, `FLUENT_OJ_OPTION_MAX_NESTING`, `FLUENT_OJ_OPTION_MODE`, and `FLUENT_OJ_OPTION_USE_TO_JSON` environment variable to configure it. https://github.com/fluent/fluentd/pull/3315 ### Bug fix * binlog_reader: Fixed a crash bug by missing "fluent/env" dependency. https://github.com/fluent/fluentd/pull/3443 * Fixed a crash bug on outputting log at the early stage when parsing config file. This is a regression since v1.13.0. If you use invalid '@' prefix parameter, remove it as a workaround. https://github.com/fluent/fluentd/pull/3451 * in_tail: Fixed a bug that when rotation is occurred, remaining lines will be discarded if the throttling feature is enabled. https://github.com/fluent/fluentd/pull/3390 * fluent-plugin-generate: Fixed a crash bug during gemspec generation. It was unexpectedly introduced by #3305, thus this bug was a regression since 1.12.3. https://github.com/fluent/fluentd/pull/3444 ### Misc * Fixed the runtime dependency version of http_parse.rb to 0.7.0. It was fixed because false positive detection is occurred frequently by security scanning tools. https://github.com/fluent/fluentd/pull/3450 ## Release v1.13.1 - 2021/06/25 ### Bug fix * out_forward: Fixed a race condition on handshake It's caused by using a same unpacker from multiple threads. https://github.com/fluent/fluentd/pull/3405 https://github.com/fluent/fluentd/pull/3406 * in_tail: Fixed to remove too much verbose debugging logs It was unexpectedly introduced by #3185 log throttling feature. https://github.com/fluent/fluentd/pull/3418 * Fixed not to echo back the provides path as is on a 404 error There was a potential cross-site scripting vector even though it is quite difficult to exploit. https://github.com/fluent/fluentd/pull/3427 ### Misc * Pretty print for Fluent::Config::Section has been supported for debugging https://github.com/fluent/fluentd/pull/3398 * CI: Dropped to run CI for Ruby 2.5 https://github.com/fluent/fluentd/pull/3412 ## Release v1.13.0 - 2021/05/29 ### Enhancement * in_tail: Handle log throttling per file feature https://github.com/fluent/fluentd/pull/3185 https://github.com/fluent/fluentd/pull/3364 https://github.com/fluent/fluentd/pull/3379 * Extend to support service discovery manager in simpler way https://github.com/fluent/fluentd/pull/3299 https://github.com/fluent/fluentd/pull/3362 * in_http: HTTP GET requests has been supported https://github.com/fluent/fluentd/pull/3373 * The log rotate settings in system configuration has been supported https://github.com/fluent/fluentd/pull/3352 ### Bug fix * Fix to disable `trace_instruction` when `RubyVM::InstructionSequence` is available. It improves compatibility with `truffleruby` some extent. https://github.com/fluent/fluentd/pull/3376 * in_tail: Safely skip files which are used by another process on Windows. It improves exception handling about `ERROR_SHARING_VIOLATION` on Windows. https://github.com/fluent/fluentd/pull/3378 * fluent-cat: the issue resending secondary file in specific format has been fixed https://github.com/fluent/fluentd/pull/3368 * in_tail: Shutdown immediately & safely even if reading huge files Note that `skip_refresh_on_startup` must be enabled. https://github.com/fluent/fluentd/pull/3380 ### Misc * example: Change a path to backup_path in counter_server correctly https://github.com/fluent/fluentd/pull/3359 * README: Update link to community forum to discuss.fluentd.org https://github.com/fluent/fluentd/pull/3360 # v1.12 ## Release v1.12.4 - 2021/05/26 ### Bug fix * in_tail: Fix a bug that refresh_watcher fails to handle file rotations https://github.com/fluent/fluentd/pull/3393 ## Release v1.12.3 - 2021/04/23 ### Enhancement * plugin_helper: Allow TLS to use keep-alive socket option https://github.com/fluent/fluentd/pull/3308 ### Bug fix * parser_csv, parser_syslog: Fix a naming conflict on parser_type https://github.com/fluent/fluentd/pull/3302 * in_tail: Fix incorrect error code & message on Windows https://github.com/fluent/fluentd/pull/3325 https://github.com/fluent/fluentd/pull/3329 https://github.com/fluent/fluentd/pull/3331 https://github.com/fluent/fluentd/pull/3337 * in_tail: Fix a crash bug on catching a short-lived log https://github.com/fluent/fluentd/pull/3328 * storage_local: Fix position file corruption issue on concurrent gracefulReloads https://github.com/fluent/fluentd/pull/3335 * Fix incorrect warnings about ${chunk_id} with out_s3 https://github.com/fluent/fluentd/pull/3339 * TLS Server: Add peer information to error log message https://github.com/fluent/fluentd/pull/3330 ### Misc * fluent-plugin-generate: add note about plugin name https://github.com/fluent/fluentd/pull/3303 * fluent-plugin-generate: Use same depended gem version with fluentd https://github.com/fluent/fluentd/pull/3305 * Fix some broken unit tests and improve CI's stability https://github.com/fluent/fluentd/pull/3304 https://github.com/fluent/fluentd/pull/3307 https://github.com/fluent/fluentd/pull/3312 https://github.com/fluent/fluentd/pull/3313 https://github.com/fluent/fluentd/pull/3314 https://github.com/fluent/fluentd/pull/3316 https://github.com/fluent/fluentd/pull/3336 * Permit to install with win32-service 2.2.0 on Windows https://github.com/fluent/fluentd/pull/3343 ## Release v1.12.2 - 2021/03/29 ### Enhancement * out_copy: Add ignore_if_prev_successes https://github.com/fluent/fluentd/pull/3190 https://github.com/fluent/fluentd/pull/3287 * Support multiple kind of timestamp format https://github.com/fluent/fluentd/pull/3252 * formatter_ltsv: suppress delimiters in output https://github.com/fluent/fluentd/pull/1666 https://github.com/fluent/fluentd/pull/3288 https://github.com/fluent/fluentd/pull/3289 ### Bug fix * in_tail: Expect ENOENT during stat https://github.com/fluent/fluentd/pull/3275 * out_forward: Prevent transferring duplicate logs on restart https://github.com/fluent/fluentd/pull/3267 https://github.com/fluent/fluentd/pull/3285 * in_tail: Handle to send rotated logs when mv is used for rotating https://github.com/fluent/fluentd/pull/3294 * fluent-plugin-config-format: Fill an uninitialized instance variable https://github.com/fluent/fluentd/pull/3297 * Fix MessagePackEventStream issue with Enumerable methods https://github.com/fluent/fluentd/pull/2116 ### Misc * Add webrick to support Ruby 3.0 https://github.com/fluent/fluentd/pull/3257 * Suggest Discourse instead of Google Groups https://github.com/fluent/fluentd/pull/3261 * Update MAINTAINERS.md https://github.com/fluent/fluentd/pull/3282 * Introduce DeepSource to check code quality https://github.com/fluent/fluentd/pull/3286 https://github.com/fluent/fluentd/pull/3259 https://github.com/fluent/fluentd/pull/3291 * Migrate to GitHub Actions and stabilize tests https://github.com/fluent/fluentd/pull/3266 https://github.com/fluent/fluentd/pull/3268 https://github.com/fluent/fluentd/pull/3281 https://github.com/fluent/fluentd/pull/3283 https://github.com/fluent/fluentd/pull/3290 ## Release v1.12.1 - 2021/02/18 ### Enhancement * out_http: Add `headers_from_placeholders` parameter https://github.com/fluent/fluentd/pull/3241 * fluent-plugin-config-format: Add `--table` option to use markdown table https://github.com/fluent/fluentd/pull/3240 * Add `--disable-shared-socket`/`disable_shared_socket` to disable ServerEngine's shared socket setup https://github.com/fluent/fluentd/pull/3250 ### Bug fix * ca_generate: Fix creating TLS certification files which include broken extensions https://github.com/fluent/fluentd/pull/3246 * test: Drop TLS 1.1 tests https://github.com/fluent/fluentd/pull/3256 * Remove old gem constraints to support Ruby 3 ### Misc * Use GitHub Actions https://github.com/fluent/fluentd/pull/3233 https://github.com/fluent/fluentd/pull/3255 ## Release v1.12.0 - 2021/01/05 ### New feature * in_tail: Add `follow_inode` to support log rotation with wild card https://github.com/fluent/fluentd/pull/3182 * in_tail: Handle linux capability https://github.com/fluent/fluentd/pull/3155 https://github.com/fluent/fluentd/pull/3162 * windows: Add win32 events alternative to unix signals https://github.com/fluent/fluentd/pull/3131 ### Enhancement * buffer: Enable metadata comparison optimization on all platforms https://github.com/fluent/fluentd/pull/3095 * fluent-plugin-config-formatter: Handle `service_discovery` type https://github.com/fluent/fluentd/pull/3178 * in_http: Add `add_query_params` parameter to add query params to event record https://github.com/fluent/fluentd/pull/3197 * inject: Support `unixtime_micros` and `unixtime_nanos` in `time_type` https://github.com/fluent/fluentd/pull/3220 * Refactoring code https://github.com/fluent/fluentd/pull/3167 https://github.com/fluent/fluentd/pull/3170 https://github.com/fluent/fluentd/pull/3180 https://github.com/fluent/fluentd/pull/3196 https://github.com/fluent/fluentd/pull/3213 https://github.com/fluent/fluentd/pull/3222 ### Bug fix * output: Prevent retry.step from being called too many times in a short time https://github.com/fluent/fluentd/pull/3203 # v1.11 ## Release v1.11.5 - 2020/11/06 ### Enhancement * formatter: Provide `newline` parameter to support `CRLF` https://github.com/fluent/fluentd/pull/3152 * out_http: adding support for intermediate certificates https://github.com/fluent/fluentd/pull/3146 * Update serverengine dependency to 2.2.2 or later ### Bug fix * Fix a bug that windows service isn't stopped gracefully https://github.com/fluent/fluentd/pull/3156 ## Release v1.11.4 - 2020/10/13 ### Enhancement * inject: Support `unixtime_millis` in `time_type` parameter https://github.com/fluent/fluentd/pull/3145 ### Bug fix * out_http: Fix broken data with `json_array true` https://github.com/fluent/fluentd/pull/3144 * output: Fix wrong logging issue for `${chunk_id}` https://github.com/fluent/fluentd/pull/3134 ## Release v1.11.3 - 2020/09/30 ### Enhancement * in_exec: Add `connect_mode` parameter to read stderr https://github.com/fluent/fluentd/pull/3108 * parser_json: Improve the performance https://github.com/fluent/fluentd/pull/3109 * log: Add `ignore_same_log_interval` parameter https://github.com/fluent/fluentd/pull/3119 * Upgrade win32 gems https://github.com/fluent/fluentd/pull/3100 * Refactoring code https://github.com/fluent/fluentd/pull/3094 https://github.com/fluent/fluentd/pull/3118 ### Bug fix * buffer: Fix calculation of timekey stats https://github.com/fluent/fluentd/pull/3018 * buffer: fix binmode usage for prevent gc https://github.com/fluent/fluentd/pull/3138 ## Release v1.11.2 - 2020/08/04 ### Enhancement * `in_dummy` renamed to `in_sample` https://github.com/fluent/fluentd/pull/3065 * Allow regular expression in filter/match directive https://github.com/fluent/fluentd/pull/3071 * Refactoring code https://github.com/fluent/fluentd/pull/3051 ### Bug fix * buffer: Fix log message for `chunk_limit_records` case https://github.com/fluent/fluentd/pull/3079 * buffer: Fix timekey optimization for non-windows platform https://github.com/fluent/fluentd/pull/3092 * cert: Raise an error for broken certificate file https://github.com/fluent/fluentd/pull/3086 * cert: Set TLS ciphers list correctly on older OpenSSL https://github.com/fluent/fluentd/pull/3093 ## Release v1.11.1 - 2020/06/22 ### Enhancement * in_http: Add `dump_error_log` parameter https://github.com/fluent/fluentd/pull/3035 * in_http: Improve time field handling https://github.com/fluent/fluentd/pull/3046 * Refactoring code https://github.com/fluent/fluentd/pull/3047 ### Bug fix * in_tail: Use actual path instead of based pattern for ignore list https://github.com/fluent/fluentd/pull/3042 * child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout https://github.com/fluent/fluentd/pull/3044 ## Release v1.11.0 - 2020/06/04 ### New feature * in_unix: Use v1 API https://github.com/fluent/fluentd/pull/2992 ### Enhancement * parser_syslog: Support any `time_format` for RFC3164 string parser https://github.com/fluent/fluentd/pull/3014 * parser_syslog: Add new parser for RFC5424 https://github.com/fluent/fluentd/pull/3015 * Refactoring code https://github.com/fluent/fluentd/pull/3019 ### Bug fix * in_gc_stat: Add `use_symbol_keys` parameter to emit string key record https://github.com/fluent/fluentd/pull/3008 # v1.10 ## Release v1.10.4 - 2020/05/12 ### Enhancement * out_http: Support single json array payload https://github.com/fluent/fluentd/pull/2973 * Refactoring https://github.com/fluent/fluentd/pull/2988 ### Bug fix * supervisor: Call `File.umask(0)` for standalone worker https://github.com/fluent/fluentd/pull/2987 * out_forward: Fix ZeroDivisionError issue with `weight 0` https://github.com/fluent/fluentd/pull/2989 ## Release v1.10.3 - 2020/05/01 ### Enhancement * record_accessor: Add `set` method https://github.com/fluent/fluentd/pull/2977 * config: Ruby DSL format is deprecated https://github.com/fluent/fluentd/pull/2958 * Refactor code https://github.com/fluent/fluentd/pull/2961 https://github.com/fluent/fluentd/pull/2962 https://github.com/fluent/fluentd/pull/2965 https://github.com/fluent/fluentd/pull/2966 https://github.com/fluent/fluentd/pull/2978 ### Bug fix * out_forward: Disable `linger_timeout` setting on Windows https://github.com/fluent/fluentd/pull/2959 * out_forward: Fix warning of service discovery manager when fluentd stops https://github.com/fluent/fluentd/pull/2974 ## Release v1.10.2 - 2020/04/15 ### Enhancement * out_copy: Add plugin_id to log message https://github.com/fluent/fluentd/pull/2934 * socket: Allow cert chains in mutual auth https://github.com/fluent/fluentd/pull/2930 * system: Add ignore_repeated_log_interval parameter https://github.com/fluent/fluentd/pull/2937 * windows: Allow to launch fluentd from whitespace included path https://github.com/fluent/fluentd/pull/2920 * Refactor code https://github.com/fluent/fluentd/pull/2935 https://github.com/fluent/fluentd/pull/2936 https://github.com/fluent/fluentd/pull/2938 https://github.com/fluent/fluentd/pull/2939 https://github.com/fluent/fluentd/pull/2946 ### Bug fix * in_syslog: Fix octet-counting mode bug https://github.com/fluent/fluentd/pull/2942 * out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set https://github.com/fluent/fluentd/pull/2943 * out_forward: Need authentication when sending tcp heartbeat with keepalive https://github.com/fluent/fluentd/pull/2945 * command: Fix fluent-debug start failure https://github.com/fluent/fluentd/pull/2948 * command: Fix regression of supervisor's worker and `--daemon` combo https://github.com/fluent/fluentd/pull/2950 ## Release v1.10.1 - 2020/04/02 ### Enhancement * command: `--daemon` and `--no-supervisor` now work together https://github.com/fluent/fluentd/pull/2912 * Refactor code https://github.com/fluent/fluentd/pull/2913 ### Bug fix * in_tail: `Fix pos_file_compaction_interval` parameter type https://github.com/fluent/fluentd/pull/2921 * in_tail: Fix seek position update after compaction https://github.com/fluent/fluentd/pull/2922 * parser_syslog: Fix regression in the `with_priority` and RFC5424 case https://github.com/fluent/fluentd/pull/2923 ### Misc * Add document for security audit https://github.com/fluent/fluentd/pull/2911 ## Release v1.10.0 - 2020/03/24 ### New feature * sd plugin: Add SRV record plugin https://github.com/fluent/fluentd/pull/2876 ### Enhancement * server: Add `cert_verifier` parameter for TLS transport https://github.com/fluent/fluentd/pull/2888 * parser_syslog: Support customized time format https://github.com/fluent/fluentd/pull/2886 * in_dummy: Delete `suspend` parameter https://github.com/fluent/fluentd/pull/2897 * Refactor code https://github.com/fluent/fluentd/pull/2858 https://github.com/fluent/fluentd/pull/2862 https://github.com/fluent/fluentd/pull/2864 https://github.com/fluent/fluentd/pull/2869 https://github.com/fluent/fluentd/pull/2870 https://github.com/fluent/fluentd/pull/2874 https://github.com/fluent/fluentd/pull/2881 https://github.com/fluent/fluentd/pull/2885 https://github.com/fluent/fluentd/pull/2894 https://github.com/fluent/fluentd/pull/2896 https://github.com/fluent/fluentd/pull/2898 https://github.com/fluent/fluentd/pull/2899 https://github.com/fluent/fluentd/pull/2900 https://github.com/fluent/fluentd/pull/2901 https://github.com/fluent/fluentd/pull/2906 ### Bug fix * out_forward: windows: Permit to specify `linger_timeout` https://github.com/fluent/fluentd/pull/2868 * parser_syslog: Fix syslog format detection https://github.com/fluent/fluentd/pull/2879 * buffer: Fix `available_buffer_space_ratio` calculation https://github.com/fluent/fluentd/pull/2882 * tls: Support CRLF based X.509 certificates https://github.com/fluent/fluentd/pull/2890 * msgpack_factory mixin: Fix performance penalty for deprecation log https://github.com/fluent/fluentd/pull/2903 # v1.9 ## Release v1.9.3 - 2020/03/05 ### Enhancement * in_tail: Emit buffered lines as `unmatched_line` at shutdown phase when `emit_unmatched_lines true` https://github.com/fluent/fluentd/pull/2837 * Specify directory mode explicitly https://github.com/fluent/fluentd/pull/2827 * server helper: Change SSLError log level to warn in accept https://github.com/fluent/fluentd/pull/2861 * Refactor code https://github.com/fluent/fluentd/pull/2829 https://github.com/fluent/fluentd/pull/2830 https://github.com/fluent/fluentd/pull/2832 https://github.com/fluent/fluentd/pull/2836 https://github.com/fluent/fluentd/pull/2838 https://github.com/fluent/fluentd/pull/2842 https://github.com/fluent/fluentd/pull/2843 ### Bug fix * buffer: Add seq to metadata that it can be unique https://github.com/fluent/fluentd/pull/2824 https://github.com/fluent/fluentd/pull/2853 * buffer: Use `Tempfile` as binmode for decompression https://github.com/fluent/fluentd/pull/2847 ### Misc * Add `.idea` to git ignore file https://github.com/fluent/fluentd/pull/2834 * appveyor: Fix tests https://github.com/fluent/fluentd/pull/2853 https://github.com/fluent/fluentd/pull/2855 * Update pem for test https://github.com/fluent/fluentd/pull/2839 ## Release v1.9.2 - 2020/02/13 ### Enhancement * in_tail: Add `pos_file_compaction_interval` parameter for auto compaction https://github.com/fluent/fluentd/pull/2805 * command: Use given encoding when RUBYOPT has `-E` https://github.com/fluent/fluentd/pull/2814 ### Bug fix * command: Accept RUBYOPT with two or more options https://github.com/fluent/fluentd/pull/2807 * command: Fix infinite loop bug when RUBYOPT is invalid https://github.com/fluent/fluentd/pull/2813 * log: serverengine's log should be formatted with the same format of fluentd https://github.com/fluent/fluentd/pull/2812 * in_http: Fix `NoMethodError` when `OPTIONS` request doesn't have 'Origin' header https://github.com/fluent/fluentd/pull/2823 * parser_syslog: Improved for parsing RFC5424 structured data in `parser_syslog` https://github.com/fluent/fluentd/pull/2816 ## Release v1.9.1 - 2020/01/31 ### Enhancement * http_server helper: Support HTTPS https://github.com/fluent/fluentd/pull/2787 * in_tail: Add `path_delimiter` to split with any char https://github.com/fluent/fluentd/pull/2796 * in_tail: Remove an entry from PositionFile when it is unwatched https://github.com/fluent/fluentd/pull/2803 * out_http: Add warning for `retryable_response_code` https://github.com/fluent/fluentd/pull/2809 * parser_syslog: Add multiline RFC5424 support https://github.com/fluent/fluentd/pull/2767 * Add TLS module to unify TLS related code https://github.com/fluent/fluentd/pull/2802 ### Bug fix * output: Add `EncodingError` to unrecoverable errors https://github.com/fluent/fluentd/pull/2808 * tls: Fix TLS version handling in secure mode https://github.com/fluent/fluentd/pull/2802 ## Release v1.9.0 - 2020/01/22 ### New feature * New light-weight config reload mechanism https://github.com/fluent/fluentd/pull/2716 * Drop ruby 2.1/2.2/2.3 support https://github.com/fluent/fluentd/pull/2750 ### Enhancement * output: Show better message for secondary warning https://github.com/fluent/fluentd/pull/2751 * Use `ext_monitor` gem if it is installed. For ruby 2.6 or earlier https://github.com/fluent/fluentd/pull/2670 * Support Ruby's Time class in msgpack serde https://github.com/fluent/fluentd/pull/2775 * Clean up code/test https://github.com/fluent/fluentd/pull/2753 https://github.com/fluent/fluentd/pull/2763 https://github.com/fluent/fluentd/pull/2764 https://github.com/fluent/fluentd/pull/2780 ### Bug fix * buffer: Disable the optimization of Metadata instance comparison on Windows https://github.com/fluent/fluentd/pull/2778 * output/buffer: Fix stage size computation https://github.com/fluent/fluentd/pull/2734 * server: Ignore Errno::EHOSTUNREACH in TLS accept to avoid fluentd restart https://github.com/fluent/fluentd/pull/2773 * server: Fix IPv6 dual stack mode issue for udp socket https://github.com/fluent/fluentd/pull/2781 * config: Support @include/include directive for spaces included path https://github.com/fluent/fluentd/pull/2780 # v1.8 ## Release v1.8.1 - 2019/12/26 ### Enhancement * in_tail: Add `path_timezone` parameter to format `path` with the specified timezone https://github.com/fluent/fluentd/pull/2719 * out_copy: Add `copy_mode` parameter. `deep_copy` parameter is now deprecated. https://github.com/fluent/fluentd/pull/2747 * supervisor: Add deprecated log for `inline_config` https://github.com/fluent/fluentd/pull/2746 ### Bug fixes * parser_ltsv: Prevent garbage result by checking `label_delimiter` https://github.com/fluent/fluentd/pull/2748 ## Release v1.8.0 - 2019/12/11 ### New feature * Add service discovery plugin and `out_forward` use it https://github.com/fluent/fluentd/pull/2541 * config: Add strict mode and support `default`/`nil` value in ruby embedded mode https://github.com/fluent/fluentd/pull/2685 ### Enhancement * formatter_csv: Support nested fields https://github.com/fluent/fluentd/pull/2643 * record_accessor helper: Make code simple and bit faster https://github.com/fluent/fluentd/pull/2660 * Relax tzinfo dependency to accept v1 https://github.com/fluent/fluentd/pull/2673 * log: Deprecate top-level match for capturing fluentd logs https://github.com/fluent/fluentd/pull/2689 * in_monitor_agent: Expose Fluentd version in REST API https://github.com/fluent/fluentd/pull/2706 * time: Accept localtime xor utc https://github.com/fluent/fluentd/pull/2720 https://github.com/fluent/fluentd/pull/2731 * formatter_stdout: Make time_format configurable in stdout format https://github.com/fluent/fluentd/pull/2721 * supervisor: create log directory when it doesn't exists https://github.com/fluent/fluentd/pull/2732 * clean up internal classes / methods / code https://github.com/fluent/fluentd/pull/2647 https://github.com/fluent/fluentd/pull/2648 https://github.com/fluent/fluentd/pull/2653 https://github.com/fluent/fluentd/pull/2654 https://github.com/fluent/fluentd/pull/2657 https://github.com/fluent/fluentd/pull/2667 https://github.com/fluent/fluentd/pull/2674 https://github.com/fluent/fluentd/pull/2677 https://github.com/fluent/fluentd/pull/2680 https://github.com/fluent/fluentd/pull/2709 https://github.com/fluent/fluentd/pull/2730 ### Bug fixes * output: Fix warning printed when chunk key placeholder not replaced https://github.com/fluent/fluentd/pull/2523 https://github.com/fluent/fluentd/pull/2733 * Fix dry-run mode https://github.com/fluent/fluentd/pull/2651 * suppress warning https://github.com/fluent/fluentd/pull/2652 * suppress keyword argument warning for ruby2.7 https://github.com/fluent/fluentd/pull/2664 * RPC: Fix debug log text https://github.com/fluent/fluentd/pull/2666 * time: Properly show class names in error message https://github.com/fluent/fluentd/pull/2671 * Fix a potential bug that ThreadError may occur on SIGUSR1 https://github.com/fluent/fluentd/pull/2678 * server helper: Ignore ECONNREFUSED in TLS accept to avoid fluentd restart https://github.com/fluent/fluentd/pull/2695 * server helper: Fix IPv6 dual stack mode issue for tcp socket. https://github.com/fluent/fluentd/pull/2697 * supervisor: Fix inline config handling https://github.com/fluent/fluentd/pull/2708 * Fix typo https://github.com/fluent/fluentd/pull/2710 https://github.com/fluent/fluentd/pull/2714 # v1.7 ## Release v1.7.4 - 2019/10/24 ### Enhancement * in_http: Add `use_204_response` parameter to return proper 204 response instead of 200. fluentd v2 will change this parameter to `true`. https://github.com/fluent/fluentd/pull/2640 ### Bug fixes * child_process helper: fix stderr blocking for discard case https://github.com/fluent/fluentd/pull/2649 * log: Fix log rotation handling on Windows https://github.com/fluent/fluentd/pull/2663 ## Release v1.7.3 - 2019/10/01 ### Enhancement * in_syslog: Replace priority_key with severity_key https://github.com/fluent/fluentd/pull/2636 ### Bug fixes * out_forward: Fix nil error after purge obsoleted sockets in socket cache https://github.com/fluent/fluentd/pull/2635 * fix typo in ChangeLog https://github.com/fluent/fluentd/pull/2633 ## Release v1.7.2 - 2019/09/19 ### Enhancement * in_tcp: Add security/client to restrict access https://github.com/fluent/fluentd/pull/2622 ### Bug fixes * buf_file/buf_file_single: fix to handle compress data during restart https://github.com/fluent/fluentd/pull/2620 * plugin: Use `__send__` to avoid conflict with user defined `send` https://github.com/fluent/fluentd/pull/2614 * buffer: reject invalid timekey at configure phase https://github.com/fluent/fluentd/pull/2615 ## Release v1.7.1 - 2019/09/08 ### Enhancement * socket helper/out_forward: Support Windows certstore to load certificates https://github.com/fluent/fluentd/pull/2601 * parser_syslog: Add faster parser for rfc3164 message https://github.com/fluent/fluentd/pull/2599 ### Bug fixes * buf_file/buf_file_single: fix to ignore placeholder based path. https://github.com/fluent/fluentd/pull/2594 * server helper: Ignore ETIMEDOUT error in SSL_accept https://github.com/fluent/fluentd/pull/2595 * buf_file: ensure to remove metadata after buffer creation failure https://github.com/fluent/fluentd/pull/2598 * buf_file_single: fix duplicated path setting check https://github.com/fluent/fluentd/pull/2600 * fix msgpack-ruby dependency to use recent feature https://github.com/fluent/fluentd/pull/2606 ## Release v1.7.0 - 2019/08/20 ### New feature * buffer: Add file_single buffer plugin https://github.com/fluent/fluentd/pull/2579 * output: Add http output plugin https://github.com/fluent/fluentd/pull/2488 ### Enhancement * buffer: Improve the performance of buffer routine https://github.com/fluent/fluentd/pull/2560 https://github.com/fluent/fluentd/pull/2563 https://github.com/fluent/fluentd/pull/2564 * output: Use Mutex instead of Monitor https://github.com/fluent/fluentd/pull/2561 * event: Add `OneEventStrea#empty?` method https://github.com/fluent/fluentd/pull/2565 * thread: Set thread name for ruby 2.3 or later https://github.com/fluent/fluentd/pull/2574 * core: Cache msgpack packer/unpacker to avoid the object allocation https://github.com/fluent/fluentd/pull/2559 * time: Use faster way to get sec and nsec https://github.com/fluent/fluentd/pull/2557 * buf_file: Reduce IO flush by removing `IO#truncate` https://github.com/fluent/fluentd/pull/2551 * in_tcp: Improve the performance for multiple event case https://github.com/fluent/fluentd/pull/2567 * in_syslog: support `source_hostname_key` and `source_address_key` for unmatched event https://github.com/fluent/fluentd/pull/2553 * formatter_csv: Improve the format performance. https://github.com/fluent/fluentd/pull/2529 * parser_csv: Add fast parser for typical cases https://github.com/fluent/fluentd/pull/2535 * out_forward: Refactor code https://github.com/fluent/fluentd/pull/2516 https://github.com/fluent/fluentd/pull/2532 ### Bug fixes * output: fix data lost on decompression https://github.com/fluent/fluentd/pull/2547 * out_exec_filter: fix non-ascii encoding issue https://github.com/fluent/fluentd/pull/2539 * in_tail: Don't call parser's configure twice https://github.com/fluent/fluentd/pull/2569 * Fix unused message handling for
parameters https://github.com/fluent/fluentd/pull/2578 * Fix comment/message typos https://github.com/fluent/fluentd/pull/2549 https://github.com/fluent/fluentd/pull/2554 https://github.com/fluent/fluentd/pull/2556 https://github.com/fluent/fluentd/pull/2566 https://github.com/fluent/fluentd/pull/2573 https://github.com/fluent/fluentd/pull/2576 https://github.com/fluent/fluentd/pull/2583 # v1.6 ## Release v1.6.3 - 2019/07/29 ### Enhancement * in_syslog: Add `emit_unmatched_lines` parameter https://github.com/fluent/fluentd/pull/2499 * buf_file: Add `path_suffix` parameter https://github.com/fluent/fluentd/pull/2524 * in_tail: Improve the performance of split lines https://github.com/fluent/fluentd/pull/2527 ### Bug fixes * http_server: Fix re-define render_json method https://github.com/fluent/fluentd/pull/2517 ## Release v1.6.2 - 2019/07/11 ### Bug fixes * http_server helper: Add title argument to support multiple servers https://github.com/fluent/fluentd/pull/2493 ## Release v1.6.1 - 2019/07/10 ### Enhancement * socket/cert: Support all private keys OpenSSL supports, not only RSA. https://github.com/fluent/fluentd/pull/2487 * output/buffer: Improve statistics method performance https://github.com/fluent/fluentd/pull/2491 ### Bug fixes * plugin_config_formatter: update new doc URL https://github.com/fluent/fluentd/pull/2481 * out_forward: Avoid zero division error when there are no available nodes https://github.com/fluent/fluentd/pull/2482 ## Release v1.6.0 - 2019/07/01 ### New feature * plugin: Add http_server helper and in_monitor_agent use it https://github.com/fluent/fluentd/pull/2447 ### Enhancement * in_monitor_agent: Add more metrics for buffer/output https://github.com/fluent/fluentd/pull/2450 * time/plugin: Add `EventTime#to_time` method for fast conversion https://github.com/fluent/fluentd/pull/2469 * socket helper/out_forward: Add connect_timeout parameter https://github.com/fluent/fluentd/pull/2467 * command: Add `--conf-encoding` option https://github.com/fluent/fluentd/pull/2453 * parser_none: Small performance optimization https://github.com/fluent/fluentd/pull/2455 ### Bug fixes * cert: Fix cert match pattern https://github.com/fluent/fluentd/pull/2466 * output: Fix forget to increment rollback count https://github.com/fluent/fluentd/pull/2462 # v1.5 ## Release v1.5.2 - 2019/06/13 ### Bug fixes * out_forward: Fix duplicated handshake bug in keepalive https://github.com/fluent/fluentd/pull/2456 ## Release v1.5.1 - 2019/06/05 ### Enhancement * in_tail: Increase read block size to reduce IO call https://github.com/fluent/fluentd/pull/2418 * in_monitor_agent: Refactor code https://github.com/fluent/fluentd/pull/2422 ### Bug fixes * out_forward: Fix socket handling of keepalive https://github.com/fluent/fluentd/pull/2434 * parser: Fix the use of name based timezone https://github.com/fluent/fluentd/pull/2421 * in_monitor_agent: Fix debug parameter handling https://github.com/fluent/fluentd/pull/2423 * command: Fix error handling of log rotation age option https://github.com/fluent/fluentd/pull/2427 * command: Fix ERB warning for ruby 2.6 or later https://github.com/fluent/fluentd/pull/2430 ## Release v1.5.0 - 2019/05/18 ### New feature * out_forward: Support keepalive feature https://github.com/fluent/fluentd/pull/2393 * in_http: Support TLS via server helper https://github.com/fluent/fluentd/pull/2395 * in_syslog: Support TLS via server helper https://github.com/fluent/fluentd/pull/2399 ### Enhancement * in_syslog: Add delimiter parameter https://github.com/fluent/fluentd/pull/2378 * in_forward: Add tag/add_tag_prefix parameters https://github.com/fluent/fluentd/pull/2396 * parser_json: Add stream_buffer_size parameter for yajl https://github.com/fluent/fluentd/pull/2381 * command: Add deprecated message to show-plugin-config option https://github.com/fluent/fluentd/pull/2401 * storage_local: Ignore empty file. Call sync after write for XFS. https://github.com/fluent/fluentd/pull/2409 ### Bug fixes * out_forward: Don't use SO_LINGER on SSL/TLS WinSock https://github.com/fluent/fluentd/pull/2398 * server helper: Fix recursive lock issue in TLSServer https://github.com/fluent/fluentd/pull/2341 * Fix typo https://github.com/fluent/fluentd/pull/2369 # v1.4 ## Release v1.4.2 - 2019/04/02 ### Enhancements * in_http: subdomain support in CORS domain https://github.com/fluent/fluentd/pull/2337 * in_monitor_agent: Expose current timekey list as a buffer metrics https://github.com/fluent/fluentd/pull/2343 * in_tcp/in_udp: Add source_address_key parameter https://github.com/fluent/fluentd/pull/2347 * in_forward: Add send_keepalive_packet parameter to check the remote connection is available or not https://github.com/fluent/fluentd/pull/2352 ### Bug fixes * out_exec_filter: Fix typo of child_respawn description https://github.com/fluent/fluentd/pull/2341 * in_tail: Create parent directories for symlink https://github.com/fluent/fluentd/pull/2353 * in_tail: Fix encoding duplication check for non-specified case https://github.com/fluent/fluentd/pull/2361 * log: Fix time format handling of plugin logger when log format is JSON https://github.com/fluent/fluentd/pull/2356 ## Release v1.4.1 - 2019/03/18 ### Enhancements * system: Add worker_id to process_name when workers is larger than 1 https://github.com/fluent/fluentd/pull/2321 * parser_regexp: Check named captures. When no named captures, configuration error is raised https://github.com/fluent/fluentd/pull/2331 ### Bug fixes * out_forward: Make tls_client_private_key_passphrase secret https://github.com/fluent/fluentd/pull/2324 * in_syslog: Check message length when read from buffer in octet counting https://github.com/fluent/fluentd/pull/2323 ## Release v1.4.0 - 2019/02/24 ### New features * multiprocess: Support syntax https://github.com/fluent/fluentd/pull/2292 * output: Work and retry_forever together https://github.com/fluent/fluentd/pull/2276 * out_file: Support placeholders in symlink_path https://github.com/fluent/fluentd/pull/2254 ### Enhancements * output: Add MessagePack unpacker error to unrecoverable error list https://github.com/fluent/fluentd/pull/2301 * output: Reduce flush delay when large timekey and small timekey_wait are specified https://github.com/fluent/fluentd/pull/2291 * config: Support embedded ruby code in section argument. https://github.com/fluent/fluentd/pull/2295 * in_tail: Improve encoding parameter handling https://github.com/fluent/fluentd/pull/2305 * in_tcp/in_udp: Add section check https://github.com/fluent/fluentd/pull/2267 ### Bug fixes * server: Ignore IOError and related errors in UDP https://github.com/fluent/fluentd/pull/2310 * server: Ignore EPIPE in TLS accept to avoid fluentd restart https://github.com/fluent/fluentd/pull/2253 # v1.3 ## Release v1.3.3 - 2019/01/06 ### Enhancements * parser_syslog: Use String#squeeze for performance improvement https://github.com/fluent/fluentd/pull/2239 * parser_syslog: Support RFC5424 timestamp without subseconds https://github.com/fluent/fluentd/pull/2240 ### Bug fixes * server: Ignore ECONNRESET in TLS accept to avoid fluentd restart https://github.com/fluent/fluentd/pull/2243 * log: Fix plugin logger ignores fluentd log event setting https://github.com/fluent/fluentd/pull/2252 ## Release v1.3.2 - 2018/12/10 ### Enhancements * out_forward: Support mutual TLS https://github.com/fluent/fluentd/pull/2187 * out_file: Create `pos_file` directory if it doesn't exist https://github.com/fluent/fluentd/pull/2223 ### Bug fixes * output: Fix logs during retry https://github.com/fluent/fluentd/pull/2203 ## Release v1.3.1 - 2018/11/27 ### Enhancements * out_forward: Separate parameter names for certificate https://github.com/fluent/fluentd/pull/2181 https://github.com/fluent/fluentd/pull/2190 * out_forward: Add `verify_connection_at_startup` parameter to check connection setting at startup phase https://github.com/fluent/fluentd/pull/2184 * config: Check right slash position in regexp type https://github.com/fluent/fluentd/pull/2176 * parser_nginx: Support multiple IPs in `http_x_forwarded_for` field https://github.com/fluent/fluentd/pull/2171 ### Bug fixes * fluent-cat: Fix retry limit handling https://github.com/fluent/fluentd/pull/2193 * record_accessor helper: Delete top level field with bracket style https://github.com/fluent/fluentd/pull/2192 * filter_record_transformer: Keep `class` method to avoid undefined method error https://github.com/fluent/fluentd/pull/2186 ## Release v1.3.0 - 2018/11/10 ### New features * output: Change thread execution control https://github.com/fluent/fluentd/pull/2170 * in_syslog: Support octet counting frame https://github.com/fluent/fluentd/pull/2147 * Use `flush_thread_count` value for `queued_chunks_limit_size` when `queued_chunks_limit_size` is not specified https://github.com/fluent/fluentd/pull/2173 ### Enhancements * output: Show backtrace for unrecoverable errors https://github.com/fluent/fluentd/pull/2149 * in_http: Implement support for CORS preflight requests https://github.com/fluent/fluentd/pull/2144 ### Bug fixes * server: Fix deadlock between on_writable and close in sockets https://github.com/fluent/fluentd/pull/2165 * output: show correct error when wrong plugin is specified for secondary https://github.com/fluent/fluentd/pull/2169 # v1.2 ## Release v1.2.6 - 2018/10/03 ### Enhancements * output: Add `disable_chunk_backup` for ignore broken chunks. https://github.com/fluent/fluentd/pull/2117 * parser_syslog: Improve regexp for RFC5424 https://github.com/fluent/fluentd/pull/2141 * in_http: Allow specifying the wildcard '*' as the CORS domain https://github.com/fluent/fluentd/pull/2139 ### Bug fixes * in_tail: Prevent thread switching in the interval between seek and read/write operations to pos_file https://github.com/fluent/fluentd/pull/2118 * parser: Handle LoadError properly for oj https://github.com/fluent/fluentd/pull/2140 ## Release v1.2.5 - 2018/08/22 ### Bug fixes * in_tail: Fix resource leak by file rotation https://github.com/fluent/fluentd/pull/2105 * fix typos ## Release v1.2.4 - 2018/08/01 ### Bug fixes * output: Consider timezone when calculate timekey https://github.com/fluent/fluentd/pull/2054 * output: Fix bug in suppress_emit_error_log_interval https://github.com/fluent/fluentd/pull/2069 * server-helper: Fix connection leak by close timing issue. https://github.com/fluent/fluentd/pull/2087 ## Release v1.2.3 - 2018/07/10 ### Enhancements * in_http: Consider `` parameters in batch mode https://github.com/fluent/fluentd/pull/2055 * in_http: Support gzip payload https://github.com/fluent/fluentd/pull/2060 * output: Improve compress performance https://github.com/fluent/fluentd/pull/2031 * in_monitor_agent: Add missing descriptions for configurable options https://github.com/fluent/fluentd/pull/2037 * parser_syslog: update regex of pid field for conformance to RFC5424 spec https://github.com/fluent/fluentd/pull/2051 ### Bug fixes * in_tail: Fix to rescue Errno::ENOENT for File.mtime() https://github.com/fluent/fluentd/pull/2063 * fluent-plugin-generate: Fix Parser plugin template https://github.com/fluent/fluentd/pull/2026 * fluent-plugin-config-format: Fix NoMethodError for some plugins https://github.com/fluent/fluentd/pull/2023 * config: Don't warn message for reserved parameters in DSL https://github.com/fluent/fluentd/pull/2034 ## Release v1.2.2 - 2018/06/12 ### Enhancements * filter_parser: Add remove_key_name_field parameter https://github.com/fluent/fluentd/pull/2012 * fluent-plugin-config-format: Dump config_argument https://github.com/fluent/fluentd/pull/2003 ### Bug fixes * in_tail: Change pos file entry handling to avoid read conflict for other plugins https://github.com/fluent/fluentd/pull/1963 * buffer: Wait for all chunks being purged before deleting @queued_num items https://github.com/fluent/fluentd/pull/2016 ## Release v1.2.1 - 2018/05/23 ### Enhancements * Counter: Add wait API to client https://github.com/fluent/fluentd/pull/1997 ### Bug fixes * in_tcp/in_udp: Fix source_hostname_key to set hostname correctly https://github.com/fluent/fluentd/pull/1976 * in_monitor_agent: Fix buffer_total_queued_size calculation https://github.com/fluent/fluentd/pull/1990 * out_file: Temporal fix for broken gzipped files with gzip and append https://github.com/fluent/fluentd/pull/1995 * test: Fix unstable backup test https://github.com/fluent/fluentd/pull/1979 * gemspec: Remove deprecated has_rdoc ## Release v1.2.0 - 2018/04/30 ### New Features * New Counter API https://github.com/fluent/fluentd/pull/1857 * output: Backup for broken chunks https://github.com/fluent/fluentd/pull/1952 * filter_grep: Support for `` and `` sections https://github.com/fluent/fluentd/pull/1897 * config: Support `regexp` type in configuration parameter https://github.com/fluent/fluentd/pull/1927 ### Enhancements * parser_nginx: Support optional `http-x-forwarded-for` field https://github.com/fluent/fluentd/pull/1932 * filter_grep: Improve the performance https://github.com/fluent/fluentd/pull/1940 ### Bug fixes * log: Fix unexpected implementation bug when log rotation setting is applied https://github.com/fluent/fluentd/pull/1957 * server helper: Close invalid socket when ssl error happens on reading https://github.com/fluent/fluentd/pull/1942 * output: Buffer chunk's unique id should be formatted as hex in the log # v1.1 ## Release v1.1.3 - 2018/04/03 ### Enhancements * output: Support negative index for tag placeholders https://github.com/fluent/fluentd/pull/1908 * buffer: Add queued_chunks_limit_size to control the number of queued chunks https://github.com/fluent/fluentd/pull/1916 * time: Make Fluent::EventTime human readable for inspect https://github.com/fluent/fluentd/pull/1915 ### Bug fixes * output: Delete empty queued_num field after purging chunks https://github.com/fluent/fluentd/pull/1919 * fluent-debug: Fix usage message of fluent-debug command https://github.com/fluent/fluentd/pull/1920 * out_forward: The node should be disabled when TLS socket for ack returns an error https://github.com/fluent/fluentd/pull/1925 ## Release v1.1.2 - 2018/03/18 ### Enhancements * filter_grep: Support pattern starts with character classes with // https://github.com/fluent/fluentd/pull/1887 ### Bug fixes * in_tail: Handle records in the correct order on file rotation https://github.com/fluent/fluentd/pull/1880 * out_forward: Fix race condition with `` on multi thread environment https://github.com/fluent/fluentd/pull/1893 * output: Prevent flushing threads consume too much CPU when retry happens https://github.com/fluent/fluentd/pull/1901 * config: Fix boolean param handling for comment without value https://github.com/fluent/fluentd/pull/1883 * test: Fix random test failures in test/plugin/test_out_forward.rb https://github.com/fluent/fluentd/pull/1881 https://github.com/fluent/fluentd/pull/1890 * command: Fix typo in binlog_reader https://github.com/fluent/fluentd/pull/1898 ## Release v1.1.1 - 2018/03/05 ### Enhancements * in_debug_agent: Support multi worker environment https://github.com/fluent/fluentd/pull/1869 * in_forward: Improve SSL setup to support mutual TLS https://github.com/fluent/fluentd/pull/1861 * buf_file: Skip and delete broken file chunks to avoid unsuccessful retry in resume https://github.com/fluent/fluentd/pull/1874 * command: Show fluentd version for debug purpose https://github.com/fluent/fluentd/pull/1839 ### Bug fixes * in_forward: Do not close connection until write is complete on failed auth PONG https://github.com/fluent/fluentd/pull/1835 * in_tail: Fix IO event race condition during shutdown https://github.com/fluent/fluentd/pull/1876 * in_http: Emit event time instead of raw time value in batch https://github.com/fluent/fluentd/pull/1850 * parser_json: Add EncodingError to rescue list for oj 3.x. https://github.com/fluent/fluentd/pull/1875 * config: Fix config_param for string type with frozen string https://github.com/fluent/fluentd/pull/1838 * timer: Fix a bug to leak non-repeating timer watchers https://github.com/fluent/fluentd/pull/1864 ## Release v1.1.0 - 2018/01/17 ### New features / Enhancements * config: Add hostname and worker_id short-cut https://github.com/fluent/fluentd/pull/1814 * parser_ltsv: Add delimiter_pattern parameter https://github.com/fluent/fluentd/pull/1802 * record_accessor helper: Support nested field deletion https://github.com/fluent/fluentd/pull/1800 * record_accessor helper: Expose internal instance `@keys` variable https://github.com/fluent/fluentd/pull/1808 * log: Improve Log#on_xxx API performance https://github.com/fluent/fluentd/pull/1809 * time: Improve time formatting performance https://github.com/fluent/fluentd/pull/1796 * command: Port certificates generating command from secure-forward https://github.com/fluent/fluentd/pull/1818 ### Bug fixes * server helper: Fix TCP + TLS degradation https://github.com/fluent/fluentd/pull/1805 * time: Fix the method for TimeFormatter#call https://github.com/fluent/fluentd/pull/1813 # v1.0 ## Release v1.0.2 - 2017/12/17 ### New features / Enhancements * Use dig_rb instead of ruby_dig to support dig method in more objects https://github.com/fluent/fluentd/pull/1794 ## Release v1.0.1 - 2017/12/14 ### New features / Enhancements * in_udp: Add receive_buffer_size parameter https://github.com/fluent/fluentd/pull/1788 * in_tail: Add enable_stat_watcher option to disable inotify events https://github.com/fluent/fluentd/pull/1775 * Relax strptime gem version ### Bug fixes * in_tail: Properly handle moved back and truncated case https://github.com/fluent/fluentd/pull/1793 * out_forward: Rebuild weight array to apply server setting properly https://github.com/fluent/fluentd/pull/1784 * fluent-plugin-config-formatter: Use v1.0 for URL https://github.com/fluent/fluentd/pull/1781 ## Release v1.0.0 - 2017/12/6 See [CNCF announcement](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :) ### New features / Enhancements * out_copy: Support ignore_error argument in `` https://github.com/fluent/fluentd/pull/1764 * server helper: Improve resource usage of TLS transport https://github.com/fluent/fluentd/pull/1764 * Disable tracepoint feature to omit unnecessary insts https://github.com/fluent/fluentd/pull/1764 ### Bug fixes * out_forward: Don't update retry state when failed to get ack response. https://github.com/fluent/fluentd/pull/1686 * plugin: Combine before_shutdown and shutdown call in one sequence. https://github.com/fluent/fluentd/pull/1763 * Add description to parsers https://github.com/fluent/fluentd/pull/1776 https://github.com/fluent/fluentd/pull/1777 https://github.com/fluent/fluentd/pull/1778 https://github.com/fluent/fluentd/pull/1779 https://github.com/fluent/fluentd/pull/1780 * filter_parser: Add parameter description https://github.com/fluent/fluentd/pull/1773 * plugin: Combine before_shutdown and shutdown call in one sequence. https://github.com/fluent/fluentd/pull/1763 # v0.14 ## Release v0.14.25 - 2017/11/29 ### New features / Enhancements * Disable tracepoint feature to omit unnecessary insts https://github.com/fluent/fluentd/pull/1764 ### Bug fixes * out_forward: Don't update retry state when failed to get ack response. https://github.com/fluent/fluentd/pull/1686 * plugin: Combine before_shutdown and shutdown call in one sequence. https://github.com/fluent/fluentd/pull/1763 ## Release v0.14.24 - 2017/11/24 ### New features / Enhancements * plugin-config-formatter: Add link to plugin helper result https://github.com/fluent/fluentd/pull/1753 * server helper: Refactor code https://github.com/fluent/fluentd/pull/1759 ### Bug fixes * supervisor: Don't call change_privilege twice https://github.com/fluent/fluentd/pull/1757 ## Release v0.14.23 - 2017/11/15 ### New features / Enhancements * in_udp: Add remove_newline parameter https://github.com/fluent/fluentd/pull/1747 ### Bug fixes * buffer: Lock buffers in order of metadata https://github.com/fluent/fluentd/pull/1722 * in_tcp: Fix log corruption under load. https://github.com/fluent/fluentd/pull/1729 * out_forward: Fix elapsed time miscalculation in tcp heartbeat https://github.com/fluent/fluentd/pull/1738 * supervisor: Fix worker pid handling during worker restart https://github.com/fluent/fluentd/pull/1739 * in_tail: Skip setup failed watcher to avoid resource leak and log bloat https://github.com/fluent/fluentd/pull/1742 * agent: Add error location to emit error logs https://github.com/fluent/fluentd/pull/1746 * command: Consider hyphen and underscore in fluent-plugin-generate arguments https://github.com/fluent/fluentd/pull/1751 ## Release v0.14.22 - 2017/11/01 ### New features / Enhancements * formatter_tsv: Add add_newline parameter https://github.com/fluent/fluentd/pull/1691 * out_file/out_secondary_file: Support ${chunk_id} placeholder. This includes extract_placeholders API change https://github.com/fluent/fluentd/pull/1708 * record_accessor: Support double quotes in bracket notation https://github.com/fluent/fluentd/pull/1716 * log: Show running ruby version in startup log https://github.com/fluent/fluentd/pull/1717 * log: Log message when chunk is created https://github.com/fluent/fluentd/pull/1718 * in_tail: Add pos_file duplication check https://github.com/fluent/fluentd/pull/1720 ### Bug fixes * parser_apache2: Delay time parser initialization https://github.com/fluent/fluentd/pull/1690 * cert_option: Improve generated certificates' conformance to X.509 specification https://github.com/fluent/fluentd/pull/1714 * buffer: Always lock chunks first to avoid deadlock https://github.com/fluent/fluentd/pull/1721 ## Release v0.14.21 - 2017/09/07 ### New features / Enhancements * filter_parser: Support record_accessor in key_name https://github.com/fluent/fluentd/pull/1654 * buffer: Support record_accessor in chunk keys https://github.com/fluent/fluentd/pull/1662 ### Bug fixes * compat_parameters: Support all syslog parser parameters https://github.com/fluent/fluentd/pull/1650 * filter_record_transformer: Don't create new keys if the original record doesn't have `keep_keys` keys https://github.com/fluent/fluentd/pull/1663 * in_tail: Fix the error when 'tag *' is configured https://github.com/fluent/fluentd/pull/1664 * supervisor: Clear previous worker pids when receive kill signals. https://github.com/fluent/fluentd/pull/1683 ## Release v0.14.20 - 2017/07/31 ### New features / Enhancements * plugin: Add record_accessor plugin helper https://github.com/fluent/fluentd/pull/1637 * log: Add format and time_format parameters to `` setting https://github.com/fluent/fluentd/pull/1644 ### Bug fixes * buf_file: Improve file handling to mitigate broken meta file https://github.com/fluent/fluentd/pull/1628 * in_syslog: Fix the description of resolve_hostname parameter https://github.com/fluent/fluentd/pull/1633 * process: Fix signal handling. Send signal to all workers https://github.com/fluent/fluentd/pull/1642 * output: Fix error message typo https://github.com/fluent/fluentd/pull/1643 ## Release v0.14.19 - 2017/07/12 ### New features / Enhancements * in_syslog: More characters are available in tag part of syslog format https://github.com/fluent/fluentd/pull/1610 * in_syslog: Add resolve_hostname parameter https://github.com/fluent/fluentd/pull/1616 * filter_grep: Support new configuration format by config_section https://github.com/fluent/fluentd/pull/1611 ### Bug fixes * output: Fix race condition of retry state in flush thread https://github.com/fluent/fluentd/pull/1623 * test: Fix typo in test_in_tail.rb https://github.com/fluent/fluentd/pull/1622 ## Release v0.14.18 - 2017/06/21 ### New features / Enhancements * parser: Add rfc5424 regex without priority https://github.com/fluent/fluentd/pull/1600 ### Bug fixes * in_tail: Fix timing issue that the excluded_path doesn't apply. https://github.com/fluent/fluentd/pull/1597 * config: Fix broken UTF-8 encoded configuration file handling https://github.com/fluent/fluentd/pull/1592 * out_forward: Don't stop heartbeat when error happen https://github.com/fluent/fluentd/pull/1602 * Fix command name typo in plugin template https://github.com/fluent/fluentd/pull/1603 ## Release v0.14.17 - 2017/05/29 ### New features / Enhancements * in_tail: Add ignore_repeated_permission_error https://github.com/fluent/fluentd/pull/1574 * server: Accept private key for TLS server without passphrase https://github.com/fluent/fluentd/pull/1575 * config: Validate workers option on standalone mode https://github.com/fluent/fluentd/pull/1577 ### Bug fixes * config: Mask all secret parameters in worker section https://github.com/fluent/fluentd/pull/1580 * out_forward: Fix ack handling https://github.com/fluent/fluentd/pull/1581 * plugin-config-format: Fix markdown format generator https://github.com/fluent/fluentd/pull/1585 ## Release v0.14.16 - 2017/05/13 ### New features / Enhancements * config: Allow null byte in double-quoted string https://github.com/fluent/fluentd/pull/1552 * parser: Support %iso8601 special case for time_format https://github.com/fluent/fluentd/pull/1562 ### Bug fixes * out_forward: Call proper method for each connection type https://github.com/fluent/fluentd/pull/1560 * in_monitor_agent: check variable buffer is a Buffer instance https://github.com/fluent/fluentd/pull/1556 * log: Add missing '<<' method to delegators https://github.com/fluent/fluentd/pull/1558 * command: uninitialized constant Fluent::Engine in fluent-binlog-reader https://github.com/fluent/fluentd/pull/1568 ## Release v0.14.15 - 2017/04/23 ### New features / Enhancements * Add `` directive https://github.com/fluent/fluentd/pull/1507 * in_tail: Do not warn that directories are unreadable in the in_tail plugin https://github.com/fluent/fluentd/pull/1540 * output: Add formatted_to_msgpack_binary? to Output plugin API https://github.com/fluent/fluentd/pull/1547 * windows: Allow the Windows Service name Fluentd runs as to be configurable https://github.com/fluent/fluentd/pull/1548 ### Bug fixes * in_http: Fix X-Forwarded-For header handling. Accept multiple headers https://github.com/fluent/fluentd/pull/1535 * Fix backward compatibility with Fluent::DetachProcess and Fluent::DetachMultiProcess https://github.com/fluent/fluentd/pull/1522 * fix typo https://github.com/fluent/fluentd/pull/1521 https://github.com/fluent/fluentd/pull/1523 https://github.com/fluent/fluentd/pull/1544 * test: Fix out_file test with timezone https://github.com/fluent/fluentd/pull/1546 * windows: Quote the file path to the Ruby bin directory when starting fluentd as a windows service https://github.com/fluent/fluentd/pull/1536 ## Release v0.14.14 - 2017/03/23 ### New features / Enhancements * in_http: Support 'application/msgpack` header https://github.com/fluent/fluentd/pull/1498 * in_udp: Add message_length_limit parameter for parameter name consistency with in_syslog https://github.com/fluent/fluentd/pull/1515 * in_monitor_agent: Start one HTTP server per worker on sequential port numbers https://github.com/fluent/fluentd/pull/1493 * in_tail: Skip the refresh of watching list on startup https://github.com/fluent/fluentd/pull/1487 * filter_parser: filter_parser: Add emit_invalid_record_to_error parameter https://github.com/fluent/fluentd/pull/1494 * parser_syslog: Support RFC5424 syslog format https://github.com/fluent/fluentd/pull/1492 * parser: Allow escape sequence in Apache access log https://github.com/fluent/fluentd/pull/1479 * config: Add actual value in the placeholder error message https://github.com/fluent/fluentd/pull/1497 * log: Add Fluent::Log#<< to support some SDKs https://github.com/fluent/fluentd/pull/1478 ### Bug fixes * Fix cleanup resource https://github.com/fluent/fluentd/pull/1483 * config: Set encoding forcefully to avoid UndefinedConversionError https://github.com/fluent/fluentd/pull/1477 * Fix Input and Output deadlock when buffer is full during startup https://github.com/fluent/fluentd/pull/1502 * config: Fix log_level handling in `` https://github.com/fluent/fluentd/pull/1501 * Fix typo in root agent error log https://github.com/fluent/fluentd/pull/1491 * storage: Fix a bug storage_create cannot accept hash as `conf` keyword argument https://github.com/fluent/fluentd/pull/1482 ## Release v0.14.13 - 2017/02/17 ### New features / Enhancements * in_tail: Add 'limit_recently_modified' to limit watch files. https://github.com/fluent/fluentd/pull/1474 * configuration: Improve 'flush_interval' handling for better message and backward compatibility https://github.com/fluent/fluentd/pull/1442 * command: Add 'fluent-plugin-generate' command https://github.com/fluent/fluentd/pull/1427 * output: Skip record when 'Output#format' returns nil https://github.com/fluent/fluentd/pull/1469 ### Bug fixes * output: Secondary calculation should consider 'retry_max_times' https://github.com/fluent/fluentd/pull/1452 * Fix regression of deprecated 'process' module https://github.com/fluent/fluentd/pull/1443 * Fix missing parser_regex require https://github.com/fluent/fluentd/issues/1458 https://github.com/fluent/fluentd/pull/1453 * Keep 'Fluent::BufferQueueLimitError' for existing plugins https://github.com/fluent/fluentd/pull/1456 * in_tail: Untracked files should be removed from watching list to avoid memory bloat https://github.com/fluent/fluentd/pull/1467 * in_tail: directories should be skipped when the ** pattern is used https://github.com/fluent/fluentd/pull/1464 * record_transformer: Revert "Use BasicObject for cleanroom" for `enable_ruby` regression. https://github.com/fluent/fluentd/pull/1461 * buf_file: handle "Too many open files" error to keep buffer and metadata pair https://github.com/fluent/fluentd/pull/1468 ## Release v0.14.12 - 2017/01/30 ### New features / Enhancements * Support multi process workers by `workers` option https://github.com/fluent/fluentd/pull/1386 * Support TLS transport security layer by server plugin helper, and forward input/output plugins https://github.com/fluent/fluentd/pull/1423 * Update internal log event handling to route log events to `@FLUENT_LOG` label if configured, suppress log events in startup/shutdown in default https://github.com/fluent/fluentd/pull/1405 * Rename buffer plugin chunk limit parameters for consistency https://github.com/fluent/fluentd/pull/1412 * Encode string values from configuration files in UTF8 https://github.com/fluent/fluentd/pull/1411 * Reorder plugin load paths to load rubygem plugins earlier than built-in plugins to overwrite them https://github.com/fluent/fluentd/pull/1410 * Clock API to control internal thread control https://github.com/fluent/fluentd/pull/1425 * Validate `config_param` options to restrict unexpected specifications https://github.com/fluent/fluentd/pull/1437 * formatter: Add `add_newline` option to get formatted lines without newlines https://github.com/fluent/fluentd/pull/1420 * in_forward: Add `ignore_network_errors_at_startup` option for automated cluster deployment https://github.com/fluent/fluentd/pull/1399 * in_forward: Close listening socket in #stop, not to accept new connection request in early stage of shutdown https://github.com/fluent/fluentd/pull/1401 * out_forward: Ensure to pack values in `str` type of msgpack https://github.com/fluent/fluentd/pull/1413 * in_tail: Add `emit_unmatched_lines` to capture lines which unmatch configured regular expressions https://github.com/fluent/fluentd/pull/1421 * in_tail: Add `open_on_every_update` to read lines from files opened in exclusive mode on Windows platform https://github.com/fluent/fluentd/pull/1409 * in_monitor_agent: Add `with_ivars` query parameter to get instance variables only for specified instance variables https://github.com/fluent/fluentd/pull/1393 * storage_local: Generate file store path using `usage`, with `root_dir` configuration https://github.com/fluent/fluentd/pull/1438 * Improve test stability https://github.com/fluent/fluentd/pull/1426 ### Bug fixes * Fix bug to ignore command line options: `--rpc-endpoint`, `--suppress-config-dump`, etc https://github.com/fluent/fluentd/pull/1398 * Fix bug to block infinitely in shutdown when buffer is full and `overflow_action` is `block` https://github.com/fluent/fluentd/pull/1396 * buf_file: Fix bug not to use `root_dir` even if configured correctly https://github.com/fluent/fluentd/pull/1417 * filter_record_transformer: Fix to use BasicObject for clean room https://github.com/fluent/fluentd/pull/1415 * filter_record_transformer: Fix bug that `remove_keys` doesn't work with `renew_time_key` https://github.com/fluent/fluentd/pull/1433 * in_monitor_agent: Fix bug to crash with NoMethodError for some output plugins https://github.com/fluent/fluentd/pull/1365 ## Release v0.14.11 - 2016/12/26 ### New features / Enhancements * Add "root_dir" parameter in `` directive to configure server root directory, used for buffer/storage paths https://github.com/fluent/fluentd/pull/1374 * Fix not to restart Fluentd processes when unrecoverable errors occur https://github.com/fluent/fluentd/pull/1359 * Show warnings in log when output flush operation takes longer time than threshold https://github.com/fluent/fluentd/pull/1370 * formatter_csv: Raise configuration error when no field names are specified https://github.com/fluent/fluentd/pull/1369 * in_syslog: Update implementation to use plugin helpers https://github.com/fluent/fluentd/pull/1382 * in_forward: Add a configuration parameter "source_address_key" https://github.com/fluent/fluentd/pull/1382 * in_monitor_agent: Add a parameter "include_retry" to get detail retry status https://github.com/fluent/fluentd/pull/1387 * Add Ruby 2.4 into supported ruby versions ### Bug fixes * Fix to set process name of supervisor process https://github.com/fluent/fluentd/pull/1380 * in_forward: Fix a bug not to handle "require_ack_response" correctly https://github.com/fluent/fluentd/pull/1389 ## Release v0.14.10 - 2016/12/14 ### New features / Enhancement * Add socket/server plugin helper to write TCP/UDP clients/servers as Fluentd plugin https://github.com/fluent/fluentd/pull/1312 https://github.com/fluent/fluentd/pull/1350 https://github.com/fluent/fluentd/pull/1356 https://github.com/fluent/fluentd/pull/1362 * Fix to raise errors when injected hostname is also specified as chunk key https://github.com/fluent/fluentd/pull/1357 * in_tail: Optimize to read lines from file https://github.com/fluent/fluentd/pull/1325 * in_monitor_agent: Add new parameter "include_config"(default: true) https://github.com/fluent/fluentd/pull/1317 * in_syslog: Add "priority_key" and "facility_key" options https://github.com/fluent/fluentd/pull/1351 * filter_record_transformer: Remove obsoleted syntax like "${message}" and not to dump records in logs https://github.com/fluent/fluentd/pull/1328 * Add an option "--time-as-integer" to fluent-cat command to send events from v0.14 fluent-cat to v0.12 fluentd https://github.com/fluent/fluentd/pull/1349 ### Bug fixes * Specify correct Oj options for newer versions (Oj 2.18.0 or later) https://github.com/fluent/fluentd/pull/1331 * TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified https://github.com/fluent/fluentd/pull/1319 * Parser plugins cannot use options for regular expressions https://github.com/fluent/fluentd/pull/1326 * Fix bugs not to raise errors to use logger in v0.12 plugins https://github.com/fluent/fluentd/pull/1344 https://github.com/fluent/fluentd/pull/1332 * Fix bug about shutting down Fluentd in Windows https://github.com/fluent/fluentd/pull/1367 * in_tail: Close files explicitly in tests https://github.com/fluent/fluentd/pull/1327 * out_forward: Fix bug not to convert buffer configurations into v0.14 parameters https://github.com/fluent/fluentd/pull/1337 * out_forward: Fix bug to raise error when "expire_dns_cache" is specified https://github.com/fluent/fluentd/pull/1346 * out_file: Fix bug to raise error about buffer chunking when it's configured as secondary https://github.com/fluent/fluentd/pull/1338 ## Release v0.14.9 - 2016/11/15 ### New features / Enhancement * filter_parser: Port fluent-plugin-parser into built-in plugin https://github.com/fluent/fluentd/pull/1191 * parser/formatter plugin helpers with default @type in plugin side https://github.com/fluent/fluentd/pull/1267 * parser: Reconstruct Parser related classes https://github.com/fluent/fluentd/pull/1286 * filter_record_transformer: Remove old behaviours https://github.com/fluent/fluentd/pull/1311 * Migrate some built-in plugins into v0.14 API https://github.com/fluent/fluentd/pull/1257 (out_file) https://github.com/fluent/fluentd/pull/1297 (out_exec, out_exec_filter) https://github.com/fluent/fluentd/pull/1306 (in_forward, out_forward) https://github.com/fluent/fluentd/pull/1308 (in_http) * test: Improve test drivers https://github.com/fluent/fluentd/pull/1302 https://github.com/fluent/fluentd/pull/1305 ### Bug fixes * log: Avoid name conflict between Fluent::Logger https://github.com/fluent/fluentd/pull/1274 * fluent-cat: Fix fluent-cat command to send sub-second precision time https://github.com/fluent/fluentd/pull/1277 * config: Fix a bug not to overwrite default value with nil https://github.com/fluent/fluentd/pull/1296 * output: Fix timezone for compat timesliced output plugins https://github.com/fluent/fluentd/pull/1307 * out_forward: fix not to raise error when out_forward is initialized as secondary https://github.com/fluent/fluentd/pull/1313 * output: Event router for secondary output https://github.com/fluent/fluentd/pull/1283 * test: fix to return the block value as expected by many rubyists https://github.com/fluent/fluentd/pull/1284 ## Release v0.14.8 - 2016/10/13 ### Bug fixes * Add msgpack_each to buffer chunks in compat-layer output plugins https://github.com/fluent/fluentd/pull/1273 ## Release v0.14.7 - 2016/10/07 ### New features / Enhancement * Support data compression in buffer plugins https://github.com/fluent/fluentd/pull/1172 * in_forward: support to transfer compressed data https://github.com/fluent/fluentd/pull/1179 * out_stdout: fix to show nanosecond resolution time https://github.com/fluent/fluentd/pull/1249 * Add option to rotate Fluentd daemon's log https://github.com/fluent/fluentd/pull/1235 * Add extract plugin helper, with symmetric time parameter support in parser/formatter and inject/extract https://github.com/fluent/fluentd/pull/1207 * Add a feature to parse/format numeric time (unix time [+ subsecond value]) https://github.com/fluent/fluentd/pull/1254 * Raise configuration errors for inconsistent ` @type single @id single @label @dummydata CONF conf_path = create_conf_file('worker_section1.conf', conf) assert Dir.exist?(@root_path) assert_log_matches( create_cmdline(conf_path, "-p", File.dirname(plugin_path)), "#0 fluentd worker is now running worker=0", "#1 fluentd worker is now running worker=1", '#1 adding source type="single"' ) end test "multiple values are set to RUBYOPT" do conf = < @type dummy tag dummy @type null CONF conf_path = create_conf_file('rubyopt_test.conf', conf) assert_log_matches( create_cmdline(conf_path), '#0 fluentd worker is now running worker=0', patterns_not_match: ['(LoadError)'], env: { 'RUBYOPT' => '-rtest-unit -rbundler/setup' }, ) end data( '-E' => '-Eutf-8', '-encoding' => '--encoding=utf-8', '-external-encoding' => '--external-encoding=utf-8', '-internal-encoding' => '--internal-encoding=utf-8', ) test "-E option is set to RUBYOPT" do |base_opt| conf = < @type dummy tag dummy @type null CONF conf_path = create_conf_file('rubyopt_test.conf', conf) opt = base_opt.dup opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT'] assert_log_matches( create_cmdline(conf_path), *opt.split(' '), patterns_not_match: ['-Eascii-8bit:ascii-8bit'], env: { 'RUBYOPT' => opt }, ) end test "without RUBYOPT" do saved_ruby_opt = ENV["RUBYOPT"] ENV["RUBYOPT"] = nil conf = < @type dummy tag dummy @type null CONF conf_path = create_conf_file('rubyopt_test.conf', conf) assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit') ensure ENV["RUBYOPT"] = saved_ruby_opt end test 'invalid values are set to RUBYOPT' do omit "hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT" if Fluent.windows? conf = < @type dummy tag dummy @type null CONF conf_path = create_conf_file('rubyopt_invalid_test.conf', conf) if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0') expected_phrase = 'ruby: invalid switch in RUBYOPT' else expected_phrase = 'Invalid option is passed to RUBYOPT' end assert_log_matches( create_cmdline(conf_path), expected_phrase, env: { 'RUBYOPT' => 'a' }, ) end # https://github.com/fluent/fluentd/issues/2915 test "ruby path contains spaces" do saved_ruby_opt = ENV["RUBYOPT"] ENV["RUBYOPT"] = nil conf = < @type dummy tag dummy @type null CONF ruby_path = ServerEngine.ruby_bin_path tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces") if Fluent.windows? tmp_ruby_path << ".bat" Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file| file.write "#{ruby_path} %*" end else FileUtils.ln_sf(ruby_path, tmp_ruby_path) end ENV["TEST_RUBY_PATH"] = tmp_ruby_path cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd") conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf) args = ["bundle", "exec", tmp_ruby_path, cmd_path, "-c", conf_path] assert_log_matches( args, 'spawn command to main:', '-Eascii-8bit:ascii-8bit' ) ensure ENV["RUBYOPT"] = saved_ruby_opt end test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do conf = < workers 2 @type dummy @id dummy tag dummy dummy {"message": "yay!"} @type null @id blackhole @type file path #{File.join(@root_path, "buf")} CONF conf_path = create_conf_file('worker_section2.conf', conf) assert_log_matches( create_cmdline(conf_path), "#0 fluentd worker is now running worker=0", "#1 fluentd worker is now running worker=1", '#1 adding match pattern="dummy" type="null"' ) end test 'success to start workers when configured plugins as a children of MultiOutput only for specific worker do not support multi worker configuration' do script = <<-EOC require 'fluent/plugin/output' module Fluent::Plugin class SingleOutput < Output Fluent::Plugin.register_output('single', self) def multi_workers_ready? false end def write(chunk) end end end EOC plugin_path = create_plugin_file('out_single.rb', script) conf = < workers 2 @type dummy @id dummy tag dummy dummy {"message": "yay!"} @type copy @type single @type single CONF conf_path = create_conf_file('worker_section3.conf', conf) assert_log_matches( create_cmdline(conf_path, "-p", File.dirname(plugin_path)), "#0 fluentd worker is now running worker=0", "#1 fluentd worker is now running worker=1", '#1 adding match pattern="dummy" type="copy"' ) end end sub_test_case 'config dump' do test 'all secret parameters in worker section is sealed' do script = <<-EOC require 'fluent/plugin/input' module Fluent::Plugin class FakeInput < Input Fluent::Plugin.register_input('fake', self) config_param :secret, :string, secret: true def multi_workers_ready?; true; end end end EOC plugin_path = create_plugin_file('in_fake.rb', script) conf = < workers 2 @type fake secret secret0 @type null @type fake secret secret1 @type null CONF conf_path = create_conf_file('secret_in_worker.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "-p", File.dirname(plugin_path)), "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"]) end end sub_test_case 'shared socket options' do test 'enable shared socket by default' do conf = "" conf_path = create_conf_file('empty.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path), patterns_not_match: ["shared socket for multiple workers is disabled"]) end test 'disable shared socket by command line option' do conf = "" conf_path = create_conf_file('empty.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"), "shared socket for multiple workers is disabled",) end test 'disable shared socket by system config' do conf = < disable_shared_socket CONF conf_path = create_conf_file('empty.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"), "shared socket for multiple workers is disabled",) end end # TODO: `patterns_not_match` can test only logs up to `pattern_list`, # so we need to fix some meaningless `patterns_not_match` conditions. sub_test_case 'log_level by command line option' do test 'info' do conf = "" conf_path = create_conf_file('empty.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path), "[info]", patterns_not_match: ["[debug]"]) end test 'debug' do conf = "" conf_path = create_conf_file('empty.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "-v"), "[debug]", patterns_not_match: ["[trace]"]) end data("Trace" => "-vv") data("Invalid low level should be treated as Trace level": "-vvv") test 'trace' do |option| conf = < @type sample tag test CONF conf_path = create_conf_file('sample.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, option), "[trace]",) end test 'warn' do omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows? conf = < @type sample tag test CONF conf_path = create_conf_file('sample.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "-q"), "[warn]", patterns_not_match: ["[info]"]) end data("Error" => "-qq") data("Fatal should be treated as Error level" => "-qqq") data("Invalid high level should be treated as Error level": "-qqqq") test 'error' do |option| # This test can run on Windows correctly, # since the process will stop automatically with an error. conf = < @type plugin_not_found tag test CONF conf_path = create_conf_file('plugin_not_found.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, option), "[error]", patterns_not_match: ["[warn]"]) end test 'system config one should not be overwritten when cmd line one is not specified' do conf = < log_level debug CONF conf_path = create_conf_file('debug.conf', conf) assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path), "[debug]") end end sub_test_case "inline_config" do test "can change log_level by --inline-config" do # Since we can't define multiple `` directives, this use-case is not recommended. # This is just for this test. inline_conf = '\nlog_level debug\n' conf_path = create_conf_file('test.conf', "") assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf), "[debug]") end end sub_test_case "plugin option" do test "should be the default value when not specifying" do conf_path = create_conf_file('test.conf', <<~CONF) @type monitor_agent CONF assert File.exist?(conf_path) cmdline = create_cmdline(conf_path) assert_log_matches(cmdline, "fluentd worker is now running") do response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json")) actual_conf = JSON.parse(response) assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"] end end data(short: "-p") data(long: "--plugin") test "can be added by specifying the option" do |option_name| conf_path = create_conf_file('test.conf', <<~CONF) @type monitor_agent CONF assert File.exist?(conf_path) cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir) assert_log_matches(cmdline, "fluentd worker is now running") do response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json")) actual_conf = JSON.parse(response) assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"] end end end test "--umask should be recognized as command line" do conf_path = create_conf_file("empty.conf", "") assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "--umask", "222"), "spawn command to main:", '"--umask", "222"', patterns_not_match: ["[error]"]) end sub_test_case "--with-source-only" do setup do omit "Not supported on Windows" if Fluent.windows? end test "should work without error" do conf_path = create_conf_file("empty.conf", "") assert File.exist?(conf_path) assert_log_matches(create_cmdline(conf_path, "--with-source-only"), "with-source-only: the emitted data will be stored in the buffer files under", "fluentd worker is now running", patterns_not_match: ["[error]"]) end end sub_test_case "zero_downtime_restart" do setup do omit "Not supported on Windows" if Fluent.windows? end def conf(udp_port, tcp_port, syslog_port) <<~CONF rpc_endpoint localhost:24444 @type monitor_agent @type udp tag test.udp port #{udp_port} @type none @type tcp tag test.tcp port #{tcp_port} @type none @type syslog tag test.syslog port #{syslog_port} @type record_transformer foo foo @type stdout CONF end def run_fluentd(config) conf_path = create_conf_file("test.conf", config) assert File.exist?(conf_path) cmdline = create_cmdline(conf_path) stdio_buf = "" execute_command(cmdline) do |pid, stdout| begin waiting(60) do while true readables, _, _ = IO.select([stdout], nil, nil, 1) next unless readables break if readables.first.eof? buf = eager_read(readables.first) stdio_buf << buf logs = buf.split("\n") yield logs break if buf.include? "finish test" end end ensure supervisor_pids = stdio_buf.scan(SUPERVISOR_PID_PATTERN) @supervisor_pid = supervisor_pids.last.first.to_i if supervisor_pids.size >= 2 stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid| @worker_pids << worker_pid.first.to_i end end end end def send_udp(port, count:, interval_sec:) count.times do |i| s = UDPSocket.new s.send("udp-#{i}", 0, "localhost", port) s.close sleep interval_sec end end def send_tcp(port, count:, interval_sec:) count.times do |i| s = TCPSocket.new("localhost", port) s.write("tcp-#{i}\n") s.close sleep interval_sec end end def send_syslog(port, count:, interval_sec:) count.times do |i| s = UDPSocket.new s.send("<6>Sep 10 00:00:00 localhost test: syslog-#{i}", 0, "localhost", port) s.close sleep interval_sec end end def send_end(port) s = TCPSocket.new("localhost", port) s.write("finish test\n") s.close end test "should restart with zero downtime (no data loss)" do udp_port, syslog_port = unused_port(2, protocol: :udp) tcp_port = unused_port(protocol: :tcp) client_threads = [] end_thread = nil records_by_type = { "udp" => [], "tcp" => [], "syslog" => [], } phase = "startup" run_fluentd(conf(udp_port, tcp_port, syslog_port)) do |logs| logs.each do |log| next unless /"message":"(udp|tcp|syslog)-(\d+)","foo":"foo"}/ =~ log type = $1 num = $2.to_i assert_true records_by_type.key?(type) records_by_type[type].append(num) end if phase == "startup" and logs.any? { |log| log.include?("fluentd worker is now running worker") } phase = "zero-downtime-restart" client_threads << Thread.new do send_udp(udp_port, count: 500, interval_sec: 0.01) end client_threads << Thread.new do send_tcp(tcp_port, count: 500, interval_sec: 0.01) end client_threads << Thread.new do send_syslog(syslog_port, count: 500, interval_sec: 0.01) end sleep 1 response = Net::HTTP.get(URI.parse("http://localhost:24444/api/processes.zeroDowntimeRestart")) assert_equal '{"ok":true}', response elsif phase == "zero-downtime-restart" and logs.any? { |log| log.include?("zero-downtime-restart: done all sequences") } phase = "flush" response = Net::HTTP.get(URI.parse("http://localhost:24444/api/plugins.flushBuffers")) assert_equal '{"ok":true}', response elsif phase == "flush" phase = "done" end_thread = Thread.new do client_threads.each(&:join) sleep 5 # make sure to flush each chunk (1s flush interval for 1chunk) send_end(tcp_port) end end end assert_equal( [(0..499).to_a, (0..499).to_a, (0..499).to_a], [ records_by_type["udp"].sort, records_by_type["tcp"].sort, records_by_type["syslog"].sort, ] ) ensure client_threads.each(&:kill) end_thread&.kill end end def create_config_include_dir_configuration(config_path, config_dir, yaml_format = false) if yaml_format conf = < config_include_dir #{config_dir} CONF end create_conf_file(config_path, conf) end sub_test_case "test additional configuration directory" do setup do FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d")) end test "disable additional configuration directory" do conf_path = create_config_include_dir_configuration("disabled_config_include_dir.conf", "") assert_log_matches(create_cmdline(conf_path), "[info]: configuration include directory is disabled") end test "inaccessible include directory error" do conf_path = create_config_include_dir_configuration("inaccessible_include.conf", "/nonexistent") assert_log_matches(create_cmdline(conf_path), "[info]: inaccessible include directory was specified") end data("include additional configuration with relative conf.d" => {"relative_path" => true}, "include additional configuration with full-path conf.d" => {"relative_path" => false}) test "additional configuration file (conf.d/child.conf) was loaded" do |option| conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" conf_path = create_config_include_dir_configuration("parent.conf", conf_dir) create_conf_file('conf.d/child.conf', "") assert_log_matches(create_cmdline(conf_path), "[info]: loading additional configuration file path=\"#{conf_dir}/child.conf\"") end end sub_test_case "test additional configuration directory (YAML)" do setup do FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d")) end test "disable additional configuration directory" do conf_path = create_config_include_dir_configuration("disabled_config_include_dir.yml", "", true) assert_log_matches(create_cmdline(conf_path), "[info]: configuration include directory is disabled") end test "inaccessible include directory error" do conf_path = create_config_include_dir_configuration("inaccessible_include.yml", "/nonexistent", true) assert_log_matches(create_cmdline(conf_path), "[info]: inaccessible include directory was specified") end data("include additional YAML configuration with relative conf.d" => {"relative_path" => true}, "include additional YAML configuration with full path conf.d" => {"relative_path" => false}) test "additional relative configuration file (conf.d/child.yml) was loaded" do |option| conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d" conf_path = create_config_include_dir_configuration("parent.yml", conf_dir, true) create_conf_file('conf.d/child.yml', "") assert_log_matches(create_cmdline(conf_path), "[info]: loading additional configuration file path=\"#{conf_dir}/child.yml\"") end end test "allow --disable-input-metrics option" do conf_path = create_conf_file('empty.conf', '') assert_log_matches( create_cmdline(conf_path, '--disable-input-metrics'), "#0 fluentd worker is now running worker=0" ) end sub_test_case "test suspicious harmful backed-up configuration" do data('suspicious .bak.conf' => 'dummy.bak.conf', 'suspicious .old.conf' => 'dummy.old.conf', 'suspicious .backup.conf' => 'dummy.backup.conf', 'suspicious .orig.conf' => 'dummy.orig.conf', 'suspicious .prev.conf' => 'dummy.prev.conf', 'suspicious .conf.conf' => 'dummy.conf.conf', 'suspicious .tmp.conf' => 'dummy.tmp.conf', 'suspicious .temp.conf' => 'dummy.temp.conf', 'suspicious .debug.conf' => 'dummy.debug.conf', 'suspicious .wip.conf' => 'dummy.wip.conf' ) test "warn suspicious backed-up file will be loaded" do |suspicious_conf| create_conf_file("dummy.conf", <<~EOF) @type forward EOF create_conf_file(suspicious_conf, <<~EOF) @type forward EOF working_dir = File.join(@tmp_dir, 'working') FileUtils.mkdir_p(working_dir) conf_path = create_conf_file("working/fluent.conf", <<~EOF) config_include_dir "" @include #{@tmp_dir}/*.conf EOF expected_warning_message = "[warn]: There is a possibility that '@include #{@tmp_dir}/*.conf' includes duplicated backed-up config file such as <#{suspicious_conf}>" assert_log_matches(create_cmdline(conf_path, '--dry-run'), expected_warning_message) end data('non suspicious bar.conf' => 'bar.conf') test "no warn message" do |non_suspicious_conf| create_conf_file("foo.conf", <<~EOF) @type forward EOF create_conf_file(non_suspicious_conf, <<~EOF) @type forward EOF working_dir = File.join(@tmp_dir, 'working') FileUtils.mkdir_p(working_dir) conf_path = create_conf_file("working/fluent.conf", <<~EOF) config_include_dir "" @include #{@tmp_dir}/*.conf EOF expected_warning_message = "[warn]: There is a possibility that '@include #{@tmp_dir}/*.conf' includes duplicated backed-up config file such as <#{non_suspicious_conf}>" assert_log_matches(create_cmdline(conf_path, '--dry-run'), "as dry run mode", patterns_not_match: [expected_warning_message]) end end sub_test_case "test suspicious harmful antivirus exclusion path" do test "warn pos_file path" do omit "skip recommendation warnings about exclusion path for antivirus" unless Fluent.windows? conf_path = create_conf_file("foo.conf", <<~EOF) @type tail tag t1 path #{@tmp_dir}/test.log pos_file #{@tmp_dir}/test.pos EOF expected_warning_message = "[warn]: Recommend adding #{@tmp_dir} to the exclusion path of your antivirus software on Windows" assert_log_matches(create_cmdline(conf_path, '--dry-run'), expected_warning_message) end test "warn storage path" do omit "skip recommendation warnings about exclusion path for antivirus" unless Fluent.windows? conf_path = create_conf_file("foo.conf", <<~EOF) @type sample tag t1 @type local path #{@tmp_dir}/test.pos EOF expected_warning_message = "[warn]: Recommend adding #{@tmp_dir} to the exclusion path of your antivirus software on Windows" assert_log_matches(create_cmdline(conf_path, '--dry-run'), expected_warning_message) end end sub_test_case "test suspicious timekey interval (1d) in out_file configuration" do data('without buffer' => {buffer: nil, buffer_arg: nil, message: :missing_buffer}, 'buffer' => {buffer: true, buffer_arg: '', message: :warning}, 'buffer time' => {buffer: true, buffer_arg: 'time', message: :warning}, 'buffer []' => {buffer: true, buffer_arg: '[]', message: nil}) test 'warn the default value of timekey (1d) is used as-is' do |data| prefix = "default timekey interval (1d) will be used" advice = "To change the output frequency, please modify the timekey value" missing_warning = "#{prefix} because of missing section. #{advice}" warning = "#{prefix}. #{advice}" conf_path = if data[:buffer] create_conf_file("warning.conf", <<~EOF) config_include_dir "" @type file tag test path #{@tmp_dir}/test.log @type file EOF else create_conf_file("warning.conf", <<~EOF) config_include_dir "" @type file tag test path #{@tmp_dir}/test.log EOF end case data[:message] when :missing_buffer assert_log_matches(create_cmdline(conf_path, '--dry-run'), missing_warning, patterns_not_match: [warning]) when :warning assert_log_matches(create_cmdline(conf_path, '--dry-run'), "warning", patterns_not_match: [missing_warning]) else assert_log_matches(create_cmdline(conf_path, '--dry-run'), patterns_not_match: [warning, missing_warning]) end end end end ================================================ FILE: test/command/test_plugin_config_formatter.rb ================================================ require_relative '../helper' require 'pathname' require 'fluent/command/plugin_config_formatter' require 'fluent/plugin/input' require 'fluent/plugin/output' require 'fluent/plugin/filter' require 'fluent/plugin/parser' require 'fluent/plugin/formatter' class TestFluentPluginConfigFormatter < Test::Unit::TestCase class FakeInput < ::Fluent::Plugin::Input ::Fluent::Plugin.register_input("fake", self) desc "path to something" config_param :path, :string end class FakeOutput < ::Fluent::Plugin::Output ::Fluent::Plugin.register_output("fake", self) desc "path to something" config_param :path, :string def process(tag, es) end end class FakeFilter < ::Fluent::Plugin::Filter ::Fluent::Plugin.register_filter("fake", self) desc "path to something" config_param :path, :string def filter(tag, time, record) end end class FakeParser < ::Fluent::Plugin::Parser ::Fluent::Plugin.register_parser("fake", self) desc "path to something" config_param :path, :string def parse(text) end end class FakeFormatter < ::Fluent::Plugin::Formatter ::Fluent::Plugin.register_formatter("fake", self) desc "path to something" config_param :path, :string def format(tag, time, record) end end class FakeStorage < ::Fluent::Plugin::Storage ::Fluent::Plugin.register_storage('fake', self) def get(key) end def fetch(key, defval) end def put(key, value) end def delete(key) end def update(key, &block) end end class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery ::Fluent::Plugin.register_sd('fake', self) desc "hostname" config_param :hostname, :string end class SimpleInput < ::Fluent::Plugin::Input ::Fluent::Plugin.register_input("simple", self) helpers :inject, :compat_parameters desc "path to something" config_param :path, :string end class ComplexOutput < ::Fluent::Plugin::Output ::Fluent::Plugin.register_output("complex", self) helpers :inject, :compat_parameters config_section :authentication, required: true, multi: false do desc "username" config_param :username, :string desc "password" config_param :password, :string, secret: true end config_section :parent do config_section :child do desc "names" config_param :names, :array desc "difficulty" config_param :difficulty, :enum, list: [:easy, :normal, :hard], default: :normal end end def process(tag, es) end end class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery ::Fluent::Plugin.register_sd('simple', self) desc "servers" config_param :servers, :array end sub_test_case "json" do data(input: [FakeInput, "input"], output: [FakeOutput, "output"], filter: [FakeFilter, "filter"], parser: [FakeParser, "parser"], formatter: [FakeFormatter, "formatter"]) test "dumped config should be valid JSON" do |(klass, type)| dumped_config = capture_stdout do FluentPluginConfigFormatter.new(["--format=json", type, "fake"]).call end expected = { path: { desc: "path to something", type: "string", required: true } } assert_equal(expected, JSON.parse(dumped_config, symbolize_names: true)[klass.name.to_sym]) end end sub_test_case "text" do test "input simple" do dumped_config = capture_stdout do FluentPluginConfigFormatter.new(["--format=txt", "input", "simple"]).call end expected = <: optional, single chunk_keys: array: ([]) @type: string: ("memory") timekey: time: (nil) timekey_wait: time: (600) timekey_use_utc: bool: (false) timekey_zone: string: ("#{Time.now.strftime('%z')}") flush_at_shutdown: bool: (nil) flush_mode: enum: (:default) flush_interval: time: (60) flush_thread_count: integer: (1) flush_thread_interval: float: (1.0) flush_thread_burst_interval: float: (1.0) delayed_commit_timeout: time: (60) overflow_action: enum: (:throw_exception) retry_forever: bool: (false) retry_timeout: time: (259200) retry_max_times: integer: (nil) retry_secondary_threshold: float: (0.8) retry_type: enum: (:exponential_backoff) retry_wait: time: (1) retry_exponential_backoff_base: float: (2) retry_max_interval: time: (nil) retry_randomize: bool: (true) : optional, single @type: string: (nil) : optional, single : optional, single : required, single username: string: (nil) password: string: (nil) : optional, multiple : optional, multiple names: array: (nil) difficulty: enum: (:normal) TEXT assert_equal(expected, dumped_config) end end sub_test_case "markdown" do test "input simple" do dumped_config = capture_stdout do FluentPluginConfigFormatter.new(["--format=markdown", "input", "simple"]).call end expected = < "sd", "normal" => "service_discovery") test "service_discovery simple" do |data| plugin_type = data dumped_config = capture_stdout do FluentPluginConfigFormatter.new(["--format=markdown", plugin_type, "simple"]).call end expected = < section (required) (single) #### username (string) (required) username #### password (string) (required) password ### \\ section (optional) (multiple) #### \\ section (optional) (multiple) ##### names (array) (required) names ##### difficulty (enum) (optional) difficulty Available values: easy, normal, hard Default value: `normal`. TEXT assert_equal(expected, dumped_config) end test "output complex (table)" do dumped_config = capture_stdout do FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call end expected = < section (required) (single) ### Configuration |parameter|type|description|default| |---|---|---|---| |username|string (required)|username|| |password|string (required)|password|| ### \\ section (optional) (multiple) #### \\ section (optional) (multiple) ### Configuration |parameter|type|description|default| |---|---|---|---| |names|array (required)|names|| |difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`| TEXT assert_equal(expected, dumped_config) end end sub_test_case "arguments" do data do hash = {} ["input", "output", "filter", "parser", "formatter", "storage", "service_discovery"].each do |type| ["txt", "json", "markdown"].each do |format| argv = ["--format=#{format}"] [ ["--verbose", "--compact"], ["--verbose"], ["--compact"] ].each do |options| hash["[#{type}] " + (argv + options).join(" ")] = argv + options + [type, "fake"] end end end hash end test "dump txt" do |argv| capture_stdout do assert_nothing_raised do FluentPluginConfigFormatter.new(argv).call end end end end end ================================================ FILE: test/command/test_plugin_generator.rb ================================================ require_relative '../helper' require 'pathname' require 'fluent/command/plugin_generator' class TestFluentPluginGenerator < Test::Unit::TestCase TEMP_DIR = "tmp/plugin_generator" setup do FileUtils.mkdir_p(TEMP_DIR) @pwd = Dir.pwd Dir.chdir(TEMP_DIR) end teardown do Dir.chdir(@pwd) FileUtils.rm_rf(TEMP_DIR) end def stub_git_process(target) stub(target).spawn do |cmd, arg1, arg2| assert_equal %w[git init .], [cmd, arg1, arg2] -1 end stub(Process).wait { |pid| assert_equal(pid, -1) } end sub_test_case "generate plugin" do data(input: ["input", "in"], output: ["output", "out"], filter: ["filter", "filter"], parser: ["parser", "parser"], formatter: ["formatter", "formatter"], storage: ["storage", "storage"]) test "generate plugin" do |(type, part)| generator = FluentPluginGenerator.new([type, "fake"]) stub_git_process(generator) capture_stdout do generator.call end plugin_base_dir = Pathname("fluent-plugin-fake") assert { plugin_base_dir.directory? } expected = [ "fluent-plugin-fake", "fluent-plugin-fake/Gemfile", "fluent-plugin-fake/LICENSE", "fluent-plugin-fake/README.md", "fluent-plugin-fake/Rakefile", "fluent-plugin-fake/fluent-plugin-fake.gemspec", "fluent-plugin-fake/lib", "fluent-plugin-fake/lib/fluent", "fluent-plugin-fake/lib/fluent/plugin", "fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb", "fluent-plugin-fake/test", "fluent-plugin-fake/test/helper.rb", "fluent-plugin-fake/test/plugin", "fluent-plugin-fake/test/plugin/test_#{part}_fake.rb", ] actual = plugin_base_dir.find.reject {|f| f.fnmatch("*/.git*") }.map(&:to_s).sort assert_equal(expected, actual) end test "no license" do generator = FluentPluginGenerator.new(["--no-license", "filter", "fake"]) stub_git_process(generator) capture_stdout do generator.call end assert { !Pathname("fluent-plugin-fake/LICENSE").exist? } assert { Pathname("fluent-plugin-fake/Gemfile").exist? } end test "unknown license" do out = capture_stdout do assert_raise(SystemExit) do FluentPluginGenerator.new(["--license=unknown", "filter", "fake"]).call end end assert { out.lines.include?("License: unknown\n") } end end sub_test_case "unify plugin name" do data("word" => ["fake", "fake"], "underscore" => ["rewrite_tag_filter", "rewrite_tag_filter"], "dash" => ["rewrite-tag-filter", "rewrite_tag_filter"]) test "plugin_name" do |(name, plugin_name)| generator = FluentPluginGenerator.new(["filter", name]) stub_git_process(generator) stub(Process).wait { |pid| assert_equal(pid, -1) } capture_stdout do generator.call end assert_equal(plugin_name, generator.__send__(:plugin_name)) end data("word" => ["fake", "fluent-plugin-fake"], "underscore" => ["rewrite_tag_filter", "fluent-plugin-rewrite-tag-filter"], "dash" => ["rewrite-tag-filter", "fluent-plugin-rewrite-tag-filter"]) test "gem_name" do |(name, gem_name)| generator = FluentPluginGenerator.new(["output", name]) stub_git_process(generator) stub(Process).wait { |pid| assert_equal(pid, -1) } capture_stdout do generator.call end assert_equal(gem_name, generator.__send__(:gem_name)) end end end ================================================ FILE: test/compat/test_calls_super.rb ================================================ require_relative '../helper' # these are Fluent::Compat::* in fact require 'fluent/input' require 'fluent/output' require 'fluent/filter' class CompatCallsSuperTest < Test::Unit::TestCase class DummyGoodInput < Fluent::Input def configure(conf); super; end def start; super; end def before_shutdown; super; end def shutdown; super; end end class DummyBadInput < Fluent::Input def configure(conf); super; end def start; end def before_shutdown; end def shutdown; end end class DummyGoodOutput < Fluent::Output def configure(conf); super; end def start; super; end def before_shutdown; super; end def shutdown; super; end end class DummyBadOutput < Fluent::Output def configure(conf); super; end def start; end def before_shutdown; end def shutdown; end end class DummyGoodFilter < Fluent::Filter def configure(conf); super; end def filter(tag, time, record); end def start; super; end def before_shutdown; super; end def shutdown; super; end end class DummyBadFilter < Fluent::Filter def configure(conf); super; end def filter(tag, time, record); end def start; end def before_shutdown; end def shutdown; end end setup do Fluent::Test.setup end sub_test_case 'old API plugin which calls super properly' do test 'Input#start, #before_shutdown and #shutdown calls all superclass methods properly' do i = DummyGoodInput.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? assert i.log.out.logs.empty? end test 'Output#start, #before_shutdown and #shutdown calls all superclass methods properly' do i = DummyGoodOutput.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? assert i.log.out.logs.empty? end test 'Filter#start, #before_shutdown and #shutdown calls all superclass methods properly' do i = DummyGoodFilter.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? assert i.log.out.logs.empty? end end sub_test_case 'old API plugin which does not call super' do test 'Input#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do i = DummyBadInput.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? logs = i.log.out.logs assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadInput") } } end test 'Output#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do i = DummyBadOutput.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? logs = i.log.out.logs assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadOutput") } } end test 'Filter#start, #before_shutdown and #shutdown calls superclass methods forcedly with logs' do i = DummyBadFilter.new i.configure(config_element()) assert i.configured? i.start assert i.started? i.before_shutdown assert i.before_shutdown? i.shutdown assert i.shutdown? logs = i.log.out.logs assert{ logs.any?{|l| l.include?("[warn]: super was not called in #start: called it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #before_shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } } assert{ logs.any?{|l| l.include?("[warn]: super was not called in #shutdown: calling it forcedly plugin=CompatCallsSuperTest::DummyBadFilter") } } end end end ================================================ FILE: test/compat/test_parser.rb ================================================ require_relative '../helper' require 'fluent/plugin/parser' class TextParserTest < ::Test::Unit::TestCase def setup Fluent::Test.setup end class MultiEventTestParser < ::Fluent::Parser include Fluent::Configurable def parse(text) 2.times { |i| record = {} record['message'] = text record['number'] = i yield Fluent::Engine.now, record } end end Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new }) def test_lookup_unknown_format assert_raise Fluent::NotFoundPluginError do Fluent::Plugin.new_parser('unknown') end end data('register_formatter' => 'known', 'register_template' => 'known_old') def test_lookup_known_parser(data) $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), '..', 'scripts')) assert_nothing_raised Fluent::ConfigError do Fluent::Plugin.new_parser(data) end $LOAD_PATH.shift end def test_parse_with_return parser = Fluent::TextParser.new parser.configure(config_element('test', '', 'format' => 'none')) _time, record = parser.parse('log message!') assert_equal({'message' => 'log message!'}, record) end def test_parse_with_block parser = Fluent::TextParser.new parser.configure(config_element('test', '', 'format' => 'none')) parser.parse('log message!') { |time, record| assert_equal({'message' => 'log message!'}, record) } end def test_multi_event_parser parser = Fluent::TextParser.new parser.configure(config_element('test', '', 'format' => 'multi_event_test')) i = 0 parser.parse('log message!') { |time, record| assert_equal('log message!', record['message']) assert_equal(i, record['number']) i += 1 } end def test_setting_estimate_current_event_value p1 = Fluent::TextParser.new assert_nil p1.estimate_current_event assert_nil p1.parser p1.configure(config_element('test', '', 'format' => 'none')) assert_equal true, p1.parser.estimate_current_event p2 = Fluent::TextParser.new assert_nil p2.estimate_current_event assert_nil p2.parser p2.estimate_current_event = false p2.configure(config_element('test', '', 'format' => 'none')) assert_equal false, p2.parser.estimate_current_event end data(ignorecase: Regexp::IGNORECASE, multiline: Regexp::MULTILINE, both: Regexp::IGNORECASE & Regexp::MULTILINE) def test_regexp_parser_config(options) source = "(?.*)" parser = Fluent::TextParser::RegexpParser.new(Regexp.new(source, options), { "dummy" => "dummy" }) regexp = parser.instance_variable_get(:@regexp) assert_equal(options, regexp.options) end end ================================================ FILE: test/config/assertions.rb ================================================ require 'test/unit/assertions' module Test::Unit::Assertions def assert_text_parsed_as(expected, actual) msg = parse_text(actual).inspect rescue 'failed' msg = "expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}" assert_block(msg) { v = parse_text(actual) if expected.is_a?(Float) v.is_a?(Float) && (v == obj || (v.nan? && obj.nan?) || (v - obj).abs < 0.000001) else v == expected end } end def assert_text_parsed_as_json(expected, actual) msg = parse_text(actual).inspect rescue 'failed' msg = "expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}" assert_block(msg) { v = JSON.parse(parse_text(actual)) v == expected } end def assert_parse_error(actual) msg = begin parse_text(actual).inspect rescue => e e.inspect end msg = "expected that #{actual.inspect} would cause a parse error but got #{msg}" assert_block(msg) { begin parse_text(actual) false rescue Fluent::ConfigParseError true end } end end ================================================ FILE: test/config/test_config_parser.rb ================================================ require_relative '../helper' require_relative "assertions" require "json" require "fluent/config/error" require "fluent/config/basic_parser" require "fluent/config/literal_parser" require "fluent/config/v1_parser" require 'fluent/config/parser' module Fluent::Config module V1TestHelper def root(*elements) if elements.first.is_a?(Fluent::Config::Element) attrs = {} else attrs = elements.shift || {} end Fluent::Config::Element.new('ROOT', '', attrs, elements) end def e(name, arg='', attrs={}, elements=[]) Fluent::Config::Element.new(name, arg, attrs, elements) end end class AllTypes include Fluent::Configurable config_param :param_string, :string config_param :param_enum, :enum, list: [:foo, :bar, :baz] config_param :param_integer, :integer config_param :param_float, :float config_param :param_size, :size config_param :param_bool, :bool config_param :param_time, :time config_param :param_hash, :hash config_param :param_array, :array config_param :param_regexp, :regexp end class TestV1Parser < ::Test::Unit::TestCase def read_config(path) path = File.expand_path(path) data = File.read(path) Fluent::Config::V1Parser.parse(data, File.basename(path), File.dirname(path)) end def parse_text(text) basepath = File.expand_path(File.dirname(__FILE__) + '/../../') Fluent::Config::V1Parser.parse(text, '(test)', basepath, nil) end include V1TestHelper extend V1TestHelper sub_test_case 'attribute parsing' do test "parses attributes" do assert_text_parsed_as(e('ROOT', '', {"k1"=>"v1", "k2"=>"v2"}), %[ k1 v1 k2 v2 ]) end test "allows attribute without value" do assert_text_parsed_as(e('ROOT', '', {"k1"=>"", "k2"=>"v2"}), %[ k1 k2 v2 ]) end test "parses attribute key always string" do assert_text_parsed_as(e('ROOT', '', {"1" => "1"}), "1 1") end data("_.%$!," => "_.%$!,", "/=~-~@\`:?" => "/=~-~@\`:?", "()*{}.[]" => "()*{}.[]") test "parses a value with symbols" do |v| assert_text_parsed_as(e('ROOT', '', {"k" => v}), "k #{v}") end test "ignores spacing around value" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a"}), " k1 a ") end test "allows spaces in value" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a b c"}), "k1 a b c") end test "parses value into empty string if only key exists" do # value parser parses empty string as true for bool type assert_text_parsed_as(e('ROOT', '', {"k1" => ""}), "k1\n") assert_text_parsed_as(e('ROOT', '', {"k1" => ""}), "k1") end sub_test_case 'non-quoted string' do test "remains text starting with '#'" do assert_text_parsed_as(e('ROOT', '', {"k1" => "#not_comment"}), " k1 #not_comment") end test "remains text just after '#'" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a#not_comment"}), " k1 a#not_comment") end test "remove text after ` #` (comment)" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a"}), " k1 a #comment") end test "does not require escaping backslash" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\\\\"}), " k1 \\\\") assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), " k1 \\") end test "remains backslash in front of a normal character" do assert_text_parsed_as(e('ROOT', '', {"k1" => '\['}), " k1 \\[") end test "does not accept escape characters" do assert_text_parsed_as(e('ROOT', '', {"k1" => '\n'}), " k1 \\n") end end sub_test_case 'double quoted string' do test "allows # in value" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a#comment"}), ' k1 "a#comment"') end test "rejects characters after double quoted string" do assert_parse_error(' k1 "a" 1') end test "requires escaping backslash" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), ' k1 "\\\\"') assert_parse_error(' k1 "\\"') end test "requires escaping double quote" do assert_text_parsed_as(e('ROOT', '', {"k1" => '"'}), ' k1 "\\""') assert_parse_error(' k1 """') assert_parse_error(' k1 ""\'') end test "removes backslash in front of a normal character" do assert_text_parsed_as(e('ROOT', '', {"k1" => '['}), ' k1 "\\["') end test "accepts escape characters" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\n"}), ' k1 "\\n"') end test "support multiline string" do assert_text_parsed_as(e('ROOT', '', {"k1" => %[line1 line2] }), %[k1 "line1 line2"] ) assert_text_parsed_as(e('ROOT', '', {"k1" => %[line1 line2] }), %[k1 "line1\\ line2"] ) assert_text_parsed_as(e('ROOT', '', {"k1" => %[line1 line2 line3] }), %[k1 "line1 line2 line3"] ) assert_text_parsed_as(e('ROOT', '', {"k1" => %[line1 line2 line3] }), %[k1 "line1 line2\\ line3"] ) end end sub_test_case 'single quoted string' do test "allows # in value" do assert_text_parsed_as(e('ROOT', '', {"k1" => "a#comment"}), " k1 'a#comment'") end test "rejects characters after single quoted string" do assert_parse_error(" k1 'a' 1") end test "requires escaping backslash" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), " k1 '\\\\'") assert_parse_error(" k1 '\\'") end test "requires escaping single quote" do assert_text_parsed_as(e('ROOT', '', {"k1" => "'"}), " k1 '\\''") assert_parse_error(" k1 '''") end test "remains backslash in front of a normal character" do assert_text_parsed_as(e('ROOT', '', {"k1" => '\\['}), " k1 '\\['") end test "does not accept escape characters" do assert_text_parsed_as(e('ROOT', '', {"k1" => "\\n"}), " k1 '\\n'") end end data( "in match" => %[ @k v ], "in source" => %[ @k v ], "in filter" => %[ @k v ], "in top-level" => ' @k v ' ) def test_rejects_at_prefix_in_the_parameter_name(data) assert_parse_error(data) end data( "in nested" => %[ @k v ] ) def test_not_reject_at_prefix_in_the_parameter_name(data) assert_nothing_raised { parse_text(data) } end end sub_test_case 'element parsing' do data( 'root' => [root, ""], "accepts empty element" => [root(e("test")), %[ ]], "accepts argument and attributes" => [root(e("test", 'var', {'key'=>"val"})), %[ key val ]], "accepts nested elements" => [root( e("test", 'var', {'key'=>'1'}, [ e('nested1'), e('nested2') ])), %[ key 1 ]], "accepts multiline json values" => [root(e("test", 'var', {'key'=>"[\"a\",\"b\",\"c\",\"d\"]"})), %[ key ["a", "b", "c", "d"] ]], "parses empty element argument to nil" => [root(e("test", '')), %[ ]], "ignores spacing around element argument" => [root(e("test", "a")), %[ ]], "accepts spacing inside element argument (for multiple tags)" => [root(e("test", "a.** b.**")), %[ ]]) def test_parse_element(data) expected, target = data assert_text_parsed_as(expected, target) end [ "**", "*.*", "1", "_.%$!", "/", "()*{}.[]", ].each do |arg| test "parses symbol element argument:#{arg}" do assert_text_parsed_as(root(e("test", arg)), %[ ]) end end data( "considers comments in element argument" => %[ ], "requires line_end after begin tag" => %[ ], "requires line_end after end tag" => %[ ]) def test_parse_error(data) assert_parse_error(data) end end sub_test_case "Embedded Ruby Code in section attributes" do setup do ENV["EMBEDDED_VAR"] = "embedded" ENV["NESTED_EMBEDDED_VAR"] = "nested-embedded" @hostname = Socket.gethostname end teardown do ENV["EMBEDDED_VAR"] = nil ENV["NESTED_EMBEDDED_VAR"] = nil end test "embedded Ruby code should be expanded" do assert_text_parsed_as(root( e("test", 'embedded', {'key'=>'1'}, [ e('nested1', 'nested-embedded'), e('nested2', "#{@hostname}") ])), <<-EOF key 1 EOF ) end end # port from test_config.rb sub_test_case '@include parsing' do TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}" TMP_DIR_WITH_SPACES = File.dirname(__FILE__) + "/tmp/folder with spaces/v1_config#{ENV['TEST_ENV_NUMBER']}" def write_config(path, data) FileUtils.mkdir_p(File.dirname(path)) File.open(path, "w") { |f| f.write data } end def prepare_config(tmp_dir) write_config "#{tmp_dir}/config_test_1.conf", %[ k1 root_config include dir/config_test_2.conf # @include #{tmp_dir}/config_test_4.conf include file://#{tmp_dir}/config_test_5.conf @include config.d/*.conf ] write_config "#{tmp_dir}/dir/config_test_2.conf", %[ k2 relative_path_include @include ../config_test_3.conf ] write_config "#{tmp_dir}/config_test_3.conf", %[ k3 relative_include_in_included_file ] write_config "#{tmp_dir}/config_test_4.conf", %[ k4 absolute_path_include ] write_config "#{tmp_dir}/config_test_5.conf", %[ k5 uri_include ] write_config "#{tmp_dir}/config.d/config_test_6.conf", %[ k6 wildcard_include_1 include normal_parameter ] write_config "#{tmp_dir}/config.d/config_test_7.conf", %[ k7 wildcard_include_2 ] write_config "#{tmp_dir}/config.d/config_test_8.conf", %[ @include ../dir/config_test_9.conf ] write_config "#{tmp_dir}/dir/config_test_9.conf", %[ k9 embedded nested nested_value include hoge ] write_config "#{tmp_dir}/config.d/00_config_test_8.conf", %[ k8 wildcard_include_3 include normal_parameter ] end data("TMP_DIR without spaces" => TMP_DIR, "TMP_DIR with spaces" => TMP_DIR_WITH_SPACES) test 'parses @include / include correctly' do |data| prepare_config(data) c = read_config("#{data}/config_test_1.conf") assert_equal('root_config', c['k1']) assert_equal('relative_path_include', c['k2']) assert_equal('relative_include_in_included_file', c['k3']) assert_equal('absolute_path_include', c['k4']) assert_equal('uri_include', c['k5']) assert_equal('wildcard_include_1', c['k6']) assert_equal('wildcard_include_2', c['k7']) assert_equal('wildcard_include_3', c['k8']) assert_equal([ 'k1', 'k2', 'k3', 'k4', 'k5', 'k8', # Because of the file name this comes first. 'k6', 'k7', ], c.keys) elem1 = c.elements.find { |e| e.name == 'elem1' } assert(elem1) assert_equal('name', elem1.arg) assert_equal('normal_parameter', elem1['include']) elem2 = c.elements.find { |e| e.name == 'elem2' } assert(elem2) assert_equal('name', elem2.arg) assert_equal('embedded', elem2['k9']) assert_not_include(elem2, 'include') elem3 = elem2.elements.find { |e| e.name == 'elem3' } assert(elem3) assert_equal('nested_value', elem3['nested']) assert_equal('hoge', elem3['include']) end # TODO: Add uri based include spec end sub_test_case '#to_s' do test 'parses dumpped configuration' do original = %q!a\\\n\r\f\b'"z! expected = %q!a\\\n\r\f\b'"z! conf = parse_text(%[k1 #{original}]) assert_equal(expected, conf['k1']) # escape check conf2 = parse_text(conf.to_s) # use dumpped configuration to check unescape assert_equal(expected, conf2.elements.first['k1']) end test 'all types' do conf = parse_text(%[ param_string "value" param_enum foo param_integer 999 param_float 55.55 param_size 4k param_bool true param_time 10m param_hash { "key1": "value1", "key2": 2 } param_array ["value1", "value2", 100] param_regexp /pattern/ ]) target = AllTypes.new.configure(conf) assert_equal(conf.to_s, target.config.to_s) expected = < param_string "value" param_enum foo param_integer 999 param_float 55.55 param_size 4k param_bool true param_time 10m param_hash {"key1":"value1","key2":2} param_array ["value1","value2",100] param_regexp /pattern/ DUMP assert_equal(expected, conf.to_s) end end end class TestV0Parser < ::Test::Unit::TestCase def parse_text(text) basepath = File.expand_path(File.dirname(__FILE__) + '/../../') Fluent::Config::Parser.parse(StringIO.new(text), '(test)', basepath) end sub_test_case "Fluent::Config::Element#to_s" do test 'all types' do conf = parse_text(%[ param_string value param_enum foo param_integer 999 param_float 55.55 param_size 4k param_bool true param_time 10m param_hash { "key1": "value1", "key2": 2 } param_array ["value1", "value2", 100] param_regexp /pattern/ ]) target = AllTypes.new.configure(conf) assert_equal(conf.to_s, target.config.to_s) expected = < param_string value param_enum foo param_integer 999 param_float 55.55 param_size 4k param_bool true param_time 10m param_hash { "key1": "value1", "key2": 2 } param_array ["value1", "value2", 100] param_regexp /pattern/ DUMP assert_equal(expected, conf.to_s) end end end end ================================================ FILE: test/config/test_configurable.rb ================================================ require_relative '../helper' require 'fluent/configurable' require 'fluent/config/element' require 'fluent/config/section' module ConfigurableSpec class Base1 include Fluent::Configurable config_param :node, :string, default: "node" config_param :flag1, :bool, default: false config_param :flag2, :bool, default: true config_param :name1, :string config_param :name2, :string config_param :name3, :string, default: "base1" config_param :name4, :string, default: "base1" config_param :opt1, :enum, list: [:foo, :bar, :baz] config_param :opt2, :enum, list: [:foo, :bar, :baz], default: :foo def get_all [@node, @flag1, @flag2, @name1, @name2, @name3, @name4] end end class Base1Safe < Base1 config_set_default :name1, "basex1" config_set_default :name2, "basex2" config_set_default :opt1, :baz end class Base1Nil < Base1 config_set_default :name1, nil config_set_default :name2, nil config_set_default :opt1, nil config_param :name5, :string, default: nil end class Base2 < Base1 config_set_default :name2, "base2" config_set_default :name4, "base2" config_set_default :opt1, :bar config_param :name5, :string config_param :name6, :string, default: "base2" config_param :opt3, :enum, list: [:a, :b] def get_all ary = super ary + [@name5, @name6] end end class Base3 < Base2 config_set_default :opt3, :a config_section :node do config_param :name, :string, default: "node" config_param :type, :string end config_section :branch, required: true, multi: true do config_argument :name, :string config_param :size, :integer, default: 10 config_section :leaf, required: false, multi: true do config_param :weight, :integer config_section :worm, param_name: 'worms', multi: true do config_param :type, :string, default: 'ladybird' end end end def get_all ary = super ary + [@branch] end end class Base4 < Base2 config_set_default :opt3, :a config_section :node, param_name: :nodes do config_argument :num, :integer config_param :name, :string, default: "node" config_param :type, :string, default: "b4" end config_section :description1, required: false, multi: false do config_argument :note, :string, default: "desc1" config_param :text, :string end config_section :description2, required: true, multi: false do config_argument :note, :string, default: "desc2" config_param :text, :string end config_section :description3, required: true, multi: true do config_argument :note, default: "desc3" do |val| "desc3: #{val}" end config_param :text, :string end def get_all ary = super ary + [@nodes, @description1, @description2, @description3] end end class Base4Safe < Base4 # config_section :node, param_name: :nodes do # config_argument :num, :integer # config_param :name, :string, default: "node" # config_param :type, :string, default: "b4" # end # config_section :description1, required: false, multi: false do # config_argument :note, :string, default: "desc1" # config_param :text, :string # end # config_section :description2, required: true, multi: false do # config_argument :note, :string, default: "desc2" # config_param :text, :string # end # config_section :description3, required: true, multi: true do # config_argument :note, default: "desc3" do |val| # "desc3: #{val}" # end # config_param :text, :string # end config_section :node do config_set_default :num, 0 end config_section :description1 do config_set_default :text, "teeeext" end config_section :description2 do config_set_default :text, nil end config_section :description3 do config_set_default :text, "yay" end end class Init0 include Fluent::Configurable config_section :sec1, init: true, multi: false do config_param :name, :string, default: 'sec1' end config_section :sec2, init: true, multi: true do config_param :name, :string, default: 'sec1' end end class Example0 include Fluent::Configurable config_param :stringvalue, :string config_param :boolvalue, :bool config_param :integervalue, :integer config_param :sizevalue, :size config_param :timevalue, :time config_param :floatvalue, :float config_param :hashvalue, :hash config_param :arrayvalue, :array end class ExampleWithAlias include Fluent::Configurable config_param :name, :string, alias: :fullname config_param :bool, :bool, alias: :flag config_section :detail, required: false, multi: false, alias: "information" do config_param :address, :string, default: "x" end def get_all [@name, @detail] end end class ExampleWithSecret include Fluent::Configurable config_param :normal_param, :string config_param :secret_param, :string, secret: true config_section :section do config_param :normal_param2, :string config_param :secret_param2, :string, secret: true end end class ExampleWithDefaultHashAndArray include Fluent::Configurable config_param :obj1, :hash, default: {} config_param :obj2, :array, default: [] end class ExampleWithSkipAccessor include Fluent::Configurable config_param :name, :string, default: 'example7', skip_accessor: true end class ExampleWithCustomSection include Fluent::Configurable config_param :name_param, :string config_section :normal_section do config_param :normal_section_param, :string end class CustomSection include Fluent::Configurable config_param :custom_section_param, :string end class AnotherElement include Fluent::Configurable end def configure(conf) super conf.elements.each do |e| next if e.name != 'custom_section' CustomSection.new.configure(e) end end end class ExampleWithIntFloat include Fluent::Configurable config_param :int1, :integer config_param :float1, :float end module Overwrite class Base include Fluent::Configurable config_param :name, :string, alias: :fullname config_param :bool, :bool, alias: :flag config_section :detail, required: false, multi: false, alias: "information" do config_param :address, :string, default: "x" end end class Required < Base config_section :detail, required: true do config_param :address, :string, default: "x" end end class Multi < Base config_section :detail, multi: true do config_param :address, :string, default: "x" end end class Alias < Base config_section :detail, alias: "information2" do config_param :address, :string, default: "x" end end class DefaultOptions < Base config_section :detail do config_param :address, :string, default: "x" end end class DetailAddressDefault < Base config_section :detail do config_param :address, :string, default: "y" end end class AddParam < Base config_section :detail do config_param :phone_no, :string end end class AddParamOverwriteAddress < Base config_section :detail do config_param :address, :string, default: "y" config_param :phone_no, :string end end end module Final # Show what is allowed in finalized sections # InheritsFinalized < Finalized < Base class Base include Fluent::Configurable config_section :appendix, multi: false, final: false do config_param :code, :string config_param :name, :string config_param :address, :string, default: "" end end class Finalized < Base # to non-finalized section # subclass can change type (code) # add default value (name) # change default value (address) # add field (age) config_section :appendix, final: true do config_param :code, :integer config_set_default :name, "y" config_set_default :address, "-" config_param :age, :integer, default: 10 end end class InheritsFinalized < Finalized # to finalized section # subclass can add default value (code) # change default value (age) # add field (phone_no) config_section :appendix do config_set_default :code, 2 config_set_default :age, 0 config_param :phone_no, :string end end # Show what is allowed/prohibited for finalized sections class FinalizedBase include Fluent::Configurable config_section :appendix, param_name: :apd, init: false, required: true, multi: false, alias: "options", final: true do config_param :name, :string end end class FinalizedBase2 include Fluent::Configurable config_section :appendix, param_name: :apd, init: false, required: false, multi: false, alias: "options", final: true do config_param :name, :string end end # subclass can change init with adding default values class OverwriteInit < FinalizedBase2 config_section :appendix, init: true do config_set_default :name, "moris" config_param :code, :integer, default: 0 end end # subclass cannot change type (name) class Subclass < FinalizedBase config_section :appendix do config_param :name, :integer end end # subclass cannot change param_name class OverwriteParamName < FinalizedBase config_section :appendix, param_name: :adx do end end # subclass cannot change final (section) class OverwriteFinal < FinalizedBase config_section :appendix, final: false do config_param :name, :integer end end # subclass cannot change required class OverwriteRequired < FinalizedBase config_section :appendix, required: false do end end # subclass cannot change multi class OverwriteMulti < FinalizedBase config_section :appendix, multi: true do end end # subclass cannot change alias class OverwriteAlias < FinalizedBase config_section :appendix, alias: "options2" do end end end module OverwriteDefaults class Owner include Fluent::Configurable config_set_default :key1, "V1" config_section :buffer do config_set_default :size_of_something, 1024 end end class SubOwner < Owner config_section :buffer do config_set_default :size_of_something, 2048 end end class NilOwner < Owner config_section :buffer do config_set_default :size_of_something, nil end end class FlatChild include Fluent::Configurable attr_accessor :owner config_param :key1, :string, default: "v1" end class BufferChild include Fluent::Configurable attr_accessor :owner configured_in :buffer config_param :size_of_something, :size, default: 128 end class BufferBase include Fluent::Configurable end class BufferSubclass < BufferBase attr_accessor :owner configured_in :buffer config_param :size_of_something, :size, default: 512 end class BufferSubSubclass < BufferSubclass end end class UnRecommended include Fluent::Configurable attr_accessor :log config_param :key1, :string, default: 'deprecated', deprecated: "key1 will be removed." config_param :key2, :string, default: 'obsoleted', obsoleted: "key2 has been removed." end end module Fluent::Config class TestConfigurable < ::Test::Unit::TestCase sub_test_case 'class defined without config_section' do sub_test_case '#initialize' do test 'create instance methods and default values by config_param and config_set_default' do obj1 = ConfigurableSpec::Base1.new assert_equal("node", obj1.node) assert_false(obj1.flag1) assert_true(obj1.flag2) assert_nil(obj1.name1) assert_nil(obj1.name2) assert_equal("base1", obj1.name3) assert_equal("base1", obj1.name4) assert_nil(obj1.opt1) assert_equal(:foo, obj1.opt2) end test 'create instance methods and default values overwritten by sub class definition' do obj2 = ConfigurableSpec::Base2.new assert_equal("node", obj2.node) assert_false(obj2.flag1) assert_true(obj2.flag2) assert_nil(obj2.name1) assert_equal("base2", obj2.name2) assert_equal("base1", obj2.name3) assert_equal("base2", obj2.name4) assert_nil(obj2.name5) assert_equal("base2", obj2.name6) assert_equal(:bar, obj2.opt1) assert_equal(:foo, obj2.opt2) end end sub_test_case '#configured_section_create' do test 'raises configuration error if required param exists but no configuration element is specified' do obj = ConfigurableSpec::Base1.new assert_raise(Fluent::ConfigError.new("'name1' parameter is required")) do obj.configured_section_create(nil) end end test 'creates root section with default values if name and config are specified with nil' do obj = ConfigurableSpec::Base1Safe.new root = obj.configured_section_create(nil) assert_equal "node", root.node assert_false root.flag1 assert_true root.flag2 assert_equal "basex1", root.name1 assert_equal "basex2", root.name2 assert_equal "base1", root.name3 assert_equal "base1", root.name4 assert_equal :baz, root.opt1 assert_equal :foo, root.opt2 end test 'creates root section with default values if name is nil and config is empty element' do obj = ConfigurableSpec::Base1Safe.new root = obj.configured_section_create(nil, config_element()) assert_equal "node", root.node assert_false root.flag1 assert_true root.flag2 assert_equal "basex1", root.name1 assert_equal "basex2", root.name2 assert_equal "base1", root.name3 assert_equal "base1", root.name4 assert_equal :baz, root.opt1 assert_equal :foo, root.opt2 end test 'creates root section with specified value if name is nil and configuration element is specified' do obj = ConfigurableSpec::Base1Safe.new root = obj.configured_section_create(nil, config_element('match', '', {'node' => "nodename", 'flag1' => 'true', 'name1' => 'fixed1', 'opt1' => 'foo'})) assert_equal "nodename", root.node assert_equal "fixed1", root.name1 assert_true root.flag1 assert_equal :foo, root.opt1 assert_true root.flag2 assert_equal "basex2", root.name2 assert_equal "base1", root.name3 assert_equal "base1", root.name4 assert_equal :foo, root.opt2 end end sub_test_case '#configure' do test 'returns configurable object itself' do b2 = ConfigurableSpec::Base2.new assert_instance_of(ConfigurableSpec::Base2, b2.configure(config_element("", "", {"name1" => "t1", "name5" => "t5", "opt3" => "a"}))) end test 'can accept frozen string' do b2 = ConfigurableSpec::Base2.new assert_instance_of(ConfigurableSpec::Base2, b2.configure(config_element("", "", {"name1" => "t1".freeze, "name5" => "t5", "opt3" => "a"}))) end test 'raise errors without any specifications for param without defaults' do b2 = ConfigurableSpec::Base2.new assert_raise(Fluent::ConfigError) { b2.configure(config_element("", "", {})) } assert_raise(Fluent::ConfigError) { b2.configure(config_element("", "", {"name1" => "t1"})) } assert_raise(Fluent::ConfigError) { b2.configure(config_element("", "", {"name5" => "t5"})) } assert_raise(Fluent::ConfigError) { b2.configure(config_element("", "", {"name1" => "t1", "name5" => "t5"})) } assert_nothing_raised { b2.configure(config_element("", "", {"name1" => "t1", "name5" => "t5", "opt3" => "a"})) } assert_equal(["node", false, true, "t1", "base2", "base1", "base2", "t5", "base2"], b2.get_all) assert_equal(:a, b2.opt3) end test 'can configure bool values' do b2a = ConfigurableSpec::Base2.new assert_nothing_raised { b2a.configure(config_element("", "", {"flag1" => "true", "flag2" => "yes", "name1" => "t1", "name5" => "t5", "opt3" => "a"})) } assert_true(b2a.flag1) assert_true(b2a.flag2) b2b = ConfigurableSpec::Base2.new assert_nothing_raised { b2b.configure(config_element("", "", {"flag1" => false, "flag2" => "no", "name1" => "t1", "name5" => "t5", "opt3" => "a"})) } assert_false(b2b.flag1) assert_false(b2b.flag2) end test 'overwrites values of defaults' do b2 = ConfigurableSpec::Base2.new b2.configure(config_element("", "", {"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"})) assert_equal("t1", b2.name1) assert_equal("t2", b2.name2) assert_equal("t3", b2.name3) assert_equal("t4", b2.name4) assert_equal("t5", b2.name5) assert_equal("base2", b2.name6) assert_equal(:foo, b2.opt1) assert_equal(:b, b2.opt3) assert_equal(["node", false, true, "t1", "t2", "t3", "t4", "t5", "base2"], b2.get_all) end test 'enum type rejects values which does not exist in list' do default = config_element("", "", {"name1" => "t1", "name2" => "t2", "name3" => "t3", "name4" => "t4", "name5" => "t5", "opt1" => "foo", "opt3" => "b"}) b2 = ConfigurableSpec::Base2.new assert_nothing_raised { b2.configure(default) } assert_raise(Fluent::ConfigError) { b2.configure(default.merge({"opt1" => "bazz"})) } assert_raise(Fluent::ConfigError) { b2.configure(default.merge({"opt2" => "fooooooo"})) } assert_raise(Fluent::ConfigError) { b2.configure(default.merge({"opt3" => "c"})) } end sub_test_case 'default values should be duplicated before touched in plugin code' do test 'default object should be dupped for cases configured twice' do x6a = ConfigurableSpec::ExampleWithDefaultHashAndArray.new assert_nothing_raised { x6a.configure(config_element("")) } assert_equal({}, x6a.obj1) assert_equal([], x6a.obj2) x6b = ConfigurableSpec::ExampleWithDefaultHashAndArray.new assert_nothing_raised { x6b.configure(config_element("")) } assert_equal({}, x6b.obj1) assert_equal([], x6b.obj2) assert { x6a.obj1.object_id != x6b.obj1.object_id } assert { x6a.obj2.object_id != x6b.obj2.object_id } x6c = ConfigurableSpec::ExampleWithDefaultHashAndArray.new assert_nothing_raised { x6c.configure(config_element("")) } assert_equal({}, x6c.obj1) assert_equal([], x6c.obj2) x6c.obj1['k'] = 'v' x6c.obj2 << 'v' assert_equal({'k' => 'v'}, x6c.obj1) assert_equal(['v'], x6c.obj2) assert_equal({}, x6a.obj1) assert_equal([], x6a.obj2) end end test 'strict value type' do default = config_element("", "", {"int1" => "1", "float1" => ""}) c = ConfigurableSpec::ExampleWithIntFloat.new assert_nothing_raised { c.configure(default) } assert_raise(Fluent::ConfigError) { c.configure(default, true) } end end test 'set nil for a parameter which has no default value' do obj = ConfigurableSpec::Base2.new conf = config_element("", "", {"name1" => nil, "name5" => "t5", "opt3" => "a"}) assert_raise(Fluent::ConfigError.new("'name1' parameter is required but nil is specified")) do obj.configure(conf) end end test 'set nil for a parameter which has non-nil default value' do obj = ConfigurableSpec::Base2.new conf = config_element("", "", {"name1" => "t1", "name3" => nil, "name5" => "t5", "opt3" => "a"}) assert_raise(Fluent::ConfigError.new("'name3' parameter is required but nil is specified")) do obj.configure(conf) end end test 'set nil for a parameter whose default value is nil' do obj = ConfigurableSpec::Base1Nil.new conf = config_element("", "", {"name5" => nil}) obj.configure(conf) assert_nil obj.name5 end test 'set nil for parameters whose default values are overwritten by nil' do obj = ConfigurableSpec::Base1Nil.new conf = config_element("", "", {"name1" => nil, "name2" => nil, "opt1" => nil}) obj.configure(conf) assert_nil obj.name1 assert_nil obj.name2 assert_nil obj.opt1 end test 'set :default' do obj = ConfigurableSpec::Base2.new conf = config_element("", "", {"name1" => "t1", "name3" => :default, "name5" => "t5", "opt3" => "a"}) obj.configure(conf) assert_equal "base1", obj.name3 end test 'set :default for a parameter which has no default value' do obj = ConfigurableSpec::Base2.new conf = config_element("", "", {"name1" => :default, "name5" => "t5", "opt3" => "a"}) assert_raise(Fluent::ConfigError.new("'name1' doesn't have default value")) do obj.configure(conf) end end test 'set :default for a parameter which has an overwritten default value' do obj = ConfigurableSpec::Base2.new conf = config_element("", "", {"name1" => "t1", "name3" => "t3", "name4" => :default, "name5" => "t5", "opt3" => "a"}) obj.configure(conf) assert_equal "base2", obj.name4 end end sub_test_case 'class defined with config_section' do sub_test_case '#initialize' do test 'create instance methods and default values as nil for params from config_section specified as non-multi' do b4 = ConfigurableSpec::Base4.new assert_nil(b4.description1) assert_nil(b4.description2) end test 'create instance methods and default values as [] for params from config_section specified as multi' do b4 = ConfigurableSpec::Base4.new assert_equal([], b4.description3) end test 'overwrite base class definition by config_section of sub class definition' do b3 = ConfigurableSpec::Base3.new assert_equal([], b3.node) end test 'create instance methods and default values by param_name' do b4 = ConfigurableSpec::Base4.new assert_equal([], b4.nodes) assert_equal("node", b4.node) end test 'create non-required and multi without any specifications' do b3 = ConfigurableSpec::Base3.new assert_false(b3.class.merged_configure_proxy.sections[:node].required?) assert_true(b3.class.merged_configure_proxy.sections[:node].multi?) end end sub_test_case '#configured_section_create' do test 'raises configuration error if required param exists but no configuration element is specified' do obj = ConfigurableSpec::Base4.new assert_raise(Fluent::ConfigError.new("'' section requires argument")) do obj.configured_section_create(:node) end assert_raise(Fluent::ConfigError.new("'text' parameter is required")) do obj.configured_section_create(:description1) end end test 'creates any defined section with default values if name is nil and config is not specified' do obj = ConfigurableSpec::Base4Safe.new node = obj.configured_section_create(:node) assert_equal 0, node.num assert_equal "node", node.name assert_equal "b4", node.type desc1 = obj.configured_section_create(:description1) assert_equal "desc1", desc1.note assert_equal "teeeext", desc1.text end test 'creates any defined section with default values if name is nil and config is empty element' do obj = ConfigurableSpec::Base4Safe.new node = obj.configured_section_create(:node, config_element()) assert_equal 0, node.num assert_equal "node", node.name assert_equal "b4", node.type desc1 = obj.configured_section_create(:description1, config_element()) assert_equal "desc1", desc1.note assert_equal "teeeext", desc1.text end test 'creates any defined section with specified value if name is nil and configuration element is specified' do obj = ConfigurableSpec::Base4Safe.new node = obj.configured_section_create(:node, config_element('node', '1', {'name' => 'node1', 'type' => 'b1'})) assert_equal 1, node.num assert_equal "node1", node.name assert_equal "b1", node.type desc1 = obj.configured_section_create(:description1, config_element('description1', 'desc one', {'text' => 't'})) assert_equal "desc one", desc1.note assert_equal "t", desc1.text end test 'creates a defined section instance even if it is defined as multi:true' do obj = ConfigurableSpec::Base4Safe.new desc3 = obj.configured_section_create(:description3) assert_equal "desc3", desc3.note assert_equal "yay", desc3.text desc3 = obj.configured_section_create(:description3, config_element('description3', 'foo')) assert_equal "desc3: foo", desc3.note assert_equal "yay", desc3.text end end sub_test_case '#configure' do BASE_ATTRS = { "name1" => "1", "name2" => "2", "name3" => "3", "name4" => "4", "name5" => "5", "name6" => "6", } test 'checks required subsections' do b3 = ConfigurableSpec::Base3.new # branch sections required assert_raise(Fluent::ConfigError) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [])) } # branch argument required msg = "'' section requires argument, in section branch" #expect{ b3.configure(e('ROOT', '', BASE_ATTRS, [e('branch', '')])) }.to raise_error(Fluent::ConfigError, msg) assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [config_element('branch', '')])) } # leaf is not required assert_nothing_raised { b3.configure(config_element('ROOT', '', BASE_ATTRS, [config_element('branch', 'branch_name')])) } # leaf weight required msg = "'weight' parameter is required, in section branch > leaf" branch1 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '10', {"weight" => 1})]) assert_nothing_raised { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch1])) } branch2 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '20')]) assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch1, branch2])) } branch3 = config_element('branch', 'branch_name', {size: 1}, [config_element('leaf', '10', {"weight" => 3}), config_element('leaf', '20')]) assert_raise(Fluent::ConfigError.new(msg)) { b3.configure(config_element('ROOT', '', BASE_ATTRS, [branch3])) } ### worm not required b4 = ConfigurableSpec::Base4.new d1 = config_element('description1', '', {"text" => "d1"}) d2 = config_element('description2', '', {"text" => "d2"}) d3 = config_element('description3', '', {"text" => "d3"}) assert_nothing_raised { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup])) } # description1 cannot be specified 2 or more msg = "'' section cannot be written twice or more" assert_raise(Fluent::ConfigError.new(msg)) { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d1.dup, d3.dup])) } # description2 cannot be specified 2 or more msg = "'' section cannot be written twice or more" assert_raise(Fluent::ConfigError.new(msg)) { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup, d2.dup])) } # description3 can be specified 2 or more assert_nothing_raised { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup, d3.dup])) } end test 'constructs configuration object tree for Base3' do conf = config_element( 'ROOT', '', BASE_ATTRS, [ config_element('node', '', {"type" => "1"}), config_element('node', '', {"name" => "node2","type" => "2"}), config_element('branch', 'b1.*', {}, []), config_element('branch', 'b2.*', {"size" => 5}, [ config_element('leaf', 'THIS IS IGNORED', {"weight" => 55}, []), config_element('leaf', 'THIS IS IGNORED', {"weight" => 50}, [ config_element('worm', '', {}) ]), config_element('leaf', 'THIS IS IGNORED', {"weight" => 50}, [ config_element('worm', '', {"type" => "w1"}), config_element('worm', '', {"type" => "w2"}) ]), ] ), config_element('branch', 'b3.*', {"size" => "503"}, [ config_element('leaf', 'THIS IS IGNORED', {"weight" => 55}, []), ] ) ], ) b3 = ConfigurableSpec::Base3.new.configure(conf) assert_not_equal("node", b3.node) # overwritten assert_equal("1", b3.name1) assert_equal("2", b3.name2) assert_equal("3", b3.name3) assert_equal("4", b3.name4) assert_equal("5", b3.name5) assert_equal("6", b3.name6) assert_instance_of(Array, b3.node) assert_equal(2, b3.node.size) assert_equal("node", b3.node[0].name) assert_equal("1", b3.node[0].type) assert_equal(b3.node[0].type, b3.node[0][:type]) assert_equal("node2", b3.node[1].name) assert_equal("2", b3.node[1].type) assert_equal(b3.node[1].type, b3.node[1][:type]) assert_instance_of(Array, b3.branch) assert_equal(3, b3.branch.size) assert_equal('b1.*', b3.branch[0].name) assert_equal(10, b3.branch[0].size) assert_equal([], b3.branch[0].leaf) assert_equal('b2.*', b3.branch[1].name) assert_equal(5, b3.branch[1].size) assert_equal(3, b3.branch[1].leaf.size) assert_equal(b3.branch[1].leaf, b3.branch[1][:leaf]) assert_equal(55, b3.branch[1].leaf[0].weight) assert_equal(0, b3.branch[1].leaf[0].worms.size) assert_equal(50, b3.branch[1].leaf[1].weight) assert_equal(1, b3.branch[1].leaf[1].worms.size) assert_equal("ladybird", b3.branch[1].leaf[1].worms[0].type) assert_equal(50, b3.branch[1].leaf[2].weight) assert_equal(2, b3.branch[1].leaf[2].worms.size) assert_equal("w1", b3.branch[1].leaf[2].worms[0].type) assert_equal("w2", b3.branch[1].leaf[2].worms[1].type) assert_equal('b3.*', b3.branch[2].name) assert_equal(503, b3.branch[2].size) assert_equal(1, b3.branch[2].leaf.size) assert_equal(55, b3.branch[2].leaf[0].weight) end test 'constructs configuration object tree for Base4' do conf = config_element( 'ROOT', '', BASE_ATTRS, [ config_element('node', '1', {"type" => "1"}), config_element('node', '2', {"name" => "node2"}), config_element('description3', '', {"text" => "dddd3-1"}), config_element('description2', 'd-2', {"text" => "dddd2"}), config_element('description1', '', {"text" => "dddd1"}), config_element('description3', 'd-3', {"text" => "dddd3-2"}), config_element('description3', 'd-3a', {"text" => "dddd3-3"}), config_element('node', '4', {"type" => "four"}), ], ) b4 = ConfigurableSpec::Base4.new.configure(conf) assert_equal("node", b4.node) assert_equal("1", b4.name1) assert_equal("2", b4.name2) assert_equal("3", b4.name3) assert_equal("4", b4.name4) assert_equal("5", b4.name5) assert_equal("6", b4.name6) assert_instance_of(Array, b4.nodes) assert_equal(3, b4.nodes.size) assert_equal(1, b4.nodes[0].num) assert_equal("node", b4.nodes[0].name) assert_equal("1", b4.nodes[0].type) assert_equal(2, b4.nodes[1].num) assert_equal("node2", b4.nodes[1].name) assert_equal("b4", b4.nodes[1].type) assert_equal(4, b4.nodes[2].num) assert_equal("node", b4.nodes[2].name) assert_equal("four", b4.nodes[2].type) # config_element('description3', '', {"text" => "dddd3-1"}), # config_element('description3', 'd-3', {"text" => "dddd3-2"}), # config_element('description3', 'd-3a', {"text" => "dddd3-3"}), # NoMethodError: undefined method `class' for :Fluent::Config::Section occurred. Should we add class method to Section? #assert_equal('Fluent::Config::Section', b4.description1.class.name) assert_equal("desc1", b4.description1.note) assert_equal("dddd1", b4.description1.text) # same with assert_equal('Fluent::Config::Section', b4.description1) #assert_equal('Fluent::Config::Section', b4.description2) assert_equal("d-2", b4.description2.note) assert_equal("dddd2", b4.description2.text) assert_instance_of(Array, b4.description3) assert_equal(3, b4.description3.size) assert_equal("desc3", b4.description3[0].note) assert_equal("dddd3-1", b4.description3[0].text) assert_equal('desc3: d-3', b4.description3[1].note) assert_equal('dddd3-2', b4.description3[1].text) assert_equal('desc3: d-3a', b4.description3[2].note) assert_equal('dddd3-3', b4.description3[2].text) end test 'checks missing of specifications' do conf0 = config_element('ROOT', '', {}, []) ex01 = ConfigurableSpec::Example0.new assert_raise(Fluent::ConfigError) { ex01.configure(conf0) } complete = config_element('ROOT', '', { "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10", "sizevalue" => "10m", "timevalue" => "100s", "floatvalue" => "1.001", "hashvalue" => '{"foo":1, "bar":2}', "arrayvalue" => '[1,"ichi"]', }) checker = lambda { |conf| ConfigurableSpec::Example0.new.configure(conf) } assert_nothing_raised { checker.call(complete) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("stringvalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("boolvalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("integervalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("sizevalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("timevalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("floatvalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("hashvalue"); checker.call(c) } assert_raise(Fluent::ConfigError) { c = complete.dup; c.delete("arrayvalue"); checker.call(c) } end test 'generates section with default values for init:true sections' do conf = config_element('ROOT', '', {}, []) init0 = ConfigurableSpec::Init0.new assert_nothing_raised { init0.configure(conf) } assert init0.sec1 assert_equal "sec1", init0.sec1.name assert_equal 1, init0.sec2.size assert_equal "sec1", init0.sec2.first.name end test 'accepts configuration values as string representation' do conf = config_element('ROOT', '', { "stringvalue" => "s1", "boolvalue" => "yes", "integervalue" => "10", "sizevalue" => "10m", "timevalue" => "10m", "floatvalue" => "1.001", "hashvalue" => '{"foo":1, "bar":2}', "arrayvalue" => '[1,"ichi"]', }) ex = ConfigurableSpec::Example0.new.configure(conf) assert_equal("s1", ex.stringvalue) assert_true(ex.boolvalue) assert_equal(10, ex.integervalue) assert_equal(10 * 1024 * 1024, ex.sizevalue) assert_equal(10 * 60, ex.timevalue) assert_equal(1.001, ex.floatvalue) assert_equal({"foo" => 1, "bar" => 2}, ex.hashvalue) assert_equal([1, "ichi"], ex.arrayvalue) end test 'accepts configuration values as ruby value representation (especially for DSL)' do conf = config_element('ROOT', '', { "stringvalue" => "s1", "boolvalue" => true, "integervalue" => 10, "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001, "hashvalue" => {"foo" => 1, "bar" => 2}, "arrayvalue" => [1,"ichi"], }) ex = ConfigurableSpec::Example0.new.configure(conf) assert_equal("s1", ex.stringvalue) assert_true(ex.boolvalue) assert_equal(10, ex.integervalue) assert_equal(10 * 1024 * 1024, ex.sizevalue) assert_equal(10 * 60, ex.timevalue) assert_equal(1.001, ex.floatvalue) assert_equal({"foo" => 1, "bar" => 2}, ex.hashvalue) assert_equal([1, "ichi"], ex.arrayvalue) end test 'gets both of true(yes) and false(no) for bool value parameter' do conf = config_element('ROOT', '', { "stringvalue" => "s1", "integervalue" => 10, "sizevalue" => 10 * 1024 * 1024, "timevalue" => 10 * 60, "floatvalue" => 1.001, "hashvalue" => {"foo" => 1, "bar" => 2}, "arrayvalue" => [1,"ichi"], }) ex0 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "true"})) assert_true(ex0.boolvalue) ex1 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "yes"})) assert_true(ex1.boolvalue) ex2 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => true})) assert_true(ex2.boolvalue) ex3 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "false"})) assert_false(ex3.boolvalue) ex4 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => "no"})) assert_false(ex4.boolvalue) ex5 = ConfigurableSpec::Example0.new.configure(conf.merge({"boolvalue" => false})) assert_false(ex5.boolvalue) end end sub_test_case '.config_section' do CONF1 = config_element('ROOT', '', { 'name' => 'tagomoris', 'bool' => true, }) CONF2 = config_element('ROOT', '', { 'name' => 'tagomoris', 'bool' => true, }, [config_element('detail', '', { 'phone_no' => "+81-00-0000-0000" }, [])]) CONF3 = config_element('ROOT', '', { 'name' => 'tagomoris', 'bool' => true, }, [config_element('detail', '', { 'address' => "Chiyoda Tokyo Japan" }, [])]) CONF4 = config_element('ROOT', '', { 'name' => 'tagomoris', 'bool' => true, }, [ config_element('detail', '', { 'address' => "Chiyoda Tokyo Japan", 'phone_no' => '+81-00-0000-0000' }, []) ]) data(conf1: CONF1, conf2: CONF2, conf3: CONF3, conf4: CONF4,) test 'base class' do |data| assert_nothing_raised { ConfigurableSpec::Overwrite::Base.new.configure(data) } end test 'subclass cannot overwrite required' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: required")) do ConfigurableSpec::Overwrite::Required.new.configure(CONF1) end end test 'subclass cannot overwrite multi' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: multi")) do ConfigurableSpec::Overwrite::Multi.new.configure(CONF1) end end test 'subclass cannot overwrite alias' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: alias")) do ConfigurableSpec::Overwrite::Alias.new.configure(CONF1) end end test 'subclass uses superclass default options' do base = ConfigurableSpec::Overwrite::Base.new.configure(CONF2) sub = ConfigurableSpec::Overwrite::DefaultOptions.new.configure(CONF2) detail_base = base.class.merged_configure_proxy.sections[:detail] detail_sub = sub.class.merged_configure_proxy.sections[:detail] detail_base_attributes = { required: detail_base.required, multi: detail_base.multi, alias: detail_base.alias, } detail_sub_attributes = { required: detail_sub.required, multi: detail_sub.multi, alias: detail_sub.alias, } assert_equal(detail_base_attributes, detail_sub_attributes) end test 'subclass can overwrite detail.address' do base = ConfigurableSpec::Overwrite::Base.new.configure(CONF2) target = ConfigurableSpec::Overwrite::DetailAddressDefault.new.configure(CONF2) expected_addresses = ["x", "y"] actual_addresses = [base.detail.address, target.detail.address] assert_equal(expected_addresses, actual_addresses) end test 'subclass can add param' do assert_raise(Fluent::ConfigError.new("'phone_no' parameter is required, in section detail")) do ConfigurableSpec::Overwrite::AddParam.new.configure(CONF3) end target = ConfigurableSpec::Overwrite::AddParam.new.configure(CONF4) expected = { address: "Chiyoda Tokyo Japan", phone_no: "+81-00-0000-0000" } actual = { address: target.detail.address, phone_no: target.detail.phone_no } assert_equal(expected, actual) end test 'subclass can add param with overwriting address' do assert_raise(Fluent::ConfigError.new("'phone_no' parameter is required, in section detail")) do ConfigurableSpec::Overwrite::AddParamOverwriteAddress.new.configure(CONF3) end target = ConfigurableSpec::Overwrite::AddParamOverwriteAddress.new.configure(CONF4) expected = { address: "Chiyoda Tokyo Japan", phone_no: "+81-00-0000-0000" } actual = { address: target.detail.address, phone_no: target.detail.phone_no } assert_equal(expected, actual) end sub_test_case 'final' do test 'base class has designed params and default values' do b = ConfigurableSpec::Final::Base.new appendix_conf = config_element('appendix', '', {"code" => "b", "name" => "base"}) b.configure(config_element('ROOT', '', {}, [appendix_conf])) assert_equal "b", b.appendix.code assert_equal "base", b.appendix.name assert_equal "", b.appendix.address end test 'subclass can change type, add default value, change default value of parameters, and add parameters to non-finalized section' do f = ConfigurableSpec::Final::Finalized.new appendix_conf = config_element('appendix', '', {"code" => 1}) f.configure(config_element('ROOT', '', {}, [appendix_conf])) assert_equal 1, f.appendix.code assert_equal 'y', f.appendix.name assert_equal "-", f.appendix.address assert_equal 10, f.appendix.age end test 'subclass can add default value, change default value of parameters, and add parameters to finalized section' do i = ConfigurableSpec::Final::InheritsFinalized.new appendix_conf = config_element('appendix', '', {"phone_no" => "00-0000-0000"}) i.configure(config_element('ROOT', '', {}, [appendix_conf])) assert_equal 2, i.appendix.code assert_equal 0, i.appendix.age assert_equal "00-0000-0000", i.appendix.phone_no end test 'finalized base class works as designed' do b = ConfigurableSpec::Final::FinalizedBase.new appendix_conf = config_element('options', '', {"name" => "moris"}) assert_nothing_raised do b.configure(config_element('ROOT', '', {}, [appendix_conf])) end assert b.apd assert_equal "moris", b.apd.name end test 'subclass can change init' do n = ConfigurableSpec::Final::OverwriteInit.new assert_nothing_raised do n.configure(config_element('ROOT', '')) end assert n.apd assert_equal "moris", n.apd.name assert_equal 0, n.apd.code end test 'subclass cannot change parameter types in finalized sections' do s = ConfigurableSpec::Final::Subclass.new appendix_conf = config_element('options', '', {"name" => "1"}) assert_nothing_raised do s.configure(config_element('ROOT', '', {}, [appendix_conf])) end assert_equal "1", s.apd.name end test 'subclass cannot change param_name of finalized section' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: param_name")) do ConfigurableSpec::Final::OverwriteParamName.new end end test 'subclass cannot change final of finalized section' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite finalized base class's config_section")) do ConfigurableSpec::Final::OverwriteFinal.new end end test 'subclass cannot change required of finalized section' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: required")) do ConfigurableSpec::Final::OverwriteRequired.new end end test 'subclass cannot change multi of finalized section' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: multi")) do ConfigurableSpec::Final::OverwriteMulti.new end end test 'subclass cannot change alias of finalized section' do assert_raise(Fluent::ConfigError.new("BUG: subclass cannot overwrite base class's config_section: alias")) do ConfigurableSpec::Final::OverwriteAlias.new end end end end end sub_test_case 'class defined with config_param/config_section having :alias' do sub_test_case '#initialize' do test 'does not create methods for alias' do ex1 = ConfigurableSpec::ExampleWithAlias.new assert_nothing_raised { ex1.name } assert_raise(NoMethodError) { ex1.fullname } assert_nothing_raised { ex1.bool } assert_raise(NoMethodError) { ex1.flag } assert_nothing_raised { ex1.detail } assert_raise(NoMethodError) { ex1.information} end end sub_test_case '#configure' do test 'provides accessible data for alias attribute keys' do ex1 = ConfigurableSpec::ExampleWithAlias.new conf = config_element('ROOT', '', { "fullname" => "foo bar", "bool" => false }, [config_element('information', '', {"address" => "Mountain View 0"})]) ex1.configure(conf) assert_equal("foo bar", ex1.name) assert_not_nil(ex1.bool) assert_false(ex1.bool) assert_not_nil(ex1.detail) assert_equal("Mountain View 0", ex1.detail.address) end end end sub_test_case 'defaults can be overwritten by owner' do test 'for feature plugin which has flat parameters with parent' do owner = ConfigurableSpec::OverwriteDefaults::Owner.new child = ConfigurableSpec::OverwriteDefaults::FlatChild.new assert_nil child.class.merged_configure_proxy.configured_in_section child.owner = owner child.configure(config_element('ROOT', '', {}, [])) assert_equal "V1", child.key1 end test 'for feature plugin which has parameters in subsection of parent' do owner = ConfigurableSpec::OverwriteDefaults::Owner.new child = ConfigurableSpec::OverwriteDefaults::BufferChild.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.owner = owner child.configure(config_element('ROOT', '', {}, [])) assert_equal 1024, child.size_of_something end test 'even in subclass of owner' do owner = ConfigurableSpec::OverwriteDefaults::SubOwner.new child = ConfigurableSpec::OverwriteDefaults::BufferChild.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.owner = owner child.configure(config_element('ROOT', '', {}, [])) assert_equal 2048, child.size_of_something end test 'default values can be overwritten with nil' do owner = ConfigurableSpec::OverwriteDefaults::NilOwner.new child = ConfigurableSpec::OverwriteDefaults::BufferChild.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.owner = owner child.configure(config_element('ROOT', '', {}, [])) assert_nil child.size_of_something end test 'the first configured_in (in the order from base class) will be applied' do child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.configure(config_element('ROOT', '', {}, [])) assert_equal 512, child.size_of_something end test 'the first configured_in is valid with owner classes' do owner = ConfigurableSpec::OverwriteDefaults::Owner.new child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.owner = owner child.configure(config_element('ROOT', '', {}, [])) assert_equal 1024, child.size_of_something end test 'the only first configured_in is valid even in subclasses of a class with configured_in' do child = ConfigurableSpec::OverwriteDefaults::BufferSubSubclass.new assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section child.configure(config_element('ROOT', '', {}, [])) assert_equal 512, child.size_of_something end end sub_test_case ':secret option' do setup do @conf = config_element('ROOT', '', { 'normal_param' => 'normal', 'secret_param' => 'secret' }, [config_element('section', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )]) @example = ConfigurableSpec::ExampleWithSecret.new @example.configure(@conf) end test 'to_s hides secret config_param' do @conf.to_s.each_line { |line| key, value = line.strip.split(' ', 2) assert_secret_param(key, value) } end test 'config returns masked configuration' do conf = @example.config conf.each_pair { |key, value| assert_secret_param(key, value) } conf.elements.each { |element| element.each_pair { |key, value| assert_secret_param(key, value) } } end def assert_secret_param(key, value) case key when 'normal_param', 'normal_param2' assert_equal 'normal', value when 'secret_param', 'secret_param2' assert_equal 'xxxxxx', value end end end sub_test_case 'unused section' do test 'get plugin name when found unknown section' do conf = config_element('ROOT', '', { 'name_param' => 'name', }, [config_element('unknown', '', {'name_param' => 'normal'} )]) example = ConfigurableSpec::ExampleWithCustomSection.new example.configure(conf) conf.elements.each { |e| assert_equal(['ROOT', nil], e.unused_in) } end test 'get an empty array when the section is defined without using config_section' do conf = config_element('ROOT', '', { 'name_param' => 'name', }, [config_element('custom_section', '', {'custom_section_param' => 'custom'} )]) example = ConfigurableSpec::ExampleWithCustomSection.new example.configure(conf) conf.elements.each { |e| assert_equal([], e.unused_in) } end test 'get an empty array when the configuration is used in another element without any sections' do conf = config_element('ROOT', '', { 'name_param' => 'name', }, [config_element('normal_section', '', {'normal_section_param' => 'normal'} )]) example = ConfigurableSpec::ExampleWithCustomSection.new example.configure(conf) ConfigurableSpec::ExampleWithCustomSection::AnotherElement.new.configure(conf) conf.elements.each { |e| assert_equal([], e.unused_in) } end end sub_test_case ':skip_accessor option' do test 'it does not create accessor methods for parameters' do @example = ConfigurableSpec::ExampleWithSkipAccessor.new @example.configure(config_element('ROOT')) assert_equal 'example7', @example.instance_variable_get(:@name) assert_raise NoMethodError do @example.name end end end sub_test_case 'non-required options for config_param' do test 'desc must be a string if specified' do assert_raise ArgumentError.new("key: desc must be a String, but Symbol") do class InvalidDescClass include Fluent::Configurable config_param :key, :string, default: '', desc: :invalid_description end end end test 'alias must be a symbol if specified' do assert_raise ArgumentError.new("key: alias must be a Symbol, but String") do class InvalidAliasClass include Fluent::Configurable config_param :key, :string, default: '', alias: 'yay' end end end test 'secret must be true or false if specified' do assert_raise ArgumentError.new("key: secret must be true or false, but NilClass") do class InvalidSecretClass include Fluent::Configurable config_param :key, :string, default: '', secret: nil end end assert_raise ArgumentError.new("key: secret must be true or false, but String") do class InvalidSecret2Class include Fluent::Configurable config_param :key, :string, default: '', secret: 'yes' end end end test 'deprecated must be a string if specified' do assert_raise ArgumentError.new("key: deprecated must be a String, but TrueClass") do class InvalidDeprecatedClass include Fluent::Configurable config_param :key, :string, default: '', deprecated: true end end end test 'obsoleted must be a string if specified' do assert_raise ArgumentError.new("key: obsoleted must be a String, but TrueClass") do class InvalidObsoletedClass include Fluent::Configurable config_param :key, :string, default: '', obsoleted: true end end end test 'value_type for hash must be a symbol' do assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do class InvalidValueTypeOfHashClass include Fluent::Configurable config_param :key, :hash, value_type: 'yay' end end end test 'value_type for array must be a symbol' do assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do class InvalidValueTypeOfArrayClass include Fluent::Configurable config_param :key, :array, value_type: 'yay' end end end test 'skip_accessor must be true or false if specified' do assert_raise ArgumentError.new("key: skip_accessor must be true or false, but NilClass") do class InvalidSkipAccessorClass include Fluent::Configurable config_param :key, :string, default: '', skip_accessor: nil end end assert_raise ArgumentError.new("key: skip_accessor must be true or false, but String") do class InvalidSkipAccessor2Class include Fluent::Configurable config_param :key, :string, default: '', skip_accessor: 'yes' end end end end sub_test_case 'enum parameters' do test 'list must be specified as an array of symbols' end sub_test_case 'deprecated/obsoleted parameters' do test 'both cannot be specified at once' do assert_raise ArgumentError.new("param1: both of deprecated and obsoleted cannot be specified at once") do class Buggy1 include Fluent::Configurable config_param :param1, :string, default: '', deprecated: 'yay', obsoleted: 'foo!' end end end test 'warned if deprecated parameter is configured' do obj = ConfigurableSpec::UnRecommended.new obj.log = Fluent::Test::TestLogger.new obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, [])) assert_equal 'yay', obj.key1 first_log = obj.log.logs.first assert{ first_log && first_log.include?("[warn]") && first_log.include?("'key1' parameter is deprecated: key1 will be removed.") } end test 'error raised if obsoleted parameter is configured' do obj = ConfigurableSpec::UnRecommended.new obj.log = Fluent::Test::TestLogger.new assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, [])) end first_log = obj.log.logs.first assert{ first_log && first_log.include?("[error]") && first_log.include?("config error in:\n\n key2 yay\n") } end sub_test_case 'logger is nil' do test 'nothing raised if deprecated parameter is configured' do obj = ConfigurableSpec::UnRecommended.new obj.log = nil obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, [])) assert_nil(obj.log) end test 'NoMethodError is not raised if obsoleted parameter is configured' do obj = ConfigurableSpec::UnRecommended.new obj.log = nil assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, [])) end assert_nil(obj.log) end end end sub_test_case '#config_param without default values cause error if section is configured as init:true' do setup do @type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) } @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'with simple config_param with default value' do class InitTestClass01 include Fluent::Configurable config_section :subsection, init: true do config_param :param1, :integer, default: 1 end end c = InitTestClass01.new c.configure(config_element('root', '')) assert_equal 1, c.subsection.size assert_equal 1, c.subsection.first.param1 end test 'with simple config_param without default value' do class InitTestClass02 include Fluent::Configurable config_section :subsection, init: true do config_param :param1, :integer end end c = InitTestClass02.new assert_raises ArgumentError.new("subsection: init is specified, but there're parameters without default values:param1") do c.configure(config_element('root', '')) end c.configure(config_element('root', '', {}, [config_element('subsection', '', {'param1' => '1'})])) assert_equal 1, c.subsection.size assert_equal 1, c.subsection.first.param1 end test 'with config_param with config_set_default' do module InitTestModule03 include Fluent::Configurable config_section :subsection, init: true do config_param :param1, :integer end end class InitTestClass03 include Fluent::Configurable include InitTestModule03 config_section :subsection do config_set_default :param1, 1 end end c = InitTestClass03.new c.configure(config_element('root', '')) assert_equal 1, c.subsection.size assert_equal 1, c.subsection.first.param1 end test 'with config_argument with default value' do class InitTestClass04 include Fluent::Configurable config_section :subsection, init: true do config_argument :param0, :string, default: 'yay' end end c = InitTestClass04.new c.configure(config_element('root', '')) assert_equal 1, c.subsection.size assert_equal 'yay', c.subsection.first.param0 end test 'with config_argument without default value' do class InitTestClass04 include Fluent::Configurable config_section :subsection, init: true do config_argument :param0, :string end end c = InitTestClass04.new assert_raise ArgumentError.new("subsection: init is specified, but default value of argument is missing") do c.configure(config_element('root', '')) end end test 'with config_argument with config_set_default' do module InitTestModule05 include Fluent::Configurable config_section :subsection, init: true do config_argument :param0, :string end end class InitTestClass05 include Fluent::Configurable include InitTestModule05 config_section :subsection do config_set_default :param0, 'foo' end end c = InitTestClass05.new c.configure(config_element('root', '')) assert_equal 1, c.subsection.size assert_equal 'foo', c.subsection.first.param0 end end sub_test_case '#config_argument' do test 'with strict_config_value' do class TestClass01 include Fluent::Configurable config_section :subsection do config_argument :param1, :integer end end c = TestClass01.new subsection = config_element('subsection', "hoge", { }) assert_raise(Fluent::ConfigError.new('param1: invalid value for Integer(): "hoge"')) do c.configure(config_element('root', '', {}, [subsection]), true) end end test 'with nil' do class TestClass02 include Fluent::Configurable config_section :subsection do config_argument :param1, :integer end end c = TestClass02.new subsection = config_element('subsection', nil, { }) assert_raise(Fluent::ConfigError.new("'' section requires argument, in section subsection")) do c.configure(config_element('root', '', {}, [subsection])) end end test 'with nil for an argument whose default value is nil' do class TestClass03 include Fluent::Configurable config_section :subsection do config_argument :param1, :integer, default: nil end end c = TestClass03.new subsection = config_element('subsection', nil, { }) c.configure(config_element('root', '', {}, [subsection])) assert_equal 1, c.subsection.size assert_equal nil, c.subsection.first.param1 end test 'with :default' do class TestClass04 include Fluent::Configurable config_section :subsection do config_argument :param1, :integer, default: 3 end end c = TestClass04.new subsection = config_element('subsection', :default, { }) c.configure(config_element('root', '', {}, [subsection])) assert_equal 1, c.subsection.size assert_equal 3, c.subsection.first.param1 end test 'with :default for an argument which does not have default value' do class TestClass05 include Fluent::Configurable config_section :subsection do config_argument :param1, :integer end end c = TestClass05.new subsection = config_element('subsection', :default, { }) assert_raise(Fluent::ConfigError.new("'param1' doesn\'t have default value")) do c.configure(config_element('root', '', {}, [subsection])) end end end end end ================================================ FILE: test/config/test_configure_proxy.rb ================================================ require_relative '../helper' require 'fluent/config/configure_proxy' module Fluent::Config class TestConfigureProxy < ::Test::Unit::TestCase setup do @type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) } end sub_test_case 'to generate a instance' do sub_test_case '#initialize' do test 'has default values' do proxy = Fluent::Config::ConfigureProxy.new('section', type_lookup: @type_lookup) assert_equal(:section, proxy.name) proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) assert_equal(:section, proxy.name) assert_nil(proxy.param_name) assert_equal(:section, proxy.variable_name) assert_false(proxy.root?) assert_nil(proxy.init) assert_nil(proxy.required) assert_false(proxy.required?) assert_nil(proxy.multi) assert_true(proxy.multi?) end test 'can specify param_name/required/multi with optional arguments' do proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: 'sections', init: true, required: false, multi: true, type_lookup: @type_lookup) assert_equal(:section, proxy.name) assert_equal(:sections, proxy.param_name) assert_equal(:sections, proxy.variable_name) assert_false(proxy.required) assert_false(proxy.required?) assert_true(proxy.multi) assert_true(proxy.multi?) proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: :sections, init: false, required: true, multi: false, type_lookup: @type_lookup) assert_equal(:section, proxy.name) assert_equal(:sections, proxy.param_name) assert_equal(:sections, proxy.variable_name) assert_true(proxy.required) assert_true(proxy.required?) assert_false(proxy.multi) assert_false(proxy.multi?) end test 'raise error if both of init and required are true' do assert_raise RuntimeError.new("init and required are exclusive") do Fluent::Config::ConfigureProxy.new(:section, init: true, required: true, type_lookup: @type_lookup) end end end sub_test_case '#merge' do test 'generate a new instance which values are overwritten by the argument object' do proxy = p1 = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) assert_equal(:section, proxy.name) assert_nil(proxy.param_name) assert_equal(:section, proxy.variable_name) assert_nil(proxy.init) assert_nil(proxy.required) assert_false(proxy.required?) assert_nil(proxy.multi) assert_true(proxy.multi?) assert_nil(proxy.configured_in_section) p2 = Fluent::Config::ConfigureProxy.new(:section, init: false, required: true, multi: false, type_lookup: @type_lookup) proxy = p1.merge(p2) assert_equal(:section, proxy.name) assert_nil(proxy.param_name) assert_equal(:section, proxy.variable_name) assert_false(proxy.init) assert_false(proxy.init?) assert_true(proxy.required) assert_true(proxy.required?) assert_false(proxy.multi) assert_false(proxy.multi?) assert_nil(proxy.configured_in_section) end test 'does not overwrite with argument object without any specifications of required/multi' do p1 = Fluent::Config::ConfigureProxy.new(:section1, param_name: :sections, type_lookup: @type_lookup) p1.configured_in_section = :subsection p2 = Fluent::Config::ConfigureProxy.new(:section2, init: false, required: true, multi: false, type_lookup: @type_lookup) p3 = Fluent::Config::ConfigureProxy.new(:section3, type_lookup: @type_lookup) proxy = p1.merge(p2).merge(p3) assert_equal(:section1, proxy.name) assert_equal(:sections, proxy.param_name) assert_equal(:sections, proxy.variable_name) assert_false(proxy.init) assert_false(proxy.init?) assert_true(proxy.required) assert_true(proxy.required?) assert_false(proxy.multi) assert_false(proxy.multi?) assert_equal :subsection, proxy.configured_in_section end test "does overwrite name of proxy for root sections which are used for plugins" do # latest plugin class shows actual plugin implementation p1 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP1'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup) p1.config_param :key1, :integer p2 = Fluent::Config::ConfigureProxy.new('Fluent::Plugin::MyP2'.to_sym, root: true, required: true, multi: false, type_lookup: @type_lookup) p2.config_param :key2, :string, default: "value2" merged = p1.merge(p2) assert_equal 'Fluent::Plugin::MyP2'.to_sym, merged.name assert_true merged.root? end end sub_test_case '#overwrite_defaults' do test 'overwrites only defaults with others defaults' do type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) } p1 = Fluent::Config::ConfigureProxy.new(:mychild, type_lookup: type_lookup) p1.configured_in_section = :child p1.config_param(:k1a, :string) p1.config_param(:k1b, :string) p1.config_param(:k2a, :integer, default: 0) p1.config_param(:k2b, :integer, default: 0) p1.config_section(:sub1) do config_param :k3, :time, default: 30 end p0 = Fluent::Config::ConfigureProxy.new(:myparent, type_lookup: type_lookup) p0.config_section(:child) do config_set_default :k1a, "v1a" config_param :k1b, :string, default: "v1b" config_set_default :k2a, 21 config_param :k2b, :integer, default: 22 config_section :sub1 do config_set_default :k3, 60 end end p1.overwrite_defaults(p0.sections[:child]) assert_equal "v1a", p1.defaults[:k1a] assert_equal "v1b", p1.defaults[:k1b] assert_equal 21, p1.defaults[:k2a] assert_equal 22, p1.defaults[:k2b] assert_equal 60, p1.sections[:sub1].defaults[:k3] end end sub_test_case '#configured_in' do test 'sets a section name which have configuration parameters of target plugin in owners configuration' do proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) proxy.configured_in(:mysection) assert_equal :mysection, proxy.configured_in_section end test 'do not permit to be called twice' do proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) proxy.configured_in(:mysection) assert_raise(ArgumentError) { proxy.configured_in(:myothersection) } end end sub_test_case '#config_param / #config_set_default / #config_argument' do setup do @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'handles configuration parameters without type as string' do @proxy.config_argument(:label) @proxy.config_param(:name) assert_equal :label, @proxy.argument[0] assert_equal :string, @proxy.argument[2][:type] assert_equal :string, @proxy.params[:name][1][:type] end data( default: [:default, nil], alias: [:alias, :alias_name_in_config], secret: [:secret, true], skip_accessor: [:skip_accessor, true], deprecated: [:deprecated, 'it is deprecated'], obsoleted: [:obsoleted, 'it is obsoleted'], desc: [:desc, "description"], ) test 'always allow options for all types' do |(option, value)| opt = {option => value} assert_nothing_raised{ @proxy.config_argument(:param0, **opt) } assert_nothing_raised{ @proxy.config_param(:p1, :string, **opt) } assert_nothing_raised{ @proxy.config_param(:p2, :enum, list: [:a, :b, :c], **opt) } assert_nothing_raised{ @proxy.config_param(:p3, :integer, **opt) } assert_nothing_raised{ @proxy.config_param(:p4, :float, **opt) } assert_nothing_raised{ @proxy.config_param(:p5, :size, **opt) } assert_nothing_raised{ @proxy.config_param(:p6, :bool, **opt) } assert_nothing_raised{ @proxy.config_param(:p7, :time, **opt) } assert_nothing_raised{ @proxy.config_param(:p8, :hash, **opt) } assert_nothing_raised{ @proxy.config_param(:p9, :array, **opt) } assert_nothing_raised{ @proxy.config_param(:pa, :regexp, **opt) } end data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array, regexp: :regexp) test 'deny list for non-enum types' do |type| assert_raise ArgumentError.new(":list is valid only for :enum type, but #{type}: arg") do @proxy.config_argument(:arg, type, list: [:a, :b]) end assert_raise ArgumentError.new(":list is valid only for :enum type, but #{type}: p1") do @proxy.config_param(:p1, type, list: [:a, :b]) end end data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, regexp: :regexp) test 'deny value_type for non-hash/array types' do |type| assert_raise ArgumentError.new(":value_type is valid only for :hash and :array, but #{type}: arg") do @proxy.config_argument(:arg, type, value_type: :string) end assert_raise ArgumentError.new(":value_type is valid only for :hash and :array, but #{type}: p1") do @proxy.config_param(:p1, type, value_type: :integer) end end data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, array: :array, regexp: :regexp) test 'deny symbolize_keys for non-hash types' do |type| assert_raise ArgumentError.new(":symbolize_keys is valid only for :hash, but #{type}: arg") do @proxy.config_argument(:arg, type, symbolize_keys: true) end assert_raise ArgumentError.new(":symbolize_keys is valid only for :hash, but #{type}: p1") do @proxy.config_param(:p1, type, symbolize_keys: true) end end data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array) test 'deny unknown options' do |type| assert_raise ArgumentError.new("unknown option 'required' for configuration parameter: arg") do @proxy.config_argument(:arg, type, required: true) end assert_raise ArgumentError.new("unknown option 'param_name' for configuration parameter: p1") do @proxy.config_argument(:p1, type, param_name: :yay) end end test 'desc gets string' do assert_nothing_raised do @proxy.config_param(:name, :string, desc: "it is description") end assert_raise ArgumentError.new("name1: desc must be a String, but Symbol") do @proxy.config_param(:name1, :string, desc: :yaaaaaaaay) end end test 'alias gets symbol' do assert_nothing_raised do @proxy.config_param(:name, :string, alias: :label) end assert_raise ArgumentError.new("name1: alias must be a Symbol, but String") do @proxy.config_param(:name1, :string, alias: 'label1') end end test 'secret gets true/false' do assert_nothing_raised do @proxy.config_param(:name1, :string, secret: false) end assert_nothing_raised do @proxy.config_param(:name2, :string, secret: true) end assert_raise ArgumentError.new("name3: secret must be true or false, but String") do @proxy.config_param(:name3, :string, secret: 'yes') end assert_raise ArgumentError.new("name4: secret must be true or false, but NilClass") do @proxy.config_param(:name4, :string, secret: nil) end end test 'symbolize_keys gets true/false' do assert_nothing_raised do @proxy.config_param(:data1, :hash, symbolize_keys: false) end assert_nothing_raised do @proxy.config_param(:data2, :hash, symbolize_keys: true) end assert_raise ArgumentError.new("data3: symbolize_keys must be true or false, but NilClass") do @proxy.config_param(:data3, :hash, symbolize_keys: nil) end end test 'value_type gets symbol' do assert_nothing_raised do @proxy.config_param(:data1, :array, value_type: :integer) end assert_raise ArgumentError.new("data2: value_type must be a Symbol, but Class") do @proxy.config_param(:data2, :array, value_type: Integer) end end test 'list gets an array of symbols' do assert_nothing_raised do @proxy.config_param(:proto1, :enum, list: [:a, :b]) end assert_raise ArgumentError.new("proto2: enum parameter requires :list of Symbols") do @proxy.config_param(:proto2, :enum, list: nil) end assert_raise ArgumentError.new("proto3: enum parameter requires :list of Symbols") do @proxy.config_param(:proto3, :enum, list: ['a', 'b']) end assert_raise ArgumentError.new("proto4: enum parameter requires :list of Symbols") do @proxy.config_param(:proto4, :enum, list: []) end end test 'deprecated gets string' do assert_nothing_raised do @proxy.config_param(:name1, :string, deprecated: "use name2 instead") end assert_raise ArgumentError.new("name2: deprecated must be a String, but TrueClass") do @proxy.config_param(:name2, :string, deprecated: true) end end test 'obsoleted gets string' do assert_nothing_raised do @proxy.config_param(:name1, :string, obsoleted: "use name2 instead") end assert_raise ArgumentError.new("name2: obsoleted must be a String, but TrueClass") do @proxy.config_param(:name2, :string, obsoleted: true) end end test 'skip_accessor gets true/false' do assert_nothing_raised do @proxy.config_param(:format1, :string, skip_accessor: false) end assert_nothing_raised do @proxy.config_param(:format2, :string, skip_accessor: true) end assert_raise ArgumentError.new("format2: skip_accessor must be true or false, but String") do @proxy.config_param(:format2, :string, skip_accessor: 'yes') end end test 'list is required for :enum' do assert_nothing_raised do @proxy.config_param(:proto1, :enum, list: [:a, :b]) end assert_raise ArgumentError.new("proto1: enum parameter requires :list of Symbols") do @proxy.config_param(:proto1, :enum, default: :a) end end test 'does not permit config_set_default for param w/ :default option' do @proxy.config_param(:name, :string, default: "name1") assert_raise(ArgumentError) { @proxy.config_set_default(:name, "name2") } end test 'does not permit default value specification twice' do @proxy.config_param(:name, :string) @proxy.config_set_default(:name, "name1") assert_raise(ArgumentError) { @proxy.config_set_default(:name, "name2") } end test 'does not permit default value specification twice, even on config_argument' do @proxy.config_param(:name, :string) @proxy.config_set_default(:name, "name1") @proxy.config_argument(:name) assert_raise(ArgumentError) { @proxy.config_argument(:name, default: "name2") } end end sub_test_case '#config_set_desc' do setup do @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'does not permit description specification twice w/ :desc option' do @proxy.config_param(:name, :string, desc: "description") assert_raise(ArgumentError) { @proxy.config_set_desc(:name, "description2") } end test 'does not permit description specification twice' do @proxy.config_param(:name, :string) @proxy.config_set_desc(:name, "description") assert_raise(ArgumentError) { @proxy.config_set_desc(:name, "description2") } end end sub_test_case '#desc' do setup do @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'permit to specify description twice' do @proxy.desc("description1") @proxy.desc("description2") @proxy.config_param(:name, :string) assert_equal("description2", @proxy.descriptions[:name]) end test 'does not permit description specification twice' do @proxy.desc("description1") assert_raise(ArgumentError) do @proxy.config_param(:name, :string, desc: "description2") end end end sub_test_case '#dump_config_definition' do setup do @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup) end test 'empty proxy' do assert_equal({}, @proxy.dump_config_definition) end test 'plain proxy w/o default value' do @proxy.config_param(:name, :string) expected = { name: { type: :string, required: true } } assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ default value' do @proxy.config_param(:name, :string, default: "name1") expected = { name: { type: :string, default: "name1", required: false } } assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ default value using config_set_default' do @proxy.config_param(:name, :string) @proxy.config_set_default(:name, "name1") expected = { name: { type: :string, default: "name1", required: false } } assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ argument' do @proxy.instance_eval do config_argument(:argname, :string) config_param(:name, :string, default: "name1") end expected = { argname: { type: :string, required: true, argument: true }, name: { type: :string, default: "name1", required: false } } assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ argument default value' do @proxy.instance_eval do config_argument(:argname, :string, default: "value") config_param(:name, :string, default: "name1") end expected = { argname: { type: :string, default: "value", required: false, argument: true }, name: { type: :string, default: "name1", required: false } } assert_equal(expected, @proxy.dump_config_definition) end test 'plain proxy w/ argument overwriting default value' do @proxy.instance_eval do config_argument(:argname, :string) config_param(:name, :string, default: "name1") config_set_default(:argname, "value1") end expected = { argname: { type: :string, default: "value1", required: false, argument: true }, name: { type: :string, default: "name1", required: false } } assert_equal(expected, @proxy.dump_config_definition) end test 'single sub proxy' do @proxy.config_section(:sub) do config_param(:name, :string, default: "name1") end expected = { sub: { alias: nil, multi: true, required: false, section: true, name: { type: :string, default: "name1", required: false } } } assert_equal(expected, @proxy.dump_config_definition) end test 'nested sub proxy' do @proxy.config_section(:sub) do config_param(:name1, :string, default: "name1") config_param(:name2, :string, default: "name2") config_section(:sub2) do config_param(:name3, :string, default: "name3") config_param(:name4, :string, default: "name4") end end expected = { sub: { alias: nil, multi: true, required: false, section: true, name1: { type: :string, default: "name1", required: false }, name2: { type: :string, default: "name2", required: false }, sub2: { alias: nil, multi: true, required: false, section: true, name3: { type: :string, default: "name3", required: false }, name4: { type: :string, default: "name4", required: false }, } } } assert_equal(expected, @proxy.dump_config_definition) end sub_test_case 'w/ description' do test 'single proxy' do @proxy.config_param(:name, :string, desc: "description for name") expected = { name: { type: :string, desc: "description for name", required: true } } assert_equal(expected, @proxy.dump_config_definition) end test 'single proxy using config_set_desc' do @proxy.config_param(:name, :string) @proxy.config_set_desc(:name, "description for name") expected = { name: { type: :string, desc: "description for name", required: true } } assert_equal(expected, @proxy.dump_config_definition) end test 'sub proxy' do @proxy.config_section(:sub) do config_param(:name1, :string, default: "name1", desc: "desc1") config_param(:name2, :string, default: "name2", desc: "desc2") config_section(:sub2) do config_param(:name3, :string, default: "name3") config_param(:name4, :string, default: "name4", desc: "desc4") end end expected = { sub: { alias: nil, multi: true, required: false, section: true, name1: { type: :string, default: "name1", desc: "desc1", required: false }, name2: { type: :string, default: "name2", desc: "desc2", required: false }, sub2: { alias: nil, multi: true, required: false, section: true, name3: { type: :string, default: "name3", required: false }, name4: { type: :string, default: "name4", desc: "desc4", required: false }, } } } assert_equal(expected, @proxy.dump_config_definition) end test 'sub proxy w/ desc method' do @proxy.config_section(:sub) do desc("desc1") config_param(:name1, :string, default: "name1") config_param(:name2, :string, default: "name2", desc: "desc2") config_section(:sub2) do config_param(:name3, :string, default: "name3") desc("desc4") config_param(:name4, :string, default: "name4") end end expected = { sub: { alias: nil, multi: true, required: false, section: true, name1: { type: :string, default: "name1", desc: "desc1", required: false }, name2: { type: :string, default: "name2", desc: "desc2", required: false }, sub2: { alias: nil, multi: true, required: false, section: true, name3: { type: :string, default: "name3", required: false }, name4: { type: :string, default: "name4", desc: "desc4", required: false }, } } } assert_equal(expected, @proxy.dump_config_definition) end end end end end end ================================================ FILE: test/config/test_dsl.rb ================================================ require_relative '../helper' require 'fluent/config/element' require "fluent/config/dsl" require 'tempfile' TMP_DIR = File.dirname(__FILE__) + "/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}" def write_config(path, data) FileUtils.mkdir_p(File.dirname(path)) File.open(path, "w") {|f| f.write data } end def prepare_config1 write_config "#{TMP_DIR}/config_test_1.conf", %[ k1 root_config include dir/config_test_2.conf # @include #{TMP_DIR}/config_test_4.conf include file://#{TMP_DIR}/config_test_5.conf @include config.d/*.conf ] write_config "#{TMP_DIR}/dir/config_test_2.conf", %[ k2 relative_path_include @include ../config_test_3.conf ] write_config "#{TMP_DIR}/config_test_3.conf", %[ k3 relative_include_in_included_file ] write_config "#{TMP_DIR}/config_test_4.conf", %[ k4 absolute_path_include ] write_config "#{TMP_DIR}/config_test_5.conf", %[ k5 uri_include ] write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[ k6 wildcard_include_1 include normal_parameter ] write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[ k7 wildcard_include_2 ] write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[ @include ../dir/config_test_9.conf ] write_config "#{TMP_DIR}/dir/config_test_9.conf", %[ k9 embedded nested nested_value include hoge ] write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[ k8 wildcard_include_3 include normal_parameter ] end def prepare_config2 write_config "#{TMP_DIR}/config_test_1.rb", DSL_CONFIG_EXAMPLE end DSL_CONFIG_EXAMPLE = %q[ worker { hostname = "myhostname" (0..9).each { |i| source { type :tail path "/var/log/httpd/access.part#{i}.log" filter ('bar.**') { type :hoge val1 "moge" val2 ["foo", "bar", "baz"] val3 10 id :hoge subsection { foo "bar" } subsection { foo "baz" } } filter ('foo.**') { type "pass" } match ('{foo,bar}.**') { type "file" path "/var/log/httpd/access.#{hostname}.#{i}.log" } } } } ] DSL_CONFIG_EXAMPLE_WITHOUT_WORKER = %q[ hostname = "myhostname" source { type :tail path "/var/log/httpd/access.part.log" element { name "foo" } match ('{foo,bar}.**') { type "file" path "/var/log/httpd/access.full.log" } } ] DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF = %q[ include "#{TMP_DIR}/config_test_1.conf" ] DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB = %q[ include "#{TMP_DIR}/config_test_1.rb" ] DSL_CONFIG_RETURNS_NON_ELEMENT = %q[ worker { } [] ] DSL_CONFIG_WRONG_SYNTAX1 = %q[ match ] DSL_CONFIG_WRONG_SYNTAX2 = %q[ match('aa','bb'){ type :null } ] DSL_CONFIG_WRONG_SYNTAX3 = %q[ match('aa','bb') ] DSL_CONFIG_WRONG_SYNTAX4 = %q[ include ] module Fluent::Config class TestDSLParser < ::Test::Unit::TestCase sub_test_case 'with worker tag on top level' do def setup @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE, 'dsl_config.rb') end sub_test_case '.parse' do test 'makes root element' do assert_equal('ROOT', @root.name) assert_predicate(@root.arg, :empty?) assert_equal(0, @root.keys.size) end test 'makes worker element for worker tag' do assert_equal(1, @root.elements.size) worker = @root.elements.first assert_equal('worker', worker.name) assert_predicate(worker.arg, :empty?) assert_equal(0, worker.keys.size) assert_equal(10, worker.elements.size) end test 'makes subsections for blocks, with variable substitution' do ele4 = @root.elements.first.elements[4] assert_equal('source', ele4.name) assert_predicate(ele4.arg, :empty?) assert_equal(2, ele4.keys.size) assert_equal('tail', ele4['@type']) assert_equal("/var/log/httpd/access.part4.log", ele4['path']) end test 'makes user-defined sections with blocks' do filter0 = @root.elements.first.elements[4].elements.first assert_equal('filter', filter0.name) assert_equal('bar.**', filter0.arg) assert_equal('hoge', filter0['@type']) assert_equal('moge', filter0['val1']) assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2']) assert_equal('10', filter0['val3']) assert_equal('hoge', filter0['@id']) assert_equal(2, filter0.elements.size) assert_equal('subsection', filter0.elements[0].name) assert_equal('bar', filter0.elements[0]['foo']) assert_equal('subsection', filter0.elements[1].name) assert_equal('baz', filter0.elements[1]['foo']) end test 'makes values with user-assigned variable substitutions' do match0 = @root.elements.first.elements[4].elements.last assert_equal('match', match0.name) assert_equal('{foo,bar}.**', match0.arg) assert_equal('file', match0['@type']) assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path']) end end end sub_test_case 'without worker tag on top level' do def setup @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_WITHOUT_WORKER, 'dsl_config_without_worker.rb') end sub_test_case '.parse' do test 'makes root element' do assert_equal('ROOT', @root.name) assert_predicate(@root.arg, :empty?) assert_equal(0, @root.keys.size) end test 'does not make worker element implicitly because DSL configuration does not support v10 compat mode' do assert_equal(1, @root.elements.size) assert_equal('source', @root.elements.first.name) refute(@root.elements.find { |e| e.name == 'worker' }) end end end sub_test_case 'with include conf' do def setup prepare_config1 @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF, 'dsl_config_for_include.conf') end test 'include config' do assert_equal('root_config', @root['k1']) assert_equal('relative_path_include', @root['k2']) assert_equal('relative_include_in_included_file', @root['k3']) assert_equal('absolute_path_include', @root['k4']) assert_equal('uri_include', @root['k5']) assert_equal('wildcard_include_1', @root['k6']) assert_equal('wildcard_include_2', @root['k7']) assert_equal('wildcard_include_3', @root['k8']) assert_equal([ 'k1', 'k2', 'k3', 'k4', 'k5', 'k8', # Because of the file name this comes first. 'k6', 'k7', ], @root.keys) elem1 = @root.elements.find { |e| e.name == 'elem1' } assert(elem1) assert_equal('name', elem1.arg) assert_equal('normal_parameter', elem1['include']) elem2 = @root.elements.find { |e| e.name == 'elem2' } assert(elem2) assert_equal('name', elem2.arg) assert_equal('embedded', elem2['k9']) assert_not_include(elem2, 'include') elem3 = elem2.elements.find { |e| e.name == 'elem3' } assert(elem3) assert_equal('nested_value', elem3['nested']) assert_equal('hoge', elem3['include']) end # TODO: Add uri based include spec end sub_test_case 'with include rb' do def setup prepare_config2 @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB, 'dsl_config_for_include.rb') end sub_test_case '.parse' do test 'makes root element' do assert_equal('ROOT', @root.name) assert_predicate(@root.arg, :empty?) assert_equal(0, @root.keys.size) end test 'makes worker element for worker tag' do assert_equal(1, @root.elements.size) worker = @root.elements.first assert_equal('worker', worker.name) assert_predicate(worker.arg, :empty?) assert_equal(0, worker.keys.size) assert_equal(10, worker.elements.size) end test 'makes subsections for blocks, with variable substitution' do ele4 = @root.elements.first.elements[4] assert_equal('source', ele4.name) assert_predicate(ele4.arg, :empty?) assert_equal(2, ele4.keys.size) assert_equal('tail', ele4['@type']) assert_equal("/var/log/httpd/access.part4.log", ele4['path']) end test 'makes user-defined sections with blocks' do filter0 = @root.elements.first.elements[4].elements.first assert_equal('filter', filter0.name) assert_equal('bar.**', filter0.arg) assert_equal('hoge', filter0['@type']) assert_equal('moge', filter0['val1']) assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2']) assert_equal('10', filter0['val3']) assert_equal('hoge', filter0['@id']) assert_equal(2, filter0.elements.size) assert_equal('subsection', filter0.elements[0].name) assert_equal('bar', filter0.elements[0]['foo']) assert_equal('subsection', filter0.elements[1].name) assert_equal('baz', filter0.elements[1]['foo']) end test 'makes values with user-assigned variable substitutions' do match0 = @root.elements.first.elements[4].elements.last assert_equal('match', match0.name) assert_equal('{foo,bar}.**', match0.arg) assert_equal('file', match0['@type']) assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path']) end end end sub_test_case 'with configuration that returns non element on top' do sub_test_case '.parse' do test 'does not crash' do Fluent::Config::DSL::Parser.parse(DSL_CONFIG_RETURNS_NON_ELEMENT, 'dsl_config_returns_non_element.rb') end end end sub_test_case 'with configuration with wrong arguments for specific elements' do sub_test_case '.parse' do test 'raises ArgumentError correctly' do assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') } assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax2') } assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax3') } assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX4, 'dsl_config_wrong_syntax4') } end end end sub_test_case 'with ruby keyword, that provides ruby Kernel module features' do sub_test_case '.parse' do test 'can get result of Kernel.open() by ruby.open()' do uname_string = `uname -a` tmpfile = Tempfile.create('fluentd-test') tmpfile.write(uname_string) tmpfile.close root = Fluent::Config::DSL::Parser.parse(< from erb').result } source { version ruby_version } } DSL worker = root.elements.first assert_equal('worker', worker.name) source = worker.elements.first assert_equal('source', source.name) assert_equal(1, source.keys.size) assert_equal("#{RUBY_VERSION} from erb", source['version']) end test 'raises NoMethodError when configuration DSL elements are written in ruby block' do conf = < "v1", "k2" => "v2"}, [ element('test', 'mydata', {'k3' => 'v3'}, []) ]) assert_equal('ROOT', e.name) assert_equal('', e.arg) assert_equal('v1', e["k1"]) assert_equal('v2', e["k2"]) assert_equal(1, e.elements.size) e.each_element('test') do |el| assert_equal('test', el.name) assert_equal('mydata', el.arg) assert_equal('v3', el["k3"]) end end test 'creates object which contains attrs, elements and unused' do e = element('ROOT', '', {"k1" => "v1", "k2" => "v2", "k4" => "v4"}, [ element('test', 'mydata', {'k3' => 'v3'}, []) ], "k3") assert_equal("k3", e.unused) assert_equal('ROOT', e.name) assert_equal('', e.arg) assert_equal('v1', e["k1"]) assert_equal('v2', e["k2"]) assert_equal('v4', e["k4"]) assert_equal(1, e.elements.size) e.each_element('test') do |el| assert_equal('test', el.name) assert_equal('mydata', el.arg) assert_equal('v3', el["k3"]) end assert_equal("k3", e.unused) end end sub_test_case "@unused" do sub_test_case '#[] has side effect for @unused' do test 'without unused argument' do e = element('ROOT', '', {"k1" => "v1", "k2" => "v2", "k4" => "v4"}, [ element('test', 'mydata', {'k3' => 'v3'}, []) ]) assert_equal(["k1", "k2", "k4"], e.unused) assert_equal('v1', e["k1"]) assert_equal(["k2", "k4"], e.unused) assert_equal('v2', e["k2"]) assert_equal(["k4"], e.unused) assert_equal('v4', e["k4"]) assert_equal([], e.unused) end test 'with unused argument' do e = element('ROOT', '', {"k1" => "v1", "k2" => "v2", "k4" => "v4"}, [ element('test', 'mydata', {'k3' => 'v3'}, []) ], ["k4"]) assert_equal(["k4"], e.unused) assert_equal('v1', e["k1"]) assert_equal(["k4"], e.unused) assert_equal('v2', e["k2"]) assert_equal(["k4"], e.unused) assert_equal('v4', e["k4"]) # only consume for "k4" assert_equal([], e.unused) end end end sub_test_case '#add_element' do test 'elements can be set by #add_element' do e = element() assert_equal [], e.elements e.add_element('e1', '') e.add_element('e2', '') assert_equal [element('e1', ''), element('e2', '')], e.elements end end sub_test_case '#==' do sub_test_case 'compare with two element objects' do test 'equal' do e1 = element('ROOT', '', {}, []) e2 = element('ROOT', '', {}, []) assert_true(e1 == e2) end data("differ args" => [Fluent::Config::Element.new('ROOT', '', {}, []), Fluent::Config::Element.new('ROOT', 'mydata', {}, [])], "differ keys" => [Fluent::Config::Element.new('ROOT', 'mydata', {}, []), Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, [])], "differ elements" => [Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, []), Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, [ Fluent::Config::Element.new('test', 'mydata', {'k3' => 'v3'}, []) ])]) test 'not equal' do |data| e1, e2 = data assert_false(e1 == e2) end end end sub_test_case '#+' do test 'can merge 2 elements: object side is primary' do e1 = element('ROOT', 'mydata', {"k1" => "v1"}, []) e2 = element('ROOT', 'mydata2', {"k1" => "ignored", "k2" => "v2"}, [ element('test', 'ext', {'k3' => 'v3'}, []) ]) e = e1 + e2 assert_equal('ROOT', e.name) assert_equal('mydata', e.arg) assert_equal('v1', e['k1']) assert_equal('v2', e['k2']) assert_equal(1, e.elements.size) e.each_element('test') do |el| assert_equal('test', el.name) assert_equal('ext', el.arg) assert_equal('v3', el["k3"]) end end end sub_test_case '#check_not_fetched' do sub_test_case 'without unused' do test 'can get attribute keys and original Config::Element' do e = element('ROOT', 'mydata', {"k1" => "v1"}, []) e.check_not_fetched { |key, elem| assert_equal("k1", key) assert_equal(e, elem) } end end sub_test_case 'with unused' do test 'can get unused marked attribute keys and original Config::Element' do e = element('ROOT', 'mydata', {"k1" => "v1", "k2" => "unused", "k3" => "k3"}) e.unused = "k2" e.check_not_fetched { |key, elem| assert_equal("k2", key) assert_equal(e, elem) } end end end sub_test_case '#has_key?' do test 'can get boolean with key name' do e = element('ROOT', 'mydata', {"k1" => "v1"}, []) assert_true(e.has_key?("k1")) assert_false(e.has_key?("noexistent")) end end sub_test_case '#to_s' do data("without v1_config" => [false, <<-CONF k1 v1 k2 "stringVal" k2 v2 CONF ], "with v1_config" => [true, <<-CONF k1 v1 k2 "stringVal" k2 v2 CONF ], ) test 'dump config element with #to_s' do |data| v1_config, expected = data e = element('ROOT', '', {'k1' => 'v1', "k2" =>"\"stringVal\""}, [ element('test', 'ext', {'k2' => 'v2'}, []) ]) e.v1_config = v1_config dump = expected assert_not_equal(e.inspect, e.to_s) assert_equal(dump, e.to_s) end test 'dump nil and default for v1' do expected = <<-CONF str1 str2 defstring CONF e = element('ROOT', '', {'str1' => nil, "str2" => :default}, []) type_lookup = ->(type){ Fluent::Configurable.lookup_type(type) } p = Fluent::Config::ConfigureProxy.new("test", type_lookup: type_lookup) p.config_param :str1, :string p.config_param :str2, :string, default: "defstring" e.corresponding_proxies << p e.v1_config = true assert_not_equal(e.inspect, e.to_s) assert_equal(expected, e.to_s) end end sub_test_case '#inspect' do test 'dump config element with #inspect' do e = element('ROOT', '', {'k1' => 'v1'}, [ element('test', 'ext', {'k2' => 'v2'}, []) ]) dump = <<-CONF \"v1\"}, elements:[\"v2\"}, elements:[]>]> CONF assert_not_equal(e.to_s, e.inspect.gsub(' => ', '=>')) assert_equal(dump.chomp, e.inspect.gsub(' => ', '=>')) end end sub_test_case 'for sections which has secret parameter' do setup do @type_lookup = ->(type){ Fluent::Configurable.lookup_type(type) } p1 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup) p1.config_param :str1, :string p1.config_param :str2, :string, secret: true p1.config_param :enum1, :enum, list: [:a, :b, :c] p1.config_param :enum2, :enum, list: [:a, :b, :c], secret: true p1.config_param :bool1, :bool p1.config_param :bool2, :bool, secret: true p1.config_param :int1, :integer p1.config_param :int2, :integer, secret: true p1.config_param :float1, :float p1.config_param :float2, :float, secret: true p2 = Fluent::Config::ConfigureProxy.new(:match, type_lookup: @type_lookup) p2.config_param :size1, :size p2.config_param :size2, :size, secret: true p2.config_param :time1, :time p2.config_param :time2, :time, secret: true p2.config_param :array1, :array p2.config_param :array2, :array, secret: true p2.config_param :hash1, :hash p2.config_param :hash2, :hash, secret: true p1.config_section :mysection do config_param :str1, :string config_param :str2, :string, secret: true config_param :enum1, :enum, list: [:a, :b, :c] config_param :enum2, :enum, list: [:a, :b, :c], secret: true config_param :bool1, :bool config_param :bool2, :bool, secret: true config_param :int1, :integer config_param :int2, :integer, secret: true config_param :float1, :float config_param :float2, :float, secret: true config_param :size1, :size config_param :size2, :size, secret: true config_param :time1, :time config_param :time2, :time, secret: true config_param :array1, :array config_param :array2, :array, secret: true config_param :hash1, :hash config_param :hash2, :hash, secret: true end params = { 'str1' => 'aaa', 'str2' => 'bbb', 'enum1' => 'a', 'enum2' => 'b', 'bool1' => 'true', 'bool2' => 'yes', 'int1' => '1', 'int2' => '2', 'float1' => '1.0', 'float2' => '0.5', 'size1' => '1k', 'size2' => '1m', 'time1' => '5m', 'time2' => '3h', 'array1' => 'a,b,c', 'array2' => 'd,e,f', 'hash1' => 'a:1,b:2', 'hash2' => 'a:2,b:4', 'unknown1' => 'yay', 'unknown2' => 'boo', } e2 = Fluent::Config::Element.new('mysection', '', params.dup, []) e2.corresponding_proxies << p1.sections.values.first @e = Fluent::Config::Element.new('match', '**', params, [e2]) @e.corresponding_proxies << p1 @e.corresponding_proxies << p2 end sub_test_case '#to_masked_element' do test 'returns a new element object which has masked values for secret parameters and elements' do e = @e.to_masked_element assert_equal 'aaa', e['str1'] assert_equal 'xxxxxx', e['str2'] assert_equal 'a', e['enum1'] assert_equal 'xxxxxx', e['enum2'] assert_equal 'true', e['bool1'] assert_equal 'xxxxxx', e['bool2'] assert_equal '1', e['int1'] assert_equal 'xxxxxx', e['int2'] assert_equal '1.0', e['float1'] assert_equal 'xxxxxx', e['float2'] assert_equal '1k', e['size1'] assert_equal 'xxxxxx', e['size2'] assert_equal '5m', e['time1'] assert_equal 'xxxxxx', e['time2'] assert_equal 'a,b,c', e['array1'] assert_equal 'xxxxxx', e['array2'] assert_equal 'a:1,b:2', e['hash1'] assert_equal 'xxxxxx', e['hash2'] assert_equal 'yay', e['unknown1'] assert_equal 'boo', e['unknown2'] e2 = e.elements.first assert_equal 'aaa', e2['str1'] assert_equal 'xxxxxx', e2['str2'] assert_equal 'a', e2['enum1'] assert_equal 'xxxxxx', e2['enum2'] assert_equal 'true', e2['bool1'] assert_equal 'xxxxxx', e2['bool2'] assert_equal '1', e2['int1'] assert_equal 'xxxxxx', e2['int2'] assert_equal '1.0', e2['float1'] assert_equal 'xxxxxx', e2['float2'] assert_equal '1k', e2['size1'] assert_equal 'xxxxxx', e2['size2'] assert_equal '5m', e2['time1'] assert_equal 'xxxxxx', e2['time2'] assert_equal 'a,b,c', e2['array1'] assert_equal 'xxxxxx', e2['array2'] assert_equal 'a:1,b:2', e2['hash1'] assert_equal 'xxxxxx', e2['hash2'] assert_equal 'yay', e2['unknown1'] assert_equal 'boo', e2['unknown2'] end end sub_test_case '#secret_param?' do test 'returns boolean which shows values of given key will be masked' do assert !@e.secret_param?('str1') assert @e.secret_param?('str2') assert !@e.elements.first.secret_param?('str1') assert @e.elements.first.secret_param?('str2') end end sub_test_case '#param_type' do test 'returns parameter type which are registered in corresponding proxy' do assert_equal :string, @e.param_type('str1') assert_equal :string, @e.param_type('str2') assert_equal :enum, @e.param_type('enum1') assert_equal :enum, @e.param_type('enum2') assert_nil @e.param_type('unknown1') assert_nil @e.param_type('unknown2') end end # sub_test_case '#dump_value' sub_test_case '#dump_value' do test 'dumps parameter_name and values with leading indentation' do assert_equal "str1 aaa\n", @e.dump_value("str1", @e["str1"], "") assert_equal "str2 xxxxxx\n", @e.dump_value("str2", @e["str2"], "") end end end sub_test_case '#set_target_worker' do test 'set target_worker_id recursively' do e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ]) e.set_target_worker_id(1) assert_equal [1], e.target_worker_ids assert_equal [1], e.elements[0].target_worker_ids assert_equal [1], e.elements[1].target_worker_ids assert_equal [1], e.elements[1].elements[0].target_worker_ids assert_equal [1], e.elements[1].elements[1].target_worker_ids end end sub_test_case '#for_every_workers?' do test 'has target_worker_id' do e = element() e.set_target_worker_id(1) assert_false e.for_every_workers? end test "doesn't have target_worker_id" do e = element() assert e.for_every_workers? end end sub_test_case '#for_this_workers?' do test 'target_worker_id == current worker_id' do e = element() e.set_target_worker_id(0) assert e.for_this_worker? end test 'target_worker_ids includes current worker_id' do e = element() e.set_target_worker_ids([0]) assert e.for_this_worker? end test 'target_worker_id != current worker_id' do e = element() e.set_target_worker_id(1) assert_false e.for_this_worker? end test 'target_worker_ids does not includes current worker_id' do e = element() e.set_target_worker_ids([1, 2]) assert_false e.for_this_worker? end test "doesn't have target_worker_id" do e = element() assert_false e.for_this_worker? end end sub_test_case '#for_another_worker?' do test 'target_worker_id == current worker_id' do e = element() e.set_target_worker_id(0) assert_false e.for_another_worker? end test 'target_worker_ids contains current worker_id' do e = element() e.set_target_worker_ids([0, 1]) assert_false e.for_another_worker? end test 'target_worker_id != current worker_id' do e = element() e.set_target_worker_id(1) assert e.for_another_worker? end test 'target_worker_ids does not contains current worker_id' do e = element() e.set_target_worker_ids([1, 2]) assert e.for_another_worker? end test "doesn't have target_worker_id" do e = element() assert_false e.for_another_worker? end end sub_test_case '#pretty_print' do test 'prints inspect to pp object' do q = PP.new e = element() e.pretty_print(q) assert_equal e.inspect, q.output end end end ================================================ FILE: test/config/test_literal_parser.rb ================================================ require_relative "../helper" require_relative 'assertions' require "fluent/config/error" require "fluent/config/literal_parser" require "fluent/config/v1_parser" require 'json' module Fluent::Config class TestLiteralParser < ::Test::Unit::TestCase def parse_text(text) basepath = File.expand_path(File.dirname(__FILE__)+'/../../') ss = StringScanner.new(text) parser = Fluent::Config::V1Parser.new(ss, basepath, "(test)", eval_context) parser.parse_literal end TestLiteralParserContext = Struct.new(:v1, :v2, :v3) def v1 :test end def v2 true end def v3 nil end def eval_context @eval_context ||= TestLiteralParserContext.new(v1, v2, v3) end sub_test_case 'boolean parsing' do def test_true assert_text_parsed_as('true', "true") end def test_false assert_text_parsed_as('false', "false") end def test_trueX assert_text_parsed_as('trueX', "trueX") end def test_falseX assert_text_parsed_as('falseX', "falseX") end end sub_test_case 'integer parsing' do test('0') { assert_text_parsed_as('0', "0") } test('1') { assert_text_parsed_as('1', "1") } test('10') { assert_text_parsed_as('10', "10") } test('-1') { assert_text_parsed_as('-1', "-1") } test('-10') { assert_text_parsed_as('-10', "-10") } test('0 ') { assert_text_parsed_as('0', "0 ") } test(' -1 ') { assert_text_parsed_as("-1", ' -1 ') } # string test('01') { assert_text_parsed_as('01', "01") } test('00') { assert_text_parsed_as('00', "00") } test('-01') { assert_text_parsed_as('-01', "-01") } test('-00') { assert_text_parsed_as('-00', "-00") } test('0x61') { assert_text_parsed_as('0x61', "0x61") } test('0s') { assert_text_parsed_as('0s', "0s") } end sub_test_case 'float parsing' do test('1.1') { assert_text_parsed_as('1.1', "1.1") } test('0.1') { assert_text_parsed_as('0.1', "0.1") } test('0.0') { assert_text_parsed_as('0.0', "0.0") } test('-1.1') { assert_text_parsed_as('-1.1', "-1.1") } test('-0.1') { assert_text_parsed_as('-0.1', "-0.1") } test('1.10') { assert_text_parsed_as('1.10', "1.10") } # string test('12e8') { assert_text_parsed_as('12e8', "12e8") } test('12.1e7') { assert_text_parsed_as('12.1e7', "12.1e7") } test('-12e8') { assert_text_parsed_as('-12e8', "-12e8") } test('-12.1e7') { assert_text_parsed_as('-12.1e7', "-12.1e7") } test('.0') { assert_text_parsed_as('.0', ".0") } test('.1') { assert_text_parsed_as('.1', ".1") } test('0.') { assert_text_parsed_as('0.', "0.") } test('1.') { assert_text_parsed_as('1.', "1.") } test('.0a') { assert_text_parsed_as('.0a', ".0a") } test('1.a') { assert_text_parsed_as('1.a', "1.a") } test('0@') { assert_text_parsed_as('0@', "0@") } end sub_test_case 'float keywords parsing' do test('NaN') { assert_text_parsed_as('NaN', "NaN") } test('Infinity') { assert_text_parsed_as('Infinity', "Infinity") } test('-Infinity') { assert_text_parsed_as('-Infinity', "-Infinity") } test('NaNX') { assert_text_parsed_as('NaNX', "NaNX") } test('InfinityX') { assert_text_parsed_as('InfinityX', "InfinityX") } test('-InfinityX') { assert_text_parsed_as('-InfinityX', "-InfinityX") } end sub_test_case 'double quoted string' do test('""') { assert_text_parsed_as("", '""') } test('"text"') { assert_text_parsed_as("text", '"text"') } test('"\\""') { assert_text_parsed_as("\"", '"\\""') } test('"\\t"') { assert_text_parsed_as("\t", '"\\t"') } test('"\\n"') { assert_text_parsed_as("\n", '"\\n"') } test('"\\r\\n"') { assert_text_parsed_as("\r\n", '"\\r\\n"') } test('"\\f\\b"') { assert_text_parsed_as("\f\b", '"\\f\\b"') } test('"\\.t"') { assert_text_parsed_as(".t", '"\\.t"') } test('"\\$t"') { assert_text_parsed_as("$t", '"\\$t"') } test('"\\"') { assert_text_parsed_as("#t", '"\\#t"') } test('"\\0"') { assert_text_parsed_as("\0", '"\\0"') } test('"\\z"') { assert_parse_error('"\\z"') } # unknown escaped character test('"\\1"') { assert_parse_error('"\\1"') } # unknown escaped character test('"t') { assert_parse_error('"t') } # non-terminated quoted character test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string test("\"t\\\nt\"") { assert_text_parsed_as("tt", "\"t\\\nt\"" ) } # multiline string test('t"') { assert_text_parsed_as('t"', 't"') } test('"."') { assert_text_parsed_as('.', '"."') } test('"*"') { assert_text_parsed_as('*', '"*"') } test('"@"') { assert_text_parsed_as('@', '"@"') } test('"\\#{test}"') { assert_text_parsed_as("\#{test}", '"\\#{test}"') } test('"$"') { assert_text_parsed_as('$', '"$"') } test('"$t"') { assert_text_parsed_as('$t', '"$t"') } test('"$}"') { assert_text_parsed_as('$}', '"$}"') } test('"\\\\"') { assert_text_parsed_as("\\", '"\\\\"') } test('"\\["') { assert_text_parsed_as("[", '"\\["') } end sub_test_case 'single quoted string' do test("''") { assert_text_parsed_as("", "''") } test("'text'") { assert_text_parsed_as("text", "'text'") } test("'\\''") { assert_text_parsed_as('\'', "'\\''") } test("'\\t'") { assert_text_parsed_as('\t', "'\\t'") } test("'\\n'") { assert_text_parsed_as('\n', "'\\n'") } test("'\\r\\n'") { assert_text_parsed_as('\r\n', "'\\r\\n'") } test("'\\f\\b'") { assert_text_parsed_as('\f\b', "'\\f\\b'") } test("'\\.t'") { assert_text_parsed_as('\.t', "'\\.t'") } test("'\\$t'") { assert_text_parsed_as('\$t', "'\\$t'") } test("'\\#t'") { assert_text_parsed_as('\#t', "'\\#t'") } test("'\\z'") { assert_text_parsed_as('\z', "'\\z'") } test("'\\0'") { assert_text_parsed_as('\0', "'\\0'") } test("'\\1'") { assert_text_parsed_as('\1', "'\\1'") } test("'t") { assert_parse_error("'t") } # non-terminated quoted character test("t'") { assert_text_parsed_as("t'", "t'") } test("'.'") { assert_text_parsed_as('.', "'.'") } test("'*'") { assert_text_parsed_as('*', "'*'") } test("'@'") { assert_text_parsed_as('@', "'@'") } test(%q['#{test}']) { assert_text_parsed_as('#{test}', %q['#{test}']) } test("'$'") { assert_text_parsed_as('$', "'$'") } test("'$t'") { assert_text_parsed_as('$t', "'$t'") } test("'$}'") { assert_text_parsed_as('$}', "'$}'") } test("'\\\\'") { assert_text_parsed_as('\\', "'\\\\'") } test("'\\['") { assert_text_parsed_as('\[', "'\\['") } end sub_test_case 'nonquoted string parsing' do test("''") { assert_text_parsed_as(nil, '') } test('text') { assert_text_parsed_as('text', 'text') } test('\"') { assert_text_parsed_as('\"', '\"') } test('\t') { assert_text_parsed_as('\t', '\t') } test('\n') { assert_text_parsed_as('\n', '\n') } test('\r\n') { assert_text_parsed_as('\r\n', '\r\n') } test('\f\b') { assert_text_parsed_as('\f\b', '\f\b') } test('\.t') { assert_text_parsed_as('\.t', '\.t') } test('\$t') { assert_text_parsed_as('\$t', '\$t') } test('\#t') { assert_text_parsed_as('\#t', '\#t') } test('\z') { assert_text_parsed_as('\z', '\z') } test('\0') { assert_text_parsed_as('\0', '\0') } test('\1') { assert_text_parsed_as('\1', '\1') } test('.') { assert_text_parsed_as('.', '.') } test('*') { assert_text_parsed_as('*', '*') } test('@') { assert_text_parsed_as('@', '@') } test('#{test}') { assert_text_parsed_as('#{test}', '#{test}') } test('$') { assert_text_parsed_as('$', '$') } test('$t') { assert_text_parsed_as('$t', '$t') } test('$}') { assert_text_parsed_as('$}', '$}') } test('\\\\') { assert_text_parsed_as('\\\\', '\\\\') } test('\[') { assert_text_parsed_as('\[', '\[') } test('#foo') { assert_text_parsed_as('#foo', '#foo') } # not comment out test('foo#bar') { assert_text_parsed_as('foo#bar', 'foo#bar') } # not comment out test(' text') { assert_text_parsed_as('text', ' text') } # remove starting spaces test(' #foo') { assert_text_parsed_as('#foo', ' #foo') } # remove starting spaces test('foo #bar') { assert_text_parsed_as('foo', 'foo #bar') } # comment out test('foo\t#bar') { assert_text_parsed_as('foo', "foo\t#bar") } # comment out test('t') { assert_text_parsed_as('t', 't') } test('T') { assert_text_parsed_as('T', 'T') } test('_') { assert_text_parsed_as('_', '_') } test('T1') { assert_text_parsed_as('T1', 'T1') } test('_2') { assert_text_parsed_as('_2', '_2') } test('t0') { assert_text_parsed_as('t0', 't0') } test('t@') { assert_text_parsed_as('t@', 't@') } test('t-') { assert_text_parsed_as('t-', 't-') } test('t.') { assert_text_parsed_as('t.', 't.') } test('t+') { assert_text_parsed_as('t+', 't+') } test('t/') { assert_text_parsed_as('t/', 't/') } test('t=') { assert_text_parsed_as('t=', 't=') } test('t,') { assert_text_parsed_as('t,', 't,') } test('0t') { assert_text_parsed_as('0t', "0t") } test('@1t') { assert_text_parsed_as('@1t', '@1t') } test('-1t') { assert_text_parsed_as('-1t', '-1t') } test('.1t') { assert_text_parsed_as('.1t', '.1t') } test(',1t') { assert_text_parsed_as(',1t', ',1t') } test('.t') { assert_text_parsed_as('.t', '.t') } test('*t') { assert_text_parsed_as('*t', '*t') } test('@t') { assert_text_parsed_as('@t', '@t') } test('{t') { assert_parse_error('{t') } # '{' begins map test('t{') { assert_text_parsed_as('t{', 't{') } test('}t') { assert_text_parsed_as('}t', '}t') } test('[t') { assert_parse_error('[t') } # '[' begins array test('t[') { assert_text_parsed_as('t[', 't[') } test(']t') { assert_text_parsed_as(']t', ']t') } test('t:') { assert_text_parsed_as('t:', 't:') } test('t;') { assert_text_parsed_as('t;', 't;') } test('t?') { assert_text_parsed_as('t?', 't?') } test('t^') { assert_text_parsed_as('t^', 't^') } test('t`') { assert_text_parsed_as('t`', 't`') } test('t~') { assert_text_parsed_as('t~', 't~') } test('t|') { assert_text_parsed_as('t|', 't|') } test('t>') { assert_text_parsed_as('t>', 't>') } test('t<') { assert_text_parsed_as('t<', 't<') } test('t(') { assert_text_parsed_as('t(', 't(') } end sub_test_case 'embedded ruby code parsing' do test('"#{v1}"') { assert_text_parsed_as("#{v1}", '"#{v1}"') } test('"#{v2}"') { assert_text_parsed_as("#{v2}", '"#{v2}"') } test('"#{v3}"') { assert_text_parsed_as("#{v3}", '"#{v3}"') } test('"#{1+1}"') { assert_text_parsed_as("2", '"#{1+1}"') } test('"#{}"') { assert_text_parsed_as("", '"#{}"') } test('"t#{v1}"') { assert_text_parsed_as("t#{v1}", '"t#{v1}"') } test('"t#{v1}t"') { assert_text_parsed_as("t#{v1}t", '"t#{v1}t"') } test('"#{"}"}"') { assert_text_parsed_as("}", '"#{"}"}"') } test('"#{#}"') { assert_parse_error('"#{#}"') } # error in embedded ruby code test("\"\#{\n=begin\n}\"") { assert_parse_error("\"\#{\n=begin\n}\"") } # error in embedded ruby code test('"#{v1}foo#{v2}"') { assert_text_parsed_as("#{v1}foo#{v2}", '"#{v1}foo#{v2}"') } test('"#{1+1}foo#{2+2}bar"') { assert_text_parsed_as("#{1+1}foo#{2+2}bar", '"#{1+1}foo#{2+2}bar"') } test('"foo#{hostname}"') { assert_text_parsed_as("foo#{Socket.gethostname}", '"foo#{hostname}"') } test('"foo#{worker_id}"') { ENV.delete('SERVERENGINE_WORKER_ID') assert_text_parsed_as("foo", '"foo#{worker_id}"') ENV['SERVERENGINE_WORKER_ID'] = '1' assert_text_parsed_as("foo1", '"foo#{worker_id}"') ENV.delete('SERVERENGINE_WORKER_ID') } test('nil') { assert_text_parsed_as(nil, '"#{raise SetNil}"') } test('default') { assert_text_parsed_as(:default, '"#{raise SetDefault}"') } test('nil helper') { assert_text_parsed_as(nil, '"#{use_nil}"') } test('default helper') { assert_text_parsed_as(:default, '"#{use_default}"') } end sub_test_case 'array parsing' do test('[]') { assert_text_parsed_as_json([], '[]') } test('[1]') { assert_text_parsed_as_json([1], '[1]') } test('[1,2]') { assert_text_parsed_as_json([1,2], '[1,2]') } test('[1, 2]') { assert_text_parsed_as_json([1,2], '[1, 2]') } test('[ 1 , 2 ]') { assert_text_parsed_as_json([1,2], '[ 1 , 2 ]') } test('[1,2,]') { assert_parse_error('[1,2,]') } # TODO: Need trailing commas support? test("[\n1\n,\n2\n]") { assert_text_parsed_as_json([1,2], "[\n1\n,\n2\n]") } test('["a"]') { assert_text_parsed_as_json(["a"], '["a"]') } test('["a","b"]') { assert_text_parsed_as_json(["a","b"], '["a","b"]') } test('[ "a" , "b" ]') { assert_text_parsed_as_json(["a","b"], '[ "a" , "b" ]') } test("[\n\"a\"\n,\n\"b\"\n]") { assert_text_parsed_as_json(["a","b"], "[\n\"a\"\n,\n\"b\"\n]") } test('["ab","cd"]') { assert_text_parsed_as_json(["ab","cd"], '["ab","cd"]') } test('["a","#{v1}"') { assert_text_parsed_as_json(["a","#{v1}"], '["a","#{v1}"]') } test('["a","#{v1}","#{v2}"]') { assert_text_parsed_as_json(["a","#{v1}","#{v2}"], '["a","#{v1}","#{v2}"]') } test('["a","#{v1} #{v2}"]') { assert_text_parsed_as_json(["a","#{v1} #{v2}"], '["a","#{v1} #{v2}"]') } test('["a","#{hostname}"]') { assert_text_parsed_as_json(["a","#{Socket.gethostname}"], '["a","#{hostname}"]') } test('["a","foo#{worker_id}"]') { ENV.delete('SERVERENGINE_WORKER_ID') assert_text_parsed_as('["a","foo"]', '["a","foo#{worker_id}"]') ENV['SERVERENGINE_WORKER_ID'] = '1' assert_text_parsed_as('["a","foo1"]', '["a","foo#{worker_id}"]') ENV.delete('SERVERENGINE_WORKER_ID') } json_array_with_js_comment = <1}, '{"a":1}') } test('{"a":1,"b":2}') { assert_text_parsed_as_json({"a"=>1,"b"=>2}, '{"a":1,"b":2}') } test('{ "a" : 1 , "b" : 2 }') { assert_text_parsed_as_json({"a"=>1,"b"=>2}, '{ "a" : 1 , "b" : 2 }') } test('{"a":1,"b":2,}') { assert_parse_error('{"a":1,"b":2,}') } # TODO: Need trailing commas support? test('{\n\"a\"\n:\n1\n,\n\"b\"\n:\n2\n}') { assert_text_parsed_as_json({"a"=>1,"b"=>2}, "{\n\"a\"\n:\n1\n,\n\"b\"\n:\n2\n}") } test('{"a":"b"}') { assert_text_parsed_as_json({"a"=>"b"}, '{"a":"b"}') } test('{"a":"b","c":"d"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, '{"a":"b","c":"d"}') } test('{ "a" : "b" , "c" : "d" }') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, '{ "a" : "b" , "c" : "d" }') } test('{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, "{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}") } test('{"a":"b","c":"#{v1}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"#{v1}"}, '{"a":"b","c":"#{v1}"}') } test('{"a":"b","#{v1}":"d"}') { assert_text_parsed_as_json({"a"=>"b","#{v1}"=>"d"}, '{"a":"b","#{v1}":"d"}') } test('{"a":"#{v1}","c":"#{v2}"}') { assert_text_parsed_as_json({"a"=>"#{v1}","c"=>"#{v2}"}, '{"a":"#{v1}","c":"#{v2}"}') } test('{"a":"b","c":"d #{v1} #{v2}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d #{v1} #{v2}"}, '{"a":"b","c":"d #{v1} #{v2}"}') } test('{"a":"#{hostname}"}') { assert_text_parsed_as_json({"a"=>"#{Socket.gethostname}"}, '{"a":"#{hostname}"}') } test('{"a":"foo#{worker_id}"}') { ENV.delete('SERVERENGINE_WORKER_ID') assert_text_parsed_as('{"a":"foo"}', '{"a":"foo#{worker_id}"}') ENV['SERVERENGINE_WORKER_ID'] = '1' assert_text_parsed_as('{"a":"foo1"}', '{"a":"foo#{worker_id}"}') ENV.delete('SERVERENGINE_WORKER_ID') } test('no quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '{"a":"b","c":"#{v1}"}') } test('single quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'#{v1}'}, '\'{"a":"b","c":"#{v1}"}\'') } test('double quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '"{\"a\":\"b\",\"c\":\"#{v1}\"}"') } json_hash_with_comment = <1,"b"=>2,"c"=>3}, json_hash_with_comment) } end end end ================================================ FILE: test/config/test_plugin_configuration.rb ================================================ require_relative '../helper' require 'fluent/plugin/input' require 'fluent/test/driver/input' module ConfigurationForPlugins class AllBooleanParams < Fluent::Plugin::Input config_param :flag1, :bool, default: true config_param :flag2, :bool, default: true config_param :flag3, :bool, default: false config_param :flag4, :bool, default: false config_section :child, param_name: :children, multi: true, required: true do config_param :flag1, :bool, default: true config_param :flag2, :bool, default: true config_param :flag3, :bool, default: false config_param :flag4, :bool, default: false end end class BooleanParamsWithoutValue < ::Test::Unit::TestCase CONFIG = < flag1 flag2 # yaaaaaaaaaay flag3 flag4 # yaaaaaaaaaay flag1 # yaaaaaaaaaay flag2 flag3 # yaaaaaaaaaay flag4 # with following whitespace flag1 flag2 flag3 flag4 CONFIG test 'create plugin via driver' do d = Fluent::Test::Driver::Input.new(AllBooleanParams) d.configure(CONFIG) assert_equal([true] * 4, [d.instance.flag1, d.instance.flag2, d.instance.flag3, d.instance.flag4]) num_of_sections = 3 assert_equal num_of_sections, d.instance.children.size assert_equal([true] * (num_of_sections * 4), d.instance.children.map{|c| [c.flag1, c.flag2, c.flag3, c.flag4]}.flatten) end end end ================================================ FILE: test/config/test_section.rb ================================================ require_relative '../helper' require 'fluent/config/section' require 'pp' module Fluent::Config class TestSection < ::Test::Unit::TestCase sub_test_case Fluent::Config::Section do sub_test_case 'class' do sub_test_case '.name' do test 'returns its full module name as String' do assert_equal('Fluent::Config::Section', Fluent::Config::Section.name) end end end sub_test_case 'instance object' do sub_test_case '#initialize' do test 'creates blank object without argument' do s = Fluent::Config::Section.new assert_equal({}, s.instance_eval{ @params }) end test 'creates object which contains specified hash object itself' do hash = { name: 'tagomoris', age: 34, send: 'email', klass: 'normal', keys: 5, } s1 = Fluent::Config::Section.new(hash) assert_equal(hash, s1.instance_eval { @params }) assert_equal("tagomoris", s1[:name]) assert_equal(34, s1[:age]) assert_equal("email", s1[:send]) assert_equal("normal", s1[:klass]) assert_equal(5, s1[:keys]) assert_equal("tagomoris", s1.name) assert_equal(34, s1.age) assert_equal("email", s1.send) assert_equal("normal", s1.klass) assert_equal(5, s1.keys) end test 'creates object which contains specified hash object itself, including fields with at prefix' do hash = { name: 'tagomoris', age: 34, send: 'email', klass: 'normal', keys: 5, } hash['@id'.to_sym] = 'myid' s1 = Fluent::Config::Section.new(hash) assert_equal('myid', s1['@id']) assert_equal('myid', s1['@id'.to_sym]) assert_equal('myid', s1.__send__('@id'.to_sym)) end test 'creates object and config element which corresponds to section object itself' do hash = { name: 'tagomoris', age: 34, send: 'email', klass: 'normal', keys: 5, } hash['@id'.to_sym] = 'myid' conf = config_element('section', '', {'name' => 'tagomoris', 'age' => 34, 'send' => 'email', 'klass' => 'normal', 'keys' => 5}) s2 = Fluent::Config::Section.new(hash, conf) assert s2.corresponding_config_element.is_a?(Fluent::Config::Element) end end sub_test_case '#class' do test 'returns class constant' do assert_equal Fluent::Config::Section, Fluent::Config::Section.new({}).class end end sub_test_case '#object_id' do test 'returns its object id' do s1 = Fluent::Config::Section.new({}) assert s1.object_id s2 = Fluent::Config::Section.new({}) assert s2.object_id assert_not_equal s1.object_id, s2.object_id end end sub_test_case '#to_h' do test 'returns internal hash itself' do hash = { name: 'tagomoris', age: 34, send: 'email', klass: 'normal', keys: 5, } s = Fluent::Config::Section.new(hash) assert_equal(hash, s.to_h) assert_instance_of(Hash, s.to_h) end end sub_test_case '#instance_of?' do test 'can judge whether it is a Section object or not' do s = Fluent::Config::Section.new assert_true(s.instance_of?(Fluent::Config::Section)) assert_false(s.instance_of?(BasicObject)) end end sub_test_case '#is_a?' do test 'can judge whether it belongs to or not' do s = Fluent::Config::Section.new assert_true(s.is_a?(Fluent::Config::Section)) assert_true(s.kind_of?(Fluent::Config::Section)) assert_true(s.is_a?(BasicObject)) end end sub_test_case '#+' do test 'can merge 2 sections: argument side is primary, internal hash is newly created' do h1 = {name: "s1", num: 10, klass: "A"} s1 = Fluent::Config::Section.new(h1) h2 = {name: "s2", klass: "A", num2: "5", num3: "8"} s2 = Fluent::Config::Section.new(h2) s = s1 + s2 assert_not_equal(h1.object_id, s.to_h.object_id) assert_not_equal(h2.object_id, s.to_h.object_id) assert_equal("s2", s.name) assert_equal(10, s.num) assert_equal("A", s.klass) assert_equal("5", s.num2) assert_equal("8", s.num3) end end sub_test_case '#to_s' do test '#to_s == #inspect' do h1 = {name: "s1", num: 10, klass: "A"} s1 = Fluent::Config::Section.new(h1) assert_equal(s1.to_s, s1.inspect) end end data("inspect" => [:inspect, true], "nil?" => [:nil?, true], "to_h" => [:to_h, true], "+" => [:+, true], "instance_of?" => [:instance_of?, true], "kind_of?" => [:kind_of?, true], "[]" => [:[], true], "respond_to?" => [:respond_to?, true], "respond_to_missing?" => [:respond_to_missing?, true], "!" => [:!, true], "!=" => [:!=, true], "==" => [:==, true], "equal?" => [:equal?, true], "instance_eval" => [:instance_eval, true], "instance_exec" => [:instance_exec, true], "method_missing" => [:method_missing, false], "singleton_method_added" => [:singleton_method_added, false], "singleton_method_removed" => [:singleton_method_removed, false], "singleton_method_undefined" => [:singleton_method_undefined, false], "no_such_method" => [:no_such_method, false]) test '#respond_to?' do |data| method, expected = data h1 = {name: "s1", num: 10, klass: "A"} s1 = Fluent::Config::Section.new(h1) assert_equal(expected, s1.respond_to?(method)) end test '#pretty_print' do q = PP.new h1 = {name: "s1", klass: "A"} s1 = Fluent::Config::Section.new(h1) s1.pretty_print(q) assert_equal s1.inspect, q.output end end end end end ================================================ FILE: test/config/test_system_config.rb ================================================ require_relative '../helper' require 'fluent/configurable' require 'fluent/config/element' require 'fluent/config/section' require 'fluent/system_config' module Fluent::Config class FakeSupervisor attr_writer :log_level def initialize(**opt) @system_config = nil @cl_opt = { workers: nil, restart_worker_interval: nil, root_dir: nil, log_level: Fluent::Log::LEVEL_INFO, suppress_interval: nil, suppress_config_dump: nil, suppress_repeated_stacktrace: nil, log_event_label: nil, log_event_verbose: nil, without_source: nil, with_source_only: nil, enable_input_metrics: nil, enable_size_metrics: nil, emit_error_log_interval: nil, file_permission: nil, dir_permission: nil, }.merge(opt) end def for_system_config opt = {} # this is copy from Supervisor#build_system_config Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param| if @cl_opt.key?(param) && !@cl_opt[param].nil? if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO # info level can't be specified via command line option. # log_level is info here, it is default value and 's log_level should be applied if exists. next end opt[param] = @cl_opt[param] end end opt end end class TestSystemConfig < ::Test::Unit::TestCase TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/system_config/#{ENV['TEST_ENV_NUMBER']}") def parse_text(text) basepath = File.expand_path(File.dirname(__FILE__) + '/../../') Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' } end test 'should not override default configurations when no parameters' do conf = parse_text(<<-EOS) EOS s = FakeSupervisor.new sc = Fluent::SystemConfig.new(conf) sc.overwrite_variables(**s.for_system_config) assert_equal(1, sc.workers) assert_equal(0, sc.restart_worker_interval) assert_nil(sc.root_dir) assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level) assert_nil(sc.suppress_repeated_stacktrace) assert_nil(sc.ignore_repeated_log_interval) assert_nil(sc.emit_error_log_interval) assert_nil(sc.suppress_config_dump) assert_nil(sc.without_source) assert_nil(sc.with_source_only) assert_true(sc.enable_input_metrics) assert_nil(sc.enable_size_metrics) assert_nil(sc.enable_msgpack_time_support) assert(!sc.enable_jit) assert_nil(sc.log.path) assert_equal(:text, sc.log.format) assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format) assert_equal(:none, sc.log.forced_stacktrace_level) end data( 'workers' => ['workers', 3], 'restart_worker_interval' => ['restart_worker_interval', 60], 'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')], 'log_level' => ['log_level', 'error'], 'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true], 'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10], 'log_event_verbose' => ['log_event_verbose', true], 'suppress_config_dump' => ['suppress_config_dump', true], 'without_source' => ['without_source', true], 'with_source_only' => ['with_source_only', true], 'strict_config_value' => ['strict_config_value', true], 'enable_msgpack_time_support' => ['enable_msgpack_time_support', true], 'enable_input_metrics' => ['enable_input_metrics', false], 'enable_size_metrics' => ['enable_size_metrics', true], 'enable_jit' => ['enable_jit', true], ) test "accepts parameters" do |(k, v)| conf = parse_text(<<-EOS) #{k} #{v} EOS s = FakeSupervisor.new sc = Fluent::SystemConfig.new(conf) sc.overwrite_variables(**s.for_system_config) if k == 'log_level' assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k)) else assert_equal(v, sc.__send__(k)) end end test "log parameters" do conf = parse_text(<<-EOS) path /tmp/fluentd.log format json time_format %Y forced_stacktrace_level info EOS s = FakeSupervisor.new sc = Fluent::SystemConfig.new(conf) sc.overwrite_variables(**s.for_system_config) assert_equal('/tmp/fluentd.log', sc.log.path) assert_equal(:json, sc.log.format) assert_equal('%Y', sc.log.time_format) assert_equal(Fluent::Log::LEVEL_INFO, sc.log.forced_stacktrace_level) end # info is removed because info level can't be specified via command line data('trace' => Fluent::Log::LEVEL_TRACE, 'debug' => Fluent::Log::LEVEL_DEBUG, 'warn' => Fluent::Log::LEVEL_WARN, 'error' => Fluent::Log::LEVEL_ERROR, 'fatal' => Fluent::Log::LEVEL_FATAL) test 'log_level is ignored when log_level related command line option is passed' do |level| conf = parse_text(<<-EOS) log_level info EOS s = FakeSupervisor.new(log_level: level) sc = Fluent::SystemConfig.new(conf) sc.overwrite_variables(**s.for_system_config) assert_equal(level, sc.log_level) end sub_test_case "log rotation" do data('daily' => "daily", 'weekly' => 'weekly', 'monthly' => 'monthly') test "strings for rotate_age" do |age| conf = parse_text(<<-EOS) rotate_age #{age} EOS sc = Fluent::SystemConfig.new(conf) assert_equal(age, sc.log.rotate_age) end test "numeric number for rotate age" do conf = parse_text(<<-EOS) rotate_age 3 EOS sc = Fluent::SystemConfig.new(conf) assert_equal(3, sc.log.rotate_age) end data(h: ['100', 100], k: ['1k', 1024], m: ['1m', 1024 * 1024], g: ['1g', 1024 * 1024 * 1024]) test "numeric and SI prefix for rotate_size" do |(label, size)| conf = parse_text(<<-EOS) rotate_size #{label} EOS sc = Fluent::SystemConfig.new(conf) assert_equal(size, sc.log.rotate_size) end end test "source-only-buffer parameters" do conf = parse_text(<<~EOS) flush_thread_count 4 overflow_action throw_exception path /tmp/source-only-buffer flush_interval 1 chunk_limit_size 100 total_limit_size 1000 compress gzip EOS s = FakeSupervisor.new sc = Fluent::SystemConfig.new(conf) sc.overwrite_variables(**s.for_system_config) assert_equal( [ 4, :throw_exception, "/tmp/source-only-buffer", 1, 100, 1000, :gzip, ], [ sc.source_only_buffer.flush_thread_count, sc.source_only_buffer.overflow_action, sc.source_only_buffer.path, sc.source_only_buffer.flush_interval, sc.source_only_buffer.chunk_limit_size, sc.source_only_buffer.total_limit_size, sc.source_only_buffer.compress, ] ) end end end ================================================ FILE: test/config/test_types.rb ================================================ require 'helper' require 'fluent/config/types' class TestConfigTypes < ::Test::Unit::TestCase include Fluent sub_test_case 'Config.size_value' do data("2k" => [2048, "2k"], "2K" => [2048, "2K"], "3m" => [3145728, "3m"], "3M" => [3145728, "3M"], "4g" => [4294967296, "4g"], "4G" => [4294967296, "4G"], "5t" => [5497558138880, "5t"], "5T" => [5497558138880, "5T"], "6" => [6, "6"]) test 'normal case' do |(expected, val)| assert_equal(expected, Config.size_value(val)) assert_equal(expected, Config.size_value(val, { strict: true })) end data("integer" => [6, 6], "hoge" => [0, "hoge"], "empty" => [0, ""]) test 'not assumed case' do |(expected, val)| assert_equal(expected, Config.size_value(val)) end test 'nil' do assert_equal(nil, Config.size_value(nil)) end data("integer" => [6, 6], "hoge" => [Fluent::ConfigError.new('name1: invalid value for Integer(): "hoge"'), "hoge"], "empty" => [Fluent::ConfigError.new('name1: invalid value for Integer(): ""'), ""]) test 'not assumed case with strict' do |(expected, val)| if expected.kind_of? Exception assert_raise(expected) do Config.size_value(val, { strict: true }, "name1") end else assert_equal(expected, Config.size_value(val, { strict: true }, "name1")) end end test 'nil with strict' do assert_equal(nil, Config.size_value(nil, { strict: true })) end end sub_test_case 'Config.time_value' do data("10s" => [10, "10s"], "10sec" => [10, "10sec"], "2m" => [120, "2m"], "3h" => [10800, "3h"], "4d" => [345600, "4d"]) test 'normal case' do |(expected, val)| assert_equal(expected, Config.time_value(val)) assert_equal(expected, Config.time_value(val, { strict: true })) end data("integer" => [4.0, 4], "float" => [0.4, 0.4], "hoge" => [0.0, "hoge"], "empty" => [0.0, ""]) test 'not assumed case' do |(expected, val)| assert_equal(expected, Config.time_value(val)) end test 'nil' do assert_equal(nil, Config.time_value(nil)) end data("integer" => [6, 6], "hoge" => [Fluent::ConfigError.new('name1: invalid value for Float(): "hoge"'), "hoge"], "empty" => [Fluent::ConfigError.new('name1: invalid value for Float(): ""'), ""]) test 'not assumed case with strict' do |(expected, val)| if expected.kind_of? Exception assert_raise(expected) do Config.time_value(val, { strict: true }, "name1") end else assert_equal(expected, Config.time_value(val, { strict: true }, "name1")) end end test 'nil with strict' do assert_equal(nil, Config.time_value(nil, { strict: true })) end end sub_test_case 'Config.bool_value' do data("true" => [true, "true"], "yes" => [true, "yes"], "empty" => [true, ""], "false" => [false, "false"], "no" => [false, "no"]) test 'normal case' do |(expected, val)| assert_equal(expected, Config.bool_value(val)) end data("true" => [true, true], "false" => [false, false], "hoge" => [nil, "hoge"], "nil" => [nil, nil], "integer" => [nil, 10]) test 'not assumed case' do |(expected, val)| assert_equal(expected, Config.bool_value(val)) end data("true" => [true, true], "false" => [false, false], "hoge" => [Fluent::ConfigError.new("name1: invalid bool value: hoge"), "hoge"], "nil" => [nil, nil], "integer" => [Fluent::ConfigError.new("name1: invalid bool value: 10"), 10]) test 'not assumed case with strict' do |(expected, val)| if expected.kind_of? Exception assert_raise(expected) do Config.bool_value(val, { strict: true }, "name1") end else assert_equal(expected, Config.bool_value(val, { strict: true }, "name1")) end end end sub_test_case 'Config.regexp_value' do data("empty" => [//, "//"], "plain" => [/regexp/, "/regexp/"], "zero width" => [/^$/, "/^$/"], "character classes" => [/[a-z]/, "/[a-z]/"], "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/']) test 'normal case' do |(expected, str)| assert_equal(expected, Config.regexp_value(str)) end data("empty" => [//, ""], "plain" => [/regexp/, "regexp"], "zero width" => [/^$/, "^$"], "character classes" => [/[a-z]/, "[a-z]"], "meta charactersx" => [/.+.*?\d\w\s\S/, '.+.*?\d\w\s\S']) test 'w/o slashes' do |(expected, str)| assert_equal(expected, Config.regexp_value(str)) end data("missing right slash" => "/regexp", "too many options" => "/regexp/imx",) test 'invalid regexp' do |(str)| assert_raise(Fluent::ConfigError.new("invalid regexp: missing right slash: #{str}")) do Config.regexp_value(str) end end test 'nil' do assert_equal nil, Config.regexp_value(nil) end end sub_test_case 'type converters for config_param definitions' do data("test" => ['test', 'test'], "1" => ['1', '1'], "spaces" => [' ', ' ']) test 'string' do |(expected, val)| assert_equal expected, Config::STRING_TYPE.call(val, {}) assert_equal Encoding::UTF_8, Config::STRING_TYPE.call(val, {}).encoding end test 'string nil' do assert_equal nil, Config::STRING_TYPE.call(nil, {}) end data('latin' => 'Märch', 'ascii' => 'ascii', 'space' => ' ', 'number' => '1', 'Hiragana' => 'あいうえお') test 'string w/ binary' do |str| actual = Config::STRING_TYPE.call(str.b, {}) assert_equal str, actual assert_equal Encoding::UTF_8, actual.encoding end data('starts_with_semicolon' => [:conor, ':conor'], 'simple_string' => [:conor, 'conor'], 'empty_string' => [nil, '']) test 'symbol' do |(expected, val)| assert_equal Config::SYMBOL_TYPE.call(val, {}), expected end data("val" => [:val, 'val'], "v" => [:v, 'v'], "value" => [:value, 'value']) test 'enum' do |(expected, val)| assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]}) end test 'enum: pick unknown choice' do assert_raises(Fluent::ConfigError.new("valid options are val,value,v but got x")) do Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) end end data("empty list" => {}, "string list" => {list: ["val", "value", "v"]}) test 'enum: invalid choices' do | list | assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")) do Config::ENUM_TYPE.call('val', list) end end test 'enum: nil' do assert_equal nil, Config::ENUM_TYPE.call(nil) end data("1" => [1, '1'], "1.0" => [1, '1.0'], "1_000" => [1000, '1_000'], "1x" => [1, '1x']) test 'integer' do |(expected, val)| assert_equal expected, Config::INTEGER_TYPE.call(val, {}) end data("integer" => [6, 6], "hoge" => [0, "hoge"], "empty" => [0, ""]) test 'integer: not assumed case' do |(expected, val)| assert_equal expected, Config::INTEGER_TYPE.call(val, {}) end test 'integer: nil' do assert_equal nil, Config::INTEGER_TYPE.call(nil, {}) end data("integer" => [6, 6], "hoge" => [Fluent::ConfigError.new('name1: invalid value for Integer(): "hoge"'), "hoge"], "empty" => [Fluent::ConfigError.new('name1: invalid value for Integer(): ""'), ""]) test 'integer: not assumed case with strict' do |(expected, val)| if expected.kind_of? Exception assert_raise(expected) do Config::INTEGER_TYPE.call(val, { strict: true }, "name1") end else assert_equal expected, Config::INTEGER_TYPE.call(val, { strict: true }, "name1") end end test 'integer: nil with strict' do assert_equal nil, Config::INTEGER_TYPE.call(nil, { strict: true }) end data("1" => [1.0, '1'], "1.0" => [1.0, '1.0'], "1.00" => [1.0, '1.00'], "1e0" => [1.0, '1e0']) test 'float' do |(expected, val)| assert_equal expected, Config::FLOAT_TYPE.call(val, {}) end data("integer" => [6, 6], "hoge" => [0, "hoge"], "empty" => [0, ""]) test 'float: not assumed case' do |(expected, val)| assert_equal expected, Config::FLOAT_TYPE.call(val, {}) end test 'float: nil' do assert_equal nil, Config::FLOAT_TYPE.call(nil, {}) end data("integer" => [6, 6], "hoge" => [Fluent::ConfigError.new('name1: invalid value for Float(): "hoge"'), "hoge"], "empty" => [Fluent::ConfigError.new('name1: invalid value for Float(): ""'), ""]) test 'float: not assumed case with strict' do |(expected, val)| if expected.kind_of? Exception assert_raise(expected) do Config::FLOAT_TYPE.call(val, { strict: true }, "name1") end else assert_equal expected, Config::FLOAT_TYPE.call(val, { strict: true }, "name1") end end test 'float: nil with strict' do assert_equal nil, Config::FLOAT_TYPE.call(nil, { strict: true }) end data("1000" => [1000, '1000'], "1k" => [1024, '1k'], "1m" => [1024*1024, '1m']) test 'size' do |(expected, val)| assert_equal expected, Config::SIZE_TYPE.call(val, {}) end data("true" => [true, 'true'], "yes" => [true, 'yes'], "no" => [false, 'no'], "false" => [false, 'false'], "TRUE" => [nil, 'TRUE'], "True" => [nil, 'True'], "Yes" => [nil, 'Yes'], "No" => [nil, 'No'], "empty" => [true, ''], "unexpected_string" => [nil, 'unexpected_string']) test 'bool' do |(expected, val)| assert_equal expected, Config::BOOL_TYPE.call(val, {}) end data("0" => [0, '0'], "1" => [1.0, '1'], "1.01" => [1.01, '1.01'], "1s" => [1, '1s'], "1," => [60, '1m'], "1h" => [3600, '1h'], "1d" => [86400, '1d']) test 'time' do |(expected, val)| assert_equal expected, Config::TIME_TYPE.call(val, {}) end data("empty" => [//, "//"], "plain" => [/regexp/, "/regexp/"], "zero width" => [/^$/, "/^$/"], "character classes" => [/[a-z]/, "/[a-z]/"], "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/']) test 'regexp' do |(expected, str)| assert_equal(expected, Config::REGEXP_TYPE.call(str, {})) end data("string and integer" => [{"x"=>"v","k"=>1}, '{"x":"v","k":1}', {}], "strings" => [{"x"=>"v","k"=>"1"}, 'x:v,k:1', {}], "w/ space" => [{"x"=>"v","k"=>"1"}, 'x:v, k:1', {}], "heading space" => [{"x"=>"v","k"=>"1"}, ' x:v, k:1 ', {}], "trailing space" => [{"x"=>"v","k"=>"1"}, 'x:v , k:1 ', {}], "multiple colons" => [{"x"=>"v:v","k"=>"1"}, 'x:v:v, k:1', {}], "symbolize keys" => [{x: "v", k: 1}, '{"x":"v","k":1}', {symbolize_keys: true}], "value_type: :string" => [{x: "v", k: "1"}, 'x:v,k:1', {symbolize_keys: true, value_type: :string}], "value_type: :string 2" => [{x: "v", k: "1"}, '{"x":"v","k":1}', {symbolize_keys: true, value_type: :string}], "value_type: :integer" => [{x: 0, k: 1}, 'x:0,k:1', {symbolize_keys: true, value_type: :integer}], "time 1" => [{"x"=>1,"y"=>60,"z"=>3600}, '{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}], "time 2" => [{"x"=>1,"y"=>60,"z"=>3600}, 'x:1s,y:1m,z:1h', {value_type: :time}]) test 'hash' do |(expected, val, opts)| assert_equal(expected, Config::HASH_TYPE.call(val, opts)) end test 'hash w/ unknown type' do assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")) do Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) end end test 'hash w/ strict option' do assert_raise(Fluent::ConfigError.new('y: invalid value for Integer(): "hoge"')) do Config::HASH_TYPE.call("x:1,y:hoge", {value_type: :integer, strict: true}) end end data('latin' => ['3:Märch', {"3"=>"Märch"}], 'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}], 'number' => ['number:1', {"number"=>"1"}], 'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}]) test 'hash w/ binary' do |(target, expected)| assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string })) end test 'hash w/ nil' do assert_equal(nil, Config::HASH_TYPE.call(nil)) end data("strings and integer" => [["1","2",1], '["1","2",1]', {}], "number strings" => [["1","2","1"], '1,2,1', {}], "alphabets" => [["a","b","c"], '["a","b","c"]', {}], "alphabets w/o quote" => [["a","b","c"], 'a,b,c', {}], "w/ spaces" => [["a","b","c"], 'a, b, c', {}], "w/ space before comma" => [["a","b","c"], 'a , b , c', {}], "comma or space w/ qupte" => [["a a","b,b"," c "], '["a a","b,b"," c "]', {}], "space in a value w/o qupte" => [["a a","b","c"], 'a a,b,c', {}], "integers" => [[1,2,1], '[1,2,1]', {}], "value_type: :integer w/ quote" => [[1,2,1], '["1","2","1"]', {value_type: :integer}], "value_type: :integer w/o quote" => [[1,2,1], '1,2,1', {value_type: :integer}]) test 'array' do |(expected, val, opts)| assert_equal(expected, Config::ARRAY_TYPE.call(val, opts)) end data('["1","2"]' => [["1","2"], '["1","2"]'], '["3"]' => [["3"], '["3"]']) test 'array w/ default values' do |(expected, val)| array_options = { default: [], } assert_equal(expected, Config::ARRAY_TYPE.call(val, array_options)) end test 'array w/ unknown type' do assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")) do Config::ARRAY_TYPE.call("1,2", {value_type: :foo}) end end test 'array w/ strict option' do assert_raise(Fluent::ConfigError.new(': invalid value for Integer(): "hoge"')) do Config::ARRAY_TYPE.call("1,hoge", {value_type: :integer, strict: true}, "name1") end end test 'array w/ nil' do assert_equal(nil, Config::ARRAY_TYPE.call(nil)) end end end ================================================ FILE: test/config/test_yaml_parser.rb ================================================ require 'helper' require 'fluent/config/yaml_parser' require 'socket' require 'json' require 'date' class YamlParserTest < Test::Unit::TestCase TMP_DIR = File.dirname(__FILE__) + "/tmp/yaml_config#{ENV['TEST_ENV_NUMBER']}" def write_config(path, data, encoding: 'utf-8') FileUtils.mkdir_p(File.dirname(path)) File.open(path, "w:#{encoding}:utf-8") {|f| f.write data } end sub_test_case 'Special YAML elements' do def test_special_yaml_elements_dollar write_config "#{TMP_DIR}/test_special_yaml_elements_dollar_source.yaml", <<~EOS config: - source: $type: dummy_type $label: dummy_label $id: dummy_id $log_level: debug $unknown: unknown EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_special_yaml_elements_dollar_source.yaml") assert_equal('source', config.elements[0].name) assert_equal('dummy_type', config.elements[0]['@type']) assert_equal('dummy_label', config.elements[0]['@label']) assert_equal('dummy_id', config.elements[0]['@id']) assert_equal('debug', config.elements[0]['@log_level']) assert_nil(config.elements[0]['@unknown']) write_config "#{TMP_DIR}/test_special_yaml_elements_dollar_match.yaml", <<~EOS config: - match: $tag: dummy_tag EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_special_yaml_elements_dollar_match.yaml") assert_equal('match', config.elements[0].name) assert_equal('dummy_tag', config.elements[0].arg) write_config "#{TMP_DIR}/test_special_yaml_elements_dollar_worker.yaml", <<~EOS config: - worker: $arg: dummy_arg config: - source: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_special_yaml_elements_dollar_worker.yaml") assert_equal('worker', config.elements[0].name) assert_equal('dummy_arg', config.elements[0].arg) end def test_embedded_ruby_code write_config "#{TMP_DIR}/test_embedded_ruby_code.yaml", <<~EOS config: - source: host: !fluent/s "#{Socket.gethostname}" EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_embedded_ruby_code.yaml") assert_equal(Socket.gethostname, config.elements[0]['host']) end def test_fluent_json_format write_config "#{TMP_DIR}/test_fluent_json_format.yaml", <<~EOS config: - source: hash_param: !fluent/json { "k": "v", "k1": 10 } EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_fluent_json_format.yaml") assert_equal({'k': 'v', 'k1': 10}.to_json, config.elements[0]['hash_param']) end end sub_test_case 'root elements' do def test_root_elements write_config "#{TMP_DIR}/test_root_elements.yaml", <<~EOS config: - source: $type: dummy system: dummy: dummy unknown: dummy: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_root_elements.yaml") assert_equal(2, config.elements.size) # the first element is system section. assert_equal("system", config.elements[0].name) # the second element is source in config section. assert_equal("source", config.elements[1].name) end def test_root_elements_only_config_section write_config "#{TMP_DIR}/test_root_elements_only_config_section.yaml", <<~EOS config: - source: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_root_elements_only_config_section.yaml") assert_equal(1, config.elements.size) assert_equal("source", config.elements[0].name) end def test_root_elements_only_system_section write_config "#{TMP_DIR}/test_root_elements_only_system_section.yaml", <<~EOS system: dummy: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_root_elements_only_system_section.yaml") assert_equal(1, config.elements.size) assert_equal("system", config.elements[0].name) end end sub_test_case 'config section' do def test_config_section_directives write_config "#{TMP_DIR}/dummy.yaml", <<~EOS - filter: $type: dummy EOS write_config "#{TMP_DIR}/test_config_section_directives.yaml", <<~EOS config: - source: $type: dummy - filter: $type: dummy - match: $tag: dummy - worker: $arg: 0 config: - source: $type: dummy - label: $name: dummy config: - filter: $type: dummy - !include dummy.yaml EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_config_section_directives.yaml") assert_equal(6, config.elements.size) assert_equal(%w(source filter match worker label filter), config.elements.map(&:name)) end def test_config_section_unknown_directives write_config "#{TMP_DIR}/test_config_section_unknown_directives.yaml", <<~EOS config: - source: $type: dummy - unknown: $type: dummy EOS # TODO: it should raise Fluent::ConfigError instead of NoMethodError, or drop unknown directives assert_raise(NoMethodError) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_config_section_unknown_directives.yaml") end end sub_test_case 'label' do def test_label_section write_config "#{TMP_DIR}/test_label_section.yaml", <<~EOS config: - label: $name: dummy_label config: - filter: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_label_section.yaml") assert_equal('label', config.elements[0].name) assert_equal('dummy_label', config.elements[0].arg) assert_equal('filter', config.elements[0].elements[0].name) end def test_label_section_missing_name write_config "#{TMP_DIR}/test_label_section_missing_name.yaml", <<~EOS config: - label: config: - filter: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_label_section_missing_name.yaml") assert_equal('label', config.elements[0].name) assert_equal('', config.elements[0].arg) assert_equal('filter', config.elements[0].elements[0].name) end def test_label_section_missing_config write_config "#{TMP_DIR}/test_label_section_missing_config.yaml", <<~EOS config: - label: $name: dummy_label EOS # TODO: it should raise Fluent::ConfigError instead of NoMethodError assert_raise(NoMethodError) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_label_section_missing_config.yaml") end end end sub_test_case 'worker' do def test_worker_section write_config "#{TMP_DIR}/test_worker_section.yaml", <<~EOS config: - worker: $arg: 0 config: - source: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_worker_section.yaml") assert_equal('worker', config.elements[0].name) assert_equal('0', config.elements[0].arg) assert_equal('source', config.elements[0].elements[0].name) end def test_worker_section_missing_arg write_config "#{TMP_DIR}/test_worker_section_missing_arg.yaml", <<~EOS config: - worker: config: - source: $type: dummy EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_worker_section_missing_arg.yaml") assert_equal('worker', config.elements[0].name) assert_equal('', config.elements[0].arg) assert_equal('source', config.elements[0].elements[0].name) end def test_worker_section_missing_config write_config "#{TMP_DIR}/test_worker_section_missing_config.yaml", <<~EOS config: - worker: $arg: 0 EOS # TODO: it should raise Fluent::ConfigError instead of NoMethodError assert_raise(NoMethodError) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_worker_section_missing_config.yaml") end end end sub_test_case 'source' do def test_source_section write_config "#{TMP_DIR}/test_source_section.yaml", <<~EOS config: - source: $type: dummy_type port: 8888 EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_source_section.yaml") assert_equal('source', config.elements[0].name) assert_equal('dummy_type', config.elements[0]['@type']) assert_equal(8888, config.elements[0]['port']) end def test_source_section_missing_type write_config "#{TMP_DIR}/test_source_section_missing_type.yaml", <<~EOS config: - source: port: 8888 EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_source_section_missing_type.yaml") assert_equal('source', config.elements[0].name) assert_equal(8888, config.elements[0]['port']) assert_nil(config.elements[0]['@type']) end end sub_test_case 'filter' do def test_filter_section write_config "#{TMP_DIR}/test_filter_section.yaml", <<~EOS config: - filter: $tag: dummy_tag $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_filter_section.yaml") assert_equal('filter', config.elements[0].name) assert_equal('dummy_tag', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_filter_section_multiple_tags write_config "#{TMP_DIR}/test_filter_section_multiple_tags_1.yaml", <<~EOS config: - filter: $tag: ['dummy_tag_A', 'dummy_tag_B'] $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_filter_section_multiple_tags_1.yaml") assert_equal('filter', config.elements[0].name) assert_equal('dummy_tag_A,dummy_tag_B', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) write_config "#{TMP_DIR}/test_filter_section_multiple_tags_2.yaml", <<~EOS config: - filter: $tag: - dummy_tag_C - dummy_tag_D $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_filter_section_multiple_tags_2.yaml") assert_equal('filter', config.elements[0].name) assert_equal('dummy_tag_C,dummy_tag_D', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_filter_section_missing_tag write_config "#{TMP_DIR}/test_filter_section_missing_tag.yaml", <<~EOS config: - filter: $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_filter_section_missing_tag.yaml") assert_equal('filter', config.elements[0].name) assert_equal('', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_filter_section_missing_type write_config "#{TMP_DIR}/test_filter_section_missing_tag.yaml", <<~EOS config: - filter: $tag: dummy_tag EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_filter_section_missing_tag.yaml") assert_equal('filter', config.elements[0].name) assert_equal('dummy_tag', config.elements[0].arg) assert_nil(config.elements[0]['@type']) end end sub_test_case 'match' do def test_match_section write_config "#{TMP_DIR}/test_match_section.yaml", <<~EOS config: - match: $tag: dummy_tag $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_match_section.yaml") assert_equal('match', config.elements[0].name) assert_equal('dummy_tag', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_match_section_multiple_tags write_config "#{TMP_DIR}/test_match_section_multiple_tags_1.yaml", <<~EOS config: - match: $tag: ['dummy_tag_A', 'dummy_tag_B'] $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_match_section_multiple_tags_1.yaml") assert_equal('match', config.elements[0].name) assert_equal('dummy_tag_A,dummy_tag_B', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) write_config "#{TMP_DIR}/test_match_section_multiple_tags_2.yaml", <<~EOS config: - match: $tag: - dummy_tag_C - dummy_tag_D $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_match_section_multiple_tags_2.yaml") assert_equal('match', config.elements[0].name) assert_equal('dummy_tag_C,dummy_tag_D', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_match_section_missing_tag write_config "#{TMP_DIR}/test_match_section_missing_tag.yaml", <<~EOS config: - match: $type: dummy_type EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_match_section_missing_tag.yaml") assert_equal('match', config.elements[0].name) assert_equal('', config.elements[0].arg) assert_equal('dummy_type', config.elements[0]['@type']) end def test_match_section_missing_type write_config "#{TMP_DIR}/test_match_section_missing_type.yaml", <<~EOS config: - match: $tag: dummy_tag EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_match_section_missing_type.yaml") assert_equal('match', config.elements[0].name) assert_equal('dummy_tag', config.elements[0].arg) assert_nil(config.elements[0]['@type']) end end sub_test_case '!include' do def test_include write_config "#{TMP_DIR}/dummy.yaml", <<~EOS - filter: $type: dummy EOS write_config "#{TMP_DIR}/test_include.yaml", <<~EOS config: - !include dummy.yaml EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_include.yaml") assert_equal(1, config.elements.size) assert_equal("filter", config.elements[0].name) # included section end def test_include_normal_config_file write_config "#{TMP_DIR}/dummy.conf", <<~EOS type dummy EOS write_config "#{TMP_DIR}/test_include_normal_config_file.yaml", <<~EOS config: - !include dummy.conf EOS # TODO: it should raise Fluent::ConfigError instead of TypeError, or parse normal config file assert_raise(TypeError) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_include_normal_config_file.yaml") end end end end def test_merge_common_parameter write_config "#{TMP_DIR}/test_merge_common_parameter.yaml", <<~EOS common_parameter: &common_parameter common_param: foobarbaz config: - match: $tag: dummy_tag_1 $type: dummy_type_1 <<: *common_parameter - match: $tag: dummy_tag_2 $type: dummy_type_2 <<: *common_parameter EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_merge_common_parameter.yaml") assert_equal(2, config.elements.size) assert_equal('foobarbaz', config.elements[0]['common_param']) assert_equal('foobarbaz', config.elements[1]['common_param']) end def test_override_merged_common_parameter write_config "#{TMP_DIR}/test_merge_common_parameter.yaml", <<~EOS common_parameter: &common_parameter common_param1: foobarbaz common_param2: 12345 config: - match: $tag: dummy_tag_1 $type: dummy_type_1 <<: *common_parameter - match: $tag: dummy_tag_2 $type: dummy_type_2 <<: *common_parameter common_param: override EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_merge_common_parameter.yaml") assert_equal(2, config.elements.size) assert_equal('foobarbaz', config.elements[0]['common_param1']) assert_equal(12345, config.elements[0]['common_param2']) assert_equal('override', config.elements[1]['common_param']) assert_equal(12345, config.elements[1]['common_param2']) end def test_merge_common_parameter_using_include write_config "#{TMP_DIR}/fluent-common.yaml", <<~EOS common_param: foobarbaz EOS write_config "#{TMP_DIR}/test_merge_common_parameter_using_include.yaml", <<~EOS config: - match: $tag: dummy_tag_1 $type: dummy_type_1 <<: !include fluent-common.yaml - match: $tag: dummy_tag_2 $type: dummy_type_2 <<: !include fluent-common.yaml EOS # TODO: Fix exception assert_raise(TypeError) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_merge_common_parameter_using_include.yaml") end end def test_unknown_anchor write_config "#{TMP_DIR}/test_unknown_anchor.yaml", <<~EOS common_parameter: &common_parameter common_param: foobarbaz config: - match: $tag: dummy_tag_1 $type: dummy_type_1 <<: *unknown_anchor EOS # TODO: it should raise Fluent::ConfigError instead of Psych::AnchorNotDefined assert_raises(Psych::AnchorNotDefined) do Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_unknown_anchor.yaml") end end def test_yaml_values write_config "#{TMP_DIR}/test_yaml_values.yaml", <<~EOS config: - source: mode: 0644 float_value: 3.40 flag_on: On flag_off: Off null: null date: 2026-01-01 timestamp: 2026-01-01 00:00:00 +09:00 symbol1: :foo symbol2: !ruby/symbol bar regexp: !ruby/regexp /^$/ EOS config = Fluent::Config::YamlParser.parse("#{TMP_DIR}/test_yaml_values.yaml") assert_equal(420, config.elements[0]['mode']) assert_equal(3.4, config.elements[0]['float_value']) assert_equal(true, config.elements[0]['flag_on']) assert_equal(false, config.elements[0]['flag_off']) assert_nil(config.elements[0]['null']) assert_equal(Date.new(2026, 1, 1), config.elements[0]['date']) assert_equal(Time.parse("2026-01-01 00:00:00 +09:00"), config.elements[0]['timestamp']) assert_equal(:foo, config.elements[0]['symbol1']) assert_equal(:bar, config.elements[0]['symbol2']) assert_equal(/^$/, config.elements[0]['regexp']) end end ================================================ FILE: test/counter/test_client.rb ================================================ require_relative '../helper' require 'fluent/counter/client' require 'fluent/counter/store' require 'fluent/counter/server' require 'flexmock/test_unit' require 'timecop' class CounterClientTest < ::Test::Unit::TestCase TEST_ADDR = '127.0.0.1' TEST_PORT = '8277' setup do # timecop isn't compatible with EventTime t = Time.parse('2016-09-22 16:59:59 +0900') Timecop.freeze(t) @now = Fluent::EventTime.now @options = { addr: TEST_ADDR, port: TEST_PORT, log: $log, } @server_name = 'server1' @scope = "worker1\tplugin1" @loop = Coolio::Loop.new @server = Fluent::Counter::Server.new(@server_name, @options).start @client = Fluent::Counter::Client.new(@loop, @options).start end teardown do Timecop.return @server.stop @client.stop end test 'Callable API' do [:establish, :init, :delete, :inc, :reset, :get].each do |m| assert_true @client.respond_to?(m) end end sub_test_case 'on_message' do setup do @future = flexmock('future') @client.instance_variable_set(:@responses, { 1 => @future }) end test 'call a set method to a corresponding object' do @future.should_receive(:set).once.with(Hash) @client.send(:on_message, { 'id' => 1 }) end test "output a warning log when passed id doesn't exist" do data = { 'id' => 2 } mock($log).warn("Receiving missing id data: #{data}") @client.send(:on_message, data) end end def extract_value_from_server(server, scope, name) store = server.instance_variable_get(:@store).instance_variable_get(:@storage).instance_variable_get(:@store) key = Fluent::Counter::Store.gen_key(scope, name) store[key] end def travel(sec) # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing # rounding errors to Time.now, we need to use Timecop.freeze() instead. Timecop.freeze(Time.now + sec) end sub_test_case 'establish' do test 'establish a scope' do @client.establish(@scope) assert_equal "#{@server_name}\t#{@scope}", @client.instance_variable_get(:@scope) end data( empty: '', invalid_string: '_scope', invalid_string2: 'Scope' ) test 'raise an error when passed scope is invalid' do |scope| assert_raise do @client.establish(scope) end end end sub_test_case 'init' do setup do @client.instance_variable_set(:@scope, @scope) end data( numeric_type: [ { name: 'key', reset_interval: 20, type: 'numeric' }, 0 ], float_type: [ { name: 'key', reset_interval: 20, type: 'float' }, 0.0 ], integer_type: [ { name: 'key', reset_interval: 20, type: 'integer' }, 0 ] ) test 'create a value' do |(param, initial_value)| assert_nil extract_value_from_server(@server, @scope, param[:name]) response = @client.init(param).get data = response.data.first assert_nil response.errors assert_equal param[:name], data['name'] assert_equal param[:reset_interval], data['reset_interval'] assert_equal param[:type], data['type'] assert_equal initial_value, data['current'] assert_equal initial_value, data['total'] v = extract_value_from_server(@server, @scope, param[:name]) assert_equal param[:name], v['name'] assert_equal param[:reset_interval], v['reset_interval'] assert_equal param[:type], v['type'] assert_equal initial_value, v['total'] assert_equal initial_value, v['current'] end test 'raise an error when @scope is nil' do @client.instance_variable_set(:@scope, nil) assert_raise 'Call `establish` method to get a `scope` before calling this method' do params = { name: 'key1', reset_interval: 10 } @client.init(params).get end end data( already_exist_key: [ { name: 'key1', reset_interval: 10 }, { 'code' => 'invalid_params', 'message' => "worker1\tplugin1\tkey1 already exists in counter" } ], missing_name: [ { reset_interval: 10 }, { 'code' => 'invalid_params', 'message' => '`name` is required' }, ], missing_reset_interval: [ { name: 'key' }, { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' }, ], invalid_name: [ { name: '\tkey' }, { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' } ] ) test 'return an error object' do |(param, expected_error)| params = { name: 'key1', reset_interval: 10 } @client.init(params).get response = @client.init(param).get errors = response.errors.first assert_empty response.data assert_equal expected_error, errors assert_raise { @client.init(param).wait } end test 'return an existing value when passed key already exists and ignore option is true' do params = { name: 'key1', reset_interval: 10 } res1 = @client.init(params).get res2 = nil assert_nothing_raised do res2 = @client.init({ name: 'key1', reset_interval: 10 }, options: { ignore: true }).get end assert_equal res1.data, res2.data end test 'return an error object and data object' do param = { name: 'key1', reset_interval: 10 } param2 = { name: 'key2', reset_interval: 10 } @client.init(param).get response = @client.init([param2, param]).get data = response.data.first error = response.errors.first assert_equal param2[:name], data['name'] assert_equal param2[:reset_interval], data['reset_interval'] assert_equal 'invalid_params', error['code'] assert_equal "#{@scope}\t#{param[:name]} already exists in counter", error['message'] end test 'return a future object when async call' do param = { name: 'key', reset_interval: 10 } r = @client.init(param) assert_true r.is_a?(Fluent::Counter::Future) assert_nil r.errors end end sub_test_case 'delete' do setup do @client.instance_variable_set(:@scope, @scope) @name = 'key' @key = Fluent::Counter::Store.gen_key(@scope, @name) @init_obj = { name: @name, reset_interval: 20, type: 'numeric' } @client.init(@init_obj).get end test 'delete a value' do assert extract_value_from_server(@server, @scope, @name) response = @client.delete(@name).get v = response.data.first assert_nil response.errors assert_equal @init_obj[:name], v['name'] assert_equal @init_obj[:type], v['type'] assert_equal @init_obj[:reset_interval], v['reset_interval'] assert_nil extract_value_from_server(@server, @scope, @name) end test 'raise an error when @scope is nil' do @client.instance_variable_set(:@scope, nil) assert_raise 'Call `establish` method to get a `scope` before calling this method' do @client.delete(@name).get end end data( key_not_found: [ 'key2', { 'code' => 'unknown_key', 'message' => "`worker1\tplugin1\tkey2` doesn't exist in counter" } ], invalid_key: [ '\tkey', { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' } ] ) test 'return an error object' do |(param, expected_error)| response = @client.delete(param).get errors = response.errors.first assert_empty response.data assert_equal expected_error, errors end test 'return an error object and data object' do unknown_name = 'key2' response = @client.delete(@name, unknown_name).get data = response.data.first error = response.errors.first assert_equal @name, data['name'] assert_equal @init_obj[:reset_interval], data['reset_interval'] assert_equal 'unknown_key', error['code'] assert_equal "`#{@scope}\t#{unknown_name}` doesn't exist in counter", error['message'] assert_nil extract_value_from_server(@server, @scope, @name) end test 'return a future object when async call' do r = @client.delete(@name) assert_true r.is_a?(Fluent::Counter::Future) assert_nil r.errors end end sub_test_case 'inc' do setup do @client.instance_variable_set(:@scope, @scope) @name = 'key' @key = Fluent::Counter::Store.gen_key(@scope, @name) @init_obj = { name: @name, reset_interval: 20, type: 'numeric' } @client.init(@init_obj).get end test 'increment a value' do v = extract_value_from_server(@server, @scope, @name) assert_equal 0, v['total'] assert_equal 0, v['current'] travel(1) inc_obj = { name: @name, value: 10 } @client.inc(inc_obj).get v = extract_value_from_server(@server, @scope, @name) assert_equal inc_obj[:value], v['total'] assert_equal inc_obj[:value], v['current'] assert_equal (@now + 1), Fluent::EventTime.new(*v['last_modified_at']) end test 'create and increment a value when force option is true' do name = 'new_key' param = { name: name, value: 11, reset_interval: 1 } assert_nil extract_value_from_server(@server, @scope, name) @client.inc(param, options: { force: true }).get v = extract_value_from_server(@server, @scope, name) assert v assert_equal param[:name], v['name'] assert_equal 1, v['reset_interval'] assert_equal param[:value], v['current'] assert_equal param[:value], v['total'] end test 'raise an error when @scope is nil' do @client.instance_variable_set(:@scope, nil) assert_raise 'Call `establish` method to get a `scope` before calling this method' do params = { name: 'name', value: 1 } @client.inc(params).get end end data( not_exist_key: [ { name: 'key2', value: 10 }, { 'code' => 'unknown_key', 'message' => "`worker1\tplugin1\tkey2` doesn't exist in counter" } ], missing_name: [ { value: 10 }, { 'code' => 'invalid_params', 'message' => '`name` is required' }, ], missing_value: [ { name: 'key' }, { 'code' => 'invalid_params', 'message' => '`value` is required' }, ], invalid_name: [ { name: '\tkey' }, { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' } ] ) test 'return an error object' do |(param, expected_error)| response = @client.inc(param).get errors = response.errors.first assert_empty response.data assert_equal expected_error, errors end test 'return an error object and data object' do parmas = [ { name: @name, value: 10 }, { name: 'unknown_key', value: 9 }, ] response = @client.inc(parmas).get data = response.data.first error = response.errors.first assert_equal @name, data['name'] assert_equal 10, data['current'] assert_equal 10, data['total'] assert_equal 'unknown_key', error['code'] assert_equal "`#{@scope}\tunknown_key` doesn't exist in counter", error['message'] end test 'return a future object when async call' do param = { name: 'key', value: 10 } r = @client.inc(param) assert_true r.is_a?(Fluent::Counter::Future) assert_nil r.errors end end sub_test_case 'get' do setup do @client.instance_variable_set(:@scope, @scope) @name = 'key' @init_obj = { name: @name, reset_interval: 20, type: 'numeric' } @client.init(@init_obj).get end test 'get a value' do v1 = extract_value_from_server(@server, @scope, @name) v2 = @client.get(@name).data.first assert_equal v1['name'], v2['name'] assert_equal v1['current'], v2['current'] assert_equal v1['total'], v2['total'] assert_equal v1['type'], v2['type'] end test 'raise an error when @scope is nil' do @client.instance_variable_set(:@scope, nil) assert_raise 'Call `establish` method to get a `scope` before calling this method' do @client.get(@name).get end end data( key_not_found: [ 'key2', { 'code' => 'unknown_key', 'message' => "`worker1\tplugin1\tkey2` doesn't exist in counter" } ], invalid_key: [ '\tkey', { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' } ] ) test 'return an error object' do |(param, expected_error)| response = @client.get(param).get errors = response.errors.first assert_empty response.data assert_equal expected_error, errors end test 'return an error object and data object' do unknown_name = 'key2' response = @client.get(@name, unknown_name).get data = response.data.first error = response.errors.first assert_equal @name, data['name'] assert_equal @init_obj[:reset_interval], data['reset_interval'] assert_equal 'unknown_key', error['code'] assert_equal "`#{@scope}\t#{unknown_name}` doesn't exist in counter", error['message'] end test 'return a future object when async call' do r = @client.get(@name) assert_true r.is_a?(Fluent::Counter::Future) assert_nil r.errors end end sub_test_case 'reset' do setup do @client.instance_variable_set(:@scope, @scope) @name = 'key' @key = Fluent::Counter::Store.gen_key(@scope, @name) @init_obj = { name: @name, reset_interval: 5, type: 'numeric' } @client.init(@init_obj).get @inc_obj = { name: @name, value: 10 } @client.inc(@inc_obj).get end test 'reset a value after `reset_interval` passed' do v1 = extract_value_from_server(@server, @scope, @name) assert_equal @inc_obj[:value], v1['total'] assert_equal @inc_obj[:value], v1['current'] assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at']) travel_sec = 6 # greater than reset_interval travel(travel_sec) v2 = @client.reset(@name).get data = v2.data.first c = data['counter_data'] assert_equal travel_sec, data['elapsed_time'] assert_true data['success'] assert_equal @inc_obj[:value], c['current'] assert_equal @inc_obj[:value], c['total'] assert_equal @now, c['last_reset_at'] v1 = extract_value_from_server(@server, @scope, @name) assert_equal 0, v1['current'] assert_equal @inc_obj[:value], v1['total'] assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_reset_at']) assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_modified_at']) end test 'return a value object before `reset_interval` passed' do v1 = extract_value_from_server(@server, @scope, @name) assert_equal @inc_obj[:value], v1['total'] assert_equal @inc_obj[:value], v1['current'] assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at']) travel_sec = 4 # less than reset_interval travel(travel_sec) v2 = @client.reset(@name).get data = v2.data.first c = data['counter_data'] assert_equal travel_sec, data['elapsed_time'] assert_equal false, data['success'] assert_equal @inc_obj[:value], c['current'] assert_equal @inc_obj[:value], c['total'] assert_equal @now, c['last_reset_at'] v1 = extract_value_from_server(@server, @scope, @name) assert_equal @inc_obj[:value], v1['current'] assert_equal @inc_obj[:value], v1['total'] assert_equal @now, Fluent::EventTime.new(*v1['last_reset_at']) end test 'raise an error when @scope is nil' do @client.instance_variable_set(:@scope, nil) assert_raise 'Call `establish` method to get a `scope` before calling this method' do @client.reset(@name).get end end data( key_not_found: [ 'key2', { 'code' => 'unknown_key', 'message' => "`worker1\tplugin1\tkey2` doesn't exist in counter" } ], invalid_key: [ '\tkey', { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' } ] ) test 'return an error object' do |(param, expected_error)| response = @client.reset(param).get errors = response.errors.first assert_empty response.data assert_equal expected_error, errors end test 'return an error object and data object' do unknown_name = 'key2' travel_sec = 6 # greater than reset_interval travel(travel_sec) response = @client.reset(@name, unknown_name).get data = response.data.first error = response.errors.first counter = data['counter_data'] assert_true data['success'] assert_equal travel_sec, data['elapsed_time'] assert_equal @name, counter['name'] assert_equal @init_obj[:reset_interval], counter['reset_interval'] assert_equal @inc_obj[:value], counter['total'] assert_equal @inc_obj[:value], counter['current'] assert_equal 'unknown_key', error['code'] assert_equal "`#{@scope}\t#{unknown_name}` doesn't exist in counter", error['message'] v1 = extract_value_from_server(@server, @scope, @name) assert_equal 0, v1['current'] assert_equal @inc_obj[:value], v1['total'] assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_reset_at']) assert_equal (@now + travel_sec), Fluent::EventTime.new(*v1['last_modified_at']) end test 'return a future object when async call' do r = @client.reset(@name) assert_true r.is_a?(Fluent::Counter::Future) assert_nil r.errors end end end ================================================ FILE: test/counter/test_error.rb ================================================ require_relative '../helper' require 'fluent/counter/error' class CounterErrorTest < ::Test::Unit::TestCase setup do @message = 'error message' end test 'invalid_params' do error = Fluent::Counter::InvalidParams.new(@message) expected = { 'code' => 'invalid_params', 'message' => @message } assert_equal expected, error.to_hash end test 'unknown_key' do error = Fluent::Counter::UnknownKey.new(@message) expected = { 'code' => 'unknown_key', 'message' => @message } assert_equal expected, error.to_hash end test 'parse_error' do error = Fluent::Counter::ParseError.new(@message) expected = { 'code' => 'parse_error', 'message' => @message } assert_equal expected, error.to_hash end test 'method_not_found' do error = Fluent::Counter::MethodNotFound.new(@message) expected = { 'code' => 'method_not_found', 'message' => @message } assert_equal expected, error.to_hash end test 'invalid_request' do error = Fluent::Counter::InvalidRequest.new(@message) expected = { 'code' => 'invalid_request', 'message' => @message } assert_equal expected, error.to_hash end test 'internal_server_error' do error = Fluent::Counter::InternalServerError.new(@message) expected = { 'code' => 'internal_server_error', 'message' => @message } assert_equal expected, error.to_hash end end ================================================ FILE: test/counter/test_mutex_hash.rb ================================================ require_relative '../helper' require 'fluent/counter/mutex_hash' require 'fluent/counter/store' require 'flexmock/test_unit' require 'timecop' class MutexHashTest < ::Test::Unit::TestCase setup do @store = {} @value = 'sample value' @counter_store_mutex = Fluent::Counter::MutexHash.new(@store) end sub_test_case 'synchronize' do test "create new mutex values if keys don't exist" do keys = ['key', 'key1'] @counter_store_mutex.synchronize(*keys) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) keys.each do |key| assert_true mhash[key].is_a?(Mutex) assert_equal @value, @store[key] end end test 'nothing to do when an empty array passed' do @counter_store_mutex.synchronize(*[]) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) assert_true mhash.empty? assert_true @store.empty? end test 'use a one mutex value when the same key specified' do key = 'key' @counter_store_mutex.synchronize(key) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) m = mhash[key] assert_true m.is_a?(Mutex) assert_equal @value, @store[key] # access the same key once again value2 = 'test value2' @counter_store_mutex.synchronize(key) do |store, k| store[k] = value2 end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) m2 = mhash[key] assert_true m2.is_a?(Mutex) assert_equal value2, @store[key] assert_equal m, m2 end end sub_test_case 'synchronize_key' do test "create new mutex values if keys don't exist" do keys = ['key', 'key1'] @counter_store_mutex.synchronize_keys(*keys) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) keys.each do |key| assert_true mhash[key].is_a?(Mutex) assert_equal @value, @store[key] end end test 'nothing to do when an empty array passed' do @counter_store_mutex.synchronize_keys(*[]) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) assert_true mhash.empty? assert_true @store.empty? end test 'use a one mutex value when the same key specified' do key = 'key' @counter_store_mutex.synchronize_keys(key) do |store, k| store[k] = @value end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) m = mhash[key] assert_true m.is_a?(Mutex) assert_equal @value, @store[key] # access the same key once again value2 = 'test value2' @counter_store_mutex.synchronize_keys(key) do |store, k| store[k] = value2 end mhash = @counter_store_mutex.instance_variable_get(:@mutex_hash) m2 = mhash[key] assert_true m2.is_a?(Mutex) assert_equal value2, @store[key] assert_equal m, m2 end end end class CleanupThreadTest < ::Test::Unit::TestCase StoreValue = Struct.new(:last_modified_at) setup do # timecop isn't compatible with EventTime t = Time.parse('2016-09-22 16:59:59 +0900') Timecop.freeze(t) @store = Fluent::Counter::Store.new @mhash = Fluent::Counter::MutexHash.new(@store) # stub sleep method to avoid waiting CLEANUP_INTERVAL ct = @mhash.instance_variable_get(:@cleanup_thread) flexstub(ct).should_receive(:sleep) end teardown do @mhash.stop Timecop.return end test 'clean up unused mutex' do name = 'key1' init_obj = { 'name' => name, 'reset_interval' => 2 } @mhash.synchronize(init_obj['name']) do @store.init(name, init_obj) end ct = @mhash.instance_variable_get(:@mutex_hash) assert ct[name] Timecop.travel(15 * 60 + 1) # 15 min @mhash.start # start cleanup sleep 1 ct = @mhash.instance_variable_get(:@mutex_hash) assert_empty ct @mhash.stop end test "don't remove when `last_modified_at` is greater than (Time.now - CLEANUP_INTERVAL)" do name = 'key1' init_obj = { 'name' => name, 'reset_interval' => 2 } @mhash.synchronize(init_obj['name']) do @store.init(name, init_obj) end ct = @mhash.instance_variable_get(:@mutex_hash) assert ct[name] @mhash.start # start cleanup sleep 1 ct = @mhash.instance_variable_get(:@mutex_hash) assert ct[name] @mhash.stop end end ================================================ FILE: test/counter/test_server.rb ================================================ require_relative '../helper' require 'fluent/counter/server' require 'fluent/counter/store' require 'fluent/time' require 'flexmock/test_unit' require 'timecop' class CounterServerTest < ::Test::Unit::TestCase setup do # timecop isn't compatible with EventTime t = Time.parse('2016-09-22 16:59:59 +0900') Timecop.freeze(t) @now = Fluent::EventTime.now @scope = "server\tworker\tplugin" @server_name = 'server1' @server = Fluent::Counter::Server.new(@server_name, opt: { log: $log }) @server.instance_eval { @server.close } end teardown do Timecop.return end def extract_value_from_counter(counter, scope, name) store = counter.instance_variable_get(:@store).instance_variable_get(:@storage).instance_variable_get(:@store) key = Fluent::Counter::Store.gen_key(scope, name) store[key] end def travel(sec) # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing # rounding errors to Time.now, we need to use Timecop.freeze() instead. Timecop.freeze(Time.now + sec) end test 'raise an error when server name is invalid' do assert_raise do Fluent::Counter::Server.new("\tinvalid_name") end end sub_test_case 'on_message' do data( establish: 'establish', init: 'init', delete: 'delete', inc: 'inc', get: 'get', reset: 'reset', ) test 'call valid methods' do |method| stub(@server).send do |_m, params, scope, options| { 'data' => [params, scope, options] } end request = { 'id' => 0, 'method' => method } expected = { 'id' => 0, 'data' => [nil, nil, nil] } assert_equal expected, @server.on_message(request) end data( missing_id: [ { 'method' => 'init' }, { 'code' => 'invalid_request', 'message' => 'Request should include `id`' } ], missing_method: [ { 'id' => 0 }, { 'code' => 'invalid_request', 'message' => 'Request should include `method`' } ], invalid_method: [ { 'id' => 0, 'method' => 'invalid_method' }, { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: invalid_method' } ] ) test 'invalid request' do |(request, error)| expected = { 'id' => request['id'], 'data' => [], 'errors' => [error], } assert_equal expected, @server.on_message(request) end test 'return an `internal_server_error` error object when an error raises in safe_run' do stub(@server).send do |_m, _params, _scope, _options| raise 'Error in safe_run' end request = { 'id' => 0, 'method' => 'init' } expected = { 'id' => request['id'], 'data' => [], 'errors' => [ { 'code' => 'internal_server_error', 'message' => 'Error in safe_run' } ] } assert_equal expected, @server.on_message(request) end test 'output an error log when passed data is not Hash' do data = 'this is not a hash' mock($log).error("Received data is not Hash: #{data}") @server.on_message(data) end end sub_test_case 'establish' do test 'establish a scope in a counter' do result = @server.send('establish', ['key'], nil, nil) expected = { 'data' => ["#{@server_name}\tkey"] } assert_equal expected, result end data( empty: [[], 'One or more `params` are required'], empty_key: [[''], '`scope` is the invalid format'], invalid_key: [['_key'], '`scope` is the invalid format'], ) test 'raise an error: invalid_params' do |(params, msg)| result = @server.send('establish', params, nil, nil) expected = { 'data' => [], 'errors' => [{ 'code' => 'invalid_params', 'message' => msg }] } assert_equal expected, result end end sub_test_case 'init' do setup do @name = 'key1' @key = Fluent::Counter::Store.gen_key(@scope, @name) end test 'create new value in a counter' do assert_nil extract_value_from_counter(@server, @scope, @name) result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {}) assert_nil result['errors'] counter = result['data'].first assert_equal 'numeric', counter['type'] assert_equal @name, counter['name'] assert_equal 0, counter['current'] assert_equal 0, counter['total'] assert_equal @now, counter['last_reset_at'] assert extract_value_from_counter(@server, @scope, @name) end data( numeric: 'numeric', integer: 'integer', float: 'float' ) test 'set the type of a counter value' do |type| result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1, 'type' => type }], @scope, {}) counter = result['data'].first assert_equal type, counter['type'] v = extract_value_from_counter(@server, @scope, @name) assert_equal type, v['type'] end data( empty: [[], 'One or more `params` are required'], missing_name: [ [{ 'rest_interval' => 20 }], '`name` is required' ], invalid_name: [ [{ 'name' => '_test', 'reset_interval' => 20 }], '`name` is the invalid format' ], missing_interval: [ [{ 'name' => 'test' }], '`reset_interval` is required' ], minus_interval: [ [{ 'name' => 'test', 'reset_interval' => -1 }], '`reset_interval` should be a positive number' ], invalid_type: [ [{ 'name' => 'test', 'reset_interval' => 1, 'type' => 'invalid_type' }], '`type` should be integer, float, or numeric' ] ) test 'return an error object: invalid_params' do |(params, msg)| result = @server.send('init', params, @scope, {}) assert_empty result['data'] error = result['errors'].first assert_equal 'invalid_params', error['code'] assert_equal msg, error['message'] end test 'return error objects when passed key already exists' do @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {}) # call `init` to same key twice result = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {}) assert_empty result['data'] error = result['errors'].first expected = { 'code' => 'invalid_params', 'message' => "#{@key} already exists in counter" } assert_equal expected, error end test 'return existing value when passed key already exists and ignore option is true' do v1 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {}) # call `init` to same key twice v2 = @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'ignore' => true }) assert_nil v2['errors'] assert_equal v1['data'], v2['data'] end test 'call `synchronize_keys` when random option is true' do mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize(@key).once @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, {}) mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize_keys(@key).once @server.send('init', [{ 'name' => @name, 'reset_interval' => 1 }], @scope, { 'random' => true }) end end sub_test_case 'delete' do setup do @name = 'key1' @key = Fluent::Counter::Store.gen_key(@scope, @name) @server.send('init', [{ 'name' => @name, 'reset_interval' => 20 }], @scope, {}) end test 'delete a value from a counter' do assert extract_value_from_counter(@server, @scope, @name) result = @server.send('delete', [@name], @scope, {}) assert_nil result['errors'] counter = result['data'].first assert_equal 0, counter['current'] assert_equal 0, counter['total'] assert_equal 'numeric', counter['type'] assert_equal @name, counter['name'] assert_equal @now, counter['last_reset_at'] assert_nil extract_value_from_counter(@server, @scope, @name) end data( empty: [[], 'One or more `params` are required'], empty_key: [[''], '`key` is the invalid format'], invalid_key: [['_key'], '`key` is the invalid format'], ) test 'return an error object: invalid_params' do |(params, msg)| result = @server.send('delete', params, @scope, {}) assert_empty result['data'] error = result['errors'].first assert_equal 'invalid_params', error['code'] assert_equal msg, error['message'] end test 'return an error object: unknown_key' do unknown_key = 'unknown_key' result = @server.send('delete', [unknown_key], @scope, {}) assert_empty result['data'] error = result['errors'].first assert_equal unknown_key, error['code'] assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message'] end test 'call `synchronize_keys` when random option is true' do mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize(@key).once @server.send('delete', [@name], @scope, {}) mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize_keys(@key).once @server.send('delete', [@name], @scope, { 'random' => true }) end end sub_test_case 'inc' do setup do @name1 = 'key1' @name2 = 'key2' @key1 = Fluent::Counter::Store.gen_key(@scope, @name1) inc_objects = [ { 'name' => @name1, 'reset_interval' => 20 }, { 'name' => @name2, 'type' => 'integer', 'reset_interval' => 20 } ] @server.send('init', inc_objects, @scope, {}) end test 'increment or decrement a value in counter' do result = @server.send('inc', [{ 'name' => @name1, 'value' => 10 }], @scope, {}) assert_nil result['errors'] counter = result['data'].first assert_equal 10, counter['current'] assert_equal 10, counter['total'] assert_equal 'numeric', counter['type'] assert_equal @name1, counter['name'] assert_equal @now, counter['last_reset_at'] c = extract_value_from_counter(@server, @scope, @name1) assert_equal 10, c['current'] assert_equal 10, c['total'] assert_equal @now, Fluent::EventTime.new(*c['last_reset_at']) assert_equal @now, Fluent::EventTime.new(*c['last_modified_at']) end test 'create new value and increment/decrement its value when `force` option is true' do new_name = 'new_key' assert_nil extract_value_from_counter(@server, @scope, new_name) v1 = @server.send('inc', [{ 'name' => new_name, 'value' => 10 }], @scope, {}) assert_empty v1['data'] error = v1['errors'].first assert_equal 'unknown_key', error['code'] assert_nil extract_value_from_counter(@server, @scope, new_name) v2 = @server.send( 'inc', [{ 'name' => new_name, 'value' => 10, 'reset_interval' => 20 }], @scope, { 'force' => true } ) assert_nil v2['errors'] counter = v2['data'].first assert_equal 10, counter['current'] assert_equal 10, counter['total'] assert_equal 'numeric', counter['type'] assert_equal new_name, counter['name'] assert_equal @now, counter['last_reset_at'] assert extract_value_from_counter(@server, @scope, new_name) end data( empty: [[], 'One or more `params` are required', {}], missing_name: [ [{ 'value' => 10 }], '`name` is required', {} ], missing_value: [ [{ 'name' => 'key1' }], '`value` is required', {} ], invalid_type: [ [{ 'name' => 'key2', 'value' => 10.0 }], '`type` is integer. You should pass integer value as a `value`', {} ], missing_reset_interval: [ [{ 'name' => 'key1', 'value' => 1 }], '`reset_interval` is required', { 'force' => true } ] ) test 'return an error object: invalid_params' do |(params, msg, opt)| result = @server.send('inc', params, @scope, opt) assert_empty result['data'] error = result['errors'].first assert_equal 'invalid_params', error['code'] assert_equal msg, error['message'] end test 'call `synchronize_keys` when random option is true' do mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize(@key1).once params = [{ 'name' => @name1, 'value' => 1 }] @server.send('inc', params, @scope, {}) mhash = @server.instance_variable_get(:@mutex_hash) mock(mhash).synchronize_keys(@key1).once @server.send('inc', params, @scope, { 'random' => true }) end end sub_test_case 'reset' do setup do @name = 'key' @travel_sec = 10 @server.send('init', [{ 'name' => @name, 'reset_interval' => 10 }], @scope, {}) @server.send('inc', [{ 'name' => @name, 'value' => 10 }], @scope, {}) end test 'reset a value in the counter' do travel(@travel_sec) result = @server.send('reset', [@name], @scope, {}) assert_nil result['errors'] data = result['data'].first assert_true data['success'] assert_equal @travel_sec, data['elapsed_time'] counter = data['counter_data'] assert_equal 10, counter['current'] assert_equal 10, counter['total'] assert_equal 'numeric', counter['type'] assert_equal @name, counter['name'] assert_equal @now, counter['last_reset_at'] v = extract_value_from_counter(@server, @scope, @name) assert_equal 0, v['current'] assert_equal 10, v['total'] assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_reset_at']) assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v['last_modified_at']) end test 'reset a value after `reset_interval` passed' do first_travel_sec = 5 travel(first_travel_sec) # jump time less than reset_interval result = @server.send('reset', [@name], @scope, {}) v = result['data'].first assert_equal false, v['success'] assert_equal first_travel_sec, v['elapsed_time'] store = extract_value_from_counter(@server, @scope, @name) assert_equal 10, store['current'] assert_equal @now, Fluent::EventTime.new(*store['last_reset_at']) # time is passed greater than reset_interval travel(@travel_sec) result = @server.send('reset', [@name], @scope, {}) v = result['data'].first assert_true v['success'] assert_equal @travel_sec + first_travel_sec, v['elapsed_time'] v1 = extract_value_from_counter(@server, @scope, @name) assert_equal 0, v1['current'] assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at']) assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at']) end data( empty: [[], 'One or more `params` are required'], empty_key: [[''], '`key` is the invalid format'], invalid_key: [['_key'], '`key` is the invalid format'], ) test 'return an error object: invalid_params' do |(params, msg)| result = @server.send('reset', params, @scope, {}) assert_empty result['data'] assert_equal 'invalid_params', result['errors'].first['code'] assert_equal msg, result['errors'].first['message'] end test 'return an error object: unknown_key' do unknown_key = 'unknown_key' result = @server.send('reset', [unknown_key], @scope, {}) assert_empty result['data'] error = result['errors'].first assert_equal unknown_key, error['code'] assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message'] end end sub_test_case 'get' do setup do @name1 = 'key1' @name2 = 'key2' init_objects = [ { 'name' => @name1, 'reset_interval' => 0 }, { 'name' => @name2, 'reset_interval' => 0 }, ] @server.send('init', init_objects, @scope, {}) end test 'get a counter value' do key = @name1 result = @server.send('get', [key], @scope, {}) assert_nil result['errors'] counter = result['data'].first assert_equal 0, counter['current'] assert_equal 0, counter['total'] assert_equal 'numeric', counter['type'] assert_equal key, counter['name'] end test 'get counter values' do result = @server.send('get', [@name1, @name2], @scope, {}) assert_nil result['errors'] counter1 = result['data'][0] assert_equal 0, counter1['current'] assert_equal 0, counter1['total'] assert_equal 'numeric', counter1['type'] assert_equal @name1, counter1['name'] counter2 = result['data'][1] assert_equal 0, counter2['current'] assert_equal 0, counter2['total'] assert_equal 'numeric', counter2['type'] assert_equal @name2, counter2['name'] end data( empty: [[], 'One or more `params` are required'], empty_key: [[''], '`key` is the invalid format'], invalid_key: [['_key'], '`key` is the invalid format'], ) test 'return an error object: invalid_params' do |(params, msg)| result = @server.send('get', params, @scope, {}) assert_empty result['data'] assert_equal 'invalid_params', result['errors'].first['code'] assert_equal msg, result['errors'].first['message'] end test 'return an error object: unknown_key' do unknown_key = 'unknown_key' result = @server.send('get', [unknown_key], @scope, {}) assert_empty result['data'] error = result['errors'].first assert_equal unknown_key, error['code'] assert_equal "`#{@scope}\t#{unknown_key}` doesn't exist in counter", error['message'] end end end class CounterCounterResponseTest < ::Test::Unit::TestCase setup do @response = Fluent::Counter::Server::Response.new @errors = [ StandardError.new('standard error'), Fluent::Counter::InternalServerError.new('internal server error') ] @now = Fluent::EventTime.now value = { 'name' => 'name', 'total' => 100, 'current' => 11, 'type' => 'numeric', 'reset_interval' => 10, 'last_reset_at' => @now, } @values = [value, 'test'] end test 'push_error' do @errors.each do |e| @response.push_error(e) end v = @response.instance_variable_get(:@errors) assert_equal @errors, v end test 'push_data' do @values.each do |v| @response.push_data v end data = @response.instance_variable_get(:@data) assert_equal @values, data end test 'to_hash' do @errors.each do |e| @response.push_error(e) end @values.each do |v| @response.push_data v end expected_errors = [ { 'code' => 'internal_server_error', 'message' => 'standard error' }, { 'code' => 'internal_server_error', 'message' => 'internal server error' } ] expected_data = @values hash = @response.to_hash assert_equal expected_errors, hash['errors'] assert_equal expected_data, hash['data'] end end ================================================ FILE: test/counter/test_store.rb ================================================ require_relative '../helper' require 'fluent/counter/store' require 'fluent/time' require 'timecop' class CounterStoreTest < ::Test::Unit::TestCase setup do @name = 'key_name' @scope = "server\tworker\tplugin" # timecop isn't compatible with EventTime t = Time.parse('2016-09-22 16:59:59 +0900') Timecop.freeze(t) @now = Fluent::EventTime.now end teardown do Timecop.return end def extract_value_from_counter(counter, key) store = counter.instance_variable_get(:@storage).instance_variable_get(:@store) store[key] end def travel(sec) # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing # rounding errors to Time.now, we need to use Timecop.freeze() instead. Timecop.freeze(Time.now + sec) end sub_test_case 'init' do setup do @reset_interval = 10 @store = Fluent::Counter::Store.new @data = { 'name' => @name, 'reset_interval' => @reset_interval } @key = Fluent::Counter::Store.gen_key(@scope, @name) end test 'create new value in the counter' do v = @store.init(@key, @data) assert_equal @name, v['name'] assert_equal @reset_interval, v['reset_interval'] v2 = extract_value_from_counter(@store, @key) v2 = @store.send(:build_response, v2) assert_equal v, v2 end test 'raise an error when a passed key already exists' do @store.init(@key, @data) assert_raise Fluent::Counter::InvalidParams do @store.init(@key, @data) end end test 'return a value when passed key already exists and a ignore option is true' do v = @store.init(@key, @data) v1 = extract_value_from_counter(@store, @key) v1 = @store.send(:build_response, v1) v2 = @store.init(@key, @data, ignore: true) assert_equal v, v2 assert_equal v1, v2 end end sub_test_case 'get' do setup do @store = Fluent::Counter::Store.new data = { 'name' => @name, 'reset_interval' => 10 } @key = Fluent::Counter::Store.gen_key(@scope, @name) @store.init(@key, data) end test 'return a value from the counter' do v = extract_value_from_counter(@store, @key) expected = @store.send(:build_response, v) assert_equal expected, @store.get(@key) end test 'return a raw value from the counter when raw option is true' do v = extract_value_from_counter(@store, @key) assert_equal v, @store.get(@key, raw: true) end test "return nil when a passed key doesn't exist" do assert_equal nil, @store.get('unknown_key') end test "raise a error when a passed key doesn't exist and raise_error option is true" do assert_raise Fluent::Counter::UnknownKey do @store.get('unknown_key', raise_error: true) end end end sub_test_case 'key?' do setup do @store = Fluent::Counter::Store.new data = { 'name' => @name, 'reset_interval' => 10 } @key = Fluent::Counter::Store.gen_key(@scope, @name) @store.init(@key, data) end test 'return true when passed key exists' do assert_true @store.key?(@key) end test "return false when passed key doesn't exist" do assert_true !@store.key?('unknown_key') end end sub_test_case 'delete' do setup do @store = Fluent::Counter::Store.new data = { 'name' => @name, 'reset_interval' => 10 } @key = Fluent::Counter::Store.gen_key(@scope, @name) @init_value = @store.init(@key, data) end test 'delete a value from the counter' do v = @store.delete(@key) assert_equal @init_value, v assert_nil extract_value_from_counter(@store, @key) end test "raise an error when passed key doesn't exist" do assert_raise Fluent::Counter::UnknownKey do @store.delete('unknown_key') end end end sub_test_case 'inc' do setup do @store = Fluent::Counter::Store.new @init_data = { 'name' => @name, 'reset_interval' => 10 } @travel_sec = 10 end data( positive: 10, negative: -10 ) test 'increment or decrement a value in the counter' do |value| key = Fluent::Counter::Store.gen_key(@scope, @name) @store.init(key, @init_data) travel(@travel_sec) v = @store.inc(key, { 'value' => value }) assert_equal value, v['total'] assert_equal value, v['current'] assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change v1 = extract_value_from_counter(@store, key) v1 = @store.send(:build_response, v1) assert_equal v, v1 end test "raise an error when passed key doesn't exist" do assert_raise Fluent::Counter::UnknownKey do @store.inc('unknown_key', { 'value' => 1 }) end end test 'raise an error when a type of passed value is incompatible with a stored value' do key1 = Fluent::Counter::Store.gen_key(@scope, @name) key2 = Fluent::Counter::Store.gen_key(@scope, 'name2') key3 = Fluent::Counter::Store.gen_key(@scope, 'name3') v1 = @store.init(key1, @init_data.merge('type' => 'integer')) v2 = @store.init(key2, @init_data.merge('type' => 'float')) v3 = @store.init(key3, @init_data.merge('type' => 'numeric')) assert_equal 'integer', v1['type'] assert_equal 'float', v2['type'] assert_equal 'numeric', v3['type'] assert_raise Fluent::Counter::InvalidParams do @store.inc(key1, { 'value' => 1.1 }) end assert_raise Fluent::Counter::InvalidParams do @store.inc(key2, { 'value' => 1 }) end assert_nothing_raised do @store.inc(key3, { 'value' => 1 }) @store.inc(key3, { 'value' => 1.0 }) end end end sub_test_case 'reset' do setup do @store = Fluent::Counter::Store.new @travel_sec = 10 @inc_value = 10 @key = Fluent::Counter::Store.gen_key(@scope, @name) @store.init(@key, { 'name' => @name, 'reset_interval' => 10 }) @store.inc(@key, { 'value' => 10 }) end test 'reset a value in the counter' do travel(@travel_sec) v = @store.reset(@key) assert_equal @travel_sec, v['elapsed_time'] assert_true v['success'] counter = v['counter_data'] assert_equal @name, counter['name'] assert_equal @inc_value, counter['total'] assert_equal @inc_value, counter['current'] assert_equal 'numeric', counter['type'] assert_equal @now, counter['last_reset_at'] assert_equal 10, counter['reset_interval'] v1 = extract_value_from_counter(@store, @key) assert_equal 0, v1['current'] assert_true v1['current'].is_a?(Integer) assert_equal @inc_value, v1['total'] assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at']) assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at']) end test 'reset a value after `reset_interval` passed' do first_travel_sec = 5 travel(first_travel_sec) # jump time less than reset_interval v = @store.reset(@key) assert_equal false, v['success'] assert_equal first_travel_sec, v['elapsed_time'] store = extract_value_from_counter(@store, @key) assert_equal 10, store['current'] assert_equal @now, Fluent::EventTime.new(*store['last_reset_at']) # time is passed greater than reset_interval travel(@travel_sec) v = @store.reset(@key) assert_true v['success'] assert_equal @travel_sec + first_travel_sec, v['elapsed_time'] v1 = extract_value_from_counter(@store, @key) assert_equal 0, v1['current'] assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at']) assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at']) end test "raise an error when passed key doesn't exist" do assert_raise Fluent::Counter::UnknownKey do @store.reset('unknown_key') end end end end ================================================ FILE: test/counter/test_validator.rb ================================================ require_relative '../helper' require 'fluent/counter/validator' class CounterValidatorTest < ::Test::Unit::TestCase data( invalid_name1: '', invalid_name3: '_', invalid_name4: 'A', invalid_name5: 'a*', invalid_name6: "a\t", invalid_name7: "\n", ) test 'invalid name' do |invalid_name| assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name) end sub_test_case 'request' do test 'return an empty array' do data = { 'id' => 0, 'method' => 'init' } errors = Fluent::Counter::Validator.request(data) assert_empty errors end data( missing_id: [ { 'method' => 'init' }, { 'code' => 'invalid_request', 'message' => 'Request should include `id`' } ], missing_method: [ { 'id' => 0 }, { 'code' => 'invalid_request', 'message' => 'Request should include `method`' } ], invalid_method: [ { 'id' => 0, 'method' => "A\t" }, { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' } ], unknown_method: [ { 'id' => 0, 'method' => 'unknown_method' }, { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' } ] ) test 'return an error array' do |(data, expected_error)| errors = Fluent::Counter::Validator.request(data) assert_equal [expected_error], errors end end sub_test_case 'call' do test "return an error hash when passed method doesn't exist" do v = Fluent::Counter::Validator.new(:unknown) success, errors = v.call(['key1']) assert_empty success assert_equal 'internal_server_error', errors.first.to_hash['code'] end end test 'validate_empty!' do v = Fluent::Counter::Validator.new(:empty) success, errors = v.call([]) assert_empty success assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors end end class CounterArrayValidatorTest < ::Test::Unit::TestCase test 'validate_key!' do ary = ['key', 100, '_'] error_expected = [ { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' }, { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' } ] v = Fluent::Counter::ArrayValidator.new(:key) valid_params, errors = v.call(ary) assert_equal ['key'], valid_params assert_equal error_expected, errors.map(&:to_hash) end end class CounterHashValidatorTest < ::Test::Unit::TestCase test 'validate_name!' do hash = [ { 'name' => 'key' }, {}, { 'name' => 10 }, { 'name' => '_' } ] error_expected = [ { 'code' => 'invalid_params', 'message' => '`name` is required' }, { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' }, { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' }, ] v = Fluent::Counter::HashValidator.new(:name) success, errors = v.call(hash) assert_equal [{ 'name' => 'key' }], success assert_equal error_expected, errors.map(&:to_hash) end test 'validate_value!' do hash = [ { 'value' => 1 }, { 'value' => -1 }, {}, { 'value' => 'str' } ] error_expected = [ { 'code' => 'invalid_params', 'message' => '`value` is required' }, { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' }, ] v = Fluent::Counter::HashValidator.new(:value) valid_params, errors = v.call(hash) assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params assert_equal error_expected, errors.map(&:to_hash) end test 'validate_reset_interval!' do hash = [ { 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }, {}, { 'reset_interval' => -1 }, { 'reset_interval' => 'str' } ] error_expected = [ { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' }, { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' }, { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' }, ] v = Fluent::Counter::HashValidator.new(:reset_interval) valid_params, errors = v.call(hash) assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash) end end ================================================ FILE: test/helper.rb ================================================ # simplecov must be loaded before any of target code if ENV['SIMPLE_COV'] require 'simplecov' if defined?(SimpleCov::SourceFile) mod = SimpleCov::SourceFile def mod.new(*args, &block) m = allocate m.instance_eval do begin initialize(*args, &block) rescue Encoding::UndefinedConversionError @src = "".force_encoding('UTF-8') end end m end end unless SimpleCov.running SimpleCov.start do add_filter '/test/' add_filter '/gems/' end end end # Some tests use Hash instead of Element for configure. # We should rewrite these tests in the future and remove this ad-hoc code class Hash def corresponding_proxies @corresponding_proxies ||= [] end def to_masked_element self end end require 'rr' require 'test/unit' require 'test/unit/rr' require 'fileutils' require 'fluent/config/element' require 'fluent/log' require 'fluent/test' require 'fluent/test/helpers' require 'fluent/plugin/base' require 'fluent/plugin_id' require 'fluent/plugin_helper' require 'fluent/msgpack_factory' require 'fluent/time' require 'serverengine' require_relative 'helpers/fuzzy_assert' require_relative 'helpers/process_extenstion' module Fluent module Plugin class TestBase < Base # a base plugin class, but not input nor output # mainly for helpers and owned plugins include PluginId include PluginLoggerMixin include PluginHelper::Mixin end end end unless defined?(Test::Unit::AssertionFailedError) class Test::Unit::AssertionFailedError < StandardError end end include Fluent::Test::Helpers def unused_port(num = 1, protocol:, bind: "0.0.0.0") case protocol when :tcp, :tls unused_port_tcp(num) when :udp unused_port_udp(num, bind: bind) when :all unused_port_tcp_udp(num) else raise ArgumentError, "unknown protocol: #{protocol}" end end def unused_port_tcp_udp(num = 1) raise "not support num > 1" if num > 1 # The default maximum number of file descriptors in macOS is 256. # It might need to set num to a smaller value than that. tcp_ports = unused_port_tcp(200) port = unused_port_udp(1, port_list: tcp_ports) raise "can't find unused port" unless port port end def unused_port_tcp(num = 1) ports = [] sockets = [] num.times do s = TCPServer.open(0) sockets << s ports << s.addr[1] end sockets.each(&:close) if num == 1 return ports.first else return *ports end end PORT_RANGE_AVAILABLE = (1024...65535) def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0") family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6 ports = [] sockets = [] use_random_port = port_list.empty? i = 0 loop do port = use_random_port ? rand(PORT_RANGE_AVAILABLE) : port_list[i] u = UDPSocket.new(family) if (u.bind(bind, port) rescue nil) ports << port sockets << u else u.close end i += 1 break if ports.size >= num break if !use_random_port && i >= port_list.size end sockets.each(&:close) if num == 1 return ports.first else return *ports end end def waiting(seconds, logs: nil, plugin: nil) begin Timeout.timeout(seconds) do yield end rescue Timeout::Error if logs STDERR.print(*logs) elsif plugin STDERR.print(*plugin.log.out.logs) end raise end end def ipv6_enabled? require 'socket' begin # Try to actually bind to an IPv6 address to verify it works sock = Socket.new(Socket::AF_INET6, Socket::SOCK_STREAM, 0) sock.bind(Socket.sockaddr_in(0, '::1')) sock.close # Also test that we can resolve IPv6 addresses # This is needed because some systems can bind but can't connect Socket.getaddrinfo('::1', nil, Socket::AF_INET6) true rescue Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, SocketError false end end dl_opts = {} dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN logdev = Fluent::Test::DummyLogDevice.new logger = ServerEngine::DaemonLogger.new(logdev, dl_opts) $log ||= Fluent::Log.new(logger) ================================================ FILE: test/helpers/fuzzy_assert.rb ================================================ require 'test/unit' class FuzzyIncludeAssertion include Test::Unit::Assertions def self.assert(expected, actual, message = nil) new(expected, actual, message).assert end def initialize(expected, actual, message) @expected = expected @actual = actual @message = message end def assert if collection? assert_same_collection else assert_same_value end end private def assert_same_value m = "expected(#{@expected}) !== actual(#{@actual.inspect})" if @message m = "#{@message}: #{m}" end assert_true(@expected === @actual, m) end def assert_same_class if @expected.class != @actual.class if (@expected.class.ancestors | @actual.class.ancestors).empty? assert_equal(@expected.class, @actual.class, @message) end end end def assert_same_collection assert_same_class assert_same_values end def assert_same_values if @expected.is_a?(Array) @expected.each_with_index do |val, i| self.class.assert(val, @actual[i], @message) end else @expected.each do |key, val| self.class.assert(val, @actual[key], "#{key}: ") end end end def collection? @actual.is_a?(Array) || @actual.is_a?(Hash) end end class FuzzyAssertion < FuzzyIncludeAssertion private def assert_same_collection super assert_same_keys end def assert_same_keys if @expected.is_a?(Array) assert_equal(@expected.size, @actual.size, "expected.size(#{@expected}) != actual.size(#{@expected})") else assert_equal(@expected.keys.sort, @actual.keys.sort) end end end module FuzzyAssert def assert_fuzzy_include(left, right, message = nil) FuzzyIncludeAssertion.new(left, right, message).assert end def assert_fuzzy_equal(left, right, message = nil) FuzzyAssertion.new(left, right, message).assert end end ================================================ FILE: test/helpers/process_extenstion.rb ================================================ require 'timecop' module Process class << self alias_method :clock_gettime_original, :clock_gettime def clock_gettime(clock_id, unit = :float_second) # now only support CLOCK_REALTIME if Process::CLOCK_REALTIME == clock_id t = Time.now case unit when :float_second t.to_i + t.nsec / 1_000_000_000.0 when :float_millisecond t.to_i * 1_000 + t.nsec / 1_000_000.0 when :float_microsecond t.to_i * 1_000_000 + t.nsec / 1_000.0 when :second t.to_i when :millisecond t.to_i * 1000 + t.nsec / 1_000_000 when :microsecond t.to_i * 1_000_000 + t.nsec / 1_000 when :nanosecond t.to_i * 1_000_000_000 + t.nsec end else Process.clock_gettime_original(clock_id, unit) end end end end ================================================ FILE: test/log/test_console_adapter.rb ================================================ require_relative '../helper' require 'fluent/log' require 'fluent/log/console_adapter' class ConsoleAdapterTest < Test::Unit::TestCase def setup @timestamp = Time.parse("2023-01-01 15:32:41 +0000") @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z") Timecop.freeze(@timestamp) @logdev = Fluent::Test::DummyLogDevice.new @logger = ServerEngine::DaemonLogger.new(@logdev) @fluent_log = Fluent::Log.new(@logger) @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log) end def teardown Timecop.return end def test_expected_log_levels assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4}, Console::Logger::LEVELS) end data(trace: [Fluent::Log::LEVEL_TRACE, :debug], debug: [Fluent::Log::LEVEL_DEBUG, :debug], info: [Fluent::Log::LEVEL_INFO, :info], warn: [Fluent::Log::LEVEL_WARN, :warn], error: [Fluent::Log::LEVEL_ERROR, :error], fatal: [Fluent::Log::LEVEL_FATAL, :fatal]) def test_reflect_log_level(data) level, expected = data @fluent_log.level = level console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log) assert_equal(Console::Logger::LEVELS[expected], console_logger.level) end data(debug: :debug, info: :info, warn: :warn, error: :error, fatal: :fatal) def test_string_subject(level) @console_logger.send(level, "subject") assert_equal(["#{@timestamp_str} [#{level}]: 0.0s: subject\n"], @logdev.logs) end data(debug: :debug, info: :info, warn: :warn, error: :error, fatal: :fatal) def test_args(level) @console_logger.send(level, "subject", 1, 2, 3) assert_equal([ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" + " | 1\n" + " | 2\n" + " | 3\n" ], @logdev.logs) end data(debug: :debug, info: :info, warn: :warn, error: :error, fatal: :fatal) def test_options(level) @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2") lines = @logdev.logs[0].split("\n") args = JSON.parse(lines[1..].collect { |str| str.sub(/\s+\|/, "") }.join("\n")) assert_equal([ 1, "#{@timestamp_str} [#{level}]: 0.0s: subject", { "kwarg1" => "opt1", "kwarg2" => "opt2" } ], [ @logdev.logs.size, lines[0], args ]) end data(debug: :debug, info: :info, warn: :warn, error: :error, fatal: :fatal) def test_block(level) @console_logger.send(level, "subject") { "block message" } assert_equal([ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" + " | block message\n" ], @logdev.logs) end data(debug: :debug, info: :info, warn: :warn, error: :error, fatal: :fatal) def test_multiple_entries(level) @console_logger.send(level, "subject1") @console_logger.send(level, "line2") assert_equal([ "#{@timestamp_str} [#{level}]: 0.0s: subject1\n", "#{@timestamp_str} [#{level}]: 0.0s: line2\n" ], @logdev.logs) end end ================================================ FILE: test/plugin/data/2010/01/20100102-030405.log ================================================ ================================================ FILE: test/plugin/data/2010/01/20100102-030406.log ================================================ ================================================ FILE: test/plugin/data/2010/01/20100102.log ================================================ ================================================ FILE: test/plugin/data/log/bar ================================================ ================================================ FILE: test/plugin/data/log/foo/bar.log ================================================ ================================================ FILE: test/plugin/data/log/foo/bar2 ================================================ ================================================ FILE: test/plugin/data/log/test.log ================================================ ================================================ FILE: test/plugin/data/log_numeric/01.log ================================================ ================================================ FILE: test/plugin/data/log_numeric/02.log ================================================ ================================================ FILE: test/plugin/data/log_numeric/12.log ================================================ ================================================ FILE: test/plugin/data/log_numeric/14.log ================================================ ================================================ FILE: test/plugin/data/sd_file/config ================================================ - 'host': 127.0.0.1 'port': 24224 'weight': 1 'name': test1 'standby': false 'username': user1 'password': pass1 'shared_key': key1 - 'host': 127.0.0.1 'port': 24225 'weight': 1 ================================================ FILE: test/plugin/data/sd_file/config.json ================================================ [ { "host": "127.0.0.1", "port": 24224, "weight": 1, "name": "test1", "standby": false, "username": "user1", "password": "pass1", "shared_key": "key1" }, { "host": "127.0.0.1", "port": 24225, "weight": 1 } ] ================================================ FILE: test/plugin/data/sd_file/config.yaml ================================================ - 'host': 127.0.0.1 'port': 24224 'weight': 1 'name': test1 'standby': false 'username': user1 'password': pass1 'shared_key': key1 - 'host': 127.0.0.1 'port': 24225 'weight': 1 ================================================ FILE: test/plugin/data/sd_file/config.yml ================================================ - 'host': 127.0.0.1 'port': 24224 'weight': 1 'name': test1 'standby': false 'username': user1 'password': pass1 'shared_key': key1 - 'host': 127.0.0.1 'port': 24225 'weight': 1 ================================================ FILE: test/plugin/data/sd_file/invalid_config.yml ================================================ - 'host': 127.0.0.1 'weight': 1 'name': test1 'standby': false 'username': user1 'password': pass1 'shared_key': key1 ================================================ FILE: test/plugin/in_tail/test_fifo.rb ================================================ require_relative '../../helper' require 'fluent/plugin/in_tail' class IntailFIFO < Test::Unit::TestCase sub_test_case '#read_line' do test 'returns lines splitting per `\n`' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["test\n", "test\n", "test\n"], lines end test 'concat line when line is separated' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) text = ("test\n" * 3 + 'test').force_encoding(Encoding::ASCII_8BIT) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["test\n", "test\n", "test\n"], lines fifo << "2\n" fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines end test 'returns lines which convert encoding' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, nil, Encoding::UTF_8) text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::UTF_8, lines[0].encoding assert_equal ["test\n", "test\n", "test\n"], lines end test 'reads lines as from_encoding' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, $log, nil, Encoding::ASCII_8BIT) text = ("test\n" * 3).force_encoding(Encoding::UTF_8) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["test\n", "test\n", "test\n"], lines end sub_test_case 'when it includes multi byte chars' do test 'handles it as ascii_8bit' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) text = ("てすと\n" * 3).force_encoding(Encoding::ASCII_8BIT) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["てすと\n", "てすと\n", "てすと\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines end test 'replaces character with ? when convert error happens' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, $log, nil, Encoding::ASCII_8BIT) text = ("てすと\n" * 3).force_encoding(Encoding::UTF_8) fifo << text lines = [] fifo.read_lines(lines) assert_equal Encoding::ASCII_8BIT, lines[0].encoding assert_equal ["???\n", "???\n", "???\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines end end test 'returns nothing when buffer is empty' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) lines = [] fifo.read_lines(lines) assert_equal [], lines text = "test\n" * 3 fifo << text fifo.read_lines(lines) assert_equal ["test\n", "test\n", "test\n"], lines lines = [] fifo.read_lines(lines) assert_equal [], lines end data('bigger than max_line_size', [ ["test test test\n" * 3], [], ]) data('less than or equal to max_line_size', [ ["test\n" * 2], ["test\n", "test\n"], ]) data('mix', [ ["test test test\ntest\ntest test test\ntest\ntest test test\n"], ["test\n", "test\n"], ]) data('mix and multiple', [ [ "test test test\ntest\n", "test", " test test\nt", "est\nt" ], ["test\n", "test\n"], ]) data('remaining data bigger than max_line_size should be discarded', [ [ "test\nlong line still not having EOL", "following texts to the previous long line\ntest\n", ], ["test\n", "test\n"], ]) test 'return lines only that size is less than or equal to max_line_size' do |(input_texts, expected)| max_line_size = 5 fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, max_line_size) lines = [] input_texts.each do |text| fifo << text.force_encoding(Encoding::ASCII_8BIT) fifo.read_lines(lines) # The size of remaining buffer (i.e. a line still not having EOL) must not exceed max_line_size. assert { fifo.buffer.bytesize <= max_line_size } end assert_equal expected, lines end end sub_test_case '#<<' do test 'does not make any change about encoding to an argument' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) text = ("test\n" * 3).force_encoding(Encoding::UTF_8) assert_equal Encoding::UTF_8, text.encoding fifo << text assert_equal Encoding::UTF_8, text.encoding end end sub_test_case '#reading_bytesize' do test 'returns buffer size' do fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log) text = "test\n" * 3 + 'test' fifo << text assert_equal text.bytesize, fifo.reading_bytesize lines = [] fifo.read_lines(lines) assert_equal ["test\n", "test\n", "test\n"], lines assert_equal 'test'.bytesize, fifo.reading_bytesize fifo << "2\n" fifo.read_lines(lines) assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines assert_equal 0, fifo.reading_bytesize end test 'returns the entire line size even if the size is over max_line_size' do max_line_size = 20 fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, $log, max_line_size) lines = [] text = "long line still not having EOL" fifo << text fifo.read_lines(lines) assert_equal [], lines assert_equal 0, fifo.buffer.bytesize assert_equal text.bytesize, fifo.reading_bytesize text2 = " following texts" fifo << text2 fifo.read_lines(lines) assert_equal [], lines assert_equal 0, fifo.buffer.bytesize assert_equal text.bytesize + text2.bytesize, fifo.reading_bytesize text3 = " end of the line\n" fifo << text3 fifo.read_lines(lines) assert_equal [], lines assert_equal 0, fifo.buffer.bytesize assert_equal 0, fifo.reading_bytesize end end end ================================================ FILE: test/plugin/in_tail/test_io_handler.rb ================================================ require_relative '../../helper' require 'fluent/plugin/in_tail' require 'fluent/plugin/metrics_local' require 'tempfile' class IntailIOHandlerTest < Test::Unit::TestCase def setup Tempfile.create('intail_io_handler') do |file| file.binmode @file = file opened_file_metrics = Fluent::Plugin::LocalMetrics.new opened_file_metrics.configure(config_element('metrics', '', {})) closed_file_metrics = Fluent::Plugin::LocalMetrics.new closed_file_metrics.configure(config_element('metrics', '', {})) rotated_file_metrics = Fluent::Plugin::LocalMetrics.new rotated_file_metrics.configure(config_element('metrics', '', {})) throttling_metrics = Fluent::Plugin::LocalMetrics.new throttling_metrics.configure(config_element('metrics', '', {})) @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics) yield end end def create_target_info Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino) end def create_watcher Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil) end test '#on_notify load file content and passed it to receive_lines method' do text = "this line is test\ntest line is test\n" @file.write(text) @file.close watcher = create_watcher update_pos = 0 stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { 0 } stub(pe).update_pos { |val| update_pos = val } pe end returned_lines = '' r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher| returned_lines << lines.join true end r.on_notify assert_equal text.bytesize, update_pos assert_equal text, returned_lines r.on_notify assert_equal text.bytesize, update_pos assert_equal text, returned_lines end sub_test_case 'when open_on_every_update is true and read_pos returns always 0' do test 'open new IO and change pos to 0 and read it' do text = "this line is test\ntest line is test\n" @file.write(text) @file.close update_pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { 0 } stub(pe).update_pos { |val| update_pos = val } pe end returned_lines = '' r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher| returned_lines << lines.join true end r.on_notify assert_equal text.bytesize, update_pos assert_equal text, returned_lines r.on_notify assert_equal text * 2, returned_lines end end sub_test_case 'when limit is 5' do test 'call receive_lines once when short line(less than 65536)' do text = "line\n" * 8 @file.write(text) @file.close update_pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { 0 } stub(pe).update_pos { |val| update_pos = val } pe end returned_lines = [] r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher| returned_lines << lines.dup true end r.on_notify assert_equal text.bytesize, update_pos assert_equal 8, returned_lines[0].size end test 'call receive_lines some times when long line(more than 65536)' do t = 'line' * (65536 / 8) text = "#{t}\n" * 8 @file.write(text) @file.close update_pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { 0 } stub(pe).update_pos { |val| update_pos = val } pe end returned_lines = [] r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher| returned_lines << lines.dup true end r.on_notify assert_equal text.bytesize, update_pos assert_equal 5, returned_lines[0].size assert_equal 3, returned_lines[1].size end end sub_test_case 'max_line_size' do test 'does not call receive_lines when line_size exceeds max_line_size' do t = 'x' * (8192) text = "#{t}\n" max_line_size = 8192 @file.write(text) @file.close update_pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos {0} stub(pe).update_pos { |val| update_pos = val } pe end returned_lines = [] r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1, max_line_size: max_line_size, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher| returned_lines << lines.dup true end r.on_notify assert_equal text.bytesize, update_pos assert_equal 0, returned_lines.size end data( "open_on_every_update false" => false, "open_on_every_update true" => true, ) test 'manage pos correctly if a long line not having EOL occurs' do |open_on_every_update| max_line_size = 20 returned_lines = [] pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { pos } stub(pe).update_pos { |val| pos = val } pe end io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new( watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1, max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update, metrics: @metrics ) do |lines, _watcher| returned_lines << lines.dup true end short_line = "short line\n" long_lines = [ "long line still not having EOL", " end of the line\n", ] @file.write(short_line) @file.write(long_lines[0]) @file.flush io_handler.on_notify assert_equal [[short_line]], returned_lines assert_equal short_line.bytesize, pos @file.write(long_lines[1]) @file.flush io_handler.on_notify assert_equal [[short_line]], returned_lines expected_size = short_line.bytesize + long_lines[0..1].map{|l| l.bytesize}.sum assert_equal expected_size, pos io_handler.close end data( "open_on_every_update false" => false, "open_on_every_update true" => true, ) test 'discards a subsequent data in a long line even if restarting occurs between' do |open_on_every_update| max_line_size = 20 returned_lines = [] pos = 0 watcher = create_watcher stub(watcher).pe do pe = 'position_file' stub(pe).read_pos { pos } stub(pe).update_pos { |val| pos = val } pe end io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new( watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1, max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update, metrics: @metrics ) do |lines, _watcher| returned_lines << lines.dup true end short_line = "short line\n" long_lines = [ "long line still not having EOL", " end of the line\n", ] @file.write(short_line) @file.write(long_lines[0]) @file.flush io_handler.on_notify assert_equal [[short_line]], returned_lines io_handler.close io_handler = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new( watcher, path: @file.path, read_lines_limit: 1000, read_bytes_limit_per_second: -1, max_line_size: max_line_size, log: $log, open_on_every_update: open_on_every_update, metrics: @metrics ) do |lines, _watcher| returned_lines << lines.dup true end @file.write(long_lines[1]) @file.flush io_handler.on_notify assert_equal [[short_line]], returned_lines io_handler.close end end end ================================================ FILE: test/plugin/in_tail/test_position_file.rb ================================================ require_relative '../../helper' require 'fluent/plugin/in_tail/position_file' require 'fluent/plugin/in_tail' require 'fileutils' require 'tempfile' class IntailPositionFileTest < Test::Unit::TestCase def setup Tempfile.create('intail_position_file_test') do |file| file.binmode @file = file yield end end UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION TEST_CONTENT = <<~EOF valid_path\t0000000000000002\t0000000000000001 inode23bit\t0000000000000000\t00000000 invalidpath100000000000000000000000000000000 unwatched\t#{UNWATCHED_STR}\t0000000000000000 EOF TEST_CONTENT_PATHS = { "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1), "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0), } TEST_CONTENT_INODES = { 1 => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1), 0 => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0), } def write_data(f, content) f.write(content) f.seek(0) end def follow_inodes_block [true, false].each do |follow_inodes| yield follow_inodes end end test '.load' do write_data(@file, TEST_CONTENT) Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log}) @file.seek(0) lines = @file.readlines assert_equal 2, lines.size assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0] assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1] end sub_test_case '#try_compact' do test 'compact invalid and convert 32 bit inode value' do write_data(@file, TEST_CONTENT) Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact @file.seek(0) lines = @file.readlines assert_equal 2, lines.size assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0] assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1] end test 'compact data if duplicated line' do write_data(@file, <<~EOF) valid_path\t0000000000000002\t0000000000000001 valid_path\t0000000000000003\t0000000000000004 EOF Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact @file.seek(0) lines = @file.readlines assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0] end test 'does not change when the file is changed' do write_data(@file, TEST_CONTENT) pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}) mock.proxy(pf).fetch_compacted_entries do |r| @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n") r end pf.try_compact @file.seek(0) lines = @file.readlines assert_equal 5, lines.size end test 'update seek position of remained position entry' do pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}) target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1) target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1) target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1) pf[target_info1] pf[target_info2] pf[target_info3] target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234) pf.unwatch(target_info1_2) pf.try_compact @file.seek(0) lines = @file.readlines assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0] assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1] assert_equal 2, lines.size target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235) target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236) pf.unwatch(target_info2_2) pf.unwatch(target_info3_2) @file.seek(0) lines = @file.readlines assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0] assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1] assert_equal 2, lines.size end test 'should ignore initial existing files on follow_inode' do write_data(@file, TEST_CONTENT) pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log}) @file.seek(0) assert_equal([], @file.readlines) @file.seek(0) write_data(@file, TEST_CONTENT) pos_file.try_compact @file.seek(0) assert_equal([ "valid_path\t0000000000000002\t0000000000000001\n", "inode23bit\t0000000000000000\t0000000000000000\n", ], @file.readlines) end end sub_test_case '#load' do test 'compact invalid and convert 32 bit inode value' do write_data(@file, TEST_CONTENT) Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log}) @file.seek(0) lines = @file.readlines assert_equal 2, lines.size assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0] assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1] end test 'compact deleted paths' do write_data(@file, TEST_CONTENT) Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log}) @file.seek(0) lines = @file.readlines assert_equal [], lines end test 'compact data if duplicated line' do write_data(@file, <<~EOF) valid_path\t0000000000000002\t0000000000000001 valid_path\t0000000000000003\t0000000000000004 EOF Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load @file.seek(0) lines = @file.readlines assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0] end end sub_test_case '#[]' do test 'return entry' do write_data(@file, TEST_CONTENT) pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log}) valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino) f = pf[valid_target_info] assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class assert_equal 2, f.read_pos assert_equal 1, f.read_inode @file.seek(0) lines = @file.readlines assert_equal 2, lines.size nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1) f = pf[nonexistent_target_info] assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class assert_equal 0, f.read_pos assert_equal 0, f.read_inode @file.seek(0) lines = @file.readlines assert_equal 3, lines.size assert_equal "nonexist_path\t0000000000000000\t0000000000000000\n", lines[2] end test 'does not change other value position if other entry try to write' do write_data(@file, TEST_CONTENT) pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log) f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)] assert_equal 0, f.read_inode assert_equal 0, f.read_pos pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2) f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)] assert_equal 0, f.read_inode assert_equal 0, f.read_pos pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2) assert_equal 1, f.read_inode assert_equal 2, f.read_pos end end sub_test_case '#unwatch' do test 'unwatch entry by path' do write_data(@file, TEST_CONTENT) pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log) inode1 = File.stat(@file).ino target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1) p1 = pf[target_info1] assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class pf.unwatch(target_info1) assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION inode2 = File.stat(@file).ino target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2) p2 = pf[target_info2] assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class assert_not_equal p1, p2 end test 'unwatch entries by inode' do write_data(@file, TEST_CONTENT) pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log) existing_targets = TEST_CONTENT_INODES.select do |inode, target_info| inode == 1 end pe_to_unwatch = pf[TEST_CONTENT_INODES[0]] pf.unwatch_removed_targets(existing_targets) assert_equal( { map_keys: [TEST_CONTENT_INODES[1].ino], unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION, }, { map_keys: pf.instance_variable_get(:@map).keys, unwatched_pe_pos: pe_to_unwatch.read_pos, } ) unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]] assert_not_equal pe_to_unwatch, unwatched_pe_retaken end end sub_test_case 'FilePositionEntry' do FILE_POS_CONTENT = <<~EOF valid_path\t0000000000000002\t0000000000000001 valid_path2\t0000000000000003\t0000000000000002 EOF def build_files(file) r = {} file.each_line do |line| m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line) path = m[1] pos = m[2].to_i(16) ino = m[3].to_i(16) seek = file.pos - line.bytesize + path.bytesize + 1 r[path] = Fluent::Plugin::TailInput::FilePositionEntry.new(@file, Mutex.new, seek, pos, ino) end r end test '#update' do write_data(@file, FILE_POS_CONTENT) fs = build_files(@file) f = fs['valid_path'] f.update(11, 10) @file.seek(0) lines = @file.readlines assert_equal 2, lines.size assert_equal "valid_path\t000000000000000a\t000000000000000b\n", lines[0] assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1] end test '#update_pos' do write_data(@file, FILE_POS_CONTENT) fs = build_files(@file) f = fs['valid_path'] f.update_pos(10) @file.seek(0) lines = @file.readlines assert_equal 2, lines.size assert_equal "valid_path\t000000000000000a\t0000000000000001\n", lines[0] assert_equal "valid_path2\t0000000000000003\t0000000000000002\n", lines[1] end test '#read_pos' do write_data(@file, FILE_POS_CONTENT) fs = build_files(@file) f = fs['valid_path'] assert_equal 2, f.read_pos f.update_pos(10) assert_equal 10, f.read_pos f.update(2, 11) assert_equal 11, f.read_pos end test '#read_inode' do write_data(@file, FILE_POS_CONTENT) fs = build_files(@file) f = fs['valid_path'] assert_equal 1, f.read_inode f.update_pos(10) assert_equal 1, f.read_inode f.update(2, 11) assert_equal 2, f.read_inode end end end ================================================ FILE: test/plugin/out_forward/test_ack_handler.rb ================================================ require_relative '../../helper' require 'fluent/test/driver/output' require 'flexmock/test_unit' require 'fluent/plugin/out_forward' require 'fluent/plugin/out_forward/ack_handler' class AckHandlerTest < Test::Unit::TestCase data( 'chunk_id is matched' => [MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }), Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS], 'chunk_id is not matched' => [MessagePack.pack({ 'ack' => 'unmatched' }), Fluent::Plugin::ForwardOutput::AckHandler::Result::CHUNKID_UNMATCHED], 'chunk_id is empty' => ['', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED], ) test 'returns chunk_id, node, sock and result status' do |args| receved, state = args ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) node = flexmock('node', host: '127.0.0.1', port: '1000') # for log chunk_id = 'chunk_id 111' ack = ack_handler.create_ack(chunk_id, node) r, w = IO.pipe begin w.write(chunk_id) mock(r).recv(anything) { |_| receved } # IO does not have recv ack.enqueue(r) a1 = a2 = a3 = a4 = nil ack_handler.collect_response(1) do |cid, n, s, ret| # This block is rescued by ack_handler so it needs to invoke assertion outside of this block a1 = cid; a2 = n; a3 = s; a4 = ret end assert_equal chunk_id, a1 assert_equal node, a2 assert_equal r, a3 assert_equal state, a4 ensure r.close rescue nil w.close rescue nil end end test 'returns nil if raise an error' do ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) node = flexmock('node', host: '127.0.0.1', port: '1000') # for log chunk_id = 'chunk_id 111' ack = ack_handler.create_ack(chunk_id, node) r, w = IO.pipe begin w.write(chunk_id) mock(r).recv(anything) { |_| raise 'unexpected error' } # IO does not have recv ack.enqueue(r) a1 = a2 = a3 = a4 = nil ack_handler.collect_response(1) do |cid, n, s, ret| # This block is rescued by ack_handler so it needs to invoke assertion outside of this block a1 = cid; a2 = n; a3 = s; a4 = ret end assert_nil a1 assert_nil a2 assert_nil a3 assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4 ensure r.close rescue nil w.close rescue nil end end test 'when ack is expired' do ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100) node = flexmock('node', host: '127.0.0.1', port: '1000') # for log chunk_id = 'chunk_id 111' ack = ack_handler.create_ack(chunk_id, node) r, w = IO.pipe begin w.write(chunk_id) mock(r).recv(anything).never ack.enqueue(r) a1 = a2 = a3 = a4 = nil ack_handler.collect_response(1) do |cid, n, s, ret| # This block is rescued by ack_handler so it needs to invoke assertion outside of this block a1 = cid; a2 = n; a3 = s; a4 = ret end assert_equal chunk_id, a1 assert_equal node, a2 assert_equal r, a3 assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4 ensure r.close rescue nil w.close rescue nil end end # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case. # If exclusive control for this case is implemented, this test may not be necessary. test 'raises no error when another thread closes a socket' do ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) node = flexmock('node', host: '127.0.0.1', port: '1000') # for log chunk_id = 'chunk_id 111' ack = ack_handler.create_ack(chunk_id, node) r, w = IO.pipe begin w.write(chunk_id) def r.recv(arg) sleep(1) # To ensure that multiple threads select the socket before closing. raise IOError, 'stream closed in another thread' if self.closed? MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }) end ack.enqueue(r) threads = [] 2.times do threads << Thread.new do ack_handler.collect_response(1) do |cid, n, s, ret| s&.close end end end assert_true threads.map{ |t| t.join(10) }.all? assert_false( $log.out.logs.any? { |log| log.include?('[error]') }, $log.out.logs.select{ |log| log.include?('[error]') }.join('\n') ) ensure r.close rescue nil w.close rescue nil end end end ================================================ FILE: test/plugin/out_forward/test_connection_manager.rb ================================================ require_relative '../../helper' require 'fluent/test/driver/output' require 'flexmock/test_unit' require 'fluent/plugin/out_forward' require 'fluent/plugin/out_forward/connection_manager' require 'fluent/plugin/out_forward/socket_cache' class ConnectionManager < Test::Unit::TestCase sub_test_case '#connect' do sub_test_case 'when socket_cache is nil' do test 'creates socket and does not close when block is not given' do cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock }, socket_cache: nil, ) mock.proxy(cm).connect_keepalive(anything).never sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) assert_equal(sock, 'sock') assert_equal(ri.state, :established) end test 'creates socket and calls close when block is given' do cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { sock = 'sock' mock(sock).close.once mock(sock).close_write.once sock }, socket_cache: nil, ) mock.proxy(cm).connect_keepalive(anything).never cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri| assert_equal(sock, 'sock') assert_equal(ri.state, :established) end end test 'when secure is true, state is helo' do cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: true, connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock }, socket_cache: nil, ) mock.proxy(cm).connect_keepalive(anything).never sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) assert_equal(sock, 'sock') assert_equal(ri.state, :helo) end test 'when passed ack' do sock = 'sock' cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { mock(sock).close.never mock(sock).close_write.never sock }, socket_cache: nil, ) mock.proxy(cm).connect_keepalive(anything).never ack = mock('ack').enqueue(sock).once.subject cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri| assert_equal(sock, 'sock') assert_equal(ri.state, :established) end end end sub_test_case 'when socket_cache exists' do test 'calls connect_keepalive' do cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) mock(cache).checkin('sock').never cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock }, socket_cache: cache, ) mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) assert_equal(sock, 'sock') assert_equal(ri.state, :established) end test 'calls connect_keepalive and closes socket with block' do cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) mock(cache).checkin('sock').once cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock); sock }, socket_cache: cache, ) mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri| assert_equal(sock, 'sock') assert_equal(ri.state, :established) end end test 'does not call dec_ref when ack is passed' do cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) mock(cache).checkin('sock').never sock = 'sock' ack = stub('ack').enqueue(sock).once.subject cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new( log: $log, secure: false, connection_factory: -> (_, _, _) { mock(sock).close.never mock(sock).close_write.never sock }, socket_cache: cache, ) mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: ack).once cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri| assert_equal(sock, 'sock') assert_equal(ri.state, :established) end end end end end ================================================ FILE: test/plugin/out_forward/test_handshake_protocol.rb ================================================ require_relative '../../helper' require 'flexmock/test_unit' require 'fluent/plugin/out_forward' require 'fluent/plugin/out_forward/handshake_protocol' require 'fluent/plugin/out_forward/connection_manager' class HandshakeProtocolTest < Test::Unit::TestCase sub_test_case '#invok when helo state' do test 'sends PING message and change state to pingpong' do hostname = 'hostname' handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: nil, username: nil) ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo) sock = StringIO.new('') handshake.invoke(sock, ri, ['HELO', {}]) assert_equal(ri.state, :pingpong) Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping| assert_equal(ping.size, 6) assert_equal(ping[0], 'PING') assert_equal(ping[1], hostname) assert(ping[2].is_a?(String)) # content is hashed value assert(ping[3].is_a?(String)) # content is hashed value assert_equal(ping[4], '') assert_equal(ping[5], '') end end test 'returns PING message with username if auth exists' do hostname = 'hostname' username = 'username' pass = 'pass' handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: pass, username: username) ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo) sock = StringIO.new('') handshake.invoke(sock, ri, ['HELO', { 'auth' => 'auth' }]) assert_equal(ri.state, :pingpong) Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping| assert_equal(ping.size, 6) assert_equal(ping[0], 'PING') assert_equal(ping[1], hostname) assert(ping[2].is_a?(String)) # content is hashed value assert(ping[3].is_a?(String)) # content is hashed value assert_equal(ping[4], username) assert_not_equal(ping[5], pass) # should be hashed end end data( lack_of_elem: ['HELO'], wrong_message: ['HELLO!', {}], ) test 'raises an error when message is' do |msg| handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil) ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo) sock = StringIO.new('') assert_raise(Fluent::Plugin::ForwardOutput::HeloError) do handshake.invoke(sock, ri, msg) end assert_equal(ri.state, :helo) end end sub_test_case '#invok when pingpong state' do test 'sends PING message and change state to pingpong' do handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil) handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '') handshake.invoke( '', ri, # 40a3.... = Digest::SHA512.new.update('ce1897b0d3dbd76b90d7fb96010dcac3').update('client_hostname').update('').update('shared_key').hexdigest ['PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'] ) assert_equal(ri.state, :established) end test 'raises an error when password and username are nil if auth exists' do handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil) ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo) assert_raise(Fluent::Plugin::ForwardOutput::PingpongError.new('username and password are required')) do handshake.invoke('', ri, ['HELO', { 'auth' => 'auth' }]) end end data( lack_of_elem: ['PONG', true, '', 'client_hostname'], wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'], error_by_server: ['PONG', false, 'error', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'], same_hostname_as_server: ['PONG', true, '', 'hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'], wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'], ) test 'raises an error when message is' do |msg| handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: '', username: '') handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '') assert_raise(Fluent::Plugin::ForwardOutput::PingpongError) do handshake.invoke('', ri, msg) end assert_equal(ri.state, :pingpong) end end end ================================================ FILE: test/plugin/out_forward/test_load_balancer.rb ================================================ require_relative '../../helper' require 'flexmock/test_unit' require 'fluent/plugin/out_forward/load_balancer' class LoadBalancerTest < Test::Unit::TestCase sub_test_case 'select_healthy_node' do test 'select healthy node' do lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) n1 = flexmock('node', :'standby?' => false, :'available?' => false, weight: 1) n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) lb.rebuild_weight_array([n1, n2]) lb.select_healthy_node do |node| assert_equal(node, n2) end lb.select_healthy_node do |node| assert_equal(node, n2) end end test 'call like round robin' do lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) lb.rebuild_weight_array([n1, n2]) lb.select_healthy_node do |node| # to handle random choice if node == n1 lb.select_healthy_node do |node| assert_equal(node, n2) end lb.select_healthy_node do |node| assert_equal(node, n1) end else lb.select_healthy_node do |node| assert_equal(node, n1) end lb.select_healthy_node do |node| assert_equal(node, n2) end end end end test 'call like round robin without weight=0 node' do lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0) lb.rebuild_weight_array([n1, n2, n3]) lb.select_healthy_node do |node| # to handle random choice if node == n1 lb.select_healthy_node do |node| assert_equal(node, n2) end lb.select_healthy_node do |node| assert_equal(node, n1) end lb.select_healthy_node do |node| assert_equal(node, n2) end else lb.select_healthy_node do |node| assert_equal(node, n1) end lb.select_healthy_node do |node| assert_equal(node, n2) end lb.select_healthy_node do |node| assert_equal(node, n1) end end end end test 'raise an error if all node are unavailable' do lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)]) assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do lb.select_healthy_node end end test 'it regards weight=0 node as unavailable' do lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)]) assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do lb.select_healthy_node end end end end ================================================ FILE: test/plugin/out_forward/test_socket_cache.rb ================================================ require_relative '../../helper' require 'fluent/plugin/out_forward/socket_cache' require 'timecop' class SocketCacheTest < Test::Unit::TestCase sub_test_case 'checkout_or' do test 'when given key does not exist' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) sock = mock!.open { 'socket' }.subject assert_equal('socket', c.checkout_or('key') { sock.open }) end test 'when given key exists' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) socket = 'socket' assert_equal(socket, c.checkout_or('key') { socket }) c.checkin(socket) sock = mock!.open.never.subject assert_equal(socket, c.checkout_or('key') { sock.open }) end test 'when given key exists but used by other' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) assert_equal('sock', c.checkout_or('key') { 'sock' }) new_sock = 'new sock' sock = mock!.open { new_sock }.subject assert_equal(new_sock, c.checkout_or('key') { sock.open }) end test "when given key's value was expired" do c = Fluent::Plugin::ForwardOutput::SocketCache.new(0, $log) assert_equal('sock', c.checkout_or('key') { 'sock' }) new_sock = 'new sock' sock = mock!.open { new_sock }.subject assert_equal(new_sock, c.checkout_or('key') { sock.open }) end test 'reuse same hash object after calling purge_obsolete_socks' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) c.checkout_or('key') { 'socket' } c.purge_obsolete_socks assert_nothing_raised(NoMethodError) do c.checkout_or('key') { 'new socket' } end end end sub_test_case 'checkin' do test 'when value exists' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) socket = 'socket' c.checkout_or('key') { socket } c.checkin(socket) assert_equal(socket, c.instance_variable_get(:@available_sockets)['key'].first.sock) assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size) assert_equal(0, c.instance_variable_get(:@inflight_sockets).size) end test 'when value does not exist' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) c.checkout_or('key') { 'sock' } c.checkin('other sock') assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size) assert_equal(1, c.instance_variable_get(:@inflight_sockets).size) end end test 'revoke' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) socket = 'socket' c.checkout_or('key') { socket } c.revoke(socket) assert_equal(1, c.instance_variable_get(:@inactive_sockets).size) assert_equal(0, c.instance_variable_get(:@inflight_sockets).size) assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size) sock = mock!.open { 1 }.subject assert_equal(1, c.checkout_or('key') { sock.open }) end sub_test_case 'clear' do test 'when value is in available_sockets' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) m = mock!.close { 'closed' }.subject m2 = mock!.close { 'closed' }.subject m3 = mock!.close { 'closed' }.subject c.checkout_or('key') { m } c.revoke(m) c.checkout_or('key') { m2 } c.checkin(m2) c.checkout_or('key2') { m3 } assert_equal(1, c.instance_variable_get(:@inflight_sockets).size) assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size) assert_equal(1, c.instance_variable_get(:@inactive_sockets).size) c.clear assert_equal(0, c.instance_variable_get(:@inflight_sockets).size) assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size) assert_equal(0, c.instance_variable_get(:@inactive_sockets).size) end end sub_test_case 'purge_obsolete_socks' do def teardown Timecop.return end test 'delete key in inactive_socks' do c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) sock = mock!.close { 'closed' }.subject c.checkout_or('key') { sock } c.revoke(sock) assert_false(c.instance_variable_get(:@inactive_sockets).empty?) c.purge_obsolete_socks assert_true(c.instance_variable_get(:@inactive_sockets).empty?) end test 'move key from available_sockets to inactive_sockets' do Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900')) c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) sock = mock!.close { 'closed' }.subject sock2 = mock!.close.never.subject stub(sock).inspect stub(sock2).inspect c.checkout_or('key') { sock } c.checkin(sock) # wait timeout Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900')) c.checkout_or('key') { sock2 } assert_equal(1, c.instance_variable_get(:@inflight_sockets).size) assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock) c.purge_obsolete_socks assert_equal(1, c.instance_variable_get(:@inflight_sockets).size) assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock) end test 'should not purge just after checkin and purge after timeout' do Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900')) c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log) sock = mock!.close.never.subject stub(sock).inspect c.checkout_or('key') { sock } Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900')) c.checkin(sock) assert_equal(1, c.instance_variable_get(:@available_sockets).size) c.purge_obsolete_socks assert_equal(1, c.instance_variable_get(:@available_sockets).size) Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900')) assert_equal(1, c.instance_variable_get(:@available_sockets).size) c.purge_obsolete_socks assert_equal(0, c.instance_variable_get(:@available_sockets).size) end end end ================================================ FILE: test/plugin/test_bare_output.rb ================================================ require_relative '../helper' require 'fluent/plugin/bare_output' require 'fluent/event' module FluentPluginBareOutputTest class DummyPlugin < Fluent::Plugin::BareOutput attr_reader :store def initialize super @store = [] end def process(tag, es) es.each do |time, record| @store << [tag, time, record] end end end end class BareOutputTest < Test::Unit::TestCase setup do Fluent::Test.setup @p = FluentPluginBareOutputTest::DummyPlugin.new end test 'has healthy lifecycle' do assert !@p.configured? @p.configure(config_element()) assert @p.configured? assert !@p.started? @p.start assert @p.start assert !@p.stopped? @p.stop assert @p.stopped? assert !@p.before_shutdown? @p.before_shutdown assert @p.before_shutdown? assert !@p.shutdown? @p.shutdown assert @p.shutdown? assert !@p.after_shutdown? @p.after_shutdown assert @p.after_shutdown? assert !@p.closed? @p.close assert @p.closed? assert !@p.terminated? @p.terminate assert @p.terminated? end test 'has plugin_id automatically generated' do assert @p.respond_to?(:plugin_id_configured?) assert @p.respond_to?(:plugin_id) @p.configure(config_element()) assert !@p.plugin_id_configured? assert @p.plugin_id assert{ @p.plugin_id != 'mytest' } end test 'has plugin_id manually configured' do @p.configure(config_element('ROOT', '', {'@id' => 'mytest'})) assert @p.plugin_id_configured? assert_equal 'mytest', @p.plugin_id end test 'has plugin logger' do assert @p.respond_to?(:log) assert @p.log # default logger original_logger = @p.log @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'})) assert(@p.log.object_id != original_logger.object_id) assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level end test 'can load plugin helpers' do assert_nothing_raised do class FluentPluginBareOutputTest::DummyPlugin2 < Fluent::Plugin::BareOutput helpers :storage end end end test 'can use metrics plugins and fallback methods' do @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'})) %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name| assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics) end assert_equal 0, @p.num_errors assert_equal 0, @p.emit_count assert_equal 0, @p.emit_size assert_equal 0, @p.emit_records end test 'can get input event stream to write' do @p.configure(config_element('ROOT')) @p.start es1 = Fluent::OneEventStream.new(event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}) es2 = Fluent::ArrayEventStream.new([ [event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}], [event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}], ]) @p.emit_events('mytest1', es1) @p.emit_events('mytest2', es2) all_events = [ ['mytest1', event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}], ['mytest2', event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}], ['mytest2', event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}], ] assert_equal all_events, @p.store end end ================================================ FILE: test/plugin/test_base.rb ================================================ require_relative '../helper' require 'tmpdir' require 'fluent/plugin/base' module FluentPluginBaseTest class DummyPlugin < Fluent::Plugin::Base end end class BaseTest < Test::Unit::TestCase setup do @p = FluentPluginBaseTest::DummyPlugin.new end test 'has methods for phases of plugin life cycle, and methods to know "super"s were correctly called or not' do assert !@p.configured? @p.configure(config_element()) assert @p.configured? assert !@p.started? @p.start assert @p.start assert !@p.stopped? @p.stop assert @p.stopped? assert !@p.before_shutdown? @p.before_shutdown assert @p.before_shutdown? assert !@p.shutdown? @p.shutdown assert @p.shutdown? assert !@p.after_shutdown? @p.after_shutdown assert @p.after_shutdown? assert !@p.closed? @p.close assert @p.closed? assert !@p.terminated? @p.terminate assert @p.terminated? end test 'can access system config' do assert @p.system_config @p.system_config_override({'process_name' => 'mytest'}) assert_equal 'mytest', @p.system_config.process_name end test 'does not have router in default' do assert !@p.has_router? end sub_test_case '#fluentd_worker_id' do test 'returns 0 in default' do assert_equal 0, @p.fluentd_worker_id end test 'returns the value specified via SERVERENGINE_WORKER_ID env variable' do pre_value = ENV['SERVERENGINE_WORKER_ID'] begin ENV['SERVERENGINE_WORKER_ID'] = 7.to_s assert_equal 7, @p.fluentd_worker_id ensure ENV['SERVERENGINE_WORKER_ID'] = pre_value end end end test 'does not have root dir in default' do assert_nil @p.plugin_root_dir end test 'is configurable by config_param and config_section' do assert_nothing_raised do class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase config_param :myparam1, :string config_section :mysection, multi: false do config_param :myparam2, :integer end end end p2 = FluentPluginBaseTest::DummyPlugin2.new assert_nothing_raised do p2.configure(config_element('ROOT', '', {'myparam1' => 'myvalue1'}, [config_element('mysection', '', {'myparam2' => 99})])) end assert_equal 'myvalue1', p2.myparam1 assert_equal 99, p2.mysection.myparam2 end test 'plugins are available with multi worker configuration in default' do assert @p.multi_workers_ready? end test 'provides #string_safe_encoding to scrub invalid sequence string with info logging' do logger = Fluent::Test::TestLogger.new m = Module.new do define_method(:log) do logger end end @p.extend m assert_equal [], logger.logs ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(".") } assert_equal ['abc?', "\u0001f"], ret assert_equal 1, logger.logs.size assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") } end test 'generates worker lock path safely' do Dir.mktmpdir("test-fluentd-lock-") do |lock_dir| ENV['FLUENTD_LOCK_DIR'] = lock_dir p = FluentPluginBaseTest::DummyPlugin.new path = p.get_lock_path("Aa\\|=~/_123") assert_equal lock_dir, File.dirname(path) assert_equal "fluentd-Aa______123.lock", File.basename(path) end end test 'can acquire inter-worker locking' do Dir.mktmpdir("test-fluentd-lock-") do |lock_dir| ENV['FLUENTD_LOCK_DIR'] = lock_dir p = FluentPluginBaseTest::DummyPlugin.new lock_path = p.get_lock_path("test_base") p.acquire_worker_lock("test_base") do # With LOCK_NB set, flock() returns `false` when the # file is already locked. File.open(lock_path, "w") do |f| assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB) end end # Lock should be release by now. In that case, flock # must return 0. File.open(lock_path, "w") do |f| assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB) end end end test '`ArgumentError` when `conf` is not `Fluent::Config::Element`' do assert_raise ArgumentError.new('BUG: type of conf must be Fluent::Config::Element, but Hash is passed.') do @p.configure({}) end end sub_test_case 'system_config.workers value after configure' do def assert_system_config_workers_value(data) conf = config_element() conf.set_target_worker_ids(data[:target_worker_ids]) @p.configure(conf) assert{ @p.system_config.workers == data[:expected] } end def stub_supervisor_mode stub(Fluent::Engine).supervisor_mode { true } stub(Fluent::Engine).worker_id { -1 } end sub_test_case 'with workers 3 ' do setup do system_config = Fluent::SystemConfig.new system_config.workers = 3 stub(Fluent::Engine).system_config { system_config } end data( 'without directive', { target_worker_ids: [], expected: 3 }, keep: true ) data( 'with ', { target_worker_ids: [0], expected: 1 }, keep: true ) data( 'with ', { target_worker_ids: [0, 1], expected: 2 }, keep: true ) data( 'with ', { target_worker_ids: [0, 1, 2], expected: 3 }, keep: true ) test 'system_config.workers value after configure' do assert_system_config_workers_value(data) end test 'system_config.workers value after configure with supervisor_mode' do stub_supervisor_mode assert_system_config_workers_value(data) end end sub_test_case 'without directive' do data( 'without directive', { target_worker_ids: [], expected: 1 }, keep: true ) data( 'with ', { target_worker_ids: [0], expected: 1 }, keep: true ) test 'system_config.workers value after configure' do assert_system_config_workers_value(data) end test 'system_config.workers value after configure with supervisor_mode' do stub_supervisor_mode assert_system_config_workers_value(data) end end end end ================================================ FILE: test/plugin/test_buf_file.rb ================================================ require_relative '../helper' require 'fluent/plugin/buf_file' require 'fluent/plugin/output' require 'fluent/unique_id' require 'fluent/system_config' require 'fluent/env' require 'msgpack' module FluentPluginFileBufferTest class DummyOutputPlugin < Fluent::Plugin::Output Fluent::Plugin.register_output('buffer_file_test_output', self) config_section :buffer do config_set_default :@type, 'file' end def multi_workers_ready? true end def write(chunk) # drop end end class DummyErrorOutputPlugin < DummyOutputPlugin def register_write(&block) instance_variable_set(:@write, block) end def initialize super @should_fail_writing = true @write = nil end def recover @should_fail_writing = false end def write(chunk) if @should_fail_writing raise "failed writing chunk" else @write ? @write.call(chunk) : nil end end def format(tag, time, record) [tag, time.to_i, record].to_json + "\n" end end end class FileBufferTest < Test::Unit::TestCase def metadata(timekey: nil, tag: nil, variables: nil, seq: 0) m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) m.seq = seq m end def write_metadata_old(path, chunk_id, metadata, size, ctime, mtime) metadata = { timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables, id: chunk_id, s: size, c: ctime, m: mtime, } File.open(path, 'wb') do |f| f.write metadata.to_msgpack end end def write_metadata(path, chunk_id, metadata, size, ctime, mtime) metadata = { timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables, seq: metadata.seq, id: chunk_id, s: size, c: ctime, m: mtime, } data = metadata.to_msgpack size = [data.size].pack('N') File.open(path, 'wb') do |f| f.write(Fluent::Plugin::Buffer::FileChunk::BUFFER_HEADER + size + data) end end sub_test_case 'non configured buffer plugin instance' do setup do Fluent::Test.setup @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__) FileUtils.rm_rf @dir FileUtils.mkdir_p @dir end test 'path should include * normally' do d = FluentPluginFileBufferTest::DummyOutputPlugin.new p = Fluent::Plugin::FileBuffer.new p.owner = d p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer.*.file')})) assert_equal File.join(@dir, 'buffer.*.file'), p.path end data('default' => [nil, 'log'], 'conf' => ['.buf', 'buf']) test 'existing directory will be used with additional default file name' do |params| conf, suffix = params d = FluentPluginFileBufferTest::DummyOutputPlugin.new p = Fluent::Plugin::FileBuffer.new p.owner = d c = {'path' => @dir} c['path_suffix'] = conf if conf p.configure(config_element('buffer', '', c)) assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path end data('default' => [nil, 'log'], 'conf' => ['.buf', 'buf']) test 'unexisting path without * handled as directory' do |params| conf, suffix = params d = FluentPluginFileBufferTest::DummyOutputPlugin.new p = Fluent::Plugin::FileBuffer.new p.owner = d c = {'path' => File.join(@dir, 'buffer')} c['path_suffix'] = conf if conf p.configure(config_element('buffer', '', c)) assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path end end sub_test_case 'buffer configurations and workers' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) FileUtils.rm_rf @bufdir Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d end test 'raise error if configured path is of existing file' do @bufpath = File.join(@bufdir, 'buf') FileUtils.mkdir_p @bufdir File.open(@bufpath, 'w'){|f| } # create and close the file assert File.exist?(@bufpath) assert File.file?(@bufpath) buf_conf = config_element('buffer', '', {'path' => @bufpath}) assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do Fluent::SystemConfig.overwrite_system_config('workers' => 4) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf])) end end end test 'raise error if fluentd is configured to use file path pattern and multi workers' do @bufpath = File.join(@bufdir, 'testbuf.*.log') buf_conf = config_element('buffer', '', {'path' => @bufpath}) assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do Fluent::SystemConfig.overwrite_system_config('workers' => 4) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf])) end end end test 'enables multi worker configuration with unexisting directory path' do assert_false File.exist?(@bufdir) buf_conf = config_element('buffer', '', {'path' => @bufdir}) assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {}, [buf_conf])) end end end test 'enables multi worker configuration with existing directory path' do FileUtils.mkdir_p @bufdir buf_conf = config_element('buffer', '', {'path' => @bufdir}) assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {}, [buf_conf])) end end end test 'enables multi worker configuration with root dir' do buf_conf = config_element('buffer', '') assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf])) end end end end sub_test_case 'buffer plugin configured only with path' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) @bufpath = File.join(@bufdir, 'testbuf.*.log') FileUtils.rm_r @bufdir if File.exist?(@bufdir) Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test 'this is persistent plugin' do assert @p.persistent? end test '#start creates directory for buffer chunks' do plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') bufdir = File.dirname(bufpath) FileUtils.rm_r bufdir if File.exist?(bufdir) assert !File.exist?(bufdir) plugin.configure(config_element('buffer', '', {'path' => bufpath})) assert !File.exist?(bufdir) plugin.start assert File.exist?(bufdir) assert{ File.stat(bufdir).mode.to_s(8).end_with?('755') } plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate FileUtils.rm_r bufdir end test '#start creates directory for buffer chunks with specified permission' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') bufdir = File.dirname(bufpath) FileUtils.rm_r bufdir if File.exist?(bufdir) assert !File.exist?(bufdir) plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => '0700'})) assert !File.exist?(bufdir) plugin.start assert File.exist?(bufdir) assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') } plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate FileUtils.rm_r bufdir end test '#start creates directory for buffer chunks with specified permission via system config' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? sysconf = {'dir_permission' => '700'} Fluent::SystemConfig.overwrite_system_config(sysconf) do plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') bufdir = File.dirname(bufpath) FileUtils.rm_r bufdir if File.exist?(bufdir) assert !File.exist?(bufdir) plugin.configure(config_element('buffer', '', {'path' => bufpath})) assert !File.exist?(bufdir) plugin.start assert File.exist?(bufdir) assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') } plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate FileUtils.rm_r bufdir end end test '#generate_chunk generates blank file chunk on path from unique_id of metadata' do m1 = metadata() c1 = @p.generate_chunk(m1) assert c1.is_a? Fluent::Plugin::Buffer::FileChunk assert_equal m1, c1.metadata assert c1.empty? assert_equal :unstaged, c1.state assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') } m2 = metadata(timekey: event_time('2016-04-17 11:15:00 -0700').to_i) c2 = @p.generate_chunk(m2) assert c2.is_a? Fluent::Plugin::Buffer::FileChunk assert_equal m2, c2.metadata assert c2.empty? assert_equal :unstaged, c2.state assert_equal Fluent::DEFAULT_FILE_PERMISSION, c2.permission assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') } c1.purge c2.purge end test '#generate_chunk generates blank file chunk with specified permission' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') bufdir = File.dirname(bufpath) FileUtils.rm_r bufdir if File.exist?(bufdir) assert !File.exist?(bufdir) plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => '0600'})) assert !File.exist?(bufdir) plugin.start m = metadata() c = plugin.generate_chunk(m) assert c.is_a? Fluent::Plugin::Buffer::FileChunk assert_equal m, c.metadata assert c.empty? assert_equal :unstaged, c.state assert_equal 0600, c.permission assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path assert{ File.stat(c.path).mode.to_s(8).end_with?('600') } c.purge plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate FileUtils.rm_r bufdir end test '#generate_chunk generates blank file chunk with specified permission with system_config' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? begin plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') bufdir = File.dirname(bufpath) FileUtils.rm_r bufdir if File.exist?(bufdir) assert !File.exist?(bufdir) plugin.configure(config_element('buffer', '', { 'path' => bufpath })) assert !File.exist?(bufdir) plugin.start m = metadata() c = nil Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do c = plugin.generate_chunk(m) end assert c.is_a? Fluent::Plugin::Buffer::FileChunk assert_equal m, c.metadata assert c.empty? assert_equal :unstaged, c.state assert_equal 0700, c.permission assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path assert{ File.stat(c.path).mode.to_s(8).end_with?('700') } c.purge plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate ensure FileUtils.rm_r bufdir end end end sub_test_case 'configured with system root directory and plugin @id' do setup do @root_dir = File.expand_path('../../tmp/buffer_file_root', __FILE__) FileUtils.rm_rf @root_dir Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end data('default' => [nil, 'log'], 'conf' => ['.buf', 'buf']) test '#start creates directory for buffer chunks' do |params| conf, suffix = params c = {} c['path_suffix'] = conf if conf Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'})) @p.configure(config_element('buffer', '', c)) end expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}") expected_buffer_dir = File.dirname(expected_buffer_path) assert_equal expected_buffer_path, @p.path assert_false Dir.exist?(expected_buffer_dir) @p.start assert Dir.exist?(expected_buffer_dir) end end sub_test_case 'there are no existing file chunks' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) @bufpath = File.join(@bufdir, 'testbuf.*.log') FileUtils.rm_r @bufdir if File.exist?(@bufdir) Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns empty buffer state' do ary = @p.resume assert_equal({}, ary[0]) assert_equal([], ary[1]) end end sub_test_case 'there are some existing file chunks' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir) @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i ) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i), 3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i ) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log") File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i ) @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log") File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i), 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i ) @bufpath = File.join(@bufdir, 'etest.*.log') Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns staged/queued chunks with metadata' do assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size stage = @p.stage m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) assert_equal @c3id, stage[m3].unique_id assert_equal 4, stage[m3].size assert_equal :staged, stage[m3].state m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i) assert_equal @c4id, stage[m4].unique_id assert_equal 3, stage[m4].size assert_equal :staged, stage[m4].state end test '#resume returns queued chunks ordered by last modified time (FIFO)' do assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size queue = @p.queue assert{ queue[0].modified_at < queue[1].modified_at } assert_equal @c1id, queue[0].unique_id assert_equal :queued, queue[0].state assert_equal event_time('2016-04-17 13:58:00 -0700').to_i, queue[0].metadata.timekey assert_nil queue[0].metadata.tag assert_nil queue[0].metadata.variables assert_equal Time.parse('2016-04-17 13:58:00 -0700').localtime, queue[0].created_at assert_equal Time.parse('2016-04-17 13:58:22 -0700').localtime, queue[0].modified_at assert_equal 4, queue[0].size assert_equal @c2id, queue[1].unique_id assert_equal :queued, queue[1].state assert_equal event_time('2016-04-17 13:59:00 -0700').to_i, queue[1].metadata.timekey assert_nil queue[1].metadata.tag assert_nil queue[1].metadata.variables assert_equal Time.parse('2016-04-17 13:59:00 -0700').localtime, queue[1].created_at assert_equal Time.parse('2016-04-17 13:59:23 -0700').localtime, queue[1].modified_at assert_equal 3, queue[1].size end end sub_test_case 'there are some existing file chunks with placeholders path' do setup do @bufdir = File.expand_path('../../tmp/buffer_${test}_file', __FILE__) FileUtils.rm_rf(@bufdir) FileUtils.mkdir_p(@bufdir) @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), 1, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i ) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c2id)}.log") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), 1, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i ) @bufpath = File.join(@bufdir, 'etest.*.log') Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end FileUtils.rm_rf(@bufdir) end test '#resume returns staged/queued chunks with metadata' do assert_equal 1, @p.stage.size assert_equal 1, @p.queue.size end end sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do setup do @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__) @worker0_dir = File.join(@bufdir, "worker0") @worker1_dir = File.join(@bufdir, "worker1") FileUtils.rm_rf @bufdir FileUtils.mkdir_p @worker0_dir FileUtils.mkdir_p @worker1_dir @bufdir_chunk_1 = Fluent::UniqueId.generate bc1 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.log") File.open(bc1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( bc1 + '.meta', @bufdir_chunk_1, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i ) @bufdir_chunk_2 = Fluent::UniqueId.generate bc2 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.log") File.open(bc2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( bc2 + '.meta', @bufdir_chunk_2, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i ) @worker_dir_chunk_1 = Fluent::UniqueId.generate wc0_1 = File.join(@worker0_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log") wc1_1 = File.join(@worker1_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log") [wc0_1, wc1_1].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( chunk_path + '.meta', @worker_dir_chunk_1, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i), 3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i ) end @worker_dir_chunk_2 = Fluent::UniqueId.generate wc0_2 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log") wc1_2 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log") [wc0_2, wc1_2].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( chunk_path + '.meta', @worker_dir_chunk_2, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i ) end @worker_dir_chunk_3 = Fluent::UniqueId.generate wc0_3 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log") wc1_3 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log") [wc0_3, wc1_3].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( chunk_path + '.meta', @worker_dir_chunk_3, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i), 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i ) end Fluent::Test.setup end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do ENV['SERVERENGINE_WORKER_ID'] = '0' buf_conf = config_element('buffer', '', {'path' => @bufdir}) @d = FluentPluginFileBufferTest::DummyOutputPlugin.new with_worker_config(workers: 2, worker_id: 0) do @d.configure(config_element('output', '', {}, [buf_conf])) end @d.start @p = @d.buffer assert_equal 2, @p.stage.size assert_equal 3, @p.queue.size stage = @p.stage m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) assert_equal @worker_dir_chunk_2, stage[m1].unique_id assert_equal 4, stage[m1].size assert_equal :staged, stage[m1].state m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i) assert_equal @worker_dir_chunk_3, stage[m2].unique_id assert_equal 3, stage[m2].size assert_equal :staged, stage[m2].state queue = @p.queue assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort assert_equal [3, 4, 4], queue.map(&:size).sort assert_equal [:queued, :queued, :queued], queue.map(&:state) end test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do buf_conf = config_element('buffer', '', {'path' => @bufdir}) @d = FluentPluginFileBufferTest::DummyOutputPlugin.new with_worker_config(workers: 2, worker_id: 1) do @d.configure(config_element('output', '', {}, [buf_conf])) end @d.start @p = @d.buffer assert_equal 2, @p.stage.size assert_equal 1, @p.queue.size stage = @p.stage m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) assert_equal @worker_dir_chunk_2, stage[m1].unique_id assert_equal 4, stage[m1].size assert_equal :staged, stage[m1].state m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i) assert_equal @worker_dir_chunk_3, stage[m2].unique_id assert_equal 3, stage[m2].size assert_equal :staged, stage[m2].state queue = @p.queue assert_equal @worker_dir_chunk_1, queue[0].unique_id assert_equal 3, queue[0].size assert_equal :queued, queue[0].state end end sub_test_case 'there are some existing file chunks with old format metadata' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir) @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata_old( p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i ) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata_old( p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i), 3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i ) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log") File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata_old( p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i ) @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log") File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata_old( p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i), 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i ) @bufpath = File.join(@bufdir, 'etest.*.log') Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns staged/queued chunks with metadata' do assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size stage = @p.stage m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) assert_equal @c3id, stage[m3].unique_id assert_equal 4, stage[m3].size assert_equal :staged, stage[m3].state m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i) assert_equal @c4id, stage[m4].unique_id assert_equal 3, stage[m4].size assert_equal :staged, stage[m4].state end end sub_test_case 'there are some existing file chunks with old format metadata file' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700')) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "etest.201604171359.q#{Fluent::UniqueId.hex(@c2id)}.log") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(p2, mtime: Time.parse('2016-04-17 13:59:30 -0700')) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "etest.201604171400.b#{Fluent::UniqueId.hex(@c3id)}.log") File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(p3, mtime: Time.parse('2016-04-17 14:00:29 -0700')) @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "etest.201604171401.b#{Fluent::UniqueId.hex(@c4id)}.log") File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(p4, mtime: Time.parse('2016-04-17 14:01:22 -0700')) @bufpath = File.join(@bufdir, 'etest.*.log') Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns queued chunks for files without metadata' do assert_equal 0, @p.stage.size assert_equal 4, @p.queue.size queue = @p.queue m = metadata() assert_equal @c1id, queue[0].unique_id assert_equal m, queue[0].metadata assert_equal 0, queue[0].size assert_equal :queued, queue[0].state assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at assert_equal @c2id, queue[1].unique_id assert_equal m, queue[1].metadata assert_equal 0, queue[1].size assert_equal :queued, queue[1].state assert_equal Time.parse('2016-04-17 13:59:30 -0700'), queue[1].modified_at assert_equal @c3id, queue[2].unique_id assert_equal m, queue[2].metadata assert_equal 0, queue[2].size assert_equal :queued, queue[2].state assert_equal Time.parse('2016-04-17 14:00:29 -0700'), queue[2].modified_at assert_equal @c4id, queue[3].unique_id assert_equal m, queue[3].metadata assert_equal 0, queue[3].size assert_equal :queued, queue[3].state assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at end end sub_test_case 'there are the same timekey metadata in stage' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) @bufpath = File.join(@bufdir, 'testbuf.*.log') FileUtils.rm_r(@bufdir) if File.exist?(@bufdir) FileUtils.mkdir_p(@bufdir) m = metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i) c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay1"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay2"}].to_json + "\n" end write_metadata(p1 + '.meta', c1id, m, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i) c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c2id)}.log") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay3"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay4"}].to_json + "\n" end m2 = m.dup_next write_metadata(p2 + '.meta', c2id, m2, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i) c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c3id)}.log") File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay5"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay6"}].to_json + "\n" end m3 = m2.dup_next write_metadata(p3 + '.meta', c3id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i) c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "testbuf.b#{Fluent::UniqueId.hex(c4id)}.log") File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay5"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay6"}].to_json + "\n" end write_metadata(p4 + '.meta', c4id, m3, 2, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i) Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) # Windows does not permit to delete files which are used in another process. # Just ignore for removing failure. File.delete(path) rescue nil end end end test '#resume returns each chunks' do s, e = @p.resume assert_equal 3, s.size assert_equal [0, 1, 2], s.keys.map(&:seq).sort assert_equal 1, e.size assert_equal [0], e.map { |e| e.metadata.seq } end end sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) FileUtils.rm_rf @bufdir FileUtils.mkdir_p @bufdir @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700')) @not_chunk = File.join(@bufdir, 'etest.20160416.log') File.open(@not_chunk, 'wb') do |f| f.write ["t1.test", event_time('2016-04-16 23:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-16 23:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-16 23:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-16 23:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end FileUtils.touch(@not_chunk, mtime: Time.parse('2016-04-17 00:00:00 -0700')) @bufpath = File.join(@bufdir, 'etest.*.log') Fluent::Test.setup @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::FileBuffer.new @p.owner = @d @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns queued chunks for files without metadata, while ignoring non-chunk looking files' do assert_equal 0, @p.stage.size assert_equal 1, @p.queue.size queue = @p.queue m = metadata() assert_equal @c1id, queue[0].unique_id assert_equal m, queue[0].metadata assert_equal 0, queue[0].size assert_equal :queued, queue[0].state assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at assert File.exist?(@not_chunk) end end sub_test_case 'there are existing broken file chunks' do setup do @id_output = 'backup_test' @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__) FileUtils.rm_rf @bufdir rescue nil FileUtils.mkdir_p @bufdir @bufpath = File.join(@bufdir, 'broken_test.*.log') Fluent::Test.setup end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end def setup_plugins(buf_conf) @d = FluentPluginFileBufferTest::DummyOutputPlugin.new @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)])) @p = @d.buffer end def create_first_chunk(mode) cid = Fluent::UniqueId.generate path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log") File.open(path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i ) return cid, path end def create_second_chunk(mode) cid = Fluent::UniqueId.generate path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log") File.open(path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end write_metadata( path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i), 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i ) return cid, path end def compare_staged_chunk(staged, id, time, num, mode) assert_equal 1, staged.size m = metadata(timekey: event_time(time).to_i) assert_equal id, staged[m].unique_id assert_equal num, staged[m].size assert_equal mode, staged[m].state end def compare_queued_chunk(queued, id, num, mode) assert_equal 1, queued.size assert_equal id, queued[0].unique_id assert_equal num, queued[0].size assert_equal mode, queued[0].state end def compare_log(plugin, msg) logs = plugin.log.out.logs assert { logs.any? { |log| log.include?(msg) } } end test '#resume backups staged empty chunk' do setup_plugins({'path' => @bufpath}) c1id, p1 = create_first_chunk('b') File.open(p1, 'wb') { |f| } # create staged empty chunk file c2id, _ = create_second_chunk('b') Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged) compare_log(@p, 'staged file chunk is empty') assert { not File.exist?(p1) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") } end test '#resume backups staged broken metadata' do setup_plugins({'path' => @bufpath}) c1id, _ = create_first_chunk('b') c2id, p2 = create_second_chunk('b') File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged) compare_log(@p, 'staged meta file is broken') assert { not File.exist?(p2) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") } end test '#resume backups enqueued empty chunk' do setup_plugins({'path' => @bufpath}) c1id, p1 = create_first_chunk('q') File.open(p1, 'wb') { |f| } # create enqueued empty chunk file c2id, _ = create_second_chunk('q') Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_queued_chunk(@p.queue, c2id, 3, :queued) compare_log(@p, 'enqueued file chunk is empty') assert { not File.exist?(p1) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") } end test '#resume backups enqueued broken metadata' do setup_plugins({'path' => @bufpath}) c1id, _ = create_first_chunk('q') c2id, p2 = create_second_chunk('q') File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_queued_chunk(@p.queue, c1id, 4, :queued) compare_log(@p, 'enqueued meta file is broken') assert { not File.exist?(p2) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") } end test '#resume backups enqueued broken metadata which has broken id, c, m fields' do setup_plugins({'path' => @bufpath}) cid, path = create_first_chunk('q') metadata = File.read(path + '.meta') File.open(path + '.meta', 'wb') { |f| f.write(metadata[0..6] + "\0" * (metadata.size - 6)) } # create enqueued broken meta file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_log(@p, 'enqueued meta file is broken') assert { not File.exist?(path) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(cid)}.log") } end test '#resume backups enqueued broken metadata by truncated' do setup_plugins({'path' => @bufpath}) cid, path = create_first_chunk('q') metadata = File.read(path + '.meta') File.open(path + '.meta', 'wb') { |f| f.write(metadata[0..-2]) } # create enqueued broken meta file with last byte truncated Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_log(@p, 'enqueued meta file is broken') assert { not File.exist?(path) } assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(cid)}.log") } end test '#resume throws away broken chunk with disable_chunk_backup' do setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true}) c1id, _ = create_first_chunk('b') c2id, p2 = create_second_chunk('b') File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged) compare_log(@p, 'staged meta file is broken') compare_log(@p, 'disable_chunk_backup is true') assert { not File.exist?(p2) } assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") } end end sub_test_case 'evacuate_chunk' do def setup Fluent::Test.setup @now = Time.local(2025, 5, 30, 17, 0, 0) @base_dir = File.expand_path("../../tmp/evacuate_chunk", __FILE__) @buf_dir = File.join(@base_dir, "buffer") @root_dir = File.join(@base_dir, "root") FileUtils.mkdir_p(@root_dir) Fluent::SystemConfig.overwrite_system_config("root_dir" => @root_dir) do Timecop.freeze(@now) yield end ensure Timecop.return FileUtils.rm_rf(@base_dir) end def start_plugin(plugin) plugin.start plugin.after_start end def stop_plugin(plugin) plugin.stop unless plugin.stopped? plugin.before_shutdown unless plugin.before_shutdown? plugin.shutdown unless plugin.shutdown? plugin.after_shutdown unless plugin.after_shutdown? plugin.close unless plugin.closed? plugin.terminate unless plugin.terminated? end def configure_output(id, chunk_key, buffer_conf) output = FluentPluginFileBufferTest::DummyErrorOutputPlugin.new output.configure( config_element('ROOT', '', {'@id' => id}, [config_element('buffer', chunk_key, buffer_conf)]) ) yield output ensure stop_plugin(output) end def wait(sec: 4) waiting(sec) do Thread.pass until yield end end def emit_events(output, tag, es) output.interrupt_flushes output.emit_events("test.1", dummy_event_stream) @now += 1 Timecop.freeze(@now) output.enqueue_thread_wait output.flush_thread_wakeup end def proceed_to_next_retry(output) @now += 1 Timecop.freeze(@now) output.flush_thread_wakeup end def dummy_event_stream Fluent::ArrayEventStream.new([ [ event_time("2025-05-30 10:00:00"), {"message" => "data1"} ], [ event_time("2025-05-30 10:10:00"), {"message" => "data2"} ], [ event_time("2025-05-30 10:20:00"), {"message" => "data3"} ], ]) end def evacuate_dir(plugin_id) File.join(@root_dir, "buffer", plugin_id) end test 'can recover by putting back evacuated chunk files' do plugin_id = "test_output" tag = "test.1" buffer_conf = { "path" => @buf_dir, "flush_mode" => "interval", "flush_interval" => "1s", "retry_type" => "periodic", "retry_max_times" => 1, "retry_randomize" => false, } # Fail flushing and reach retry limit configure_output(plugin_id, "tag", buffer_conf) do |output| start_plugin(output) emit_events(output, tag, dummy_event_stream) wait { output.write_count == 1 and output.num_errors == 1 } proceed_to_next_retry(output) wait { output.write_count == 2 and output.num_errors == 2 } wait { Dir.empty?(@buf_dir) } # Assert evacuated files evacuated_files = Dir.children(evacuate_dir(plugin_id)).map do |child_name| File.join(evacuate_dir(plugin_id), child_name) end assert { evacuated_files.size == 2 } # .log and .log.meta # Put back evacuated chunk files for recovery FileUtils.move(evacuated_files, @buf_dir) end # Restart plugin to load the chunk files that were put back written_data = [] configure_output(plugin_id, "tag", buffer_conf) do |output| output.recover output.register_write do |chunk| written_data << chunk.read end start_plugin(output) wait { not written_data.empty? } end # Assert the recovery success assert { written_data.length == 1 } expected_records = [] dummy_event_stream.each do |(time, record)| expected_records << [tag, time.to_i, record] end actual_records = StringIO.open(written_data.first) do |io| io.each_line.map do |line| JSON.parse(line) end end assert_equal(expected_records, actual_records) end end end ================================================ FILE: test/plugin/test_buf_file_single.rb ================================================ require_relative '../helper' require 'fluent/plugin/buf_file_single' require 'fluent/plugin/output' require 'fluent/unique_id' require 'fluent/system_config' require 'fluent/env' require 'fluent/test/driver/output' require 'msgpack' module FluentPluginFileSingleBufferTest class DummyOutputPlugin < Fluent::Plugin::Output Fluent::Plugin.register_output('buf_file_single_test', self) config_section :buffer do config_set_default :@type, 'file_single' end def multi_workers_ready? true end def write(chunk) # drop end end class DummyOutputMPPlugin < Fluent::Plugin::Output Fluent::Plugin.register_output('buf_file_single_mp_test', self) config_section :buffer do config_set_default :@type, 'file_single' end def formatted_to_msgpack_binary? true end def multi_workers_ready? true end def write(chunk) # drop end end class DummyErrorOutputPlugin < DummyOutputPlugin def register_write(&block) instance_variable_set(:@write, block) end def initialize super @should_fail_writing = true @write = nil end def recover @should_fail_writing = false end def write(chunk) if @should_fail_writing raise "failed writing chunk" else @write ? @write.call(chunk) : nil end end def format(tag, time, record) [tag, time.to_i, record].to_json + "\n" end end end class FileSingleBufferTest < Test::Unit::TestCase def metadata(timekey: nil, tag: 'testing', variables: nil) Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) end PATH = File.expand_path('../../tmp/buffer_file_single_dir', __FILE__) TAG_CONF = %[ @type file_single path #{PATH} ] FIELD_CONF = %[ @type file_single path #{PATH} ] setup do Fluent::Test.setup @d = nil @bufdir = PATH FileUtils.rm_rf(@bufdir) rescue nil FileUtils.mkdir_p(@bufdir) end teardown do FileUtils.rm_rf(@bufdir) rescue nil end def create_driver(conf = TAG_CONF, klass = FluentPluginFileSingleBufferTest::DummyOutputPlugin) Fluent::Test::Driver::Output.new(klass).configure(conf) end sub_test_case 'configuration' do test 'path has "fsb" prefix and "buf" suffix by default' do @d = create_driver p = @d.instance.buffer assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path end data('text based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputPlugin, :text], 'msgpack based chunk' => [FluentPluginFileSingleBufferTest::DummyOutputMPPlugin, :msgpack]) test 'detect chunk_format' do |param| klass, expected = param @d = create_driver(TAG_CONF, klass) p = @d.instance.buffer assert_equal expected, p.chunk_format end test '"prefix.*.suffix" path will be replaced with default' do @d = create_driver(%[ @type file_single path #{@bufdir}/foo.*.bar ]) p = @d.instance.buffer assert_equal File.join(@bufdir, 'fsb.*.buf'), p.path end end sub_test_case 'buffer configurations and workers' do setup do @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new end test 'enables multi worker configuration with unexisting directory path' do FileUtils.rm_rf(@bufdir) buf_conf = config_element('buffer', '', {'path' => @bufdir}) assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {}, [buf_conf])) end end end test 'enables multi worker configuration with existing directory path' do FileUtils.mkdir_p @bufdir buf_conf = config_element('buffer', '', {'path' => @bufdir}) assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {}, [buf_conf])) end end end test 'enables multi worker configuration with root dir' do buf_conf = config_element('buffer', '') assert_nothing_raised do Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf])) end end end end test 'raise config error when using same file path' do d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new Fluent::SystemConfig.overwrite_system_config({}) do d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })])) end any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass| stub(klass).called_in_test? { false } end err = assert_raise(Fluent::ConfigError) do Fluent::SystemConfig.overwrite_system_config({}) do d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })])) end end assert_match(/plugin already uses same buffer path/, err.message) end sub_test_case 'buffer plugin configured only with path' do setup do @bufpath = File.join(@bufdir, 'testbuf.*.buf') FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir) @d = create_driver @p = @d.instance.buffer end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end test 'this is persistent plugin' do assert @p.persistent? end test '#start creates directory for buffer chunks' do @d = create_driver @p = @d.instance.buffer FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir) assert !File.exist?(@bufdir) @p.start assert File.exist?(@bufdir) assert { File.stat(@bufdir).mode.to_s(8).end_with?('755') } end test '#start creates directory for buffer chunks with specified permission' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? @d = create_driver(%[ @type file_single path #{PATH} dir_permission 700 ]) @p = @d.instance.buffer FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir) assert !File.exist?(@bufdir) @p.start assert File.exist?(@bufdir) assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') } end test '#start creates directory for buffer chunks with specified permission via system config' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? sysconf = {'dir_permission' => '700'} Fluent::SystemConfig.overwrite_system_config(sysconf) do @d = create_driver @p = @d.instance.buffer FileUtils.rm_r @bufdir if File.exist?(@bufdir) assert !File.exist?(@bufdir) @p.start assert File.exist?(@bufdir) assert { File.stat(@bufdir).mode.to_s(8).end_with?('700') } end end test '#generate_chunk generates blank file chunk on path with unique_id and tag' do FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir) m1 = metadata() c1 = @p.generate_chunk(m1) assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk assert_equal m1, c1.metadata assert c1.empty? assert_equal :unstaged, c1.state assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') } c1.purge end test '#generate_chunk generates blank file chunk on path with unique_id and field key' do FileUtils.mkdir_p(@bufdir) unless File.exist?(@bufdir) @d = create_driver(FIELD_CONF) @p = @d.instance.buffer m1 = metadata(tag: nil, variables: {:k => 'foo_bar'}) c1 = @p.generate_chunk(m1) assert c1.is_a? Fluent::Plugin::Buffer::FileSingleChunk assert_equal m1, c1.metadata assert c1.empty? assert_equal :unstaged, c1.state assert_equal Fluent::DEFAULT_FILE_PERMISSION, c1.permission assert_equal File.join(@bufdir, "fsb.foo_bar.b#{Fluent::UniqueId.hex(c1.unique_id)}.buf"), c1.path c1.purge end test '#generate_chunk generates blank file chunk with specified permission' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? @d = create_driver(%[ @type file_single path #{PATH} file_permission 600 ]) @p = @d.instance.buffer FileUtils.rm_r @bufdir if File.exist?(@bufdir) assert !File.exist?(@bufdir) @p.start m = metadata() c = @p.generate_chunk(m) assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk assert_equal m, c.metadata assert c.empty? assert_equal :unstaged, c.state assert_equal 0600, c.permission assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path assert{ File.stat(c.path).mode.to_s(8).end_with?('600') } c.purge end test '#generate_chunk generates blank file chunk with specified permission with system_config' do omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? @d = create_driver(%[ @type file_single path #{PATH} ]) @p = @d.instance.buffer FileUtils.rm_r @bufdir if File.exist?(@bufdir) assert !File.exist?(@bufdir) @p.start m = metadata() c = nil Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do c = @p.generate_chunk(m) end assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk assert_equal m, c.metadata assert c.empty? assert_equal :unstaged, c.state assert_equal 0700, c.permission assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path assert{ File.stat(c.path).mode.to_s(8).end_with?('700') } c.purge end end sub_test_case 'configured with system root directory and plugin @id' do setup do @root_dir = File.expand_path('../../tmp/buffer_file_single_root', __FILE__) FileUtils.rm_rf(@root_dir) @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new @p = nil end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end test '#start creates directory for buffer chunks' do Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [config_element('buffer', '', {})])) @p = @d.buffer end expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "fsb.*.buf") expected_buffer_dir = File.dirname(expected_buffer_path) assert_equal expected_buffer_path, @d.buffer.path assert_false Dir.exist?(expected_buffer_dir) @p.start assert Dir.exist?(expected_buffer_dir) end end sub_test_case 'buffer plugin configuration errors' do data('tag and key' => 'tag,key', 'multiple keys' => 'key1,key2') test 'invalid chunk keys' do |param| assert_raise Fluent::ConfigError do @d = create_driver(%[ @type file_single path #{PATH} calc_num_records false ]) end end test 'path is not specified' do assert_raise Fluent::ConfigError do @d = create_driver(%[ @type file_single ]) end end end sub_test_case 'there are no existing file chunks' do setup do FileUtils.rm_rf(@bufdir) if File.exist?(@bufdir) @d = create_driver @p = @d.instance.buffer @p.start end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns empty buffer state' do ary = @p.resume assert_equal({}, ary[0]) assert_equal([], ary[1]) end end sub_test_case 'there are some existing file chunks' do setup do @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end t = Time.now - 50000 File.utime(t, t, p1) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end t = Time.now - 40000 File.utime(t, t, p2) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf") File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf") File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns staged/queued chunks with metadata' do @d = create_driver @p = @d.instance.buffer @p.start assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size stage = @p.stage m3 = metadata() assert_equal @c3id, stage[m3].unique_id assert_equal 4, stage[m3].size assert_equal :staged, stage[m3].state m4 = metadata(tag: 'foo') assert_equal @c4id, stage[m4].unique_id assert_equal 3, stage[m4].size assert_equal :staged, stage[m4].state end test '#resume returns queued chunks ordered by last modified time (FIFO)' do @d = create_driver @p = @d.instance.buffer @p.start assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size queue = @p.queue assert{ queue[0].modified_at <= queue[1].modified_at } assert_equal @c1id, queue[0].unique_id assert_equal :queued, queue[0].state assert_equal 'testing', queue[0].metadata.tag assert_nil queue[0].metadata.variables assert_equal 4, queue[0].size assert_equal @c2id, queue[1].unique_id assert_equal :queued, queue[1].state assert_equal 'testing', queue[1].metadata.tag assert_nil queue[1].metadata.variables assert_equal 3, queue[1].size end test '#resume returns staged/queued chunks but skips size calculation by calc_num_records' do @d = create_driver(%[ @type file_single path #{PATH} calc_num_records false ]) @p = @d.instance.buffer @p.start assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size stage = @p.stage m3 = metadata() assert_equal @c3id, stage[m3].unique_id assert_equal 0, stage[m3].size assert_equal :staged, stage[m3].state m4 = metadata(tag: 'foo') assert_equal @c4id, stage[m4].unique_id assert_equal 0, stage[m4].size assert_equal :staged, stage[m4].state end end sub_test_case 'there are some existing file chunks with placeholders path' do setup do @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__) FileUtils.rm_rf(@buf_ph_dir) FileUtils.mkdir_p(@buf_ph_dir) @c1id = Fluent::UniqueId.generate p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf") File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" end t = Time.now - 50000 File.utime(t, t, p1) @c2id = Fluent::UniqueId.generate p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf") File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" end end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end FileUtils.rm_rf(@buf_ph_dir) end test '#resume returns staged/queued chunks with metadata' do @d = create_driver(%[ @type file_single path #{@buf_ph_dir} ]) @p = @d.instance.buffer @p.start assert_equal 1, @p.stage.size assert_equal 1, @p.queue.size end end sub_test_case 'there are some existing msgpack file chunks' do setup do packer = Fluent::MessagePackFactory.packer @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf") File.open(p1, 'wb') do |f| packer.write(["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}]) packer.write(["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}]) packer.write(["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}]) packer.write(["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}]) f.write packer.full_pack end t = Time.now - 50000 File.utime(t, t, p1) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@c2id)}.buf") File.open(p2, 'wb') do |f| packer.write(["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}]) packer.write(["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}]) packer.write(["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}]) f.write packer.full_pack end t = Time.now - 40000 File.utime(t, t, p2) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(@c3id)}.buf") File.open(p3, 'wb') do |f| packer.write(["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}]) packer.write(["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}]) packer.write(["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}]) packer.write(["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}]) f.write packer.full_pack end @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(@c4id)}.buf") File.open(p4, 'wb') do |f| packer.write(["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}]) packer.write(["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}]) packer.write(["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}]) f.write packer.full_pack end end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end if @bufdir Dir.glob(File.join(@bufdir, '*')).each do |path| next if ['.', '..'].include?(File.basename(path)) File.delete(path) end end end test '#resume returns staged/queued chunks with msgpack format' do @d = create_driver(%[ @type file_single path #{PATH} chunk_format msgpack ]) @p = @d.instance.buffer @p.start assert_equal 2, @p.stage.size assert_equal 2, @p.queue.size stage = @p.stage m3 = metadata() assert_equal @c3id, stage[m3].unique_id assert_equal 4, stage[m3].size assert_equal :staged, stage[m3].state m4 = metadata(tag: 'foo') assert_equal @c4id, stage[m4].unique_id assert_equal 3, stage[m4].size assert_equal :staged, stage[m4].state end end sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do setup do @worker0_dir = File.join(@bufdir, "worker0") @worker1_dir = File.join(@bufdir, "worker1") FileUtils.rm_rf(@bufdir) FileUtils.mkdir_p(@worker0_dir) FileUtils.mkdir_p(@worker1_dir) @bufdir_chunk_1 = Fluent::UniqueId.generate bc1 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.buf") File.open(bc1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end @bufdir_chunk_2 = Fluent::UniqueId.generate bc2 = File.join(@bufdir, "fsb.testing.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.buf") File.open(bc2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" end @worker_dir_chunk_1 = Fluent::UniqueId.generate wc0_1 = File.join(@worker0_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf") wc1_1 = File.join(@worker1_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.buf") [wc0_1, wc1_1].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end end @worker_dir_chunk_2 = Fluent::UniqueId.generate wc0_2 = File.join(@worker0_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf") wc1_2 = File.join(@worker1_dir, "fsb.foo.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.buf") [wc0_2, wc1_2].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" end end @worker_dir_chunk_3 = Fluent::UniqueId.generate wc0_3 = File.join(@worker0_dir, "fsb.bar.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf") wc1_3 = File.join(@worker1_dir, "fsb.baz.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.buf") [wc0_3, wc1_3].each do |chunk_path| File.open(chunk_path, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" end end end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do ENV['SERVERENGINE_WORKER_ID'] = '0' buf_conf = config_element('buffer', '', {'path' => @bufdir}) @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new with_worker_config(workers: 2, worker_id: 0) do @d.configure(config_element('output', '', {}, [buf_conf])) end @d.start @p = @d.buffer assert_equal 2, @p.stage.size assert_equal 3, @p.queue.size stage = @p.stage m1 = metadata(tag: 'testing') assert_equal @worker_dir_chunk_2, stage[m1].unique_id assert_equal 4, stage[m1].size assert_equal :staged, stage[m1].state m2 = metadata(tag: 'bar') assert_equal @worker_dir_chunk_3, stage[m2].unique_id assert_equal 3, stage[m2].size assert_equal :staged, stage[m2].state queue = @p.queue assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort assert_equal [3, 4, 4], queue.map(&:size).sort assert_equal [:queued, :queued, :queued], queue.map(&:state) end test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do buf_conf = config_element('buffer', '', {'path' => @bufdir}) @d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new with_worker_config(workers: 2, worker_id: 1) do @d.configure(config_element('output', '', {}, [buf_conf])) end @d.start @p = @d.buffer assert_equal 2, @p.stage.size assert_equal 1, @p.queue.size stage = @p.stage m1 = metadata(tag: 'foo') assert_equal @worker_dir_chunk_2, stage[m1].unique_id assert_equal 4, stage[m1].size assert_equal :staged, stage[m1].state m2 = metadata(tag: 'baz') assert_equal @worker_dir_chunk_3, stage[m2].unique_id assert_equal 3, stage[m2].size assert_equal :staged, stage[m2].state queue = @p.queue assert_equal @worker_dir_chunk_1, queue[0].unique_id assert_equal 3, queue[0].size assert_equal :queued, queue[0].state end end sub_test_case 'there are existing broken file chunks' do setup do FileUtils.rm_rf(@bufdir) rescue nil FileUtils.mkdir_p(@bufdir) end teardown do return unless @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end test '#resume backups empty chunk' do id_output = 'backup_test' @d = create_driver(%[ @id #{id_output} @type file_single path #{PATH} ]) @p = @d.instance.buffer c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf") File.open(p1, 'wb') { |f| } # create empty chunk file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end assert { not File.exist?(p1) } assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") } end test '#resume throws away broken chunk with disable_chunk_backup' do id_output = 'backup_test' @d = create_driver(%[ @id #{id_output} @type file_single path #{PATH} disable_chunk_backup true ]) @p = @d.instance.buffer c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf") File.open(p1, 'wb') { |f| } # create empty chunk file Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do @p.start end assert { not File.exist?(p1) } assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") } end end sub_test_case 'evacuate_chunk' do def setup Fluent::Test.setup @now = Time.local(2025, 5, 30, 17, 0, 0) @base_dir = File.expand_path("../../tmp/evacuate_chunk", __FILE__) @buf_dir = File.join(@base_dir, "buffer") @root_dir = File.join(@base_dir, "root") FileUtils.mkdir_p(@root_dir) Fluent::SystemConfig.overwrite_system_config("root_dir" => @root_dir) do Timecop.freeze(@now) yield end ensure Timecop.return FileUtils.rm_rf(@base_dir) end def start_plugin(plugin) plugin.start plugin.after_start end def stop_plugin(plugin) plugin.stop unless plugin.stopped? plugin.before_shutdown unless plugin.before_shutdown? plugin.shutdown unless plugin.shutdown? plugin.after_shutdown unless plugin.after_shutdown? plugin.close unless plugin.closed? plugin.terminate unless plugin.terminated? end def configure_output(id, chunk_key, buffer_conf) output = FluentPluginFileSingleBufferTest::DummyErrorOutputPlugin.new output.configure( config_element('ROOT', '', {'@id' => id}, [config_element('buffer', chunk_key, buffer_conf)]) ) yield output ensure stop_plugin(output) end def wait(sec: 4) waiting(sec) do Thread.pass until yield end end def emit_events(output, tag, es) output.interrupt_flushes output.emit_events("test.1", dummy_event_stream) @now += 1 Timecop.freeze(@now) output.enqueue_thread_wait output.flush_thread_wakeup end def proceed_to_next_retry(output) @now += 1 Timecop.freeze(@now) output.flush_thread_wakeup end def dummy_event_stream Fluent::ArrayEventStream.new([ [ event_time("2025-05-30 10:00:00"), {"message" => "data1"} ], [ event_time("2025-05-30 10:10:00"), {"message" => "data2"} ], [ event_time("2025-05-30 10:20:00"), {"message" => "data3"} ], ]) end def evacuate_dir(plugin_id) File.join(@root_dir, "buffer", plugin_id) end test 'can recover by putting back evacuated chunk files' do plugin_id = "test_output" tag = "test.1" buffer_conf = { "path" => @buf_dir, "flush_mode" => "interval", "flush_interval" => "1s", "retry_type" => "periodic", "retry_max_times" => 1, "retry_randomize" => false, } # Fail flushing and reach retry limit configure_output(plugin_id, "tag", buffer_conf) do |output| start_plugin(output) emit_events(output, tag, dummy_event_stream) wait { output.write_count == 1 and output.num_errors == 1 } proceed_to_next_retry(output) wait { output.write_count == 2 and output.num_errors == 2 } wait { Dir.empty?(@buf_dir) } # Assert evacuated files evacuated_files = Dir.children(evacuate_dir(plugin_id)).map do |child_name| File.join(evacuate_dir(plugin_id), child_name) end assert { evacuated_files.size == 1 } # .log # Put back evacuated chunk files for recovery FileUtils.move(evacuated_files, @buf_dir) end # Restart plugin to load the chunk files that were put back written_data = [] configure_output(plugin_id, "tag", buffer_conf) do |output| output.recover output.register_write do |chunk| written_data << chunk.read end start_plugin(output) wait { not written_data.empty? } end # Assert the recovery success assert { written_data.length == 1 } expected_records = [] dummy_event_stream.each do |(time, record)| expected_records << [tag, time.to_i, record] end actual_records = StringIO.open(written_data.first) do |io| io.each_line.map do |line| JSON.parse(line) end end assert_equal(expected_records, actual_records) end end end ================================================ FILE: test/plugin/test_buf_memory.rb ================================================ require_relative '../helper' require 'fluent/plugin/buf_memory' require 'fluent/plugin/output' require 'flexmock/test_unit' module FluentPluginMemoryBufferTest class DummyOutputPlugin < Fluent::Plugin::Output end end class MemoryBufferTest < Test::Unit::TestCase setup do Fluent::Test.setup @d = FluentPluginMemoryBufferTest::DummyOutputPlugin.new @p = Fluent::Plugin::MemoryBuffer.new @p.owner = @d end test 'this is non persistent plugin' do assert !@p.persistent? end test '#resume always returns empty stage and queue' do ary = @p.resume assert_equal({}, ary[0]) assert_equal([], ary[1]) end test '#generate_chunk returns memory chunk instance' do m1 = Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil) c1 = @p.generate_chunk(m1) assert c1.is_a? Fluent::Plugin::Buffer::MemoryChunk assert_equal m1, c1.metadata require 'time' t2 = Time.parse('2016-04-08 19:55:00 +0900').to_i m2 = Fluent::Plugin::Buffer::Metadata.new(t2, 'test.tag', {k1: 'v1', k2: 0}) c2 = @p.generate_chunk(m2) assert c2.is_a? Fluent::Plugin::Buffer::MemoryChunk assert_equal m2, c2.metadata end end ================================================ FILE: test/plugin/test_buffer.rb ================================================ require_relative '../helper' require 'fluent/plugin/buffer' require 'fluent/plugin/buffer/memory_chunk' require 'fluent/plugin/compressable' require 'fluent/plugin/buffer/chunk' require 'fluent/event' require 'flexmock/test_unit' require 'fluent/log' require 'fluent/plugin_id' require 'time' module FluentPluginBufferTest class DummyOutputPlugin < Fluent::Plugin::Base include Fluent::PluginId include Fluent::PluginLoggerMixin end class DummyMemoryChunkError < StandardError; end class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk attr_reader :append_count, :rollbacked, :closed, :purged, :chunk attr_accessor :failing def initialize(metadata, compress: :text) super @append_count = 0 @rollbacked = false @closed = false @purged = false @failing = false end def concat(data, size) @append_count += 1 raise DummyMemoryChunkError if @failing super end def rollback super @rollbacked = true end def close super @closed = true end def purge super @purged = true end end class DummyPlugin < Fluent::Plugin::Buffer def create_metadata(timekey=nil, tag=nil, variables=nil) Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) end def create_chunk(metadata, data) c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) c.append(data) c.commit c end def create_chunk_es(metadata, es) c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) c.concat(es.to_msgpack_stream, es.size) c.commit c end def resume dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) staged = { dm2 => create_chunk(dm2, ["b" * 100]).staged!, dm3 => create_chunk(dm3, ["c" * 100]).staged!, } queued = [ create_chunk(dm0, ["0" * 100]).enqueued!, create_chunk(dm1, ["a" * 100]).enqueued!, create_chunk(dm1, ["a" * 3]).enqueued!, ] return staged, queued end def generate_chunk(metadata) DummyMemoryChunk.new(metadata, compress: @compress) end end end class BufferTest < Test::Unit::TestCase def create_buffer(hash) buffer_conf = config_element('buffer', '', hash, []) owner = FluentPluginBufferTest::DummyOutputPlugin.new owner.configure(config_element('ROOT', '', {}, [ buffer_conf ])) p = FluentPluginBufferTest::DummyPlugin.new p.owner = owner p.configure(buffer_conf) p end def create_metadata(timekey=nil, tag=nil, variables=nil) Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) end def create_chunk(metadata, data) c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) c.append(data) c.commit c end def create_chunk_es(metadata, es) c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) c.concat(es.to_msgpack_stream, es.size) c.commit c end setup do Fluent::Test.setup end sub_test_case 'using base buffer class' do setup do buffer_conf = config_element('buffer', '', {}, []) owner = FluentPluginBufferTest::DummyOutputPlugin.new owner.configure(config_element('ROOT', '', {}, [ buffer_conf ])) p = Fluent::Plugin::Buffer.new p.owner = owner p.configure(buffer_conf) @p = p end test 'default persistency is false' do assert !@p.persistent? end test 'chunk bytes limit is 8MB, and total bytes limit is 512MB' do assert_equal 8*1024*1024, @p.chunk_limit_size assert_equal 512*1024*1024, @p.total_limit_size end test 'chunk records limit is ignored in default' do assert_nil @p.chunk_limit_records end test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_limit_size' do assert_equal 512*1024*1024, @p.total_limit_size assert_equal 0, @p.stage_size assert_equal 0, @p.queue_size assert @p.storable? @p.stage_size = 256 * 1024 * 1024 @p.queue_size = 256 * 1024 * 1024 - 1 assert @p.storable? @p.queue_size = 256 * 1024 * 1024 assert !@p.storable? end test '#resume must be implemented by subclass' do assert_raise NotImplementedError do @p.resume end end test '#generate_chunk must be implemented by subclass' do assert_raise NotImplementedError do @p.generate_chunk(Object.new) end end end sub_test_case 'with default configuration and dummy implementation' do setup do @p = create_buffer({'queued_chunks_limit_size' => 100}) @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) @dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) @p.start end test '#start resumes buffer states and update queued numbers per metadata' do plugin = create_buffer({}) assert_equal({}, plugin.stage) assert_equal([], plugin.queue) assert_equal({}, plugin.dequeued) assert_equal({}, plugin.queued_num) assert_equal 0, plugin.stage_size assert_equal 0, plugin.queue_size assert_equal [], plugin.timekeys # @p is started plugin assert_equal [@dm2,@dm3], @p.stage.keys assert_equal "b" * 100, @p.stage[@dm2].read assert_equal "c" * 100, @p.stage[@dm3].read assert_equal 200, @p.stage_size assert_equal 3, @p.queue.size assert_equal "0" * 100, @p.queue[0].read assert_equal "a" * 100, @p.queue[1].read assert_equal "a" * 3, @p.queue[2].read assert_equal 203, @p.queue_size # staged, queued assert_equal 1, @p.queued_num[@dm0] assert_equal 2, @p.queued_num[@dm1] end test '#close closes all chunks in dequeued, enqueued and staged' do dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil) cx = create_chunk(dmx, ["x" * 1024]) @p.dequeued[cx.unique_id] = cx staged_chunks = @p.stage.values.dup queued_chunks = @p.queue.dup @p.close assert cx.closed assert{ staged_chunks.all?{|c| c.closed } } assert{ queued_chunks.all?{|c| c.closed } } end test '#terminate initializes all internal states' do dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil) cx = create_chunk(dmx, ["x" * 1024]) @p.dequeued[cx.unique_id] = cx @p.close @p.terminate assert_nil @p.stage assert_nil @p.queue assert_nil @p.dequeued assert_nil @p.queued_num assert_nil @p.stage_length_metrics assert_nil @p.stage_size_metrics assert_nil @p.queue_length_metrics assert_nil @p.queue_size_metrics assert_nil @p.available_buffer_space_ratios_metrics assert_nil @p.total_queued_size_metrics assert_nil @p.newest_timekey_metrics assert_nil @p.oldest_timekey_metrics assert_equal [], @p.timekeys end test '#queued_records returns total number of size in all chunks in queue' do assert_equal 3, @p.queue.size r0 = @p.queue[0].size assert_equal 1, r0 r1 = @p.queue[1].size assert_equal 1, r1 r2 = @p.queue[2].size assert_equal 1, r2 assert_equal (r0+r1+r2), @p.queued_records end test '#queued? returns queue has any chunks or not without arguments' do assert @p.queued? @p.queue.reject!{|_c| true } assert !@p.queued? end test '#queued? returns queue has chunks for specified metadata with an argument' do assert @p.queued?(@dm0) assert @p.queued?(@dm1) assert !@p.queued?(@dm2) end test '#enqueue_chunk enqueues a chunk on stage with specified metadata' do assert_equal 2, @p.stage.size assert_equal [@dm2,@dm3], @p.stage.keys assert_equal 3, @p.queue.size assert_nil @p.queued_num[@dm2] assert_equal 200, @p.stage_size assert_equal 203, @p.queue_size @p.enqueue_chunk(@dm2) assert_equal [@dm3], @p.stage.keys assert_equal @dm2, @p.queue.last.metadata assert_equal 1, @p.queued_num[@dm2] assert_equal 100, @p.stage_size assert_equal 303, @p.queue_size end test '#enqueue_chunk ignores empty chunks' do assert_equal 3, @p.queue.size m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) c = create_chunk(m, ['']) @p.stage[m] = c assert @p.stage[m].empty? assert !c.closed @p.enqueue_chunk(m) assert_nil @p.stage[m] assert_equal 3, @p.queue.size assert_nil @p.queued_num[m] assert c.closed end test '#enqueue_chunk calls #enqueued! if chunk responds to it' do assert_equal 3, @p.queue.size m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) c = create_chunk(m, ['c' * 256]) callback_called = false (class << c; self; end).module_eval do define_method(:enqueued!){ callback_called = true } end @p.stage[m] = c @p.enqueue_chunk(m) assert_equal c, @p.queue.last assert callback_called end test '#enqueue_all enqueues chunks on stage which given block returns true with' do m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m1, ['c' * 256]) @p.stage[m1] = c1 m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i) c2 = create_chunk(m2, ['c' * 256]) @p.stage[m2] = c2 assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) @p.enqueue_all{ |m, c| m.timekey < Time.parse('2016-04-11 16:41:00 +0000').to_i } assert_equal [m2], @p.stage.keys assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1], @p.queue.map(&:metadata) end test '#enqueue_all enqueues all chunks on stage without block' do m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m1, ['c' * 256]) @p.stage[m1] = c1 m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i) c2 = create_chunk(m2, ['c' * 256]) @p.stage[m2] = c2 assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) @p.enqueue_all assert_equal [], @p.stage.keys assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1,m2], @p.queue.map(&:metadata) end test '#dequeue_chunk dequeues a chunk from queue if a chunk exists' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) m1 = @p.dequeue_chunk assert_equal @dm0, m1.metadata assert_equal @dm0, @p.dequeued[m1.unique_id].metadata m2 = @p.dequeue_chunk assert_equal @dm1, m2.metadata assert_equal @dm1, @p.dequeued[m2.unique_id].metadata m3 = @p.dequeue_chunk assert_equal @dm1, m3.metadata assert_equal @dm1, @p.dequeued[m3.unique_id].metadata m4 = @p.dequeue_chunk assert_nil m4 end test '#takeback_chunk resumes a chunk from dequeued to queued at the head of queue, and returns true' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) m1 = @p.dequeue_chunk assert_equal @dm0, m1.metadata assert_equal @dm0, @p.dequeued[m1.unique_id].metadata assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({m1.unique_id => m1}, @p.dequeued) assert @p.takeback_chunk(m1.unique_id) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) end test '#purge_chunk removes a chunk specified by argument id from dequeued chunks' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) m0 = @p.dequeue_chunk m1 = @p.dequeue_chunk assert @p.takeback_chunk(m0.unique_id) assert_equal [@dm0,@dm1], @p.queue.map(&:metadata) assert_equal({m1.unique_id => m1}, @p.dequeued) assert !m1.purged @p.purge_chunk(m1.unique_id) assert m1.purged assert_equal [@dm0,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) end test '#purge_chunk removes an argument metadata if no chunks exist on stage or in queue' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) m0 = @p.dequeue_chunk assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({m0.unique_id => m0}, @p.dequeued) assert !m0.purged @p.purge_chunk(m0.unique_id) assert m0.purged assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) end test '#takeback_chunk returns false if specified chunk_id is already purged' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) m0 = @p.dequeue_chunk assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({m0.unique_id => m0}, @p.dequeued) assert !m0.purged @p.purge_chunk(m0.unique_id) assert m0.purged assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) assert !@p.takeback_chunk(m0.unique_id) assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) assert_equal({}, @p.dequeued) end test '#clear_queue! removes all chunks in queue, but leaves staged chunks' do qchunks = @p.queue.dup assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal 2, @p.stage.size assert_equal({}, @p.dequeued) @p.clear_queue! assert_equal [], @p.queue assert_equal 0, @p.queue_size assert_equal 2, @p.stage.size assert_equal({}, @p.dequeued) assert{ qchunks.all?{ |c| c.purged } } end test '#write returns immediately if argument data is empty array' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) @p.write({m => []}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys end test '#write returns immediately if argument data is empty event stream' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) @p.write({m => Fluent::ArrayEventStream.new([])}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys end test '#write raises BufferOverflowError if buffer is not storable' do @p.stage_size = 256 * 1024 * 1024 @p.queue_size = 256 * 1024 * 1024 m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) assert_raise Fluent::Plugin::Buffer::BufferOverflowError do @p.write({m => ["x" * 256]}) end end test '#write stores data into an existing chunk with metadata specified' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys dm3data = @p.stage[@dm3].read.dup prev_stage_size = @p.stage_size assert_equal 1, @p.stage[@dm3].append_count @p.write({@dm3 => ["x" * 256, "y" * 256, "z" * 256]}) assert_equal 2, @p.stage[@dm3].append_count assert_equal (dm3data + ("x" * 256) + ("y" * 256) + ("z" * 256)), @p.stage[@dm3].read assert_equal (prev_stage_size + 768), @p.stage_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys end test '#write creates new chunk and store data into it if there are no chunks for specified metadata' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i assert !@p.timekeys.include?(timekey) prev_stage_size = @p.stage_size m = @p.metadata(timekey: timekey) @p.write({m => ["x" * 256, "y" * 256, "z" * 256]}) assert_equal 1, @p.stage[m].append_count assert_equal ("x" * 256 + "y" * 256 + "z" * 256), @p.stage[m].read assert_equal (prev_stage_size + 768), @p.stage_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys @p.update_timekeys assert @p.timekeys.include?(timekey) end test '#write tries to enqueue and store data into a new chunk if existing chunk is full' do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal 0.95, @p.chunk_full_threshold assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) row = "x" * 1024 * 1024 small_row = "x" * 1024 * 512 @p.write({m => [row] * 7 + [small_row]}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 1, @p.stage[m].append_count @p.write({m => [row]}) assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 1, @p.stage[m].append_count assert_equal 1024*1024, @p.stage[m].bytesize assert_equal 3, @p.queue.last.append_count # 1 -> write (2) -> write_step_by_step (3) assert @p.queue.last.rollbacked end test '#write rollbacks if commit raises errors' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) row = "x" * 1024 @p.write({m => [row] * 8}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys target_chunk = @p.stage[m] assert_equal 1, target_chunk.append_count assert !target_chunk.rollbacked (class << target_chunk; self; end).module_eval do define_method(:commit){ raise "yay" } end assert_raise RuntimeError.new("yay") do @p.write({m => [row]}) end assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 2, target_chunk.append_count assert target_chunk.rollbacked assert_equal row * 8, target_chunk.read end test '#write w/ format raises BufferOverflowError if buffer is not storable' do @p.stage_size = 256 * 1024 * 1024 @p.queue_size = 256 * 1024 * 1024 m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "xxxxxxxxxxxxxx"} ] ]) assert_raise Fluent::Plugin::Buffer::BufferOverflowError do @p.write({m => es}, format: ->(e){e.to_msgpack_stream}) end end test '#write w/ format stores data into an existing chunk with metadata specified' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys dm3data = @p.stage[@dm3].read.dup prev_stage_size = @p.stage_size assert_equal 1, @p.stage[@dm3].append_count es = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 128}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "y" * 128}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "z" * 128}], ] ) @p.write({@dm3 => es}, format: ->(e){e.to_msgpack_stream}) assert_equal 2, @p.stage[@dm3].append_count assert_equal (dm3data + es.to_msgpack_stream), @p.stage[@dm3].read assert_equal (prev_stage_size + es.to_msgpack_stream.bytesize), @p.stage_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys end test '#write w/ format creates new chunk and store data into it if there are not chunks for specified metadata' do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i assert !@p.timekeys.include?(timekey) m = @p.metadata(timekey: timekey) es = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}], ] ) @p.write({m => es}, format: ->(e){e.to_msgpack_stream}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 1, @p.stage[m].append_count @p.update_timekeys assert @p.timekeys.include?(timekey) end test '#write w/ format tries to enqueue and store data into a new chunk if existing chunk does not have enough space' do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) es = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}], ] ) @p.write({m => es}, format: ->(e){e.to_msgpack_stream}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 1, @p.stage[m].append_count es2 = Fluent::OneEventStream.new(event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 1024}) @p.write({m => es2}, format: ->(e){e.to_msgpack_stream}) assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 1, @p.stage[m].append_count assert_equal es2.to_msgpack_stream.bytesize, @p.stage[m].bytesize assert_equal 2, @p.queue.last.append_count # 1 -> write (2) -> rollback&enqueue assert @p.queue.last.rollbacked end test '#write w/ format enqueues chunk if it is already full after adding data' do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) es = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], # 1024 * 1024 bytes as msgpack stream [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], ] ) @p.write({m => es}, format: ->(e){e.to_msgpack_stream}) assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys assert_equal 1, @p.queue.last.append_count end test '#write w/ format rollbacks if commit raises errors' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) es = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}], [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}], ] ) @p.write({m => es}, format: ->(e){e.to_msgpack_stream}) assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys target_chunk = @p.stage[m] assert_equal 1, target_chunk.append_count assert !target_chunk.rollbacked (class << target_chunk; self; end).module_eval do define_method(:commit){ raise "yay" } end es2 = Fluent::ArrayEventStream.new( [ [event_time('2016-04-11 16:40:04 +0000'), {"message" => "z" * 1024 * 128}], ] ) assert_raise RuntimeError.new("yay") do @p.write({m => es2}, format: ->(e){e.to_msgpack_stream}) end assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3,m], @p.stage.keys assert_equal 2, target_chunk.append_count assert target_chunk.rollbacked assert_equal es.to_msgpack_stream, target_chunk.read end test '#write writes many metadata and data pairs at once' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys row = "x" * 1024 @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] }) assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys end test '#write does not commit on any chunks if any append operation on chunk fails' do assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys row = "x" * 1024 @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] }) assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys dm2_size = @p.stage[@dm2].size assert !@p.stage[@dm2].rollbacked dm3_size = @p.stage[@dm3].size assert !@p.stage[@dm3].rollbacked assert{ @p.stage[@dm0].size == 3 } assert !@p.stage[@dm0].rollbacked assert{ @p.stage[@dm1].size == 2 } assert !@p.stage[@dm1].rollbacked meta_list = [@dm0, @dm1, @dm2, @dm3].sort @p.stage[meta_list.last].failing = true assert_raise(FluentPluginBufferTest::DummyMemoryChunkError) do @p.write({ @dm2 => [row], @dm3 => [row], @dm0 => [row, row, row], @dm1 => [row, row] }) end assert{ @p.stage[@dm2].size == dm2_size } assert @p.stage[@dm2].rollbacked assert{ @p.stage[@dm3].size == dm3_size } assert @p.stage[@dm3].rollbacked assert{ @p.stage[@dm0].size == 3 } assert @p.stage[@dm0].rollbacked assert{ @p.stage[@dm1].size == 2 } assert @p.stage[@dm1].rollbacked end test '#compress returns :text' do assert_equal :text, @p.compress end # https://github.com/fluent/fluentd/issues/3089 test "closed chunk should not be committed" do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal 0.95, @p.chunk_full_threshold purge_count = 0 stub.proxy(@p).generate_chunk(anything) do |chunk| stub.proxy(chunk).purge do |result| purge_count += 1 result end stub.proxy(chunk).commit do |result| assert_false(chunk.closed?) result end stub.proxy(chunk).rollback do |result| assert_false(chunk.closed?) result end chunk end m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) small_row = "x" * 1024 * 400 big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it doesn't cause BufferOverFlowError. # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry` @p.write({m => [small_row] * 40 + [big_row] + ["x"]}) # Above event stream will be splitted twice by `Buffer#write_step_by_step` # # 1. `write_once`: 42 [events] * 1 [stream] # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream] # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams] # # The problematic data is built in the 2nd stage. # In the 2nd stage, 5 streams are packed in a chunk. # ((1024 * 400) [bytes] * 4 [events] * 5 [streams] = 8192000 [bytes] < `chunk_limit_size` (8MB)). # So 3 chunks are used to store all data. # The 1st chunk is already staged by `write_once`. # The 2nd & 3rd chunks are newly created as unstaged. # The 3rd chunk is purged before `ShouldRetry`, it's no problem: # https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L850 # The 2nd chunk is purged in `rescue ShouldRetry`: # https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L862 # It causes the issue described in https://github.com/fluent/fluentd/issues/3089#issuecomment-1811839198 assert_equal 2, purge_count end # https://github.com/fluent/fluentd/issues/4446 test "#write_step_by_step keeps chunks kept in locked in entire #write process" do assert_equal 8 * 1024 * 1024, @p.chunk_limit_size assert_equal 0.95, @p.chunk_full_threshold mon_enter_counts_by_chunk = {} mon_exit_counts_by_chunk = {} stub.proxy(@p).generate_chunk(anything) do |chunk| stub(chunk).mon_enter do enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0) exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0) mon_enter_counts_by_chunk[chunk] = enter_count # Assert that chunk is passed to &block of write_step_by_step before exiting the lock. # (i.e. The lock count must be 2 greater than the exit count). # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block. if chunk.staged? lock_in_block = enter_count == 3 assert_equal(enter_count - 2, exit_count) if lock_in_block else lock_in_block = enter_count == 2 assert_equal(enter_count - 2, exit_count) if lock_in_block end end stub(chunk).mon_exit do exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0) mon_exit_counts_by_chunk[chunk] = exit_count end chunk end m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) small_row = "x" * 1024 * 400 big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it doesn't cause BufferOverFlowError. # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry` @p.write({m => [small_row] * 40 + [big_row] + ["x"]}) # Above event stream will be splitted twice by `Buffer#write_step_by_step` # # 1. `write_once`: 42 [events] * 1 [stream] # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream] # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams] # # Example of staged chunk lock behavior: # # 1. mon_enter in write_step_by_step # 2. ShouldRetry occurs # 3. mon_exit in write_step_by_step # 4. mon_enter again in write_step_by_step (retry) # 5. passed to &block of write_step_by_step # 6. mon_enter in the block (write) # 7. mon_exit in write_step_by_step # 8. mon_exit in write assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values) end end sub_test_case 'standard format with configuration for test with lower chunk limit size' do setup do @p = create_buffer({"chunk_limit_size" => 1_280_000}) @format = ->(e){e.to_msgpack_stream} @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) # 1 record is 128bytes in msgpack stream @es0 = es0 = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:01 +0000'), {"message" => "x" * (128 - 22)}] ] * 5000) (class << @p; self; end).module_eval do define_method(:resume) { staged = { dm0 => create_chunk_es(dm0, es0).staged!, } queued = [] return staged, queued } end @p.start end test '#write appends event stream into staged chunk' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 1000) @p.write({@dm0 => es}, format: @format) assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal (@es0.to_msgpack_stream + es.to_msgpack_stream), @p.stage[@dm0].read end test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 8000) @p.write({@dm0 => es}, format: @format) assert_equal [@dm0], @p.stage.keys assert_equal [@dm0], @p.queue.map(&:metadata) assert_equal (es.to_msgpack_stream), @p.stage[@dm0].read end test '#write writes event stream into many chunks excluding staged chunk if event stream is larger than chunk limit size' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 45000) @p.write({@dm0 => es}, format: @format) # metadata whose seq is 4 is created, but overwrite with original metadata(seq=0) for next use of this chunk https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L357 assert_equal [@dm0], @p.stage.keys assert_equal 5400, @p.stage[@dm0].size assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata) assert_equal [5000, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ... # 9900 * 4 + 5400 == 45000 end test '#dequeue_chunk succeeds when chunk is splited' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 45000) @p.write({@dm0 => es}, format: @format) @p.enqueue_all(true) dequeued_chunks = Array.new(6) { |e| @p.dequeue_chunk } # splits: 45000 / 100 => 450 * ... assert_equal [5000, 9900, 9900, 9900, 9900, 5400], dequeued_chunks.map(&:size) assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0, @dm0], dequeued_chunks.map(&:metadata) end test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}] ]) assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do @p.write({@dm0 => es}, format: @format) end end data( first_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]), intermediate_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]), last_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]]), multiple_chunks: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}], [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]]) ) test '#write exceeds chunk_limit_size, raise BufferChunkOverflowError, but not lost whole messages' do |(es)| assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size nth = [] es.entries.each_with_index do |entry, index| if entry.last["message"].size == @p.chunk_limit_size nth << index end end messages = [] nth.each do |n| messages << "a 1280025 bytes record (nth: #{n}) is larger than buffer chunk limit size (1280000)" end assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages.join(", ")) do @p.write({@dm0 => es}, format: @format) end # message a and b are concatenated and staged staged_messages = Fluent::MessagePackFactory.msgpack_unpacker.feed_each(@p.stage[@dm0].chunk).collect do |record| record.last end assert_equal([2, [{"message" => "a"}, {"message" => "b"}]], [@p.stage[@dm0].size, staged_messages]) # only es0 message is queued assert_equal [@dm0], @p.queue.map(&:metadata) assert_equal [5000], @p.queue.map(&:size) end test "confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError" do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) timestamp = event_time('2016-04-11 16:00:02 +0000') es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "a" * 1_000_000}], [timestamp, {"message" => "b" * 1_000_000}], [timestamp, {"message" => "c" * 1_000_000}]]) # https://github.com/fluent/fluentd/issues/1849 # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before. # It should not be raised and message a,b,c should be stored into 3 chunks. assert_nothing_raised do @p.write({@dm0 => es}, format: @format) end messages = [] # pick up first letter to check whether chunk is queued in expected order 3.times do |index| chunk = @p.queue[index] es = Fluent::MessagePackEventStream.new(chunk.chunk) es.ensure_unpacked! records = es.instance_eval{ @unpacked_records } records.each do |record| messages << record["message"][0] end end es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk) es.ensure_unpacked! staged_message = es.instance_eval{ @unpacked_records }.first["message"] # message a and b are queued, message c is staged assert_equal([ [@dm0], "c" * 1_000_000, [@dm0, @dm0, @dm0], [5000, 1, 1], [["x"] * 5000, "a", "b"].flatten ], [ @p.stage.keys, staged_message, @p.queue.map(&:metadata), @p.queue.map(&:size), messages ]) end end sub_test_case 'custom format with configuration for test with lower chunk limit size' do setup do @p = create_buffer({"chunk_limit_size" => 1_280_000}) @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) @row = "x" * 128 @data0 = data0 = [@row] * 5000 (class << @p; self; end).module_eval do define_method(:resume) { staged = { dm0 => create_chunk(dm0, data0).staged!, } queued = [] return staged, queued } end @p.start end test '#write appends event stream into staged chunk' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size data = [@row] * 1000 @p.write({@dm0 => data}) assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal (@row * 6000), @p.stage[@dm0].read end test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) staged_chunk_object_id = @p.stage[@dm0].object_id assert_equal 1_280_000, @p.chunk_limit_size data = [@row] * 8000 @p.write({@dm0 => data}) assert_equal [@dm0], @p.queue.map(&:metadata) assert_equal [staged_chunk_object_id], @p.queue.map(&:object_id) assert_equal [@dm0], @p.stage.keys assert_equal [9800], @p.queue.map(&:size) assert_equal 3200, @p.stage[@dm0].size # 9800 + 3200 == 5000 + 8000 end test '#write writes event stream into many chunks including staging chunk if event stream is larger than chunk limit size' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) staged_chunk_object_id = @p.stage[@dm0].object_id assert_equal 1_280_000, @p.chunk_limit_size assert_equal 5000, @p.stage[@dm0].size data = [@row] * 45000 @p.write({@dm0 => data}) assert_equal staged_chunk_object_id, @p.queue.first.object_id assert_equal [@dm0], @p.stage.keys assert_equal 900, @p.stage[@dm0].size assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata) assert_equal [9500, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ... ##### 900 + 9500 + 9900 * 4 == 5000 + 45000 end test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = ["x" * 1_280_000 + "x" * 300] assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do @p.write({@dm0 => es}) end end test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do assert_equal [@dm0], @p.stage.keys assert_equal [], @p.queue.map(&:metadata) assert_equal 1_280_000, @p.chunk_limit_size es = ["a" * 1_000_000, "b" * 1_000_000, "c" * 1_000_000] assert_nothing_raised do @p.write({@dm0 => es}) end queue_messages = @p.queue.collect do |chunk| # collect first character of each message chunk.chunk[0] end assert_equal([ [@dm0], 1, "c", [@dm0, @dm0, @dm0], [5000, 1, 1], ["x", "a", "b"] ], [ @p.stage.keys, @p.stage[@dm0].size, @p.stage[@dm0].chunk[0], @p.queue.map(&:metadata), @p.queue.map(&:size), queue_messages ]) end end sub_test_case 'with configuration for test with lower limits' do setup do @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240}) @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) (class << @p; self; end).module_eval do define_method(:resume) { staged = { dm2 => create_chunk(dm2, ["b" * 128] * 7).staged!, dm3 => create_chunk(dm3, ["c" * 128] * 5).staged!, } queued = [ create_chunk(dm0, ["0" * 128] * 8).enqueued!, create_chunk(dm0, ["0" * 128] * 8).enqueued!, create_chunk(dm0, ["0" * 128] * 8).enqueued!, create_chunk(dm0, ["0" * 128] * 8).enqueued!, create_chunk(dm0, ["0" * 128] * 8).enqueued!, create_chunk(dm1, ["a" * 128] * 8).enqueued!, create_chunk(dm1, ["a" * 128] * 8).enqueued!, create_chunk(dm1, ["a" * 128] * 8).enqueued!, # 8th queued chunk create_chunk(dm1, ["a" * 128] * 3).enqueued!, ] return staged, queued } end @p.start end test '#storable? returns false when too many data exist' do assert_equal [@dm0,@dm0,@dm0,@dm0,@dm0,@dm1,@dm1,@dm1,@dm1], @p.queue.map(&:metadata) assert_equal [@dm2,@dm3], @p.stage.keys assert_equal 128*8*8+128*3, @p.queue_size assert_equal 128*7+128*5, @p.stage_size assert @p.storable? dm3 = @p.metadata(timekey: @dm3.timekey) @p.write({dm3 => ["c" * 128]}) assert_equal 10240, (@p.stage_size + @p.queue_size) assert !@p.storable? end test '#chunk_size_over? returns true if chunk size is bigger than limit' do m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m, ["a" * 128] * 8) assert !@p.chunk_size_over?(c1) c2 = create_chunk(m, ["a" * 128] * 9) assert @p.chunk_size_over?(c2) c3 = create_chunk(m, ["a" * 128] * 8 + ["a"]) assert @p.chunk_size_over?(c3) end test '#chunk_size_full? returns true if chunk size is enough big against limit' do m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m, ["a" * 128] * 7) assert !@p.chunk_size_full?(c1) c2 = create_chunk(m, ["a" * 128] * 8) assert @p.chunk_size_full?(c2) assert_equal 0.95, @p.chunk_full_threshold c3 = create_chunk(m, ["a" * 128] * 6 + ["a" * 64]) assert !@p.chunk_size_full?(c3) end end sub_test_case 'with configuration includes chunk_limit_records' do setup do @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_limit_records" => 6}) @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) (class << @p; self; end).module_eval do define_method(:resume) { staged = { dm2 => create_chunk(dm2, ["b" * 128] * 1).staged!, dm3 => create_chunk(dm3, ["c" * 128] * 2).staged!, } queued = [ create_chunk(dm0, ["0" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 3).enqueued!, ] return staged, queued } end @p.start end test '#chunk_size_over? returns true if too many records exists in a chunk even if its bytes is less than limit' do assert_equal 6, @p.chunk_limit_records m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m, ["a" * 128] * 6) assert_equal 6, c1.size assert !@p.chunk_size_over?(c1) c2 = create_chunk(m, ["a" * 128] * 7) assert @p.chunk_size_over?(c2) c3 = create_chunk(m, ["a" * 128] * 6 + ["a"]) assert @p.chunk_size_over?(c3) end test '#chunk_size_full? returns true if enough many records exists in a chunk even if its bytes is less than limit' do assert_equal 6, @p.chunk_limit_records m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) c1 = create_chunk(m, ["a" * 128] * 5) assert_equal 5, c1.size assert !@p.chunk_size_full?(c1) c2 = create_chunk(m, ["a" * 128] * 6) assert @p.chunk_size_full?(c2) c3 = create_chunk(m, ["a" * 128] * 5 + ["a"]) assert @p.chunk_size_full?(c3) end end sub_test_case 'with configuration includes queue_limit_length' do setup do @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_limit_length" => 5}) @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) (class << @p; self; end).module_eval do define_method(:resume) { staged = { dm2 => create_chunk(dm2, ["b" * 128] * 1).staged!, dm3 => create_chunk(dm3, ["c" * 128] * 2).staged!, } queued = [ create_chunk(dm0, ["0" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 6).enqueued!, create_chunk(dm1, ["a" * 128] * 3).enqueued!, ] return staged, queued } end @p.start end test '#configure will overwrite standard configuration if queue_limit_length' do assert_equal 1024, @p.chunk_limit_size assert_equal 5, @p.queue_limit_length assert_equal (1024*5), @p.total_limit_size end end sub_test_case 'when compress is gzip' do setup do @p = create_buffer({'compress' => 'gzip'}) @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) end test '#compress returns :gzip' do assert_equal :gzip, @p.compress end test 'create decompressable chunk' do chunk = @p.generate_chunk(create_metadata) assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable) end test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do @p = create_buffer({'compress' => 'gzip', 'chunk_limit_size' => 70}) timestamp = event_time('2016-04-11 16:00:02 +0000') es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}], # overflow [timestamp, {"message" => "aaa"}], [timestamp, {"message" => "bbb"}]]) assert_equal [], @p.queue.map(&:metadata) assert_equal 70, @p.chunk_limit_size # calculate the actual boundary value. it varies on machine c = @p.generate_chunk(create_metadata) c.append(Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}]]), compress: :gzip) overflow_bytes = c.bytesize messages = "concatenated/appended a #{overflow_bytes} bytes record (nth: 0) is larger than buffer chunk limit size (70)" assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages) do # test format == nil && compress == :gzip @p.write({@dm0 => es}) end # message a and b occupies each chunks in full, so both of messages are queued (no staged chunk) assert_equal([2, [@dm0, @dm0], [1, 1], nil], [@p.queue.size, @p.queue.map(&:metadata), @p.queue.map(&:size), @p.stage[@dm0]]) end end sub_test_case '#statistics' do setup do @p = create_buffer({ "total_limit_size" => 1024 }) dm = create_metadata(Time.parse('2020-03-13 16:00:00 +0000').to_i, nil, nil) (class << @p; self; end).module_eval do define_method(:resume) { queued = [create_chunk(dm, ["a" * (1024 - 102)]).enqueued!] return {}, queued } end @p.start end test 'returns available_buffer_space_ratios' do assert_equal 10.0, @p.statistics['buffer']['available_buffer_space_ratios'] end end end ================================================ FILE: test/plugin/test_buffer_chunk.rb ================================================ require_relative '../helper' require 'fluent/plugin/buffer/chunk' class BufferChunkTest < Test::Unit::TestCase sub_test_case 'blank buffer chunk' do test 'has generated unique id, given metadata, created_at and modified_at' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) assert{ chunk.unique_id.bytesize == 16 } assert{ chunk.metadata.object_id == meta.object_id } assert{ chunk.created_at.is_a? Time } assert{ chunk.modified_at.is_a? Time } assert chunk.unstaged? assert !chunk.staged? assert !chunk.queued? assert !chunk.closed? end test 'has many methods for chunks, but not implemented' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) assert chunk.respond_to?(:append) assert chunk.respond_to?(:concat) assert chunk.respond_to?(:commit) assert chunk.respond_to?(:rollback) assert chunk.respond_to?(:bytesize) assert chunk.respond_to?(:size) assert chunk.respond_to?(:length) assert chunk.respond_to?(:empty?) assert chunk.respond_to?(:read) assert chunk.respond_to?(:open) assert chunk.respond_to?(:write_to) assert_raise(NotImplementedError){ chunk.append([]) } assert_raise(NotImplementedError){ chunk.concat(nil, 0) } assert_raise(NotImplementedError){ chunk.commit } assert_raise(NotImplementedError){ chunk.rollback } assert_raise(NotImplementedError){ chunk.bytesize } assert_raise(NotImplementedError){ chunk.size } assert_raise(NotImplementedError){ chunk.length } assert_raise(NotImplementedError){ chunk.empty? } assert_raise(NotImplementedError){ chunk.read } assert_raise(NotImplementedError){ chunk.open(){} } assert_raise(NotImplementedError){ chunk.write_to(nil) } assert !chunk.respond_to?(:msgpack_each) end test 'has method #each and #msgpack_each only when extended by ChunkMessagePackEventStreamer' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) assert !chunk.respond_to?(:each) assert !chunk.respond_to?(:msgpack_each) chunk.extend Fluent::ChunkMessagePackEventStreamer assert chunk.respond_to?(:each) assert chunk.respond_to?(:msgpack_each) end test 'unpacker arg is not implemented for ChunkMessagePackEventStreamer' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) chunk.extend Fluent::ChunkMessagePackEventStreamer unpacker = Fluent::MessagePackFactory.thread_local_msgpack_unpacker assert_raise(NotImplementedError){ chunk.each(unpacker: unpacker) } assert_raise(NotImplementedError){ chunk.msgpack_each(unpacker: unpacker) } end test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) assert_raise(ArgumentError){ chunk.read(compressed: :gzip) } assert_raise(ArgumentError){ chunk.open(compressed: :gzip){} } assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :gzip) } assert_raise(ArgumentError){ chunk.append(nil, compress: :gzip) } end test 'some methods raise ArgumentError with an option of `compressed: :zstd` and without extending Compressble`' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta) assert_raise(ArgumentError){ chunk.read(compressed: :zstd) } assert_raise(ArgumentError){ chunk.open(compressed: :zstd){} } assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :zstd) } assert_raise(ArgumentError){ chunk.append(nil, compress: :zstd) } end end class TestChunk < Fluent::Plugin::Buffer::Chunk attr_accessor :data def initialize(meta) super @data = '' end def size @data.size end def open(**kwargs) require 'stringio' io = StringIO.new(@data) yield io end end sub_test_case 'minimum chunk implements #size and #open' do test 'chunk lifecycle' do c = TestChunk.new(Object.new) assert c.unstaged? assert !c.staged? assert !c.queued? assert !c.closed? assert c.writable? c.staged! assert !c.unstaged? assert c.staged? assert !c.queued? assert !c.closed? assert c.writable? c.enqueued! assert !c.unstaged? assert !c.staged? assert c.queued? assert !c.closed? assert !c.writable? c.close assert !c.unstaged? assert !c.staged? assert !c.queued? assert c.closed? assert !c.writable? end test 'chunk can be unstaged' do c = TestChunk.new(Object.new) assert c.unstaged? assert !c.staged? assert !c.queued? assert !c.closed? assert c.writable? c.staged! assert !c.unstaged? assert c.staged? assert !c.queued? assert !c.closed? assert c.writable? c.unstaged! assert c.unstaged? assert !c.staged? assert !c.queued? assert !c.closed? assert c.writable? c.enqueued! assert !c.unstaged? assert !c.staged? assert c.queued? assert !c.closed? assert !c.writable? c.close assert !c.unstaged? assert !c.staged? assert !c.queued? assert c.closed? assert !c.writable? end test 'can respond to #empty? correctly' do c = TestChunk.new(Object.new) assert_equal 0, c.size assert c.empty? end test 'can write its contents to io object' do c = TestChunk.new(Object.new) c.data << "my data\nyour data\n" io = StringIO.new c.write_to(io) assert "my data\nyour data\n", io.to_s end test 'can feed objects into blocks with unpacking msgpack if ChunkMessagePackEventStreamer is included' do require 'msgpack' c = TestChunk.new(Object.new) c.extend Fluent::ChunkMessagePackEventStreamer c.data << MessagePack.pack(['my data', 1]) c.data << MessagePack.pack(['your data', 2]) ary = [] c.msgpack_each do |obj| ary << obj end assert_equal ['my data', 1], ary[0] assert_equal ['your data', 2], ary[1] end end sub_test_case 'when compress is gzip' do test 'create decompressable chunk' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :gzip) assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable) end end sub_test_case 'when compress is zstd' do test 'create decompressable chunk' do meta = Object.new chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :zstd) assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::ZstdDecompressable) end end end ================================================ FILE: test/plugin/test_buffer_file_chunk.rb ================================================ require_relative '../helper' require 'fluent/plugin/buffer/file_chunk' require 'fluent/plugin/compressable' require 'fluent/unique_id' require 'fileutils' require 'msgpack' require 'time' require 'timecop' class BufferFileChunkTest < Test::Unit::TestCase include Fluent::Plugin::Compressable setup do @klass = Fluent::Plugin::Buffer::FileChunk @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__) FileUtils.rm_r @chunkdir rescue nil FileUtils.mkdir_p @chunkdir end teardown do Timecop.return end Metadata = Fluent::Plugin::Buffer::Metadata def gen_metadata(timekey: nil, tag: nil, variables: nil) Metadata.new(timekey, tag, variables) end def read_metadata_file(path) File.open(path, 'rb') do |f| chunk = f.read if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data(4) return nil end data = nil if chunk.slice(0, 2) == Fluent::Plugin::Buffer::FileChunk::BUFFER_HEADER size = chunk.slice(2, 4).unpack('N').first if size data = MessagePack.unpack(chunk.slice(6, size), symbolize_keys: true) end else # old type data = MessagePack.unpack(chunk, symbolize_keys: true) end data end end def gen_path(path) File.join(@chunkdir, path) end def gen_test_chunk_id require 'time' now = Time.parse('2016-04-07 14:31:33 +0900') u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)` u3 = 2979763054 # one of rand(0xffffffff) u4 = 438020492 # ditto [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN') # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c" end def hex_id(id) id.unpack('N*').map{|n| n.to_s(16)}.join end sub_test_case 'classmethods' do data( correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged], correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued], incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :unknown], incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :unknown], output_file: ['/mydir/mypath/myfile.20160716.log', :unknown], ) test 'can .assume_chunk_state' do |data| path, expected = data assert_equal expected, @klass.assume_chunk_state(path) end test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do assert_equal gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_stage_chunk_path(gen_path("mychunk.*.log"), gen_test_chunk_id) assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id) end assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("mychunk.*"), gen_test_chunk_id) end assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("*.log"), gen_test_chunk_id) end end test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do assert_equal( gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_queued_chunk_path(gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), gen_test_chunk_id) ) end test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do assert_equal( gen_path("mychunk.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), @klass.generate_queued_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id) ) assert_equal( gen_path("mychunk.q55555555555555555555555555555555.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), @klass.generate_queued_chunk_path(gen_path("mychunk.q55555555555555555555555555555555.log"), gen_test_chunk_id) ) end test '.unique_id_from_path recreates unique_id from file path to assume unique_id for v0.12 chunks' do assert_equal gen_test_chunk_id, @klass.unique_id_from_path(gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log")) end end sub_test_case 'newly created chunk' do setup do @chunk_path = File.join(@chunkdir, 'test.*.log') @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :create) end def gen_chunk_path(prefix, unique_id) File.join(@chunkdir, "test.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.log") end teardown do if @c @c.purge rescue nil end if File.exist? @chunk_path File.unlink @chunk_path end end test 'creates new files for chunk and metadata with specified path & permission' do assert{ @c.unique_id.size == 16 } assert_equal gen_chunk_path('b', @c.unique_id), @c.path assert File.exist?(gen_chunk_path('b', @c.unique_id)) assert{ File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) } assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta') assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) } assert_equal :unstaged, @c.state assert @c.empty? end test 'can #append, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'can #concat, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'has its contents in binary (ascii-8bit)' do data1 = "aaa bbb ccc".force_encoding('utf-8') @c.append([data1]) @c.commit assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding } content = @c.read assert_equal Encoding::ASCII_8BIT, content.encoding end test 'has #bytesize and #size' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.commit assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size end test 'can #rollback to revert non-committed data' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert_equal '', File.open(@c.path, 'rb'){|f| f.read } d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read } end test 'can #rollback to revert non-committed data from #concat' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert_equal '', File.open(@c.path, 'rb'){|f| f.read } d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read } end test 'can store its data by #close' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit content = @c.read unique_id = @c.unique_id size = @c.size created_at = @c.created_at modified_at = @c.modified_at @c.close assert_equal content, File.open(@c.path, 'rb'){|f| f.read } stored_meta = { timekey: nil, tag: nil, variables: nil, seq: 0, id: unique_id, s: size, c: created_at.to_i, m: modified_at.to_i, } assert_equal stored_meta, read_metadata_file(@c.path + '.meta') end test 'deletes all data by #purge' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit @c.purge assert @c.empty? assert_equal 0, @c.bytesize assert_equal 0, @c.size assert !File.exist?(@c.path) assert !File.exist?(@c.path + '.meta') end test 'can #open its contents as io' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit lines = [] @c.open do |io| assert io io.readlines.each do |l| lines << l end end assert_equal d1.to_json + "\n", lines[0] assert_equal d2.to_json + "\n", lines[1] assert_equal d3.to_json + "\n", lines[2] assert_equal d4.to_json + "\n", lines[3] end test '#write_metadata tries to store metadata on file' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit expected = { timekey: nil, tag: nil, variables: nil, seq: 0, id: @c.unique_id, s: @c.size, c: @c.created_at.to_i, m: @c.modified_at.to_i, } assert_equal expected, read_metadata_file(@c.path + '.meta') d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) # append does write_metadata dummy_now = Time.parse('2016-04-07 16:59:59 +0900') Timecop.freeze(dummy_now) @c.write_metadata expected = { timekey: nil, tag: nil, variables: nil, seq: 0, id: @c.unique_id, s: @c.size, c: @c.created_at.to_i, m: dummy_now.to_i, } assert_equal expected, read_metadata_file(@c.path + '.meta') @c.commit expected = { timekey: nil, tag: nil, variables: nil, seq: 0, id: @c.unique_id, s: @c.size, c: @c.created_at.to_i, m: @c.modified_at.to_i, } assert_equal expected, read_metadata_file(@c.path + '.meta') content = @c.read unique_id = @c.unique_id size = @c.size created_at = @c.created_at modified_at = @c.modified_at @c.close assert_equal content, File.open(@c.path, 'rb'){|f| f.read } stored_meta = { timekey: nil, tag: nil, variables: nil, seq: 0, id: unique_id, s: size, c: created_at.to_i, m: modified_at.to_i, } assert_equal stored_meta, read_metadata_file(@c.path + '.meta') end end test 'ensure to remove metadata file if #write_metadata raise an error because of disk full' do chunk_path = File.join(@chunkdir, 'test.*.log') stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass| stub(klass).write_metadata(anything) do |v| raise 'disk full' end end err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create) end assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta')) assert_match(/create buffer metadata/, err.message) end sub_test_case 'chunk with file for staged chunk' do setup do @chunk_id = gen_test_chunk_id @chunk_path = File.join(@chunkdir, "test_staged.b#{hex_id(@chunk_id)}.log") @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join File.open(@chunk_path, 'wb') do |f| f.write @d end @metadata = { timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0, id: @chunk_id, s: 4, c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:13 +0900').to_i, } File.open(@chunk_path + '.meta', 'wb') do |f| f.write @metadata.to_msgpack end @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged) end teardown do if @c @c.purge rescue nil end [@chunk_path, @chunk_path + '.meta', @enqueued_path, @enqueued_path + '.meta'].each do |path| File.unlink path if File.exist? path end end test 'can load as staged chunk from file with metadata' do assert_equal @chunk_path, @c.path assert_equal :staged, @c.state assert_nil @c.metadata.timekey assert_equal 'testing', @c.metadata.tag assert_equal({k: "x"}, @c.metadata.variables) assert_equal 4, @c.size assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at content = @c.read assert_equal @d, content end test 'can be enqueued' do stage_path = @c.path queue_path = @enqueued_path assert File.exist?(stage_path) assert File.exist?(stage_path + '.meta') assert !File.exist?(queue_path) assert !File.exist?(queue_path + '.meta') @c.enqueued! assert_equal queue_path, @c.path assert !File.exist?(stage_path) assert !File.exist?(stage_path + '.meta') assert File.exist?(queue_path) assert File.exist?(queue_path + '.meta') assert_nil @c.metadata.timekey assert_equal 'testing', @c.metadata.tag assert_equal({k: "x"}, @c.metadata.variables) assert_equal 4, @c.size assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at assert_equal @d, File.open(@c.path, 'rb'){|f| f.read } assert_equal @metadata, read_metadata_file(@c.path + '.meta') end test '#write_metadata tries to store metadata on file with non-committed data' do d5 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} d5s = d5.to_json + "\n" @c.append([d5s]) metadata = { timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0, id: @chunk_id, s: 4, c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:13 +0900').to_i, } assert_equal metadata, read_metadata_file(@c.path + '.meta') @c.write_metadata metadata = { timekey: nil, tag: 'testing', variables: {k: "x"}, seq: 0, id: @chunk_id, s: 5, c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:38 +0900').to_i, } dummy_now = Time.parse('2016-04-07 17:44:38 +0900') Timecop.freeze(dummy_now) @c.write_metadata assert_equal metadata, read_metadata_file(@c.path + '.meta') end test '#file_rename can rename chunk files even in windows, and call callback with file size' do data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc" testing_file1 = gen_path('rename1.test') testing_file2 = gen_path('rename2.test') f = File.open(testing_file1, 'wb', @c.permission) f.set_encoding(Encoding::ASCII_8BIT) f.sync = true f.binmode f.write data pos = f.pos assert f.binmode? assert f.sync assert_equal data.bytesize, f.size io = nil @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io }) assert io if Fluent.windows? assert{ f != io } else assert_equal f, io end assert_equal Encoding::ASCII_8BIT, io.external_encoding assert io.sync assert io.binmode? assert_equal data.bytesize, io.size assert_equal pos, io.pos assert_equal '', io.read io.rewind assert_equal data, io.read end end sub_test_case 'chunk with file for enqueued chunk' do setup do @chunk_id = gen_test_chunk_id @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join File.open(@enqueued_path, 'wb') do |f| f.write @d end @dummy_timekey = Time.parse('2016-04-07 17:40:00 +0900').to_i @metadata = { timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}, seq: 0, id: @chunk_id, s: 4, c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:13 +0900').to_i, } File.open(@enqueued_path + '.meta', 'wb') do |f| f.write @metadata.to_msgpack end @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @enqueued_path, :queued) end teardown do if @c @c.purge rescue nil end [@enqueued_path, @enqueued_path + '.meta'].each do |path| File.unlink path if File.exist? path end end test 'can load as queued chunk (read only) with metadata' do assert @c assert_equal @chunk_id, @c.unique_id assert_equal :queued, @c.state assert_equal gen_metadata(timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}), @c.metadata assert_equal Time.at(@metadata[:c]), @c.created_at assert_equal Time.at(@metadata[:m]), @c.modified_at assert_equal @metadata[:s], @c.size assert_equal @d.bytesize, @c.bytesize assert_equal @d, @c.read assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do @c.append(["queued chunk is read only"]) end assert_raise IOError do @c.instance_eval{ @chunk }.write "chunk io is opened as read only" end end end sub_test_case 'chunk with queued chunk file of v0.12, without metadata' do setup do @chunk_id = gen_test_chunk_id @chunk_path = File.join(@chunkdir, "test_v12.2016040811.q#{hex_id(@chunk_id)}.log") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join File.open(@chunk_path, 'wb') do |f| f.write @d end @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :queued) end teardown do if @c @c.purge rescue nil end File.unlink @chunk_path if File.exist? @chunk_path end test 'can load as queued chunk from file without metadata' do assert @c assert_equal :queued, @c.state assert_equal @chunk_id, @c.unique_id assert_equal gen_metadata, @c.metadata assert_equal @d.bytesize, @c.bytesize assert_equal 0, @c.size assert_equal @d, @c.read assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do @c.append(["queued chunk is read only"]) end assert_raise IOError do @c.instance_eval{ @chunk }.write "chunk io is opened as read only" end end end sub_test_case 'chunk with staged chunk file of v0.12, without metadata' do setup do @chunk_id = gen_test_chunk_id @chunk_path = File.join(@chunkdir, "test_v12.2016040811.b#{hex_id(@chunk_id)}.log") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join File.open(@chunk_path, 'wb') do |f| f.write @d end @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged) end teardown do if @c @c.purge rescue nil end File.unlink @chunk_path if File.exist? @chunk_path end test 'can load as queued chunk from file without metadata even if it was loaded as staged chunk' do assert @c assert_equal :queued, @c.state assert_equal @chunk_id, @c.unique_id assert_equal gen_metadata, @c.metadata assert_equal @d.bytesize, @c.bytesize assert_equal 0, @c.size assert_equal @d, @c.read assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do @c.append(["queued chunk is read only"]) end assert_raise IOError do @c.instance_eval{ @chunk }.write "chunk io is opened as read only" end end end sub_test_case 'compressed buffer' do setup do @src = 'text data for compressing' * 5 @gzipped_src = compress(@src) @zstded_src = compress(@src, type: :zstd) end test '#append with compress option writes compressed data to chunk when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip) c.append([@src, @src], compress: :gzip) c.commit # check chunk is compressed assert c.read(compressed: :gzip).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit comressed_data = c.open(compressed: :gzip) do |io| v = io.read assert_equal @gzipped_src, v v end assert_equal @gzipped_src, comressed_data end test '#write_to writes decompressed data when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :gzip) assert_equal @gzipped_src, io.string end test '#append with compress option writes compressed data to chunk when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd) c.append([@src, @src], compress: :zstd) c.commit # check chunk is compressed assert c.read(compressed: :zstd).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit comressed_data = c.open(compressed: :zstd) do |io| v = io.read assert_equal @zstded_src, v v end assert_equal @zstded_src, comressed_data end test '#write_to writes decompressed data when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :zstd) assert_equal @zstded_src, io.string end end end ================================================ FILE: test/plugin/test_buffer_file_single_chunk.rb ================================================ require_relative '../helper' require 'fluent/plugin/buffer/file_single_chunk' require 'fluent/plugin/compressable' require 'fluent/unique_id' require 'fileutils' require 'msgpack' require 'time' class BufferFileSingleChunkTest < Test::Unit::TestCase include Fluent::Plugin::Compressable setup do @klass = Fluent::Plugin::Buffer::FileSingleChunk @chunkdir = File.expand_path('../../tmp/buffer_file_single_chunk', __FILE__) FileUtils.rm_r(@chunkdir) rescue nil FileUtils.mkdir_p(@chunkdir) end Metadata = Struct.new(:timekey, :tag, :variables) def gen_metadata(timekey: nil, tag: 'testing', variables: nil) Metadata.new(timekey, tag, variables) end def gen_path(path) File.join(@chunkdir, path) end def gen_test_chunk_id now = Time.parse('2016-04-07 14:31:33 +0900') u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)` u3 = 2979763054 # one of rand(0xffffffff) u4 = 438020492 # ditto [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN') # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c" end def hex_id(id) id.unpack('N*').map { |n| n.to_s(16) }.join end sub_test_case 'classmethods' do data( correct_staged: ['/mydir/mypath/fsb.b00ff.buf', :staged], correct_queued: ['/mydir/mypath/fsb.q00ff.buf', :queued], incorrect_staged: ['/mydir/mypath/fsb.b00ff.buf/unknown', :unknown], incorrect_queued: ['/mydir/mypath/fsb.q00ff.buf/unknown', :unknown], output_file: ['/mydir/mypath/fsb.20160716.buf', :unknown], ) test 'can .assume_chunk_state' do |data| path, expected = data assert_equal expected, @klass.assume_chunk_state(path) end test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do assert_equal gen_path("fsb.foo.b52fde6425d7406bdb19b936e1a1ba98c.buf"), @klass.generate_stage_chunk_path(gen_path("fsb.*.buf"), 'foo', gen_test_chunk_id) assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("fsb.buf"), 'foo', gen_test_chunk_id) end assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("fsb.*"), 'foo', gen_test_chunk_id) end assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do @klass.generate_stage_chunk_path(gen_path("*.buf"), 'foo', gen_test_chunk_id) end end test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do assert_equal( gen_path("fsb.q52fde6425d7406bdb19b936e1a1ba98c.buf"), @klass.generate_queued_chunk_path(gen_path("fsb.b52fde6425d7406bdb19b936e1a1ba98c.buf"), gen_test_chunk_id) ) end test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do assert_equal( gen_path("fsb.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), @klass.generate_queued_chunk_path(gen_path("fsb.buf"), gen_test_chunk_id) ) assert_equal( gen_path("fsb.q55555555555555555555555555555555.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), @klass.generate_queued_chunk_path(gen_path("fsb.q55555555555555555555555555555555.buf"), gen_test_chunk_id) ) end data('1 word tag' => 'foo', '2 words tag' => 'test.log', 'empty' => '') test '.unique_id_and_key_from_path recreates unique_id and key from file path' do |key| path = @klass.unique_id_and_key_from_path(gen_path("fsb.#{key}.q52fde6425d7406bdb19b936e1a1ba98c.buf")) assert_equal [gen_test_chunk_id, key], path end end sub_test_case 'newly created chunk' do setup do @path_conf = File.join(@chunkdir, 'fsb.*.buf') @chunk_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(gen_test_chunk_id)}.buf") @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @path_conf, :create, nil) end def gen_chunk_path(prefix, unique_id) File.join(@chunkdir, "fsb.testing.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.buf") end teardown do if @c @c.purge rescue nil end if File.exist?(@chunk_path) File.unlink(@chunk_path) end end test 'creates new files for chunk and metadata with specified path & permission' do assert_equal 16, @c.unique_id.size assert_equal gen_chunk_path('b', @c.unique_id), @c.path assert File.exist?(gen_chunk_path('b', @c.unique_id)) assert { File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) } assert_equal :unstaged, @c.state assert @c.empty? end test 'can #append, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit ds = @c.read.split("\n").select { |d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit ds = @c.read.split("\n").select{|d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'can #concat, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) @c.commit ds = @c.read.split("\n").select{|d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) @c.commit ds = @c.read.split("\n").select { |d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'has its contents in binary (ascii-8bit)' do data1 = "aaa bbb ccc".force_encoding('utf-8') @c.append([data1]) @c.commit assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding } assert_equal Encoding::ASCII_8BIT, @c.read.encoding end test 'has #bytesize and #size' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.commit assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size end test 'can #rollback to revert non-committed data' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert_equal '', File.read(@c.path) d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path) end test 'can #rollback to revert non-committed data from #concat' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert_equal '', File.read(@c.path) d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path) end test 'can store its data by #close' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit content = @c.read @c.close assert_equal content, File.read(@c.path) end test 'deletes all data by #purge' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit @c.purge assert @c.empty? assert_equal 0, @c.bytesize assert_equal 0, @c.size assert !File.exist?(@c.path) end test 'can #open its contents as io' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit lines = [] @c.open do |io| assert io io.readlines.each do |l| lines << l end end assert_equal d1.to_json + "\n", lines[0] assert_equal d2.to_json + "\n", lines[1] assert_equal d3.to_json + "\n", lines[2] assert_equal d4.to_json + "\n", lines[3] end end sub_test_case 'chunk with file for staged chunk' do setup do @chunk_id = gen_test_chunk_id @staged_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(@chunk_id)}.buf") @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1, @d2, @d3, @d4].map{ |d| d.to_json + "\n" }.join File.write(@staged_path, @d, :mode => 'wb') @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @staged_path, :staged, nil) end teardown do if @c @c.purge rescue nil end [@staged_path, @enqueued_path].each do |path| File.unlink(path) if File.exist?(path) end end test 'can load as staged chunk from file with metadata' do assert_equal @staged_path, @c.path assert_equal :staged, @c.state assert_nil @c.metadata.timekey assert_equal 'testing', @c.metadata.tag assert_nil @c.metadata.variables assert_equal 0, @c.size assert_equal @d, @c.read @c.restore_size(:text) assert_equal 4, @c.size end test 'can be enqueued' do stage_path = @c.path queue_path = @enqueued_path assert File.exist?(stage_path) assert !File.exist?(queue_path) @c.enqueued! assert_equal queue_path, @c.path assert !File.exist?(stage_path) assert File.exist?(queue_path) assert_nil @c.metadata.timekey assert_equal 'testing', @c.metadata.tag assert_nil @c.metadata.variables assert_equal 0, @c.size assert_equal @d, File.read(@c.path) @c.restore_size(:text) assert_equal 4, @c.size end test '#file_rename can rename chunk files even in windows, and call callback with file size' do data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc" testing_file1 = gen_path('rename1.test') testing_file2 = gen_path('rename2.test') f = File.open(testing_file1, 'wb', @c.permission) f.set_encoding(Encoding::ASCII_8BIT) f.sync = true f.binmode f.write data pos = f.pos assert f.binmode? assert f.sync assert_equal data.bytesize, f.size io = nil @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io }) assert io if Fluent.windows? assert { f != io } else assert_equal f, io end assert_equal Encoding::ASCII_8BIT, io.external_encoding assert io.sync assert io.binmode? assert_equal data.bytesize, io.size assert_equal pos, io.pos assert_equal '', io.read io.rewind assert_equal data, io.read end end sub_test_case 'chunk with file for enqueued chunk' do setup do @chunk_id = gen_test_chunk_id @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join File.write(@enqueued_path, @d, :mode => 'wb') @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @enqueued_path, :queued, nil) end teardown do if @c @c.purge rescue nil end File.unlink(@enqueued_path) if File.exist?(@enqueued_path) end test 'can load as queued chunk (read only) with metadata' do assert @c assert_equal @chunk_id, @c.unique_id assert_equal :queued, @c.state stat = File.stat(@enqueued_path) assert_equal stat.ctime.to_i, @c.created_at.to_i assert_equal stat.mtime.to_i, @c.modified_at.to_i assert_equal 0, @c.size assert_equal @d.bytesize, @c.bytesize assert_equal @d, @c.read @c.restore_size(:text) assert_equal 4, @c.size assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do @c.append(["queued chunk is read only"]) end assert_raise IOError do @c.instance_eval{ @chunk }.write "chunk io is opened as read only" end end end sub_test_case 'chunk with queued chunk file' do setup do @chunk_id = gen_test_chunk_id @chunk_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf") @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join File.write(@chunk_path, @d, :mode => 'wb') @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @chunk_path, :queued, nil) end teardown do if @c @c.purge rescue nil end File.unlink(@chunk_path) if File.exist?(@chunk_path) end test 'can load as queued chunk' do assert @c assert_equal :queued, @c.state assert_equal @chunk_id, @c.unique_id assert_equal gen_metadata, @c.metadata assert_equal @d.bytesize, @c.bytesize assert_equal 0, @c.size assert_equal @d, @c.read assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do @c.append(["queued chunk is read only"]) end assert_raise IOError do @c.instance_eval{ @chunk }.write "chunk io is opened as read only" end end end sub_test_case 'compressed buffer' do setup do @src = 'text data for compressing' * 5 @gzipped_src = compress(@src) @zstded_src = compress(@src, type: :zstd) end test '#append with compress option writes compressed data to chunk when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip) c.append([@src, @src], compress: :gzip) c.commit # check chunk is compressed assert c.read(compressed: :gzip).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit comressed_data = c.open(compressed: :gzip) do |io| v = io.read assert_equal @gzipped_src, v v end assert_equal @gzipped_src, comressed_data end test '#write_to writes decompressed data when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is gzip' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :gzip) assert_equal @gzipped_src, io.string end test '#append with compress option writes compressed data to chunk when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd) c.append([@src, @src], compress: :zstd) c.commit # check chunk is compressed assert c.read(compressed: :zstd).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit comressed_data = c.open(compressed: :zstd) do |io| v = io.read assert_equal @zstded_src, v v end assert_equal @zstded_src, comressed_data end test '#write_to writes decompressed data when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is zstd' do c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :zstd) assert_equal @zstded_src, io.string end end end ================================================ FILE: test/plugin/test_buffer_memory_chunk.rb ================================================ require_relative '../helper' require 'fluent/plugin/buffer/memory_chunk' require 'fluent/plugin/compressable' require 'json' class BufferMemoryChunkTest < Test::Unit::TestCase include Fluent::Plugin::Compressable setup do @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new) end test 'has blank chunk initially' do assert @c.empty? assert_equal '', @c.instance_eval{ @chunk } assert_equal 0, @c.instance_eval{ @chunk_bytes } assert_equal 0, @c.instance_eval{ @adding_bytes } assert_equal 0, @c.instance_eval{ @adding_size } end test 'can #append, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'can #concat, #commit and #read it' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 2, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) @c.commit content = @c.read ds = content.split("\n").select{|d| !d.empty? } assert_equal 4, ds.size assert_equal d1, JSON.parse(ds[0]) assert_equal d2, JSON.parse(ds[1]) assert_equal d3, JSON.parse(ds[2]) assert_equal d4, JSON.parse(ds[3]) end test 'has its contents in binary (ascii-8bit)' do data1 = "aaa bbb ccc".force_encoding('utf-8') @c.append([data1]) @c.commit assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.encoding } content = @c.read assert_equal Encoding::ASCII_8BIT, content.encoding end test 'has #bytesize and #size' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.commit assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size end test 'can #rollback to revert non-committed data' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size end test 'can #rollback to revert non-committed data from #concat' do assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"].join @c.concat(data, 2) assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size @c.rollback assert @c.empty? assert @c.empty? d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize assert_equal 2, @c.size first_bytesize = @c.bytesize d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2) assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize assert_equal 4, @c.size @c.rollback assert_equal first_bytesize, @c.bytesize assert_equal 2, @c.size end test 'does nothing for #close' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit content = @c.read @c.close assert_equal content, @c.read end test 'deletes all data by #purge' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit @c.purge assert @c.empty? assert_equal 0, @c.bytesize assert_equal 0, @c.size assert_equal '', @c.read end test 'can #open its contents as io' do d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} data = [d1.to_json + "\n", d2.to_json + "\n"] @c.append(data) @c.commit d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} @c.append([d3.to_json + "\n", d4.to_json + "\n"]) @c.commit lines = [] @c.open do |io| assert io io.readlines.each do |l| lines << l end end assert_equal d1.to_json + "\n", lines[0] assert_equal d2.to_json + "\n", lines[1] assert_equal d3.to_json + "\n", lines[2] assert_equal d4.to_json + "\n", lines[3] end sub_test_case 'compressed buffer' do setup do @src = 'text data for compressing' * 5 @gzipped_src = compress(@src) @zstded_src = compress(@src, type: :zstd) end test '#append with compress option writes compressed data to chunk when compress is gzip' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip) c.append([@src, @src], compress: :gzip) c.commit # check chunk is compressed assert c.read(compressed: :gzip).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is gzip' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit comressed_data = c.open(compressed: :gzip) do |io| v = io.read assert_equal @gzipped_src, v v end assert_equal @gzipped_src, comressed_data end test '#write_to writes decompressed data when compress is gzip' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is gzip' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip) c.concat(@gzipped_src, @src.size) c.commit assert_equal @src, c.read assert_equal @gzipped_src, c.read(compressed: :gzip) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :gzip) assert_equal @gzipped_src, io.string end test '#append with compress option writes compressed data to chunk when compress is zstd' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd) c.append([@src, @src], compress: :zstd) c.commit # check chunk is compressed assert c.read(compressed: :zstd).size < [@src, @src].join("").size assert_equal @src + @src, c.read end test '#open passes io object having decompressed data to a block when compress is zstd' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit decomressed_data = c.open do |io| v = io.read assert_equal @src, v v end assert_equal @src, decomressed_data end test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit comressed_data = c.open(compressed: :zstd) do |io| v = io.read assert_equal @zstded_src, v v end assert_equal @zstded_src, comressed_data end test '#write_to writes decompressed data when compress is zstd' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new c.write_to(io) assert_equal @src, io.string end test '#write_to with compressed option writes compressed data when compress is zstd' do c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd) c.concat(@zstded_src, @src.size) c.commit assert_equal @src, c.read assert_equal @zstded_src, c.read(compressed: :zstd) io = StringIO.new io.set_encoding(Encoding::ASCII_8BIT) c.write_to(io, compressed: :zstd) assert_equal @zstded_src, io.string end end end ================================================ FILE: test/plugin/test_compressable.rb ================================================ require_relative '../helper' require 'fluent/plugin/compressable' class CompressableTest < Test::Unit::TestCase include Fluent::Plugin::Compressable def compress_assert_equal(expected, actual) e = Zlib::GzipReader.new(StringIO.new(expected)).read a = Zlib::GzipReader.new(StringIO.new(actual)).read assert_equal(e, a) end sub_test_case '#compress' do setup do @src = 'text data for compressing' * 5 @gzipped_src = compress(@src) end test 'compress data' do assert compress(@src).size < @src.size assert_not_equal @gzipped_src, @src end test 'write compressed data to IO with output_io option' do io = StringIO.new compress(@src, output_io: io) compress_assert_equal @gzipped_src, io.string end end sub_test_case '#decompress' do setup do @src = 'text data for compressing' * 5 @gzipped_src = compress(@src) end test 'decompress compressed data' do assert_equal @src, decompress(@gzipped_src) end test 'write decompressed data to IO with output_io option' do io = StringIO.new decompress(@gzipped_src, output_io: io) assert_equal @src, io.string end test 'return decompressed string with output_io option' do io = StringIO.new(@gzipped_src) assert_equal @src, decompress(input_io: io) end test 'decompress multiple compressed data' do src1 = 'text data' src2 = 'text data2' gzipped_src = compress(src1) + compress(src2) assert_equal src1 + src2, decompress(gzipped_src) end test 'decompress with input_io and output_io' do input_io = StringIO.new(@gzipped_src) output_io = StringIO.new decompress(input_io: input_io, output_io: output_io) assert_equal @src, output_io.string end test 'decompress multiple compressed data with input_io and output_io' do src1 = 'text data' src2 = 'text data2' gzipped_src = compress(src1) + compress(src2) input_io = StringIO.new(gzipped_src) output_io = StringIO.new decompress(input_io: input_io, output_io: output_io) assert_equal src1 + src2, output_io.string end test 'return the received value as it is with empty string or nil' do assert_equal nil, decompress assert_equal nil, decompress(nil) assert_equal '', decompress('') assert_equal '', decompress('', output_io: StringIO.new) end test 'decompress large zstd compressed data' do src1 = SecureRandom.random_bytes(1024) src2 = SecureRandom.random_bytes(1024) src3 = SecureRandom.random_bytes(1024) zstd_compressed_data = compress(src1, type: :zstd) + compress(src2, type: :zstd) + compress(src3, type: :zstd) assert_equal src1 + src2 + src3, decompress(zstd_compressed_data, type: :zstd) end test 'decompress large zstd compressed data with input_io and output_io' do src1 = SecureRandom.random_bytes(1024) src2 = SecureRandom.random_bytes(1024) src3 = SecureRandom.random_bytes(1024) zstd_compressed_data = compress(src1, type: :zstd) + compress(src2, type: :zstd) + compress(src3, type: :zstd) input_io = StringIO.new(zstd_compressed_data) output_io = StringIO.new output_io.set_encoding(src1.encoding) decompress(input_io: input_io, output_io: output_io, type: :zstd) assert_equal src1 + src2 + src3, output_io.string end end end ================================================ FILE: test/plugin/test_file_util.rb ================================================ require_relative '../helper' require 'fluent/plugin/file_util' require 'fileutils' class FileUtilTest < Test::Unit::TestCase def setup FileUtils.rm_rf(TEST_DIR) FileUtils.mkdir_p(TEST_DIR) end TEST_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/file_util") sub_test_case 'writable?' do test 'file exists and writable' do FileUtils.touch("#{TEST_DIR}/test_file") assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_file") end test 'file exists and not writable' do FileUtils.touch("#{TEST_DIR}/test_file") File.chmod(0444, "#{TEST_DIR}/test_file") assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file") end test 'directory exists' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir") end test 'file does not exist and parent directory is writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file") end test 'file does not exist and parent directory is not writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") File.chmod(0444, "#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file") end test 'parent directory does not exist' do FileUtils.rm_rf("#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file") end test 'parent file (not directory) exists' do FileUtils.touch("#{TEST_DIR}/test_file") assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file/foo") end end sub_test_case 'writable_p?' do test 'file exists and writable' do FileUtils.touch("#{TEST_DIR}/test_file") assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file") end test 'file exists and not writable' do FileUtils.touch("#{TEST_DIR}/test_file") File.chmod(0444, "#{TEST_DIR}/test_file") assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file") end test 'directory exists' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir") end test 'parent directory exists and writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/test_file") end test 'parent directory exists and not writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") File.chmod(0555, "#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/test_file") end test 'parent of parent (of parent ...) directory exists and writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/foo/bar/baz") end test 'parent of parent (of parent ...) directory exists and not writable' do FileUtils.mkdir_p("#{TEST_DIR}/test_dir") File.chmod(0555, "#{TEST_DIR}/test_dir") assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/foo/bar/baz") end test 'parent of parent (of parent ...) file (not directory) exists' do FileUtils.touch("#{TEST_DIR}/test_file") assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file/foo/bar/baz") end end end ================================================ FILE: test/plugin/test_filter.rb ================================================ require_relative '../helper' require 'fluent/plugin/filter' require 'fluent/event' require 'flexmock/test_unit' module FluentPluginFilterTest class DummyPlugin < Fluent::Plugin::Filter end class NumDoublePlugin < Fluent::Plugin::Filter def filter(tag, time, record) r = record.dup r["num"] = r["num"].to_i * 2 r end end class IgnoreForNumPlugin < Fluent::Plugin::Filter def filter(tag, time, record) if record["num"].is_a? Numeric nil else record end end end class RaiseForNumPlugin < Fluent::Plugin::Filter def filter(tag, time, record) if record["num"].is_a? Numeric raise "Value of num is Number!" end record end end class NumDoublePluginWithTime < Fluent::Plugin::Filter def filter_with_time(tag, time, record) r = record.dup r["num"] = r["num"].to_i * 2 [time, r] end end class IgnoreForNumPluginWithTime < Fluent::Plugin::Filter def filter_with_time(tag, time, record) if record["num"].is_a? Numeric nil else [time, record] end end end class InvalidPlugin < Fluent::Plugin::Filter # Because of implementing `filter_with_time` and `filter` methods def filter_with_time(tag, time, record); end def filter(tag, time, record); end end end class FilterPluginTest < Test::Unit::TestCase DummyRouter = Struct.new(:emits) do def emit_error_event(tag, time, record, error) self.emits << [tag, time, record, error] end end setup do @p = nil end teardown do if @p @p.stop unless @p.stopped? @p.before_shutdown unless @p.before_shutdown? @p.shutdown unless @p.shutdown? @p.after_shutdown unless @p.after_shutdown? @p.close unless @p.closed? @p.terminate unless @p.terminated? end end sub_test_case 'for basic dummy plugin' do setup do Fluent::Test.setup end test 'plugin does not define #filter raises error' do assert_raise NotImplementedError do FluentPluginFilterTest::DummyPlugin.new end end end sub_test_case 'normal filter plugin' do setup do Fluent::Test.setup @p = FluentPluginFilterTest::NumDoublePlugin.new end test 'has healthy lifecycle' do assert !@p.configured? @p.configure(config_element) assert @p.configured? assert !@p.started? @p.start assert @p.start assert !@p.stopped? @p.stop assert @p.stopped? assert !@p.before_shutdown? @p.before_shutdown assert @p.before_shutdown? assert !@p.shutdown? @p.shutdown assert @p.shutdown? assert !@p.after_shutdown? @p.after_shutdown assert @p.after_shutdown? assert !@p.closed? @p.close assert @p.closed? assert !@p.terminated? @p.terminate assert @p.terminated? end test 'has plugin_id automatically generated' do assert @p.respond_to?(:plugin_id_configured?) assert @p.respond_to?(:plugin_id) @p.configure(config_element) assert !@p.plugin_id_configured? assert @p.plugin_id assert{ @p.plugin_id != 'mytest' } end test 'has plugin_id manually configured' do @p.configure(config_element('ROOT', '', {'@id' => 'mytest'})) assert @p.plugin_id_configured? assert_equal 'mytest', @p.plugin_id end test 'has plugin logger' do assert @p.respond_to?(:log) assert @p.log # default logger original_logger = @p.log @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'})) assert(@p.log.object_id != original_logger.object_id) assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level end test 'can load plugin helpers' do assert_nothing_raised do class FluentPluginFilterTest::DummyPlugin2 < Fluent::Plugin::Filter helpers :storage end end end test 'can use metrics plugins and fallback methods' do @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'})) %w[emit_size_metrics emit_records_metrics].each do |metric_name| assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics) end assert_equal 0, @p.emit_size assert_equal 0, @p.emit_records end test 'are available with multi worker configuration in default' do assert @p.multi_workers_ready? end test 'filters events correctly' do test_es = [ [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}], ] @p.configure(config_element) es = @p.filter_stream('testing', test_es) assert es.is_a? Fluent::EventStream ary = [] es.each do |time, r| ary << [time, r] end assert_equal 3, ary.size assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0] assert_equal "Hello filters!", ary[0][1]["message"] assert_equal 2, ary[0][1]["num"] assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0] assert_equal 4, ary[1][1]["num"] assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0] assert_equal 6, ary[2][1]["num"] end end sub_test_case 'filter plugin returns nil for some records' do setup do Fluent::Test.setup @p = FluentPluginFilterTest::IgnoreForNumPlugin.new end test 'filter_stream ignores records which #filter return nil' do test_es = [ [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Ignored, yay!"}], [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}], ] @p.configure(config_element) es = @p.filter_stream('testing', test_es) assert es.is_a? Fluent::EventStream ary = [] es.each do |time, r| ary << [time, r] end assert_equal 2, ary.size assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0] assert_equal "Hello filters!", ary[0][1]["message"] assert_equal "1", ary[0][1]["num"] assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0] assert_equal "3", ary[1][1]["num"] end end sub_test_case 'filter plugin raises error' do setup do Fluent::Test.setup @p = FluentPluginFilterTest::RaiseForNumPlugin.new end test 'has router and can emit events to error streams' do assert @p.has_router? @p.configure(config_element) assert @p.router @p.router = DummyRouter.new([]) test_es = [ [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Hello error router!"}], [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}], ] es = @p.filter_stream('testing', test_es) assert es.is_a? Fluent::EventStream ary = [] es.each do |time, r| ary << [time, r] end assert_equal 2, ary.size assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0] assert_equal "Hello filters!", ary[0][1]["message"] assert_equal "1", ary[0][1]["num"] assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0] assert_equal "3", ary[1][1]["num"] assert_equal 1, @p.router.emits.size error_emits = @p.router.emits assert_equal "testing", error_emits[0][0] assert_equal event_time('2016-04-19 13:01:03 -0700'), error_emits[0][1] assert_equal({"num" => 2, "message" => "Hello error router!"}, error_emits[0][2]) assert{ error_emits[0][3].is_a? RuntimeError } assert_equal "Value of num is Number!", error_emits[0][3].message end end sub_test_case 'filter plugins that is implemented `filter_with_time`' do setup do Fluent::Test.setup @p = FluentPluginFilterTest::NumDoublePluginWithTime.new end test 'filters events correctly' do test_es = [ [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}], ] es = @p.filter_stream('testing', test_es) assert es.is_a? Fluent::EventStream ary = [] es.each do |time, r| ary << [time, r] end assert_equal 3, ary.size assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0] assert_equal "Hello filters!", ary[0][1]["message"] assert_equal 2, ary[0][1]["num"] assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0] assert_equal 4, ary[1][1]["num"] assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0] assert_equal 6, ary[2][1]["num"] end end sub_test_case 'filter plugin that is implemented `filter_with_time` and returns nil for some records' do setup do Fluent::Test.setup @p = FluentPluginFilterTest::IgnoreForNumPluginWithTime.new end test 'filter_stream ignores records which #filter_with_time return nil' do test_es = [ [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}], [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Ignored, yay!"}], [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}], ] @p.configure(config_element) es = @p.filter_stream('testing', test_es) assert es.is_a? Fluent::EventStream ary = [] es.each do |time, r| ary << [time, r] end assert_equal 2, ary.size assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0] assert_equal "Hello filters!", ary[0][1]["message"] assert_equal "1", ary[0][1]["num"] assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0] assert_equal "3", ary[1][1]["num"] end end sub_test_case 'filter plugins that is implemented both `filter_with_time` and `filter`' do setup do Fluent::Test.setup end test 'raises DuplicatedImplementError' do assert_raise do FluentPluginFilterTest::InvalidPlugin.new end end end end ================================================ FILE: test/plugin/test_filter_grep.rb ================================================ require_relative '../helper' require 'fluent/plugin/filter_grep' require 'fluent/test/driver/filter' class GrepFilterTest < Test::Unit::TestCase include Fluent setup do Fluent::Test.setup @time = event_time end def create_driver(conf = '') Fluent::Test::Driver::Filter.new(Fluent::Plugin::GrepFilter).configure(conf) end sub_test_case 'configure' do test 'check default' do d = create_driver assert_empty(d.instance.regexps) assert_empty(d.instance.excludes) end test "regexpN can contain a space" do d = create_driver(%[regexp1 message foo]) d.instance._regexp_and_conditions.each { |value| assert_equal(Regexp.compile(/ foo/), value.pattern) } end test "excludeN can contain a space" do d = create_driver(%[exclude1 message foo]) d.instance._exclude_or_conditions.each { |value| assert_equal(Regexp.compile(/ foo/), value.pattern) } end sub_test_case "duplicate key" do test "flat" do conf = %[ regexp1 message test regexp2 message test2 ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "section" do conf = %[ key message pattern test key message pattern test2 ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "mix" do conf = %[ regexp1 message test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "and/regexp" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "and/regexp, and/regexp" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "regexp, and/regexp" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "and/exclude" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "and/exclude, and/exclude" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test "exclude, or/exclude" do conf = %[ key message pattern test key message pattern test ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end end sub_test_case "pattern with slashes" do test "start with character classes" do conf = %[ key message pattern /[a-z]test/ key message pattern /[A-Z]test/ ] d = create_driver(conf) assert_equal(/[a-z]test/, d.instance.regexps.first.pattern) assert_equal(/[A-Z]test/, d.instance.excludes.first.pattern) end end sub_test_case "and/or section" do test " section cannot include both and " do conf = %[ key message pattern /test/ key level pattern /debug/ ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end test " section cannot include both and " do conf = %[ key message pattern /test/ key level pattern /debug/ ] assert_raise(Fluent::ConfigError) do create_driver(conf) end end end end sub_test_case 'filter_stream' do def messages [ "2013/01/13T07:02:11.124202 INFO GET /ping", "2013/01/13T07:02:13.232645 WARN POST /auth", "2013/01/13T07:02:21.542145 WARN GET /favicon.ico", "2013/01/13T07:02:43.632145 WARN POST /login", ] end def filter(config, msgs) d = create_driver(config) d.run { msgs.each { |msg| d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg}) } } d.filtered_records end test 'empty config' do filtered_records = filter('', messages) assert_equal(4, filtered_records.size) end test 'regexpN' do filtered_records = filter('regexp1 message WARN', messages) assert_equal(3, filtered_records.size) assert_block('only WARN logs') do filtered_records.all? { |r| !r['message'].include?('INFO') } end end test 'excludeN' do filtered_records = filter('exclude1 message favicon', messages) assert_equal(3, filtered_records.size) assert_block('remove favicon logs') do filtered_records.all? { |r| !r['message'].include?('favicon') } end end test 'regexps' do conf = %[ key message pattern WARN ] filtered_records = filter(conf, messages) assert_equal(3, filtered_records.size) assert_block('only WARN logs') do filtered_records.all? { |r| !r['message'].include?('INFO') } end end test 'excludes' do conf = %[ key message pattern favicon ] filtered_records = filter(conf, messages) assert_equal(3, filtered_records.size) assert_block('remove favicon logs') do filtered_records.all? { |r| !r['message'].include?('favicon') } end end sub_test_case 'with invalid sequence' do def messages [ "\xff".force_encoding('UTF-8'), ] end test "don't raise an exception" do assert_nothing_raised { filter(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')]) } end end sub_test_case "and/or section" do def records [ { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" }, { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" }, { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" }, { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" }, ] end def filter(conf, records) d = create_driver(conf) d.run do records.each do |record| d.feed("filter.test", @time, record) end end d.filtered_records end test "basic and/regexp" do conf = %[ key level pattern ^INFO$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(0), filtered_records) end test "basic or/exclude" do conf = %[ key level pattern ^INFO$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(1, 3), filtered_records) end test "basic or/regexp" do conf = %[ key level pattern ^INFO$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(0, 2), filtered_records) end test "basic and/exclude" do conf = %[ key level pattern ^INFO$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(1, 2, 3), filtered_records) end sub_test_case "and/or combo" do def records [ { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" }, { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" }, { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" }, { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" }, { "time" => "2013/01/13T07:02:44.959307", "level" => "ERROR", "method" => "POST", "path" => "/login" }, { "time" => "2013/01/13T07:02:45.444992", "level" => "ERROR", "method" => "GET", "path" => "/ping" }, { "time" => "2013/01/13T07:02:51.247941", "level" => "WARN", "method" => "GET", "path" => "/info" }, { "time" => "2013/01/13T07:02:53.108366", "level" => "WARN", "method" => "POST", "path" => "/ban" }, ] end test "and/regexp, or/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(4), filtered_records) end test "and/regexp, and/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records) end test "or/regexp, and/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(0, 1, 3, 4, 5, 7), filtered_records) end test "or/regexp, or/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(4), filtered_records) end test "regexp, and/regexp" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key path pattern ^/login$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(3, 4), filtered_records) end test "regexp, or/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(4), filtered_records) end test "regexp, and/exclude" do conf = %[ key level pattern ^ERROR|WARN$ key method pattern ^GET|POST$ key level pattern ^WARN$ key method pattern ^GET$ ] filtered_records = filter(conf, records) assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records) end end end end sub_test_case 'nested keys' do def messages [ {"nest1" => {"nest2" => "INFO"}}, {"nest1" => {"nest2" => "WARN"}}, {"nest1" => {"nest2" => "WARN"}} ] end def filter(config, msgs) d = create_driver(config) d.run { msgs.each { |msg| d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg}) } } d.filtered_records end test 'regexps' do conf = %[ key $.message.nest1.nest2 pattern WARN ] filtered_records = filter(conf, messages) assert_equal(2, filtered_records.size) assert_block('only 2 nested logs') do filtered_records.all? { |r| r['message']['nest1']['nest2'] == 'WARN' } end end test 'excludes' do conf = %[ key $.message.nest1.nest2 pattern WARN ] filtered_records = filter(conf, messages) assert_equal(1, filtered_records.size) assert_block('only 2 nested logs') do filtered_records.all? { |r| r['message']['nest1']['nest2'] == 'INFO' } end end end sub_test_case 'grep non-string jsonable values' do def filter(msg, config = 'regexp1 message 0') d = create_driver(config) d.run do d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg}) end d.filtered_records end data( 'array' => ["0"], 'hash' => ["0" => "0"], 'integer' => 0, 'float' => 0.1) test "value" do |data| filtered_records = filter(data) assert_equal(1, filtered_records.size) end test "value boolean" do filtered_records = filter(true, %[regexp1 message true]) assert_equal(1, filtered_records.size) end end end ================================================ FILE: test/plugin/test_filter_parser.rb ================================================ require_relative '../helper' require 'timecop' require 'fluent/test/driver/filter' require 'fluent/plugin/filter_parser' class ParserFilterTest < Test::Unit::TestCase def setup Fluent::Test.setup @tag = 'test' @default_time = Time.parse('2010-05-04 03:02:01 UTC') Timecop.freeze(@default_time) end def teardown super Timecop.return end def assert_equal_parsed_time(expected, actual) if expected.is_a?(Integer) assert_equal(expected, actual.to_i) else assert_equal_event_time(expected, actual) end end ParserError = Fluent::Plugin::Parser::ParserError CONFIG = %[ key_name message reserve_data true @type regexp expression /^(?.)(?.) (? ] def create_driver(conf=CONFIG) Fluent::Test::Driver::Filter.new(Fluent::Plugin::ParserFilter).configure(conf) end def test_configure assert_raise(Fluent::ConfigError) { create_driver('') } assert_raise(Fluent::NotFoundPluginError) { create_driver %[ key_name foo @type unknown_format_that_will_never_be_implemented ] } assert_nothing_raised { create_driver %[ key_name foo @type regexp expression /(?.)/ ] } assert_nothing_raised { create_driver %[ key_name foo @type json ] } assert_nothing_raised { create_driver %[ key_name foo format json ] } assert_nothing_raised { create_driver %[ key_name foo @type ltsv ] } assert_nothing_raised { create_driver %[ key_name message @type regexp expression /^col1=(?.+) col2=(?.+)$/ ] } d = create_driver %[ key_name foo @type regexp expression /(?.)/ ] assert_false d.instance.reserve_data end # CONFIG = %[ # remove_prefix test # add_prefix parsed # key_name message # format /^(?.)(?.) (?