• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

fluent / fluent-plugin-opensearch / 14873730023

07 May 2025 02:22AM UTC coverage: 91.339% (-0.006%) from 91.345%
14873730023

Pull #156

github

web-flow
Merge 1576e49ae into 4b6ff1a28
Pull Request #156: Fix memory usage

1160 of 1270 relevant lines covered (91.34%)

155.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.21
/lib/fluent/plugin/out_opensearch.rb
1
# SPDX-License-Identifier: Apache-2.0
2
#
3
# The fluent-plugin-opensearch Contributors require contributions made to
4
# this file be licensed under the Apache-2.0 license or a
5
# compatible open source license.
6
#
7
# Modifications Copyright fluent-plugin-opensearch Contributors. See
8
# GitHub history for details.
9
#
10
# Licensed to Uken Inc. under one or more contributor
11
# license agreements. See the NOTICE file distributed with
12
# this work for additional information regarding copyright
13
# ownership. Uken Inc. licenses this file to you under
14
# the Apache License, Version 2.0 (the "License"); you may
15
# not use this file except in compliance with the License.
16
# You may obtain a copy of the License at
17
#
18
#   http://www.apache.org/licenses/LICENSE-2.0
19
#
20
# Unless required by applicable law or agreed to in writing,
21
# software distributed under the License is distributed on an
22
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23
# KIND, either express or implied.  See the License for the
24
# specific language governing permissions and limitations
25
# under the License.
26

27
require 'date'
1✔
28
require 'excon'
1✔
29
require 'opensearch'
1✔
30
require 'set'
1✔
31
require 'json'
1✔
32
require 'uri'
1✔
33
require 'base64'
1✔
34
begin
35
  require 'strptime'
1✔
36
rescue LoadError
37
end
38
require 'resolv'
1✔
39

40
require 'fluent/plugin/output'
1✔
41
require 'fluent/event'
1✔
42
require 'fluent/error'
1✔
43
require 'fluent/time'
1✔
44
require 'fluent/unique_id'
1✔
45
require 'fluent/log-ext'
1✔
46
require 'zlib'
1✔
47
require_relative 'opensearch_constants'
1✔
48
require_relative 'opensearch_error'
1✔
49
require_relative 'opensearch_error_handler'
1✔
50
require_relative 'opensearch_index_template'
1✔
51
require_relative 'opensearch_tls'
1✔
52
require_relative 'opensearch_fallback_selector'
1✔
53
begin
54
  require_relative 'oj_serializer'
1✔
55
rescue LoadError
56
end
57
require 'aws-sdk-core'
1✔
58
require 'faraday_middleware/aws_sigv4'
1✔
59
require 'faraday/excon'
1✔
60

61
module Fluent::Plugin
1✔
62
  class OpenSearchOutput < Output
1✔
63
    class RecoverableRequestFailure < StandardError; end
1✔
64
    class UnrecoverableRequestFailure < Fluent::UnrecoverableError; end
1✔
65
    class RetryStreamEmitFailure < StandardError; end
1✔
66

67
    # MissingIdFieldError is raised for records that do not
68
    # include the field for the unique record identifier
69
    class MissingIdFieldError < StandardError; end
1✔
70

71
    # RetryStreamError privides a stream to be
72
    # put back in the pipeline for cases where a bulk request
73
    # failed (e.g some records succeed while others failed)
74
    class RetryStreamError < StandardError
1✔
75
      attr_reader :retry_stream
1✔
76
      def initialize(retry_stream)
1✔
77
        @retry_stream = retry_stream
8✔
78
      end
79
    end
80

81
    RequestInfo = Struct.new(:host, :index, :target_index, :alias)
1✔
82

83
    attr_reader :template_names
1✔
84
    attr_reader :ssl_version_options
1✔
85
    attr_reader :compressable_connection
1✔
86

87
    helpers :event_emitter, :compat_parameters, :record_accessor, :timer
1✔
88

89
    Fluent::Plugin.register_output('opensearch', self)
1✔
90

91
    DEFAULT_BUFFER_TYPE = "memory"
1✔
92
    DEFAULT_OPENSEARCH_VERSION = 1
1✔
93
    DEFAULT_TYPE_NAME = "_doc".freeze
1✔
94
    DEFAULT_RELOAD_AFTER = -1
1✔
95
    DEFAULT_TARGET_BULK_BYTES = -1
1✔
96
    DEFAULT_POLICY_ID = "logstash-policy"
1✔
97

98
    config_param :host, :string,  :default => 'localhost'
1✔
99
    config_param :port, :integer, :default => 9200
1✔
100
    config_param :user, :string, :default => nil
1✔
101
    config_param :password, :string, :default => nil, :secret => true
1✔
102
    config_param :path, :string, :default => nil
1✔
103
    config_param :scheme, :enum, :list => [:https, :http], :default => :http
1✔
104
    config_param :hosts, :string, :default => nil
1✔
105
    config_param :target_index_key, :string, :default => nil
1✔
106
    config_param :time_key_format, :string, :default => nil
1✔
107
    config_param :time_precision, :integer, :default => 9
1✔
108
    config_param :include_timestamp, :bool, :default => false
1✔
109
    config_param :logstash_format, :bool, :default => false
1✔
110
    config_param :logstash_prefix, :string, :default => "logstash"
1✔
111
    config_param :logstash_prefix_separator, :string, :default => '-'
1✔
112
    config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
1✔
113
    config_param :utc_index, :bool, :default => true
1✔
114
    config_param :suppress_type_name, :bool, :default => false
1✔
115
    config_param :index_name, :string, :default => "fluentd"
1✔
116
    config_param :id_key, :string, :default => nil
1✔
117
    config_param :write_operation, :string, :default => "index"
1✔
118
    config_param :parent_key, :string, :default => nil
1✔
119
    config_param :routing_key, :string, :default => nil
1✔
120
    config_param :request_timeout, :time, :default => 5
1✔
121
    config_param :reload_connections, :bool, :default => true
1✔
122
    config_param :reload_on_failure, :bool, :default => false
1✔
123
    config_param :retry_tag, :string, :default=>nil
1✔
124
    config_param :resurrect_after, :time, :default => 60
1✔
125
    config_param :time_key, :string, :default => nil
1✔
126
    config_param :time_key_exclude_timestamp, :bool, :default => false
1✔
127
    config_param :ssl_verify , :bool, :default => true
1✔
128
    config_param :client_key, :string, :default => nil
1✔
129
    config_param :client_cert, :string, :default => nil
1✔
130
    config_param :client_key_pass, :string, :default => nil, :secret => true
1✔
131
    config_param :ca_file, :string, :default => nil
1✔
132
    config_param :remove_keys, :string, :default => nil
1✔
133
    config_param :remove_keys_on_update, :string, :default => ""
1✔
134
    config_param :remove_keys_on_update_key, :string, :default => nil
1✔
135
    config_param :flatten_hashes, :bool, :default => false
1✔
136
    config_param :flatten_hashes_separator, :string, :default => "_"
1✔
137
    config_param :template_name, :string, :default => nil
1✔
138
    config_param :template_file, :string, :default => nil
1✔
139
    config_param :template_overwrite, :bool, :default => false
1✔
140
    config_param :customize_template, :hash, :default => nil
1✔
141
    config_param :index_date_pattern, :string, :default => "now/d"
1✔
142
    config_param :index_separator, :string, :default => "-"
1✔
143
    config_param :application_name, :string, :default => "default"
1✔
144
    config_param :templates, :hash, :default => nil
1✔
145
    config_param :max_retry_putting_template, :integer, :default => 10
1✔
146
    config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
1✔
147
    config_param :fail_on_detecting_os_version_retry_exceed, :bool, :default => true
1✔
148
    config_param :max_retry_get_os_version, :integer, :default => 15
1✔
149
    config_param :include_tag_key, :bool, :default => false
1✔
150
    config_param :tag_key, :string, :default => 'tag'
1✔
151
    config_param :time_parse_error_tag, :string, :default => 'opensearch_plugin.output.time.error'
1✔
152
    config_param :reconnect_on_error, :bool, :default => false
1✔
153
    config_param :pipeline, :string, :default => nil
1✔
154
    config_param :with_transporter_log, :bool, :default => false
1✔
155
    config_param :emit_error_for_missing_id, :bool, :default => false
1✔
156
    config_param :sniffer_class_name, :string, :default => nil
1✔
157
    config_param :selector_class_name, :string, :default => nil
1✔
158
    config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
1✔
159
    config_param :include_index_in_url, :bool, :default => false
1✔
160
    config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
1✔
161
    config_param :http_backend_excon_nonblock, :bool, :default => true
1✔
162
    config_param :validate_client_version, :bool, :default => false
1✔
163
    config_param :prefer_oj_serializer, :bool, :default => false
1✔
164
    config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "rejected_execution_exception"]
1✔
165
    config_param :unrecoverable_record_types, :array, :default => ["json_parse_exception"]
1✔
166
    config_param :emit_error_label_event, :bool, :default => true
1✔
167
    config_param :verify_os_version_at_startup, :bool, :default => true
1✔
168
    config_param :default_opensearch_version, :integer, :default => DEFAULT_OPENSEARCH_VERSION
1✔
169
    config_param :log_os_400_reason, :bool, :default => false
1✔
170
    config_param :custom_headers, :hash, :default => {}
1✔
171
    config_param :suppress_doc_wrap, :bool, :default => false
1✔
172
    config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
1✔
173
    config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
1✔
174
    config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
1✔
175
    config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
1✔
176
    config_param :truncate_caches_interval, :time, :default => nil
1✔
177
    config_param :use_legacy_template, :bool, :default => true
1✔
178
    config_param :catch_transport_exception_on_retry, :bool, :default => true
1✔
179
    config_param :target_index_affinity, :bool, :default => false
1✔
180

181
    config_section :metadata, param_name: :metainfo, multi: false do
1✔
182
      config_param :include_chunk_id, :bool, :default => false
1✔
183
      config_param :chunk_id_key, :string, :default => "chunk_id".freeze
1✔
184
    end
185

186
    config_section :endpoint, multi: false do
1✔
187
      config_param :region, :string
1✔
188
      config_param :url do |c|
1✔
189
        c.chomp("/")
4✔
190
      end
191
      config_param :access_key_id, :string, :default => ""
1✔
192
      config_param :secret_access_key, :string, :default => "", secret: true
1✔
193
      config_param :assume_role_arn, :string, :default => nil
1✔
194
      config_param :ecs_container_credentials_relative_uri, :string, :default => nil #Set with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable value
1✔
195
      config_param :assume_role_session_name, :string, :default => "fluentd"
1✔
196
      config_param :assume_role_web_identity_token_file, :string, :default => nil
1✔
197
      config_param :sts_credentials_region, :string, :default => nil
1✔
198
      config_param :refresh_credentials_interval, :time, :default => "5h"
1✔
199
      config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
1✔
200
    end
201

202
    config_section :buffer do
1✔
203
      config_set_default :@type, DEFAULT_BUFFER_TYPE
1✔
204
      config_set_default :chunk_keys, ['tag']
1✔
205
      config_set_default :timekey_use_utc, true
1✔
206
    end
207

208
    include Fluent::OpenSearchIndexTemplate
1✔
209
    include Fluent::Plugin::OpenSearchConstants
1✔
210
    include Fluent::Plugin::OpenSearchTLS
1✔
211

212
    def initialize
1✔
213
      super
635✔
214
    end
215

216
    ######################################################################################################
217
    # This creating AWS credentials code part is heavily based on fluent-plugin-aws-elasticsearch-service:
218
    # https://github.com/atomita/fluent-plugin-aws-elasticsearch-service/blob/master/lib/fluent/plugin/out_aws-elasticsearch-service.rb#L73-L134
219
    ######################################################################################################
220
    def aws_credentials(conf)
1✔
221
      credentials = nil
3✔
222
      unless conf[:access_key_id].empty? || conf[:secret_access_key].empty?
3✔
223
        credentials = Aws::Credentials.new(conf[:access_key_id], conf[:secret_access_key])
3✔
224
      else
225
        if conf[:assume_role_arn].nil?
×
226
          aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
×
227
          if aws_container_credentials_relative_uri.nil?
×
228
            credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
×
229
            credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
×
230
            credentials ||= Aws::ECSCredentials.new.credentials
×
231
          else
232
            credentials = Aws::ECSCredentials.new({
×
233
                            credential_path: aws_container_credentials_relative_uri
234
                          }).credentials
235
          end
236
        else
237
          if conf[:assume_role_web_identity_token_file].nil?
×
238
            credentials = Aws::AssumeRoleCredentials.new({
×
239
                            role_arn: conf[:assume_role_arn],
240
                            role_session_name: conf[:assume_role_session_name],
241
                            region: sts_creds_region(conf)
242
                          }).credentials
243
          else
244
            credentials = Aws::AssumeRoleWebIdentityCredentials.new({
×
245
                            role_arn: conf[:assume_role_arn],
246
                            web_identity_token_file: conf[:assume_role_web_identity_token_file],
247
                            region: sts_creds_region(conf)
248
                          }).credentials
249
          end
250
        end
251
      end
252
      raise "No valid AWS credentials found." unless credentials.set?
3✔
253

254
      credentials
3✔
255
    end
256

257
    def sts_creds_region(conf)
1✔
258
      conf[:sts_credentials_region] || conf[:region]
×
259
    end
260
    ###############################
261
    # AWS credential part is ended.
262
    ###############################
263

264
    def configure(conf)
1✔
265
      compat_parameters_convert(conf, :buffer)
426✔
266

267
      super
426✔
268

269
      if @endpoint
425✔
270
        # here overrides default value of reload_connections to false because
271
        # AWS Elasticsearch Service doesn't return addresses of nodes and Elasticsearch client
272
        # fails to reload connections properly. This ends up "temporarily failed to flush the buffer"
273
        # error repeating forever. See this discussion for details:
274
        # https://discuss.elastic.co/t/elasitcsearch-ruby-raises-cannot-get-new-connection-from-pool-error/36252
275
        @reload_connections = false
4✔
276
      end
277

278
      if placeholder_substitution_needed_for_template?
425✔
279
        # nop.
280
      elsif not @buffer_config.chunk_keys.include? "tag" and
376✔
281
        not @buffer_config.chunk_keys.include? "_index"
282
        raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required."
2✔
283
      end
284
      @time_parser = create_time_parser
423✔
285
      @backend_options = backend_options
423✔
286
      @ssl_version_options = set_tls_minmax_version_config(@ssl_version, @ssl_max_version, @ssl_min_version)
423✔
287

288
      if @remove_keys
423✔
289
        @remove_keys = @remove_keys.split(/\s*,\s*/)
5✔
290
      end
291

292
      if @target_index_key && @target_index_key.is_a?(String)
423✔
293
        @target_index_key = @target_index_key.split '.'
6✔
294
      end
295

296
      if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
423✔
297
        @remove_keys_on_update = @remove_keys_on_update.split ','
423✔
298
      end
299

300
      raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
423✔
301
      raise Fluent::ConfigError, "'max_retry_get_os_version' must be greater than or equal to zero." if @max_retry_get_os_version < 0
422✔
302

303
      # Dump log when using host placeholders and template features at same time.
304
      valid_host_placeholder = placeholder?(:host_placeholder, @host)
421✔
305
      if valid_host_placeholder && (@template_name && @template_file || @templates)
421✔
306
        if @verify_os_version_at_startup
5✔
307
          raise Fluent::ConfigError, "host placeholder, template installation, and verify OpenSearch version at startup are exclusive feature at same time. Please specify verify_os_version_at_startup as `false` when host placeholder and template installation are enabled."
1✔
308
        end
309
        log.info "host placeholder and template installation makes your OpenSearch cluster a bit slow down(beta)."
4✔
310
      end
311

312
      @template_names = []
420✔
313
      if !dry_run?
420✔
314
        if @template_name && @template_file
420✔
315
          if @logstash_format || placeholder_substitution_needed_for_template?
65✔
316
            class << self
6✔
317
              alias_method :template_installation, :template_installation_actual
6✔
318
            end
319
          else
320
            template_installation_actual(@template_name, @customize_template, @application_name, @index_name)
59✔
321
          end
322
        end
323
        if @templates
375✔
324
          retry_operate(@max_retry_putting_template,
8✔
325
                        @fail_on_putting_template_retry_exceed,
326
                        @catch_transport_exception_on_retry) do
327
            templates_hash_install(@templates, @template_overwrite)
8✔
328
          end
329
        end
330
      end
331

332
      @truncate_mutex = Mutex.new
373✔
333
      if @truncate_caches_interval
373✔
334
        timer_execute(:out_opensearch_truncate_caches, @truncate_caches_interval) do
×
335
          log.info('Clean up the indices and template names cache')
×
336

337
          @truncate_mutex.synchronize {
×
338
            @template_names.clear
×
339
          }
340
        end
341
      end
342
      # If AWS credentials is set, consider to expire credentials information forcibly before expired.
343
      @credential_mutex = Mutex.new
373✔
344
      if @endpoint
373✔
345
        @_aws_credentials = aws_credentials(@endpoint)
4✔
346

347
        if @endpoint.refresh_credentials_interval
4✔
348
          timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
4✔
349
            log.debug('Recreate the AWS credentials')
1✔
350

351
            @credential_mutex.synchronize do
1✔
352
              @_os = nil
1✔
353
              begin
354
                @_aws_credentials = aws_credentials(@endpoint)
1✔
355
              rescue => e
356
                log.error("Failed to get new AWS credentials: #{e}")
1✔
357
              end
358
            end
359
          end
360
        end
361
      end
362

363
      @serializer_class = nil
373✔
364
      begin
365
        require 'oj'
373✔
366
        @dump_proc = Oj.method(:dump)
373✔
367
        if @prefer_oj_serializer
373✔
368
          @serializer_class = Fluent::Plugin::Serializer::Oj
×
369
          OpenSearch::API.settings[:serializer] = Fluent::Plugin::Serializer::Oj
×
370
        end
371
      rescue LoadError
372
        @dump_proc = Yajl.method(:dump)
×
373
      end
374

375
      raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
373✔
376

377
      if @user && m = @user.match(/%{(?<user>.*)}/)
372✔
378
        @user = URI.encode_www_form_component(m["user"])
2✔
379
      end
380
      if @password && m = @password.match(/%{(?<password>.*)}/)
372✔
381
        @password = URI.encode_www_form_component(m["password"])
2✔
382
      end
383

384
      @transport_logger = nil
372✔
385
      if @with_transporter_log
372✔
386
        @transport_logger = log
2✔
387
        log_level = conf['@log_level'] || conf['log_level']
2✔
388
        log.warn "Consider to specify log_level with @log_level." unless log_level
2✔
389
      end
390
      # Specify @sniffer_class before calling #client.
391
      # #detect_os_major_version uses #client.
392
      @sniffer_class = nil
372✔
393
      begin
394
        @sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
372✔
395
      rescue Exception => ex
396
        raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
×
397
      end
398

399
      @selector_class = nil
372✔
400
      begin
401
        @selector_class = Object.const_get(@selector_class_name) if @selector_class_name
372✔
402
      rescue Exception => ex
403
        raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
×
404
      end
405

406
      @last_seen_major_version = if major_version = handle_last_seen_os_major_version
372✔
407
                                   major_version
364✔
408
                                 else
409
                                   @default_opensearch_version
5✔
410
                                 end
411

412
      if @validate_client_version && !dry_run?
369✔
413
        if @last_seen_major_version != client_library_version.to_i
2✔
414
          raise Fluent::ConfigError, <<-EOC
1✔
415
            Detected OpenSearch #{@last_seen_major_version} but you use OpenSearch client #{client_library_version}.
416
            Please consider to use #{@last_seen_major_version}.x series OpenSearch client.
417
          EOC
418
        end
419
      end
420

421
      if @last_seen_major_version >= 1
368✔
422
        case @ssl_version
368✔
423
        when :SSLv23, :TLSv1, :TLSv1_1
424
          if @scheme == :https
1✔
425
            log.warn "Detected OpenSearch 1.x or above and enabled insecure security:
1✔
426
                      You might have to specify `ssl_version TLSv1_2` in configuration."
427
          end
428
        end
429
      end
430

431
      if @ssl_version && @scheme == :https
368✔
432
        if !@http_backend_excon_nonblock
33✔
433
          log.warn "TLS handshake will be stucked with block connection.
1✔
434
                    Consider to set `http_backend_excon_nonblock` as true"
435
        end
436
      end
437

438
      # Consider missing the prefix of "$." in nested key specifiers.
439
      @id_key = convert_compat_id_key(@id_key) if @id_key
368✔
440
      @parent_key = convert_compat_id_key(@parent_key) if @parent_key
368✔
441
      @routing_key = convert_compat_id_key(@routing_key) if @routing_key
368✔
442

443
      @routing_key_name = configure_routing_key_name
368✔
444
      @meta_config_map = create_meta_config_map
368✔
445
      @current_config = nil
368✔
446
      @compressable_connection = false
368✔
447

448
      @ignore_exception_classes = @ignore_exceptions.map do |exception|
368✔
449
        unless Object.const_defined?(exception)
3✔
450
          log.warn "Cannot find class #{exception}. Will ignore it."
×
451

452
          nil
×
453
        else
454
          Object.const_get(exception)
3✔
455
        end
456
      end.compact
457

458
      if @bulk_message_request_threshold < 0
368✔
459
        class << self
367✔
460
          alias_method :split_request?, :split_request_size_uncheck?
367✔
461
        end
462
      else
463
        class << self
1✔
464
          alias_method :split_request?, :split_request_size_check?
1✔
465
        end
466
      end
467
    end
468

469
    def dry_run?
1✔
470
      if Fluent::Engine.respond_to?(:dry_run_mode)
811✔
471
        Fluent::Engine.dry_run_mode
×
472
      elsif Fluent::Engine.respond_to?(:supervisor_mode)
811✔
473
        Fluent::Engine.supervisor_mode
811✔
474
      end
475
    end
476

477
    def placeholder?(name, param)
1✔
478
      placeholder_validities = []
3,367✔
479
      placeholder_validators(name, param).each do |v|
3,367✔
480
        begin
481
          v.validate!
4,008✔
482
          placeholder_validities << true
98✔
483
        rescue Fluent::ConfigError => e
484
          log.debug("'#{name} #{param}' is tested built-in placeholder(s) but there is no valid placeholder(s). error: #{e}")
3,910✔
485
          placeholder_validities << false
3,910✔
486
        end
487
      end
488
      placeholder_validities.include?(true)
3,367✔
489
    end
490

491
    def emit_error_label_event?
1✔
492
      !!@emit_error_label_event
5✔
493
    end
494

495
    def compression
1✔
496
      !(@compression_level == :no_compression)
266✔
497
    end
498

499
    def compression_strategy
1✔
500
      case @compression_level
4✔
501
      when :default_compression
502
        Zlib::DEFAULT_COMPRESSION
1✔
503
      when :best_compression
504
        Zlib::BEST_COMPRESSION
2✔
505
      when :best_speed
506
        Zlib::BEST_SPEED
1✔
507
      else
508
        Zlib::NO_COMPRESSION
×
509
      end
510
    end
511

512
    def backend_options
1✔
513
      case @http_backend
423✔
514
      when :excon
515
        { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
423✔
516
      when :typhoeus
517
        require 'faraday/typhoeus'
×
518
        { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
×
519
      end
520
    rescue LoadError => ex
521
      log.error_backtrace(ex.backtrace)
×
522
      raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
×
523
    end
524

525
    def handle_last_seen_os_major_version
1✔
526
      if @verify_os_version_at_startup && !dry_run?
372✔
527
        retry_operate(@max_retry_get_os_version,
368✔
528
                      @fail_on_detecting_os_version_retry_exceed,
529
                      @catch_transport_exception_on_retry) do
530
          detect_os_major_version
373✔
531
        end
532
      else
533
        nil
534
      end
535
    end
536

537
    def detect_os_major_version
1✔
538
      @_os_info ||= client.info
2✔
539
      begin
540
        unless version = @_os_info.dig("version", "number")
1✔
541
          version = @default_opensearch_version
×
542
        end
543
      rescue NoMethodError => e
544
        log.warn "#{@_os_info} can not dig version information. Assuming OpenSearch #{@default_opensearch_version}", error: e
×
545
        version = @default_opensearch_version
×
546
      end
547
      version.to_i
1✔
548
    end
549

550
    def client_library_version
1✔
551
      OpenSearch::VERSION
×
552
    end
553

554
    def configure_routing_key_name
1✔
555
      'routing'.freeze
368✔
556
    end
557

558
    def convert_compat_id_key(key)
1✔
559
      if key.include?('.') && !key.start_with?('$[')
33✔
560
        key = "$.#{key}" unless key.start_with?('$.')
6✔
561
      end
562
      key
33✔
563
    end
564

565
    def create_meta_config_map
1✔
566
      result = []
368✔
567
      result << [record_accessor_create(@id_key), '_id'] if @id_key
368✔
568
      result << [record_accessor_create(@parent_key), '_parent'] if @parent_key
368✔
569
      result << [record_accessor_create(@routing_key), @routing_key_name] if @routing_key
368✔
570
      result
368✔
571
    end
572

573
    # once fluent v0.14 is released we might be able to use
574
    # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
575
    # [sec,nsec] where as we want something we can call `strftime` on...
576
    def create_time_parser
1✔
577
      if @time_key_format
423✔
578
        begin
579
          # Strptime doesn't support all formats, but for those it does it's
580
          # blazingly fast.
581
          strptime = Strptime.new(@time_key_format)
9✔
582
          Proc.new { |value|
8✔
583
            value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
8✔
584
            strptime.exec(value).to_datetime
8✔
585
          }
586
        rescue
587
          # Can happen if Strptime doesn't recognize the format; or
588
          # if strptime couldn't be required (because it's not installed -- it's
589
          # ruby 2 only)
590
          Proc.new { |value|
1✔
591
            value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
1✔
592
            DateTime.strptime(value, @time_key_format)
1✔
593
          }
594
        end
595
      else
596
        Proc.new { |value|
414✔
597
          value = convert_numeric_time_into_string(value) if value.is_a?(Numeric)
1,013✔
598
          DateTime.parse(value)
1,013✔
599
        }
600
      end
601
    end
602

603
    def convert_numeric_time_into_string(numeric_time, time_key_format = "%Y-%m-%d %H:%M:%S.%N%z")
1✔
604
      numeric_time_parser = Fluent::NumericTimeParser.new(:float)
2✔
605
      Time.at(numeric_time_parser.parse(numeric_time).to_r).strftime(time_key_format)
2✔
606
    end
607

608
    def parse_time(value, event_time, tag)
1✔
609
      @time_parser.call(value)
1,022✔
610
    rescue => e
611
      if emit_error_label_event?
2✔
612
        router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
2✔
613
      end
614
      return Time.at(event_time).to_datetime
2✔
615
    end
616

617
    def client(host = nil, compress_connection = false)
1✔
618
      # check here to see if we already have a client connection for the given host
619
      connection_options = get_connection_options(host)
416✔
620

621
      @_os = nil unless is_existing_connection(connection_options[:hosts])
416✔
622
      @_os = nil unless @compressable_connection == compress_connection
416✔
623

624
      @_os ||= begin
416✔
625
        @compressable_connection = compress_connection
220✔
626
        @current_config = connection_options[:hosts].clone
220✔
627
        adapter_conf = if @endpoint
220✔
628
                         lambda do |f|
×
629
                           f.request(
×
630
                             :aws_sigv4,
631
                             service: @endpoint.aws_service_name.to_s,
632
                             region: @endpoint.region,
633
                             credentials: @_aws_credentials,
634
                           )
635

636
                           f.adapter @http_backend, @backend_options
×
637
                         end
638
                       else
639
                         lambda {|f| f.adapter @http_backend, @backend_options }
451✔
640
                       end
641

642
        local_reload_connections = @reload_connections
220✔
643
        if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
220✔
644
          local_reload_connections = @reload_after
2✔
645
        end
646

647
        gzip_headers = if compress_connection
220✔
648
                         {'Content-Encoding' => 'gzip'}
3✔
649
                       else
650
                         {}
217✔
651
                       end
652
        headers = {}.merge(@custom_headers)
220✔
653
                    .merge(gzip_headers)
654
        ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
220✔
655

656
        transport = OpenSearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
220✔
657
                                                                            options: {
658
                                                                              reload_connections: local_reload_connections,
659
                                                                              reload_on_failure: @reload_on_failure,
660
                                                                              resurrect_after: @resurrect_after,
661
                                                                              logger: @transport_logger,
662
                                                                              transport_options: {
663
                                                                                headers: headers,
664
                                                                                request: { timeout: @request_timeout },
665
                                                                                ssl: ssl_options,
666
                                                                              },
667
                                                                              http: {
668
                                                                                user: @user,
669
                                                                                password: @password,
670
                                                                                scheme: @scheme
671
                                                                              },
672
                                                                              sniffer_class: @sniffer_class,
673
                                                                              serializer_class: @serializer_class,
674
                                                                              selector_class: @selector_class,
675
                                                                              compression: compress_connection,
676
                                                                            }), &adapter_conf)
677
        OpenSearch::Client.new transport: transport
220✔
678
      end
679
    end
680

681
    def get_escaped_userinfo(host_str)
1✔
682
      if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
23✔
683
        m["scheme"] +
4✔
684
          URI.encode_www_form_component(m["user"]) +
685
          ':' +
686
          URI.encode_www_form_component(m["password"]) +
687
          m["path"]
688
      else
689
        host_str
19✔
690
      end
691
    end
692

693
    def get_connection_options(con_host=nil)
1✔
694

695
      hosts = if @endpoint # For AWS OpenSearch Service
444✔
696
        uri = URI(@endpoint.url)
×
697
        host = %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
×
698
          hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
×
699
          hash
×
700
        end
701
        [host]
×
702
      elsif con_host || @hosts
444✔
703
        (con_host || @hosts).split(',').map do |host_str|
193✔
704
          # Support legacy hosts format host:port,host:port,host:port...
705
          if host_str.match(%r{^[^:]+(\:\d+)?$})
221✔
706
            {
707
              host:   host_str.split(':')[0],
198✔
708
              port:   (host_str.split(':')[1] || @port).to_i,
198✔
709
              scheme: @scheme.to_s
710
            }
711
          else
712
            # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
713
            uri = URI(get_escaped_userinfo(host_str))
23✔
714
            %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
22✔
715
              hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
66✔
716
              hash
66✔
717
            end
718
          end
719
        end.compact
720
      else
721
        if Resolv::IPv6::Regex.match(@host)
251✔
722
          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
4✔
723
        else
724
          [{host: @host, port: @port, scheme: @scheme.to_s}]
247✔
725
        end
726
      end.each do |host|
727
        host.merge!(user: @user, password: @password) if !host[:user] && @user
471✔
728
        host.merge!(path: @path) if !host[:path] && @path
471✔
729
      end
730

731
      {
732
        hosts: hosts
443✔
733
      }
734
    end
735

736
    def connection_options_description(con_host=nil)
1✔
737
      get_connection_options(con_host)[:hosts].map do |host_info|
6✔
738
        attributes = host_info.dup
6✔
739
        attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
6✔
740
        attributes.inspect
6✔
741
      end.join(', ')
742
    end
743

744
    # append_record_to_messages adds a record to the bulk message
745
    # payload to be submitted to OpenSearch.  Records that do
746
    # not include '_id' field are skipped when 'write_operation'
747
    # is configured for 'create' or 'update'
748
    #
749
    # returns 'true' if record was appended to the bulk message
750
    #         and 'false' otherwise
751
    def append_record_to_messages(op, meta, header, record, msgs)
1✔
752
      case op
2,164✔
753
      when UPDATE_OP, UPSERT_OP
754
        if meta.has_key?(ID_FIELD)
21✔
755
          header[UPDATE_OP] = meta
19✔
756
          msgs << @dump_proc.call(header) << BODY_DELIMITER
19✔
757
          msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
19✔
758
          return true
19✔
759
        end
760
      when CREATE_OP
761
        if meta.has_key?(ID_FIELD)
18✔
762
          header[CREATE_OP] = meta
9✔
763
          msgs << @dump_proc.call(header) << BODY_DELIMITER
9✔
764
          msgs << @dump_proc.call(record) << BODY_DELIMITER
9✔
765
          return true
9✔
766
        end
767
      when INDEX_OP
768
        header[INDEX_OP] = meta
2,125✔
769
        msgs << @dump_proc.call(header) << BODY_DELIMITER
2,125✔
770
        msgs << @dump_proc.call(record) << BODY_DELIMITER
2,125✔
771
        return true
2,125✔
772
      end
773
      return false
11✔
774
    end
775

776
    def update_body(record, op)
1✔
777
      update = remove_keys(record)
19✔
778
      if @suppress_doc_wrap
19✔
779
        return update
4✔
780
      end
781
      body = {"doc".freeze => update}
15✔
782
      if op == UPSERT_OP
15✔
783
        if update == record
7✔
784
          body["doc_as_upsert".freeze] = true
2✔
785
        else
786
          body[UPSERT_OP] = record
5✔
787
        end
788
      end
789
      body
15✔
790
    end
791

792
    def remove_keys(record)
1✔
793
      keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
19✔
794
      record.delete(@remove_keys_on_update_key)
19✔
795
      return record unless keys.any?
19✔
796
      record = record.dup
6✔
797
      keys.each { |key| record.delete(key) }
13✔
798
      record
6✔
799
    end
800

801
    def flatten_record(record, prefix=[])
1✔
802
      ret = {}
4✔
803
      if record.is_a? Hash
4✔
804
        record.each { |key, value|
2✔
805
          ret.merge! flatten_record(value, prefix + [key.to_s])
3✔
806
        }
807
      elsif record.is_a? Array
2✔
808
        # Don't mess with arrays, leave them unprocessed
809
        ret.merge!({prefix.join(@flatten_hashes_separator) => record})
1✔
810
      else
811
        return {prefix.join(@flatten_hashes_separator) => record}
1✔
812
      end
813
      ret
3✔
814
    end
815

816
    def expand_placeholders(chunk)
1✔
817
      logstash_prefix = extract_placeholders(@logstash_prefix, chunk)
134✔
818
      logstash_dateformat = extract_placeholders(@logstash_dateformat, chunk)
134✔
819
      index_name = extract_placeholders(@index_name, chunk)
134✔
820
      if @template_name
134✔
821
        template_name = extract_placeholders(@template_name, chunk)
6✔
822
      else
823
        template_name = nil
128✔
824
      end
825
      if @customize_template
134✔
826
        customize_template = @customize_template.each_with_object({}) { |(key, value), hash| hash[key] = extract_placeholders(value, chunk) }
12✔
827
      else
828
        customize_template = nil
130✔
829
      end
830
      if @application_name
134✔
831
        application_name = extract_placeholders(@application_name, chunk)
134✔
832
      else
833
        application_name = nil
×
834
      end
835
      if @pipeline
134✔
836
        pipeline = extract_placeholders(@pipeline, chunk)
4✔
837
      else
838
        pipeline = nil
130✔
839
      end
840
      return logstash_prefix, logstash_dateformat, index_name, template_name, customize_template, application_name, pipeline
134✔
841
    end
842

843
    def multi_workers_ready?
1✔
844
      true
×
845
    end
846

847
    def inject_chunk_id_to_record_if_needed(record, chunk_id)
1✔
848
      if @metainfo&.include_chunk_id
2,150✔
849
        record[@metainfo.chunk_id_key] = chunk_id
2✔
850
        record
2✔
851
      else
852
        record
2,148✔
853
      end
854
    end
855

856
    def write(chunk)
1✔
857
      bulk_message_count = Hash.new { |h,k| h[k] = 0 }
265✔
858
      bulk_message = Hash.new { |h,k| h[k] = '' }
268✔
859
      header = {}
134✔
860
      meta = {}
134✔
861

862
      tag = chunk.metadata.tag
134✔
863
      chunk_id = dump_unique_id_hex(chunk.unique_id)
134✔
864
      extracted_values = expand_placeholders(chunk)
134✔
865
      host = if @hosts
134✔
866
               extract_placeholders(@hosts, chunk)
2✔
867
             else
868
               extract_placeholders(@host, chunk)
132✔
869
             end
870

871
      affinity_target_indices = get_affinity_target_indices(chunk)
134✔
872
      chunk.msgpack_each do |time, record|
134✔
873
        next unless record.is_a? Hash
2,150✔
874

875
        record = inject_chunk_id_to_record_if_needed(record, chunk_id)
2,150✔
876

877
        begin
878
          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
2,150✔
879
          info = if @include_index_in_url
2,149✔
880
                   RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
1✔
881
                 else
882
                   RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
2,148✔
883
                 end
884

885
          if split_request?(bulk_message, info)
2,149✔
886
            bulk_message.each do |info, msgs|
1✔
887
              send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
1✔
888
            ensure
889
              msgs.clear
1✔
890
              # Clear bulk_message_count for this info.
891
              bulk_message_count[info] = 0;
1✔
892
            end
893
          end
894

895
          if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
2,149✔
896
            bulk_message_count[info] += 1;
2,142✔
897
          else
898
            if @emit_error_for_missing_id
7✔
899
              raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
2✔
900
            else
901
              log.on_debug { log.debug("Dropping record because its missing an '_id' field and write_operation is #{@write_operation}: #{record}") }
7✔
902
            end
903
          end
904
        rescue => e
905
          if emit_error_label_event?
3✔
906
            router.emit_error_event(tag, time, record, e)
3✔
907
          end
908
        end
909
      end
910

911
      bulk_message.each do |info, msgs|
134✔
912
        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
134✔
913
      ensure
914
        msgs.clear
134✔
915
      end
916
    end
917

918
    def target_index_affinity_enabled?()
1✔
919
      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
139✔
920
    end
921

922
    def get_affinity_target_indices(chunk)
1✔
923
      indices = Hash.new
139✔
924
      if target_index_affinity_enabled?()
139✔
925
        id_key_accessor = record_accessor_create(@id_key)
5✔
926
        ids = Set.new
5✔
927
        chunk.msgpack_each do |time, record|
5✔
928
          next unless record.is_a? Hash
7✔
929
          begin
930
            ids << id_key_accessor.call(record)
7✔
931
          end
932
        end
933
        log.debug("Find affinity target_indices by quering on OpenSearch (write_operation #{@write_operation}) for ids: #{ids.to_a}")
5✔
934
        options = {
935
          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
5✔
936
        }
937
        query = {
938
          'query' => { 'ids' => { 'values' => ids.to_a } },
5✔
939
          '_source' => false,
940
          'sort' => [
941
            {"_index" => {"order" => "desc"}}
942
         ]
943
        }
944
        result = client.search(options.merge(:body => Yajl.dump(query)))
5✔
945
        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
946
        result['hits']['hits'].each do |hit|
5✔
947
          indices[hit["_id"]] = hit["_index"]
8✔
948
          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
8✔
949
        end
950
      end
951
      indices
139✔
952
    end
953

954
    def split_request?(bulk_message, info)
1✔
955
      # For safety.
956
    end
957

958
    def split_request_size_check?(bulk_message, info)
1✔
959
      bulk_message[info].size > @bulk_message_request_threshold
2✔
960
    end
961

962
    def split_request_size_uncheck?(bulk_message, info)
1✔
963
      false
2,148✔
964
    end
965

966
    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
1✔
967
      logstash_prefix, logstash_dateformat, index_name, _template_name, _customize_template, application_name, pipeline = extracted_values
2,165✔
968

969
      if @flatten_hashes
2,165✔
970
        record = flatten_record(record)
1✔
971
      end
972

973
      dt = nil
2,165✔
974
      if @logstash_format || @include_timestamp
2,165✔
975
        if record.has_key?(TIMESTAMP_FIELD)
39✔
976
          rts = record[TIMESTAMP_FIELD]
7✔
977
          dt = parse_time(rts, time, tag)
7✔
978
        elsif record.has_key?(@time_key)
32✔
979
          rts = record[@time_key]
6✔
980
          dt = parse_time(rts, time, tag)
6✔
981
          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision) unless @time_key_exclude_timestamp
6✔
982
        else
983
          dt = Time.at(time).to_datetime
26✔
984
          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
26✔
985
        end
986
      end
987

988
      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
2,165✔
989
      if target_index_parent && target_index_parent[target_index_child_key]
2,165✔
990
        target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
4✔
991
      elsif @logstash_format
2,161✔
992
        dt = dt.new_offset(0) if @utc_index
33✔
993
        target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
33✔
994
        target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
33✔
995
      else
996
        target_index_alias = target_index = index_name
2,128✔
997
      end
998

999
      # Change target_index to lower-case since OpenSearch doesn't
1000
      # allow upper-case characters in index names.
1001
      target_index = target_index.downcase
2,165✔
1002
      target_index_alias = target_index_alias.downcase
2,164✔
1003
      if @include_tag_key
2,164✔
1004
        record[@tag_key] = tag
1✔
1005
      end
1006

1007
      # If affinity target indices map has value for this particular id, use it as target_index
1008
      if !affinity_target_indices.empty?
2,164✔
1009
        id_accessor = record_accessor_create(@id_key)
6✔
1010
        id_value = id_accessor.call(record)
6✔
1011
        if affinity_target_indices.key?(id_value)
6✔
1012
          target_index = affinity_target_indices[id_value]
6✔
1013
        end
1014
      end
1015

1016
      if @suppress_type_name || @last_seen_major_version >= 2
2,164✔
1017
        target_type = nil
2✔
1018
      else
1019
        # OpenSearch only supports "_doc".
1020
        target_type = DEFAULT_TYPE_NAME
2,162✔
1021
      end
1022

1023
      meta.clear
2,164✔
1024
      meta["_index".freeze] = target_index
2,164✔
1025
      meta["_type".freeze] = target_type unless target_type.nil?
2,164✔
1026
      meta["_alias".freeze] = target_index_alias
2,164✔
1027

1028
      if @pipeline
2,164✔
1029
        meta["pipeline".freeze] = pipeline
4✔
1030
      end
1031

1032
      @meta_config_map.each do |record_accessor, meta_key|
2,164✔
1033
        if raw_value = record_accessor.call(record)
51✔
1034
          meta[meta_key] = raw_value
40✔
1035
        end
1036
      end
1037

1038
      if @remove_keys
2,164✔
1039
        @remove_keys.each { |key| record.delete(key) }
13✔
1040
      end
1041

1042
      return [meta, header, record]
2,164✔
1043
    end
1044

1045
    # returns [parent, child_key] of child described by path array in record's tree
1046
    # returns [nil, child_key] if path doesnt exist in record
1047
    def get_parent_of(record, path)
1✔
1048
      parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
6✔
1049
      [parent_object, path[-1]]
6✔
1050
    end
1051

1052
    # gzip compress data
1053
    def gzip(string)
1✔
1054
      wio = StringIO.new("w")
3✔
1055
      w_gz = Zlib::GzipWriter.new(wio, strategy = compression_strategy)
3✔
1056
      w_gz.write(string)
3✔
1057
      w_gz.close
3✔
1058
      wio.string
3✔
1059
    end
1060

1061
    def placeholder_substitution_needed_for_template?
1✔
1062
      need_substitution = placeholder?(:host, @host.to_s) ||
516✔
1063
        placeholder?(:index_name, @index_name.to_s) ||
1064
        placeholder?(:template_name, @template_name.to_s) ||
1065
        @customize_template&.values&.any? { |value| placeholder?(:customize_template, value.to_s) } ||
24✔
1066
        placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
1067
        placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
1068
        placeholder?(:application_name, @application_name.to_s) ||
1069
      log.debug("Need substitution: #{need_substitution}")
1070
      need_substitution
516✔
1071
    end
1072

1073
    def template_installation(template_name, customize_template, application_name, target_index, host)
1✔
1074
      # for safety.
1075
    end
1076

1077
    def template_installation_actual(template_name, customize_template, application_name, target_index, host=nil)
1✔
1078
      if template_name && @template_file
66✔
1079
        if !@logstash_format && @template_names.include?(template_name)
66✔
1080
          log.debug("Template #{template_name} already exists (cached)")
×
1081
        else
1082
          retry_operate(@max_retry_putting_template,
66✔
1083
                        @fail_on_putting_template_retry_exceed,
1084
                        @catch_transport_exception_on_retry) do
1085
            if customize_template
81✔
1086
              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, host, target_index, @index_separator)
8✔
1087
            else
1088
              template_install(template_name, @template_file, @template_overwrite, host, target_index, @index_separator)
73✔
1089
            end
1090
          end
1091
          @template_names << template_name
21✔
1092
        end
1093
      end
1094
    end
1095

1096
    # send_bulk given a specific bulk request, the original tag,
1097
    # chunk, and bulk_message_count
1098
    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
1✔
1099
      _logstash_prefix, _logstash_dateformat, index_name, template_name, customize_template, application_name, _pipeline  = extracted_values
132✔
1100
      template_installation(template_name, customize_template, application_name, index_name, info.host)
132✔
1101

1102
      begin
1103

1104
        log.on_trace { log.trace "bulk request: #{data}" }
132✔
1105

1106
        prepared_data = if compression
132✔
1107
                          gzip(data)
3✔
1108
                        else
1109
                          data
129✔
1110
                        end
1111

1112
        response = client(info.host, compression).bulk body: prepared_data, index: info.index
132✔
1113
        log.on_trace { log.trace "bulk response: #{response}" }
124✔
1114

1115
        if response['errors']
124✔
1116
          error = Fluent::Plugin::OpenSearchErrorHandler.new(self)
5✔
1117
          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
5✔
1118
        end
1119
      rescue RetryStreamError => e
13✔
1120
        log.trace "router.emit_stream for retry stream doing..."
5✔
1121
        emit_tag = @retry_tag ? @retry_tag : tag
5✔
1122
        # check capacity of buffer space
1123
        if retry_stream_retryable?
5✔
1124
          router.emit_stream(emit_tag, e.retry_stream)
4✔
1125
        else
1126
          raise RetryStreamEmitFailure, "buffer is full."
1✔
1127
        end
1128
        log.trace "router.emit_stream for retry stream done."
4✔
1129
      rescue => e
1130
        ignore = @ignore_exception_classes.any? { |clazz| e.class <= clazz }
11✔
1131

1132
        log.warn "Exception ignored in tag #{tag}: #{e.class.name} #{e.message}" if ignore
8✔
1133

1134
        @_os = nil if @reconnect_on_error
8✔
1135
        @_os_info = nil if @reconnect_on_error
8✔
1136

1137
        raise UnrecoverableRequestFailure if ignore && @exception_backup
8✔
1138

1139
        # FIXME: identify unrecoverable errors and raise UnrecoverableRequestFailure instead
1140
        raise RecoverableRequestFailure, "could not push logs to OpenSearch cluster (#{connection_options_description(info.host)}): #{e.message}" unless ignore
6✔
1141
      end
1142
    end
1143

1144
    def retry_stream_retryable?
1✔
1145
      @buffer.storable?
4✔
1146
    end
1147

1148
    def is_existing_connection(host)
1✔
1149
      # check if the host provided match the current connection
1150
      return false if @_os.nil?
416✔
1151
      return false if @current_config.nil?
200✔
1152
      return false if host.length != @current_config.length
199✔
1153

1154
      for i in 0...host.length
199✔
1155
        if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
203✔
1156
          return false
1✔
1157
        end
1158
      end
1159

1160
      return true
198✔
1161
    end
1162
  end
1163
end
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc