From 4d54cceab462a6e8f39d48bd72beba9b796a97be Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Thu, 17 Jun 2021 19:46:03 +0200 Subject: [PATCH 1/4] Feat: check target is set in ECS mode (#49) if `target => option` is not present in ECS mode we log a warning (at info level) + also includes some (debug) logging improvements --- CHANGELOG.md | 4 ++++ docs/index.asciidoc | 20 ++++++++++++++++++ lib/logstash/filters/json.rb | 21 ++++++++++++------ logstash-filter-json.gemspec | 4 +++- spec/filters/json_spec.rb | 41 ++++++++++++++++++++++++++++++++---- 5 files changed, 79 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f10ed..4eb0a03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 3.2.0 + - Feat: check target is set in ECS mode [#49](https://github.com/logstash-plugins/logstash-filter-json/pull/49) + - Refactor: logging improvements to print event details in debug mode + ## 3.1.0 - Added better error handling, preventing some classes of malformed inputs from crashing the pipeline. diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d63aa19..40cb5f1 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -36,6 +36,13 @@ If the parsed data contains a `@timestamp` field, the plugin will try to use it parsing fails, the field will be renamed to `_@timestamp` and the event will be tagged with a `_timestampparsefailure`. +[id="plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) + +The plugin behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set. + +TIP: Set the `target` option to avoid potential schema conflicts. + [id="plugins-{type}s-{plugin}-options"] ==== JSON Filter Configuration Options @@ -44,6 +51,7 @@ This plugin supports the following configuration options plus the <> | <>|No | <> |<>|No | <> |<>|Yes | <> |<>|No @@ -55,6 +63,18 @@ filter plugins.   +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is <> +* Supported values are: +** `disabled`: does not use ECS-compatible field names +** `v1`: Elastic Common Schema compliant behavior (warns when `target` isn't set) + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <> for detailed information. + [id="plugins-{type}s-{plugin}-skip_on_invalid_json"] ===== `skip_on_invalid_json` diff --git a/lib/logstash/filters/json.rb b/lib/logstash/filters/json.rb index 05fe0ed..e63dae2 100644 --- a/lib/logstash/filters/json.rb +++ b/lib/logstash/filters/json.rb @@ -3,6 +3,9 @@ require "logstash/namespace" require "logstash/json" require "logstash/timestamp" +require 'logstash/plugin_mixins/ecs_compatibility_support' +require 'logstash/plugin_mixins/ecs_compatibility_support/target_check' +require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter' # This is a JSON parsing filter. It takes an existing field which contains JSON and # expands it into an actual data structure within the Logstash event. @@ -21,6 +24,11 @@ # `_timestampparsefailure`. class LogStash::Filters::Json < LogStash::Filters::Base + include LogStash::PluginMixins::ECSCompatibilitySupport + include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck + + extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter + config_name "json" # The configuration for the JSON filter: @@ -53,7 +61,7 @@ class LogStash::Filters::Json < LogStash::Filters::Base # data structure in the `target` field. # # NOTE: if the `target` field already exists, it will be overwritten! - config :target, :validate => :string + config :target, :validate => :field_reference # Append values to the `tags` field when there has been no # successful match @@ -67,7 +75,7 @@ def register end def filter(event) - @logger.debug? && @logger.debug("Running json filter", :event => event) + @logger.debug? && @logger.debug("Running json filter", :event => event.to_hash) source = event.get(@source) return unless source @@ -99,6 +107,7 @@ def filter(event) begin timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil rescue LogStash::TimestampParserError => e + @logger.debug("Failed to coerce timestamp", :timestamp => parsed_timestamp, :message => e.message) timestamp = nil end @@ -120,11 +129,11 @@ def filter(event) filter_matched(event) - @logger.debug? && @logger.debug("Event after json filter", :event => event) + @logger.debug? && @logger.debug("Event after json filter", :event => event.to_hash) rescue => ex - meta = { :exception => ex.message, :source => @source, :raw => source} - meta[:backtrace] = ex.backtrace if logger.debug? - logger.warn('Exception caught in json filter', meta) + meta = { :source => @source, :raw => source, :exception => ex } + meta[:backtrace] = ex.backtrace if @logger.debug? + @logger.warn('Exception caught in json filter', meta) _do_tag_on_failure(event) end diff --git a/logstash-filter-json.gemspec b/logstash-filter-json.gemspec index d9941cb..2b7ae34 100644 --- a/logstash-filter-json.gemspec +++ b/logstash-filter-json.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-json' - s.version = '3.1.0' + s.version = '3.2.0' s.licenses = ['Apache License (2.0)'] s.summary = "Parses JSON events" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -21,6 +21,8 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3' + s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'insist' diff --git a/spec/filters/json_spec.rb b/spec/filters/json_spec.rb index a687362..085aa63 100644 --- a/spec/filters/json_spec.rb +++ b/spec/filters/json_spec.rb @@ -155,12 +155,14 @@ end end - context "using message field source" do + subject(:filter) { LogStash::Filters::Json.new(config) } - subject(:filter) { LogStash::Filters::Json.new(config) } + let(:config) { {"source" => "message"} } + let(:event) { LogStash::Event.new("message" => message) } - let(:config) { {"source" => "message"} } - let(:event) { LogStash::Event.new("message" => message) } + let(:logger) { filter.logger } + + context "using message field source" do before(:each) do filter.register @@ -224,6 +226,37 @@ end end + describe "target" do + + let(:logger) { filter.logger } + + context 'not set in ECS mode' do + let(:config) { { "source" => "message", 'ecs_compatibility' => 'v1' } } + let(:message) { ' { "foo": "bar" } ' } + + it "works" do + filter.register # a warning logged + + filter.filter(event) + expect( event.get('foo') ).to eql 'bar' + end + end + + context 'set in ECS mode' do + let(:config) { { "source" => "message", 'ecs_compatibility' => 'v1', 'target' => '[baz]' } } + let(:message) { ' { "foo": "bar" } ' } + + it "works" do + filter.register # no warning logged + + filter.filter(event) + expect( event.include?('foo') ).to be false + expect( event.get('baz') ).to eql 'foo' => 'bar' + end + end + + end + describe "parse mixture of json an non-json content (skip_on_invalid_json)" do subject(:filter) { LogStash::Filters::Json.new(config) } From 0c2b6c4b97b04cc47a9310871b9ee8b654c1a147 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 11 May 2023 00:01:46 +0100 Subject: [PATCH 2/4] exclude Logstash 8.3+ from fail tests Field Reference can handle special characters in 8.3.0 --- spec/filters/json_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/filters/json_spec.rb b/spec/filters/json_spec.rb index 085aa63..345221e 100644 --- a/spec/filters/json_spec.rb +++ b/spec/filters/json_spec.rb @@ -62,8 +62,9 @@ end logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION) - - if (Gem::Requirement.create('>= 7.0').satisfied_by?(logstash_version) || + + # Field Reference can handle special characters since 8.3.0 + if ((Gem::Requirement.create('>= 7.0').satisfied_by?(logstash_version) && Gem::Requirement.create('< 8.3').satisfied_by?(logstash_version)) || (Gem::Requirement.create('~> 6.4').satisfied_by?(logstash_version) && LogStash::SETTINGS.get('config.field_reference.parser') == 'STRICT')) describe "unsupported field name using `target`" do config <<-CONFIG From 30260c35878d5dafd37a0467e0a1e9cd1f4d58bf Mon Sep 17 00:00:00 2001 From: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Date: Thu, 11 May 2023 10:08:20 +0000 Subject: [PATCH 3/4] bump to 3.2.1 --- CHANGELOG.md | 3 +++ logstash-filter-json.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb0a03..c096362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.2.1 + - Fix tag on failure test [#52](https://github.com/logstash-plugins/logstash-filter-json/pull/52) + ## 3.2.0 - Feat: check target is set in ECS mode [#49](https://github.com/logstash-plugins/logstash-filter-json/pull/49) - Refactor: logging improvements to print event details in debug mode diff --git a/logstash-filter-json.gemspec b/logstash-filter-json.gemspec index 2b7ae34..a746775 100644 --- a/logstash-filter-json.gemspec +++ b/logstash-filter-json.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-json' - s.version = '3.2.0' + s.version = '3.2.1' s.licenses = ['Apache License (2.0)'] s.summary = "Parses JSON events" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 4370ca699b51b3a2efd1feb4ba17e0a86a3c08e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20C=C3=A1mara?= Date: Fri, 8 May 2026 11:58:16 +0200 Subject: [PATCH 4/4] [skip ci] Add version file, update gemspec, and add Changelog & Version Update workflow trigger --- .../workflows/changelog-version-update.yml | 38 +++++++++++++++++++ logstash-filter-json.gemspec | 2 +- version | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/changelog-version-update.yml create mode 100644 version diff --git a/.github/workflows/changelog-version-update.yml b/.github/workflows/changelog-version-update.yml new file mode 100644 index 0000000..49db876 --- /dev/null +++ b/.github/workflows/changelog-version-update.yml @@ -0,0 +1,38 @@ +# This workflow automates changelog entry updates and version bumps for pull requests +# in logstash-plugins repositories. It is triggered when a PR is opened or merged, +# and delegates to the reusable pr-prepare workflow which reads /pr_prepare comments +# to determine the type of changelog entry and the appropriate version bump. +# Can also be triggered manually via workflow_dispatch for missed or late updates. + +name: Changelog & Version Update + +on: + pull_request_target: + types: [opened, closed] + workflow_dispatch: + inputs: + command: + description: 'Type of update' + required: true + type: choice + options: + - release:patch + - release:minor + - release:major + - changelog:add + pr_number: + description: 'PR number to link in the changelog entry' + required: true + type: string + changelog_entry: + description: 'Custom changelog entry (defaults to PR title if omitted)' + required: false + type: string + +jobs: + changelog: + uses: logstash-plugins/.github/.github/workflows/pr-prepare.yml@main + with: + command: ${{ inputs.command }} + pr_number: ${{ inputs.pr_number }} + changelog_entry: ${{ inputs.changelog_entry }} diff --git a/logstash-filter-json.gemspec b/logstash-filter-json.gemspec index a746775..aadff0a 100644 --- a/logstash-filter-json.gemspec +++ b/logstash-filter-json.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-json' - s.version = '3.2.1' + s.version = ::File.read('version').split("\n").first s.licenses = ['Apache License (2.0)'] s.summary = "Parses JSON events" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/version b/version new file mode 100644 index 0000000..e4604e3 --- /dev/null +++ b/version @@ -0,0 +1 @@ +3.2.1