From 09693720e62b5b2bedfbc7c9d1019958f568a2fe Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 00:02:00 +0100 Subject: [PATCH 01/10] Add ability to ignore certain paths --- README.md | 2 ++ lib/searchyll/configuration.rb | 5 +++++ lib/searchyll/indexer.rb | 11 ++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a80a420..3838c80 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ elasticsearch: default_type: "post" # Optional. Default type is "post". custom_settings: _es_settings.yml # Optional. No default. Relative to your src folder custom_mappings: _es_mappings.yml # Optional. No default. Relative to your src folder + ignore: # Optional. No default. + - /news/* ``` ### Custom Settings File Example diff --git a/lib/searchyll/configuration.rb b/lib/searchyll/configuration.rb index 504a03f..526015e 100644 --- a/lib/searchyll/configuration.rb +++ b/lib/searchyll/configuration.rb @@ -69,6 +69,11 @@ def elasticsearch_default_type site.config['elasticsearch']['default_type'] || 'post' end + # Getter for the ignore regex + def elasticsearch_ignore + site.config['elasticsearch']['ignore'] || [] + end + # Getter for es mapping def elasticsearch_mapping_path site.config['elasticsearch']['custom_mappings'] diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index f1bc0cc..e3e4283 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -20,6 +20,7 @@ class Indexer attr_accessor :timestamp attr_accessor :uri attr_accessor :working + attr_accessor :ignore_regex # Initialize a basic indexer, with a Jekyll site configuration, waiting # to be supplied with documents for indexing. @@ -30,11 +31,19 @@ def initialize(configuration) self.working = true self.timestamp = Time.now self.batch_size = BATCH_SIZE + + # Compute a regex for detecting paths to ignore + escaped = (configuration.elasticsearch_ignore.map {|i| Regexp.escape(i).gsub('\*','.+?')}).join('|') + self.ignore_regex = Regexp.new "^(#{escaped})$", Regexp::IGNORECASE end # Public: Add new documents for batch indexing. def <<(doc) - queue << doc + if doc['url'] =~ self.ignore_regex + #puts %( ...ignoring) + else + queue << doc + end end # Public: start the indexer and wait for documents to index. From d2b30442ec39e74ed620f86e00cd986706a7cc32 Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 00:12:05 +0100 Subject: [PATCH 02/10] Use page.url instead of page.name as doc id --- lib/searchyll.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/searchyll.rb b/lib/searchyll.rb index db89e0b..5e8d29e 100644 --- a/lib/searchyll.rb +++ b/lib/searchyll.rb @@ -38,7 +38,7 @@ if (indexer = indexers[page.site]) indexer << ({ - "id" => page.name, + "id" => page.url, "url" => page.url, "text" => nokogiri_doc.xpath("//article//text()").to_s.gsub(/\s+/, " ") }).merge(page.data) From 002e00a6736ded4f8285d2116e047a0e269fbcca Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 00:13:35 +0100 Subject: [PATCH 03/10] Whoops, wrong method order --- lib/searchyll/indexer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index ab7d8ea..3b0ae55 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -173,7 +173,7 @@ def es_bulk_insert!(http, batch) bulk_insert.content_type = 'application/x-ndjson' bulk_insert.body = batch.map do |doc| [{ index: {} }.to_json, doc.to_json].join("\n") - end.force_encoding('ascii-8bit').join("\n") + "\n" + end.join("\n").force_encoding('ascii-8bit') + "\n" res = http.request(bulk_insert) if !res.kind_of?(Net::HTTPSuccess) $stderr.puts "Elasticsearch returned an error when performing bulk insert: " + res.message + " " + res.body From f743b636b277468bf18d71e30a9fa0624d694184 Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 00:26:25 +0100 Subject: [PATCH 04/10] Split the alias update call in two calls --- lib/searchyll/indexer.rb | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index f1bc0cc..e21ea09 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -221,19 +221,38 @@ def finalize_replication(http) # hot swap the index into the canonical alias def finalize_aliases(http) update_aliases = http_post('/_aliases') + + # perform removal and addition in two different calls so that + # the second one is performed even if the first one fails + if !old_indices.empty? + update_aliases.body = { + actions: [ + { remove: { + index: old_indices.join(','), + alias: configuration.elasticsearch_index_base_name + } } + ] + }.to_json + res = http.request(update_aliases) + if !res.kind_of?(Net::HTTPSuccess) + $stderr.puts "Elasticsearch returned an error when removing old aliases: " + res.message + " " + res.body + exit + end + end + update_aliases.body = { actions: [ - { remove: { - index: old_indices.join(','), - alias: configuration.elasticsearch_index_base_name - } }, { add: { index: elasticsearch_index_name, alias: configuration.elasticsearch_index_base_name } } ] }.to_json - http.request(update_aliases) + res = http.request(update_aliases) + if !res.kind_of?(Net::HTTPSuccess) + $stderr.puts "Elasticsearch returned an error when assigning the new alias: " + res.message + " " + res.body + exit + end end # delete old indices after a successful reindexing run From 07129176f9cd806f9f9fce9ccfc31759ac5694e3 Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 08:41:41 +0100 Subject: [PATCH 05/10] Add notice to the README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3838c80..5dd2f82 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +> This repository is a fork of https://github.com/omc/searchyll used for indexing the [developers.italia.it](https://github.com/italia/developers.italia.it) website. All the changes were contributed upstream, so when all our [pull requests](https://github.com/omc/searchyll/pulls?utf8=✓&q=is%3Apr+author%3Aalranel+) are merged we can get rid of this fork. + # Searchyll Search for Jekyll apps. A plugin for indexing your pages into a search engine. From b84fb14ce0ded1751fc55ed98d829859d0fdb892 Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Wed, 6 Feb 2019 12:42:24 +0100 Subject: [PATCH 06/10] Do not exit if we fail to remove old aliases --- lib/searchyll/indexer.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index 9104be3..561acf1 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -251,7 +251,6 @@ def finalize_aliases(http) res = http.request(update_aliases) if !res.kind_of?(Net::HTTPSuccess) $stderr.puts "Elasticsearch returned an error when removing old aliases: " + res.message + " " + res.body - exit end end From bff2ea2b561c2647e4a2765e9da67546329b51e1 Mon Sep 17 00:00:00 2001 From: Alessandro Ranellucci Date: Sat, 11 May 2019 23:25:37 +0200 Subject: [PATCH 07/10] Several bugfixes: prevent random failures caused by mutating timestamp; cache the list of old indices; exit with a non-zero code in case of fatal Elasticsearch errors; emit a warning upon index deletion failure --- lib/searchyll/indexer.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index 561acf1..d9666da 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -106,7 +106,7 @@ def working? # A versioned index name, based on the time of the indexing run. # Will be later added to an alias for hot reindexing. def elasticsearch_index_name - "#{configuration.elasticsearch_index_base_name}-#{timestamp.strftime('%Y%m%d%H%M%S')}" + @elasticsearch_index_name ||= "#{configuration.elasticsearch_index_base_name}-#{timestamp.strftime('%Y%m%d%H%M%S')}" end # Prepare an HTTP connection @@ -177,7 +177,7 @@ def es_bulk_insert!(http, batch) res = http.request(bulk_insert) if !res.kind_of?(Net::HTTPSuccess) $stderr.puts "Elasticsearch returned an error when performing bulk insert: " + res.message + " " + res.body - exit + exit(1) end end @@ -195,12 +195,12 @@ def current_batch # List the indices currently in the cluster, caching the call in an ivar def old_indices - # return if defined?(@old_indices) + return @old_indices if defined?(@old_indices) resp = http_start { |h| h.request(http_get('/_cat/indices?h=index')) } indices = JSON.parse(resp.body).map { |i| i['index'] } indices = indices.select { |i| i =~ /\A#{configuration.elasticsearch_index_base_name}/ } indices -= [elasticsearch_index_name] - # @old_indices = indices + @old_indices = indices indices end @@ -265,16 +265,19 @@ def finalize_aliases(http) res = http.request(update_aliases) if !res.kind_of?(Net::HTTPSuccess) $stderr.puts "Elasticsearch returned an error when assigning the new alias: " + res.message + " " + res.body - exit + exit(1) end end # delete old indices after a successful reindexing run def finalize_cleanup(http) return if old_indices.nil? || old_indices.empty? - cleanup_indices = http_delete("/#{old_indices.join(',')}") puts %( Old indices: #{old_indices.join(', ')}) - http.request(cleanup_indices) + cleanup_indices = http_delete("/#{old_indices.join(',')}") + res = http.request(cleanup_indices) + if !res.kind_of?(Net::HTTPSuccess) + $stderr.puts "Elasticsearch returned an error when deleting old indices: " + res.message + " " + res.body + end end end end From 05499aa8c548b355e757caf323c2c0b7a5068c15 Mon Sep 17 00:00:00 2001 From: Alessandro Sebastiani Date: Thu, 19 Mar 2020 23:41:37 +0100 Subject: [PATCH 08/10] managing es authentication both from env var and uri params --- README.md | 12 ++++++++++++ lib/searchyll/configuration.rb | 6 ++++++ lib/searchyll/indexer.rb | 12 ++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5dd2f82..05e8fb9 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,18 @@ elasticsearch: - /news/* ``` +### ES Auth +Elasticsearch authentication can be provided both by passing user and password as URI parameter: +```yaml +url: https://user:pass@someurl.com +``` +Or by easily setting those ENV variables: +```sh +ELASTICSEARCH_URL=http://localhost:9200/ +ELASTICSEARCH_USER=elastic +ELASTICSEARCH_UPASS=changeme +``` + ### Custom Settings File Example It should be written to be plugged into the `settings` slot of a [create index](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html) call diff --git a/lib/searchyll/configuration.rb b/lib/searchyll/configuration.rb index 1d30d33..8361577 100644 --- a/lib/searchyll/configuration.rb +++ b/lib/searchyll/configuration.rb @@ -11,6 +11,12 @@ def elasticsearch_url ENV['BONSAI_URL'] || ENV['ELASTICSEARCH_URL'] || ((site.config||{})['elasticsearch']||{})['url'] end + def elasticsearch_user + ENV['ELASTICSEARCH_USER'] + end + def elasticsearch_pass + ENV['ELASTICSEARCH_PASS'] + end def valid? elasticsearch_url && !elasticsearch_url.empty? && elasticsearch_url.start_with?('http') diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index d9666da..f967135 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -19,6 +19,8 @@ class Indexer attr_accessor :queue attr_accessor :timestamp attr_accessor :uri + attr_accessor :es_user + attr_accessor :es_pass attr_accessor :working attr_accessor :ignore_regex @@ -27,6 +29,8 @@ class Indexer def initialize(configuration) self.configuration = configuration self.uri = URI(configuration.elasticsearch_url) + self.es_user = configuration.elasticsearch_user + self.es_pass = configuration.elasticsearch_pass self.queue = Queue.new self.working = true self.timestamp = Time.now @@ -159,8 +163,12 @@ def http_request(klass, path) req = klass.new(path) req.content_type = 'application/json' req['Accept'] = 'application/json' - # Append auth credentials if the exist - req.basic_auth(uri.user, uri.password) if uri.user && uri.password + # Append auth credentials if they exist + # it trying to get them from env and then from + # elasticsearch uri itself + user = es_user || uri.user + pass = es_pass || uri.password + req.basic_auth(user, pass) if user && pass req end From ceb68300634f3ef553538c0ce3a6f313be17885c Mon Sep 17 00:00:00 2001 From: Luca Prete Date: Fri, 27 Mar 2020 10:49:27 +0100 Subject: [PATCH 09/10] Fix typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05e8fb9..438e308 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Or by easily setting those ENV variables: ```sh ELASTICSEARCH_URL=http://localhost:9200/ ELASTICSEARCH_USER=elastic -ELASTICSEARCH_UPASS=changeme +ELASTICSEARCH_PASS=changeme ``` ### Custom Settings File Example From c9f4ade0248869eca43ceb18dae991af8ee89fe1 Mon Sep 17 00:00:00 2001 From: Raffaele Vitiello Date: Wed, 7 Jan 2026 13:14:03 +0100 Subject: [PATCH 10/10] fix: add OpenSearch/Elasticsearch 7+ compatibility Remove deprecated type-based mappings and bulk endpoint format. - Remove type wrapper from mappings (was: {"mappings": {"_doc": {...}}}) Now uses flat structure: {"mappings": {...}} - Remove type from bulk endpoint (was: /{index}/_doc/_bulk) Now uses: /{index}/_bulk These changes maintain backward compatibility with Elasticsearch 7+ while adding support for OpenSearch. --- lib/searchyll/indexer.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/searchyll/indexer.rb b/lib/searchyll/indexer.rb index f967135..ca54047 100644 --- a/lib/searchyll/indexer.rb +++ b/lib/searchyll/indexer.rb @@ -130,8 +130,8 @@ def prepare_index } if configuration.elasticsearch_mapping - payload['mappings'] = {} - payload['mappings'].store(configuration.elasticsearch_default_type, configuration.elasticsearch_mapping) + # OpenSearch/ES 7+ doesn't support type in mappings - use flat structure + payload['mappings'] = configuration.elasticsearch_mapping end json_payload = payload.to_json @@ -177,7 +177,8 @@ def http_request(klass, path) # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html def es_bulk_insert!(http, batch) return if batch.empty? - bulk_insert = http_post("/#{elasticsearch_index_name}/#{configuration.elasticsearch_default_type}/_bulk") + # OpenSearch/ES 7+ doesn't support type in bulk endpoint + bulk_insert = http_post("/#{elasticsearch_index_name}/_bulk") bulk_insert.content_type = 'application/x-ndjson' bulk_insert.body = batch.map do |doc| [{ index: {} }.to_json, doc.to_json].join("\n")