From ac699fe5f94dbde8d58a3ff3fad457e8544d7ad3 Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 11:40:25 -0400 Subject: [PATCH 1/3] First work towards supporting tokenization parameters --- app/controllers/search_controller.rb | 3 ++- app/helpers/application_helper.rb | 3 ++- app/models/enhancer.rb | 3 ++- app/models/query_builder.rb | 3 ++- app/models/timdex_search.rb | 6 ++++++ 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 58e02d8e..a97821b5 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -225,7 +225,8 @@ def query_primo(per_page, offset) end def execute_geospatial_query(query) - query = query.except('queryMode') + query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold', + 'semanticShortQueryMaxTokens') if query['geobox'] == 'true' && query[:geodistance] == 'true' TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index b085ceb8..9be0d2fc 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -20,7 +20,8 @@ def index_page_title def results_page_title(query, character_limit = 50) return index_page_title unless query.present? - ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode] + ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold + semanticMustBoostThreshold semanticShortQueryMaxTokens] terms = query.reject { |term| ignored_terms.include? term }.values.join(' ') terms = "#{terms.first(character_limit)}..." if terms.length > character_limit "#{terms} | #{index_page_title}" diff --git a/app/models/enhancer.rb b/app/models/enhancer.rb index b21099bf..7ee32a4a 100644 --- a/app/models/enhancer.rb +++ b/app/models/enhancer.rb @@ -1,7 +1,8 @@ class Enhancer attr_accessor :enhanced_query - QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze + QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title + queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 76491ca0..5095583a 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -1,7 +1,8 @@ class QueryBuilder attr_reader :query - QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType].freeze + QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType + semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb index d8141baf..c8ceea43 100644 --- a/app/models/timdex_search.rb +++ b/app/models/timdex_search.rb @@ -28,6 +28,9 @@ class TimdexSearch < TimdexBase $sourceFilter: [String!] $subjectsFilter: [String!] $useGlobalScoring: Boolean + $semanticDropBoostThreshold: Float + $semanticMustBoostThreshold: Float + $semanticShortQueryMaxTokens: Int ) { search( searchterm: $q @@ -54,6 +57,9 @@ class TimdexSearch < TimdexBase sourceFilter: $sourceFilter subjectsFilter: $subjectsFilter useGlobalScoring: $useGlobalScoring + semanticDropBoostThreshold: $semanticDropBoostThreshold + semanticMustBoostThreshold: $semanticMustBoostThreshold + semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens ) { hits records { From b3adbae4bb82e92202046699087db389a85f850e Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 14:53:19 -0400 Subject: [PATCH 2/3] Remove tokenizer tuning parameters --- app/models/timdex_search.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb index c8ceea43..d8141baf 100644 --- a/app/models/timdex_search.rb +++ b/app/models/timdex_search.rb @@ -28,9 +28,6 @@ class TimdexSearch < TimdexBase $sourceFilter: [String!] $subjectsFilter: [String!] $useGlobalScoring: Boolean - $semanticDropBoostThreshold: Float - $semanticMustBoostThreshold: Float - $semanticShortQueryMaxTokens: Int ) { search( searchterm: $q @@ -57,9 +54,6 @@ class TimdexSearch < TimdexBase sourceFilter: $sourceFilter subjectsFilter: $subjectsFilter useGlobalScoring: $useGlobalScoring - semanticDropBoostThreshold: $semanticDropBoostThreshold - semanticMustBoostThreshold: $semanticMustBoostThreshold - semanticShortQueryMaxTokens: $semanticShortQueryMaxTokens ) { hits records { From 85b912065b8dc0238efe6ebd15ac84a3c94c7bdd Mon Sep 17 00:00:00 2001 From: Matthew Bernhardt Date: Wed, 22 Jul 2026 16:16:01 -0400 Subject: [PATCH 3/3] Move tokenization parameters to their own extraction method --- app/models/query_builder.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 5095583a..c86f5ef3 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -1,13 +1,13 @@ class QueryBuilder attr_reader :query - QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType - semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze + QUERY_PARAMS = %w[q citation contributors fundingInformation identifiers locations subjects title booleanType].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude geodistanceLongitude geodistanceDistance].freeze VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze + TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze def initialize(enhanced_query) @query = {} @@ -23,6 +23,7 @@ def initialize(enhanced_query) extract_geosearch(enhanced_query) extract_filters(enhanced_query) evaluate_query_mode(enhanced_query) + extract_tokenization_params(enhanced_query) @query['index'] = ENV.fetch('TIMDEX_INDEX', nil) @query['booleanType'] = enhanced_query[:booleanType] @query.compact! @@ -60,6 +61,12 @@ def extract_filters(enhanced_query) end end + def extract_tokenization_params(enhanced_query) + TOKENIZATION_PARAMS.each do |tp| + @query[tp] = enhanced_query[tp.to_sym]&.strip.to_f if enhanced_query[tp.to_sym].present? + end + end + # The GraphQL API requires that lat/long in geospatial fields be floats def coerce_to_float?(geo_param) geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude')