diff --git a/src/plugin/interface/ob_plugin_ftparser_intf.h b/src/plugin/interface/ob_plugin_ftparser_intf.h index c1eafdb8d..c9b3f4141 100644 --- a/src/plugin/interface/ob_plugin_ftparser_intf.h +++ b/src/plugin/interface/ob_plugin_ftparser_intf.h @@ -112,6 +112,8 @@ class ObFTParserParam final : public ObFTParserParamExport { ObFTParserParamExport::reset(); allocator_ = nullptr; + metadata_allocator_ = nullptr; + scratch_allocator_ = nullptr; ngram_token_size_ = NGRAM_TOKEN_SIZE; } @@ -119,6 +121,8 @@ class ObFTParserParam final : public ObFTParserParamExport public: common::ObIAllocator *allocator_ = nullptr; + common::ObIAllocator *metadata_allocator_ = nullptr; + common::ObIAllocator *scratch_allocator_ = nullptr; // ik parser params ObFTIKParam ik_param_; @@ -137,6 +141,11 @@ class ObITokenIterator int64_t &word_len, int64_t &char_cnt, int64_t &word_freq) = 0; + virtual int reuse_parser(const char *fulltext, const int64_t fulltext_len) + { + UNUSEDx(fulltext, fulltext_len); + return OB_NOT_SUPPORTED; + } DECLARE_PURE_VIRTUAL_TO_STRING; }; diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index 41535301c..43122644b 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -4227,6 +4227,13 @@ int ObDDLOperator::drop_table( ObObjectType::VIEW))) { LOG_WARN("failed to delete_schema_object_dependency", K(ret), K(1UL), K(table_schema.get_table_id())); + } else if (table_schema.is_index_table() + && OB_FAIL(ObDependencyInfo::delete_schema_object_dependency( + trans, + table_schema.get_table_id(), + table_schema.get_schema_version(), + ObObjectType::INDEX))) { + LOG_WARN("failed to delete index dependency", K(ret), K(table_schema.get_table_id())); } if (OB_FAIL(ret)) { diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 8d4089715..c0d6d48f4 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -65,6 +65,7 @@ #include "rootserver/parallel_ddl/ob_ddl_helper.h" #include "storage/ddl/ob_ddl_alter_auto_part_attr.h" #include "sql/resolver/ddl/ob_vec_index_builder_util.h" +#include "sql/resolver/ddl/ob_fts_index_builder_util.h" #include "rootserver/direct_load/ob_direct_load_partition_exchange.h" #include "rootserver/truncate_info/ob_truncate_info_service.h" #include "share/truncate_info/ob_truncate_info_util.h" @@ -2281,6 +2282,9 @@ int ObDDLService::create_index_or_mlog_table_in_trans( } else if (OB_ISNULL(sql_trans) && OB_FAIL(trans.start(sql_proxy_, refreshed_schema_version))) { LOG_WARN("failed to start trans", KR(ret), K(refreshed_schema_version)); + } else if (OB_FAIL(ObFtsIndexBuilderUtil::try_load_and_lock_dictionary_tables( + table_schema, trans))) { + LOG_WARN("fail to load fulltext dictionaries", KR(ret), K(table_schema)); } else if (OB_FAIL(ddl_operator.create_table(table_schema, trans, ddl_stmt_str))) { @@ -13931,7 +13935,7 @@ int ObDDLService::alter_table_in_trans(obcall::ObAlterTableArg &alter_table_arg, need_update_index_table))) { LOG_WARN("failed to set new table options", K(ret), K(new_table_schema), K(*orig_table_schema), K(ret)); - } else { + } else if (!orig_table_schema->is_fulltext_dict()) { new_table_schema.set_table_flags(alter_table_schema.get_table_flags()); } } diff --git a/src/rootserver/parallel_ddl/ob_drop_table_helper.cpp b/src/rootserver/parallel_ddl/ob_drop_table_helper.cpp index 8fbf2fd65..444c3e046 100644 --- a/src/rootserver/parallel_ddl/ob_drop_table_helper.cpp +++ b/src/rootserver/parallel_ddl/ob_drop_table_helper.cpp @@ -1382,6 +1382,21 @@ int ObDropTableHelper::drop_table_(const ObTableSchema &table_schema, const ObSt int64_t new_schema_version = OB_INVALID_VERSION; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("fail to check inner stat", KR(ret)); + } else if (table_schema.is_fulltext_dict()) { + ObArray dependencies; + if (OB_FAIL(ObDependencyInfo::collect_dep_infos( + table_schema.get_table_id(), *sql_proxy_, dependencies))) { + LOG_WARN("fail to collect dictionary dependencies", K(ret), K(table_schema)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < dependencies.count(); ++i) { + if (ObObjectType::INDEX == dependencies.at(i).get_dep_obj_type()) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "dropping a referenced FULLTEXT_DICT table is"); + } + } + } + } + if (OB_FAIL(ret)) { } else if (OB_ISNULL(schema_service_impl = schema_service_->get_schema_service())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("schema service impl is null", KR(ret)); diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index f60fccd35..05a270075 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -184,6 +184,7 @@ static const uint64_t OB_MIN_ID = 0;//used for lower_bound #define EXTERNAL_TABLE_AUTO_REFRESH_INTERVAL_FLAG (INT64_C(1) << 3) #define EXTERNAL_TABLE_AUTO_REFRESH_FLAG_OFFSET 2 #define EXTERNAL_TABLE_AUTO_REFRESH_FLAG_BITS 2 +#define FULLTEXT_DICT_FLAG (INT64_C(1) << 5) // schema array size static const int64_t SCHEMA_SMALL_MALLOC_BLOCK_SIZE = 64; diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index c5ba78464..0b3158955 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -2756,6 +2756,42 @@ int ObTableSchema::set_view_definition(const common::ObString &view_definition) return view_schema_.set_view_definition(view_definition); } +int ObTableSchema::check_fulltext_dict_structure() const +{ + int ret = OB_SUCCESS; + const ObColumnSchemaV2 *word_column = nullptr; + if (!is_fulltext_dict()) { + ret = OB_INVALID_ARGUMENT; + } else if (!is_index_organized_table()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT table must use ORGANIZATION INDEX"); + } else if (CHARSET_UTF8MB4 != get_charset_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT table charset must be utf8mb4"); + } else if (1 != get_column_count() + || OB_ISNULL(word_column = get_column_schema_by_idx(0))) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT table must contain only the word column"); + } else if (0 != word_column->get_column_name_str().case_compare("word")) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT table column must be named word"); + } else if (ObVarcharType != word_column->get_data_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT word column must be VARCHAR"); + } else if (word_column->get_data_length() < 1 || word_column->get_data_length() > 500) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT word column length must be between 1 and 500"); + } else if (CHARSET_UTF8MB4 != word_column->get_charset_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT word column charset must be utf8mb4"); + } else if (!word_column->is_rowkey_column() || 1 != word_column->get_rowkey_position() + || 1 != get_rowkey_column_num()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT word column must be the primary key"); + } + return ret; +} + int ObTableSchema::set_parser_name_and_properties( const common::ObString &parser_name, const common::ObString &parser_properties) diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 8aafbfb19..6c46b0e36 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1413,6 +1413,9 @@ class ObTableSchema : public ObSimpleTableSchemaV2 int set_external_properties(const common::ObString &format) { return deep_copy_str(format, external_properties_); } void set_external_table_auto_refresh(const int64_t flag) { table_flags_ |= (flag << EXTERNAL_TABLE_AUTO_REFRESH_FLAG_OFFSET); } inline void set_user_specified_partition_for_external_table() { table_flags_ |= EXTERNAL_TABLE_USER_SPECIFIED_PARTITION_FLAG; } + inline void set_fulltext_dict() { table_flags_ |= FULLTEXT_DICT_FLAG; } + inline bool is_fulltext_dict() const { return (table_flags_ & FULLTEXT_DICT_FLAG) != 0; } + int check_fulltext_dict_structure() const; template int add_column(const ColumnType &column); int delete_column(const common::ObString &column_name); diff --git a/src/sql/das/ob_das_domain_utils.cpp b/src/sql/das/ob_das_domain_utils.cpp index 5d416a9a7..79dd19fc1 100644 --- a/src/sql/das/ob_das_domain_utils.cpp +++ b/src/sql/das/ob_das_domain_utils.cpp @@ -41,8 +41,25 @@ ObObjDatumMapType ObFTIndexRowCache::FTS_DOC_WORD_TYPES[] = {OBJ_DATUM_STRING, O ObExprOperatorType ObFTIndexRowCache::FTS_INDEX_EXPR_TYPE[] = {T_FUN_SYS_WORD_SEGMENT, T_FUN_SYS_DOC_ID, T_FUN_SYS_WORD_COUNT, T_FUN_SYS_DOC_LENGTH}; ObExprOperatorType ObFTIndexRowCache::FTS_DOC_WORD_EXPR_TYPE[] = {T_FUN_SYS_DOC_ID, T_FUN_SYS_WORD_SEGMENT, T_FUN_SYS_WORD_COUNT, T_FUN_SYS_DOC_LENGTH}; +static constexpr int64_t FT_WORD_DOC_COL_CNT = 4; +static constexpr int64_t FT_MIN_WORD_BUCKET_COUNT = 2; +static constexpr int64_t FT_MAX_WORD_BUCKET_COUNT = 997; +static constexpr int64_t FT_ESTIMATED_BYTES_PER_TOKEN = 10; + +static int64_t estimate_ft_word_bucket_count(const int64_t fulltext_length) +{ + return MIN(MAX(fulltext_length / FT_ESTIMATED_BYTES_PER_TOKEN, + FT_MIN_WORD_BUCKET_COUNT), + FT_MAX_WORD_BUCKET_COUNT); +} + ObFTIndexRowCache::ObFTIndexRowCache() - : rows_(), + : words_(), + word_iter_(), + word_end_iter_(), + row_(), + doc_id_datum_(), + doc_length_(0), row_idx_(0), is_fts_index_aux_(true), helper_(), @@ -70,6 +87,8 @@ int ObFTIndexRowCache::init( LOG_WARN("failed to create merge memctx", K(ret)); } else if (OB_FAIL(helper_.init(&(merge_memctx_->get_arena_allocator()), parser_name, parser_properties))) { LOG_WARN("fail to init full-text parser helper", K(ret)); + } else if (OB_FAIL(row_.init(FT_WORD_DOC_COL_CNT))) { + LOG_WARN("fail to init streaming fulltext index row", K(ret), K(FT_WORD_DOC_COL_CNT)); } else { row_idx_ = 0; is_fts_index_aux_ = is_fts_index_aux; @@ -86,58 +105,87 @@ int ObFTIndexRowCache::segment(const common::ObObjMeta &ft_obj_meta, const ObString &fulltext) { int ret = OB_SUCCESS; + const int64_t ft_word_bkt_cnt = estimate_ft_word_bucket_count(fulltext.length()); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_)); } else if (FALSE_IT(reuse())) { - } else if (OB_FAIL(ObDASDomainUtils::generate_fulltext_word_rows(merge_memctx_->get_arena_allocator(), - &helper_, - ft_obj_meta, - doc_id_datum, - fulltext, - is_fts_index_aux_, - rows_))) { - LOG_WARN("fail to generate fulltext word rows", K(ret), K(helper_), K(is_fts_index_aux_)); + } else if (0 == fulltext.length()) { + ret = OB_ITER_END; + } else if (OB_FAIL(words_.create(ft_word_bkt_cnt, common::ObMemAttr("FTWordMap")))) { + LOG_WARN("fail to create fulltext word map", K(ret), K(ft_word_bkt_cnt)); + } else if (OB_FAIL(helper_.segment(ft_obj_meta, + fulltext.ptr(), + fulltext.length(), + doc_length_, + words_))) { + LOG_WARN("fail to segment fulltext", K(ret), K(helper_), K(ft_obj_meta), K(fulltext)); + } else if (0 == words_.size()) { + ret = OB_ITER_END; } else { + doc_id_datum_ = doc_id_datum; + word_iter_ = words_.begin(); + word_end_iter_ = words_.end(); row_idx_ = 0; } - LOG_TRACE("word segment", K(ret), K(row_idx_), K(rows_.count()), K(doc_id_datum), K(fulltext)); + LOG_TRACE("word segment", K(ret), K(row_idx_), K(words_.size()), K(doc_id_datum), K(fulltext)); return ret; } int ObFTIndexRowCache::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; + row = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_)); - } else if (row_idx_ >= rows_.count()) { + } else if (!words_.created() || word_iter_ == word_end_iter_) { ret = OB_ITER_END; } else { - row = (rows_[row_idx_]); + const storage::ObFTWord &ft_word = word_iter_->first; + const int64_t word_cnt = word_iter_->second; + const int64_t word_idx = is_fts_index_aux_ ? 0 : 1; + const int64_t doc_id_idx = is_fts_index_aux_ ? 1 : 0; + row_.reuse(); + row_.storage_datums_[word_idx].set_datum(ft_word.get_word()); + row_.storage_datums_[doc_id_idx].shallow_copy_from_datum(doc_id_datum_); + row_.storage_datums_[2].set_uint(word_cnt); + row_.storage_datums_[3].set_uint(doc_length_); + row = &row_; + ++word_iter_; ++row_idx_; } - LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(rows_.count())); + LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(words_.size())); return ret; } void ObFTIndexRowCache::reset() { - rows_.reset(); - row_idx_ = 0; - is_fts_index_aux_ = true; + reuse(); + row_.reset(); helper_.reset(); if (OB_NOT_NULL(merge_memctx_)) { DESTROY_CONTEXT(merge_memctx_); merge_memctx_ = nullptr; } + is_fts_index_aux_ = true; is_inited_ = false; } void ObFTIndexRowCache::reuse() { - rows_.reuse(); + if (words_.created()) { + const int tmp_ret = words_.destroy(); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN_RET(tmp_ret, "fail to destroy fulltext word map"); + } + } + word_iter_ = storage::ObFTWordMap::const_iterator(); + word_end_iter_ = storage::ObFTWordMap::const_iterator(); + doc_id_datum_.reset(); + doc_length_ = 0; row_idx_ = 0; + row_.reuse(); if (OB_NOT_NULL(merge_memctx_)) { merge_memctx_->reset_remain_one_page(); } diff --git a/src/sql/das/ob_das_domain_utils.h b/src/sql/das/ob_das_domain_utils.h index c1295c365..ad3a32b1c 100644 --- a/src/sql/das/ob_das_domain_utils.h +++ b/src/sql/das/ob_das_domain_utils.h @@ -48,10 +48,16 @@ class ObFTIndexRowCache final int get_next_row(blocksstable::ObDatumRow *&row); void reset(); void reuse(); - TO_STRING_KV(K_(row_idx), K_(is_fts_index_aux), K_(helper), K_(is_inited), K_(rows)); + TO_STRING_KV(K_(row_idx), K_(is_fts_index_aux), K_(helper), K_(is_inited), + K_(doc_length), "word_count", words_.size()); private: lib::MemoryContext merge_memctx_; - ObDomainIndexRow rows_; + storage::ObFTWordMap words_; + storage::ObFTWordMap::const_iterator word_iter_; + storage::ObFTWordMap::const_iterator word_end_iter_; + blocksstable::ObDatumRow row_; + common::ObDatum doc_id_datum_; + int64_t doc_length_; uint64_t row_idx_; bool is_fts_index_aux_; storage::ObFTParseHelper helper_; diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 6a8b4c8d3..8a3b5040b 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -35,6 +35,9 @@ #include "sql/engine/cmd/ob_timezone_importer.h" #include "sql/engine/cmd/ob_srs_importer.h" #include "share/ob_internal_table_change_notifier.h" +#include "share/schema/ob_schema_getter_guard.h" +#include "storage/fts/ob_fts_plugin_helper.h" +#include "storage/fts/dict/ob_ft_dict_hub.h" namespace oceanbase { @@ -552,6 +555,46 @@ int ObClearMergeErrorExecutor::execute(ObExecContext &ctx, ObClearMergeErrorStmt return ret; } +int ObRefreshFulltextDictExecutor::execute(ObExecContext &ctx, + ObRefreshFulltextDictStmt &stmt) +{ + int ret = OB_SUCCESS; + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObDatabaseSchema *database_schema = nullptr; + const share::schema::ObTableSchema *table_schema = nullptr; + storage::ObFTDictHub *dict_hub = nullptr; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_database_schema(stmt.get_database_name(), database_schema))) { + LOG_WARN("fail to get dictionary database", K(ret), K(stmt)); + } else if (OB_ISNULL(database_schema)) { + ret = OB_ERR_BAD_DATABASE; + } else if (OB_FAIL(schema_guard.get_table_schema(database_schema->get_database_id(), + stmt.get_table_name(), + false, + table_schema))) { + LOG_WARN("fail to get dictionary table", K(ret), K(stmt)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + } else if (!table_schema->is_fulltext_dict()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "table is not a FULLTEXT_DICT table"); + } else if (OB_FAIL(table_schema->check_fulltext_dict_structure())) { + LOG_WARN("invalid fulltext dictionary structure", K(ret), KPC(table_schema)); + } else if (OB_FAIL(storage::ObFTParsePluginData::instance().get_dict_hub(dict_hub))) { + LOG_WARN("fail to get fulltext dictionary hub", K(ret)); + } else if (OB_ISNULL(dict_hub)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(dict_hub->refresh_user_dict(stmt.get_database_name(), + stmt.get_table_name(), + table_schema->get_table_id()))) { + LOG_WARN("fail to refresh fulltext dictionary", K(ret), K(stmt)); + } + return ret; +} + int ObUpgradeVirtualSchemaExecutor ::execute( ObExecContext &ctx, ObUpgradeVirtualSchemaStmt &stmt) { diff --git a/src/sql/engine/cmd/ob_alter_system_executor.h b/src/sql/engine/cmd/ob_alter_system_executor.h index 9638da6df..6a0a7c123 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.h +++ b/src/sql/engine/cmd/ob_alter_system_executor.h @@ -64,6 +64,8 @@ DEF_SIMPLE_EXECUTOR(ObSetConfig); DEF_SIMPLE_EXECUTOR(ObClearMergeError); +DEF_SIMPLE_EXECUTOR(ObRefreshFulltextDict); + DEF_SIMPLE_EXECUTOR(ObUpgradeVirtualSchema); diff --git a/src/sql/engine/expr/ob_expr_tokenize.cpp b/src/sql/engine/expr/ob_expr_tokenize.cpp index b913f7d05..cf3495bc5 100644 --- a/src/sql/engine/expr/ob_expr_tokenize.cpp +++ b/src/sql/engine/expr/ob_expr_tokenize.cpp @@ -237,7 +237,11 @@ int ObExprTokenize::parse_param(const ObExpr &expr, LOG_WARN("Fail to parse parser params.", K(ret)); } else if (OB_FAIL(parse_parser_properties(expr, ctx, temp_allocator, param))) { LOG_WARN("Fail to parse parser params.", K(ret)); - } else if (OB_FAIL(param.reform_parser_properties(param.properties_))) { + } else if (OB_FAIL(param.reform_parser_properties( + param.properties_, + OB_ISNULL(ctx.exec_ctx_.get_my_session()) + ? ObString() + : ctx.exec_ctx_.get_my_session()->get_database_name()))) { LOG_WARN("Fail to reform parser params.", K(ret)); } else if (OB_FAIL(param.try_load_dictionary_for_ik())) { LOG_WARN("fail to try load dictionary for ik", K(ret)); @@ -415,7 +419,8 @@ int ObExprTokenize::parse_parser_properties(const ObExpr &expr, return ret; } -int ObExprTokenize::TokenizeParam::reform_parser_properties(const ObString &properties) +int ObExprTokenize::TokenizeParam::reform_parser_properties(const ObString &properties, + const ObString &database_name) { int ret = OB_SUCCESS; storage::ObFTParserJsonProps parser_properties; @@ -425,6 +430,50 @@ int ObExprTokenize::TokenizeParam::reform_parser_properties(const ObString &prop } else if (OB_FAIL(parser_properties.parse_from_valid_str(properties))) { LOG_WARN("fail to parse properties", K(ret)); LOG_USER_ERROR(OB_INVALID_ARGUMENT, "parser properties invalid."); + } else { + const char *property_names[] = {ObFTSLiteral::CONFIG_NAME_DICT_TABLE, + ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE, + ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE}; + for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(property_names); ++i) { + ObString table_name; + if (0 == ObString(property_names[i]).compare(ObFTSLiteral::CONFIG_NAME_DICT_TABLE)) { + ret = parser_properties.config_get_dict_table(table_name); + } else if (0 == ObString(property_names[i]).compare(ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE)) { + ret = parser_properties.config_get_stopword_table(table_name); + } else { + ret = parser_properties.config_get_quantifier_table(table_name); + } + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else if (OB_SUCC(ret) && nullptr == table_name.find('.')) { + ObString qualified_name; + if (database_name.empty()) { + ret = OB_ERR_NO_DB_SELECTED; + } else { + const int64_t length = database_name.length() + 1 + table_name.length(); + char *buffer = static_cast(allocator_.alloc(length)); + if (OB_ISNULL(buffer)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + MEMCPY(buffer, database_name.ptr(), database_name.length()); + buffer[database_name.length()] = '.'; + MEMCPY(buffer + database_name.length() + 1, table_name.ptr(), table_name.length()); + qualified_name.assign_ptr(buffer, length); + } + } + if (OB_FAIL(ret)) { + LOG_WARN("fail to qualify TOKENIZE dictionary table", K(ret), K(table_name)); + } else if (0 == ObString(property_names[i]).compare(ObFTSLiteral::CONFIG_NAME_DICT_TABLE)) { + ret = parser_properties.config_set_dict_table(qualified_name); + } else if (0 == ObString(property_names[i]).compare(ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE)) { + ret = parser_properties.config_set_stopword_table(qualified_name); + } else { + ret = parser_properties.config_set_quantifier_table(qualified_name); + } + } + } + } + if (OB_FAIL(ret)) { } else if (OB_FAIL(parser_properties.rebuild_props_for_ddl(parser_name_, ObCollationType::CS_TYPE_UTF8MB4_BIN, true))) { diff --git a/src/sql/engine/expr/ob_expr_tokenize.h b/src/sql/engine/expr/ob_expr_tokenize.h index 2cc424636..28708c334 100644 --- a/src/sql/engine/expr/ob_expr_tokenize.h +++ b/src/sql/engine/expr/ob_expr_tokenize.h @@ -63,7 +63,8 @@ class ObExprTokenize : public ObStringExprOperator int parse_json_param(const ObIJsonBase *obj); // check and reform parser properties to standard format - int reform_parser_properties(const ObString &properties); + int reform_parser_properties(const ObString &properties, + const ObString &database_name); int try_load_dictionary_for_ik(); public: diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 398409f35..80c0d07a9 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -535,6 +535,10 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObClearMergeErrorStmt, ObClearMergeErrorExecutor); break; } + case stmt::T_REFRESH_FULLTEXT_DICT: { + DEFINE_EXECUTE_CMD(ObRefreshFulltextDictStmt, ObRefreshFulltextDictExecutor); + break; + } case stmt::T_UPGRADE_VIRTUAL_SCHEMA: { DEFINE_EXECUTE_CMD(ObUpgradeVirtualSchemaStmt, ObUpgradeVirtualSchemaExecutor); break; diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index fe19bc70c..8905e75b1 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -250,6 +250,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"deterministic", DETERMINISTIC}, {"dense_rank", DENSE_RANK}, {"diagnostics", DIAGNOSTICS}, + {"dict", DICT}, {"dict_table", DICT_TABLE}, {"diff", DIFF}, {"disallow", DISALLOW}, @@ -360,6 +361,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"frozen", FROZEN}, {"full", FULL}, {"fulltext", FULLTEXT}, + {"fulltext_dict", FULLTEXT_DICT}, {"function", FUNCTION}, {"following", FOLLOWING}, {"general", GENERAL}, diff --git a/src/sql/parser/ob_item_type.h b/src/sql/parser/ob_item_type.h index 295d49f95..c264943fd 100644 --- a/src/sql/parser/ob_item_type.h +++ b/src/sql/parser/ob_item_type.h @@ -2881,6 +2881,8 @@ typedef enum ObItemType T_FORK_DATABASE = 4917, T_DIFF_TABLE = 4918, T_MERGE_TABLE = 4919, + T_FULLTEXT_DICT = 4920, + T_REFRESH_FULLTEXT_DICT = 4921, T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index e395c5299..86cfb775e 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -292,7 +292,7 @@ END_P SET_VAR DELIMITER CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT DAG DATA DATAFILE DATA_DISK_SIZE DATA_SOURCE DATA_TABLE_ID DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPT - DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS DICT_TABLE + DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS DICT_TABLE DICT DIFF DIRECTORY DISABLE DISALLOW DISCARD DISK DISKGROUP DO DOT DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DUPLICATE_READ_CONSISTENCY DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT DEMAND DELETE_INSERT DYNAMIC_PARTITION_POLICY @@ -302,7 +302,7 @@ END_P SET_VAR DELIMITER FAIL FAILOVER FAST FAULTS FILE_BLOCK_SIZE FIELDS FILEX FINAL_COUNT FIRST FIRST_VALUE FIXED FLUSH FOLLOWER FORMAT FOUND FORK FREEZE FREQUENCY FUNCTION FOLLOWING FLASHBACK FULL FRAGMENTATION FROZEN FILE_ID FILTER - FIELD_OPTIONALLY_ENCLOSED_BY FIELD_DELIMITER FIELD_ENCLOSED_BY FILE_EXTENSION + FIELD_OPTIONALLY_ENCLOSED_BY FIELD_DELIMITER FIELD_ENCLOSED_BY FILE_EXTENSION FULLTEXT_DICT GENERAL GEOMETRY GEOMCOLLECTION GEOMETRYCOLLECTION GET_FORMAT GLOBAL GRANTS GRANULARITY GROUP_CONCAT GROUPING GTS GLOBAL_NAME GLOBAL_ALIAS @@ -6943,6 +6943,11 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2); /* make bison mute*/ malloc_non_terminal_node($$, result->malloc_pool_, T_COMMENT, 1, $3); } +| FULLTEXT_DICT opt_equal_mark STRING_VALUE +{ + (void)($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_FULLTEXT_DICT, 1, $3); +} | TABLEGROUP opt_equal_mark relation_name_or_string { (void)($2) ; /* make bison mute */ @@ -18457,6 +18462,31 @@ alter_with_opt_hint SYSTEM REFRESH IO CALIBRATION opt_storage_name opt_calibrati malloc_non_terminal_node($$, result->malloc_pool_, T_REFRESH_IO_CALIBRATION, 3, $6, $7, NULL); } | +alter_with_opt_hint SYSTEM REFRESH FULLTEXT DICT relation_name +{ + (void)($1); + ParseNode *dict_relation = NULL; + malloc_non_terminal_node(dict_relation, result->malloc_pool_, T_RELATION_FACTOR, 2, NULL, $6); + malloc_non_terminal_node($$, result->malloc_pool_, T_REFRESH_FULLTEXT_DICT, 1, dict_relation); +} +| +alter_with_opt_hint SYSTEM REFRESH FULLTEXT DICT relation_name '.' relation_name +{ + (void)($1); + ParseNode *dict_relation = NULL; + malloc_non_terminal_node(dict_relation, result->malloc_pool_, T_RELATION_FACTOR, 2, $6, $8); + malloc_non_terminal_node($$, result->malloc_pool_, T_REFRESH_FULLTEXT_DICT, 1, dict_relation); +} +| +alter_with_opt_hint SYSTEM REFRESH FULLTEXT DICT id_dot_id +{ + (void)($1); + ParseNode *dict_relation = NULL; + malloc_non_terminal_node(dict_relation, result->malloc_pool_, T_RELATION_FACTOR, 2, + $6->children_[0], $6->children_[1]); + malloc_non_terminal_node($$, result->malloc_pool_, T_REFRESH_FULLTEXT_DICT, 1, dict_relation); +} +| alter_with_opt_hint SYSTEM opt_set alter_system_set_parameter_actions { (void)($1); @@ -22176,6 +22206,8 @@ ACCESS_INFO | DEMAND | DIAGNOSTICS | DICT_TABLE +| DICT +| FULLTEXT_DICT | DIFF | DIRECTORY | DISABLE diff --git a/src/sql/printer/ob_schema_printer.cpp b/src/sql/printer/ob_schema_printer.cpp index 7fd3092d8..106e40424 100644 --- a/src/sql/printer/ob_schema_printer.cpp +++ b/src/sql/printer/ob_schema_printer.cpp @@ -1516,6 +1516,12 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t SHARE_SCHEMA_LOG(WARN, "fail to print collate", K(ret), K(table_schema)); } } + if (OB_SUCC(ret) && !is_for_table_status && !is_index_tbl + && !is_no_table_options(sql_mode) && table_schema.is_fulltext_dict()) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "FULLTEXT_DICT = 'Y' "))) { + SHARE_SCHEMA_LOG(WARN, "fail to print fulltext dictionary option", K(ret), K(table_schema)); + } + } if (OB_SUCC(ret) && table_schema.is_fts_index() && !is_no_key_options(sql_mode) && !table_schema.get_parser_name_str().empty()) { if (OB_FAIL(ObFtsIndexSchemaPrinter::print_fts_parser_info(table_schema, strict_compat_, buf, buf_len, pos))) { diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index e6cc5e192..d024d5064 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -1719,6 +1719,49 @@ int ObAlterSystemResolverUtil::get_and_verify_tenant_name( return ret; } +int ObRefreshFulltextDictResolver::resolve(const ParseNode &parse_tree) +{ + int ret = OB_SUCCESS; + const ParseNode *relation = nullptr; + ObSQLSessionInfo *session = params_.session_info_; + if (T_REFRESH_FULLTEXT_DICT != parse_tree.type_ + || parse_tree.num_child_ != 1 + || OB_ISNULL(parse_tree.children_) + || OB_ISNULL(relation = parse_tree.children_[0]) + || relation->num_child_ < 2 + || OB_ISNULL(relation->children_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid REFRESH FULLTEXT DICT parse tree", K(ret), K(parse_tree.type_)); + } else if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + } else { + ObRefreshFulltextDictStmt *stmt = create_stmt(); + if (OB_ISNULL(stmt)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else if (OB_ISNULL(relation->children_[1])) { + ret = OB_INVALID_ARGUMENT; + } else { + ObString database_name = session->get_database_name(); + if (OB_NOT_NULL(relation->children_[0])) { + database_name.assign_ptr(relation->children_[0]->str_value_, + relation->children_[0]->str_len_); + } + ObString table_name(relation->children_[1]->str_len_, + relation->children_[1]->str_value_); + if (database_name.empty()) { + ret = OB_ERR_NO_DB_SELECTED; + } else if (table_name.empty()) { + ret = OB_INVALID_ARGUMENT; + } else { + stmt->set_database_name(database_name); + stmt->set_table_name(table_name); + stmt_ = stmt; + } + } + } + return ret; +} + int ObUpgradeVirtualSchemaResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index 6ba4b355d..b70a56304 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -106,6 +106,8 @@ DEF_SIMPLE_CMD_RESOLVER(ObReloadGtsResolver); DEF_SIMPLE_CMD_RESOLVER(ObClearMergeErrorResolver); +DEF_SIMPLE_CMD_RESOLVER(ObRefreshFulltextDictResolver); + DEF_SIMPLE_CMD_RESOLVER(ObUpgradeVirtualSchemaResolver); DEF_SIMPLE_CMD_RESOLVER(ObCancelTaskResolver); diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index 741447f9e..d3f6646a8 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -241,6 +241,25 @@ class ObUpgradeVirtualSchemaStmt : public ObSystemCmdStmt virtual ~ObUpgradeVirtualSchemaStmt() {} }; +class ObRefreshFulltextDictStmt : public ObSystemCmdStmt +{ +public: + ObRefreshFulltextDictStmt() + : ObSystemCmdStmt(stmt::T_REFRESH_FULLTEXT_DICT), database_name_(), table_name_() + {} + virtual ~ObRefreshFulltextDictStmt() {} + + const common::ObString &get_database_name() const { return database_name_; } + const common::ObString &get_table_name() const { return table_name_; } + void set_database_name(const common::ObString &name) { database_name_ = name; } + void set_table_name(const common::ObString &name) { table_name_ = name; } + + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(database_name), K_(table_name)); +private: + common::ObString database_name_; + common::ObString table_name_; +}; + class ObCancelTaskStmt : public ObSystemCmdStmt { public: diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 52084e159..dd11ebcf1 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -26,6 +26,7 @@ #include "share/table/ob_ttl_util.h" #include "rootserver/ob_partition_exchange.h" #include "observer/vector_index/ob_vector_index_util.h" +#include "share/schema/ob_dependency_info.h" namespace oceanbase { @@ -36,6 +37,37 @@ using namespace common; using namespace obcall; namespace sql { +namespace +{ +bool changes_fulltext_dict_structure(const ParseNode &action_list) +{ + bool changes_structure = false; + for (int64_t i = 0; !changes_structure && i < action_list.num_child_; ++i) { + const ParseNode *action = action_list.children_[i]; + if (OB_ISNULL(action)) { + } else if (T_ALTER_TABLE_OPTION == action->type_ + && action->num_child_ > 0 + && OB_NOT_NULL(action->children_) + && OB_NOT_NULL(action->children_[0]) + && T_TABLE_RENAME == action->children_[0]->type_) { + changes_structure = true; + } else if (T_ALTER_COLUMN_OPTION == action->type_ + && action->num_child_ > 0 + && OB_NOT_NULL(action->children_) + && OB_NOT_NULL(action->children_[0])) { + const ObItemType column_action = action->children_[0]->type_; + changes_structure = T_COLUMN_ADD == column_action + || T_COLUMN_ADD_WITH_LOB_PARAMS == column_action + || T_COLUMN_DROP == column_action + || T_COLUMN_CHANGE == column_action + || T_COLUMN_MODIFY == column_action + || T_COLUMN_RENAME == column_action; + } + } + return changes_structure; +} +} // namespace + ObAlterTableResolver::ObAlterTableResolver(ObResolverParams ¶ms) : ObDDLResolver(params), table_schema_(NULL), @@ -177,7 +209,26 @@ int ObAlterTableResolver::resolve(const ParseNode &parse_tree) } //resolve action list if (OB_SUCCESS == ret && NULL != parse_tree.children_[ACTION_LIST]){ - if (OB_FAIL(resolve_action_list(*(parse_tree.children_[ACTION_LIST])))) { + bool has_dictionary_dependency = false; + if (table_schema_->is_fulltext_dict()) { + ObArray dependencies; + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(ObDependencyInfo::collect_dep_infos( + table_schema_->get_table_id(), *GCTX.sql_proxy_, dependencies))) { + LOG_WARN("fail to collect dictionary dependencies", K(ret), KPC(table_schema_)); + } else { + for (int64_t i = 0; !has_dictionary_dependency && i < dependencies.count(); ++i) { + has_dictionary_dependency = ObObjectType::INDEX == dependencies.at(i).get_dep_obj_type(); + } + } + } + if (OB_SUCC(ret) && has_dictionary_dependency + && changes_fulltext_dict_structure(*(parse_tree.children_[ACTION_LIST]))) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "structural ALTER on a referenced FULLTEXT_DICT table is"); + } else if (OB_FAIL(resolve_action_list(*(parse_tree.children_[ACTION_LIST])))) { SQL_RESV_LOG(WARN, "failed to resolve action list.", K(ret)); } else if (alter_table_bitset_.has_member(obcall::ObAlterTableArg::LOCALITY) && alter_table_bitset_.has_member(obcall::ObAlterTableArg::TABLEGROUP_NAME)) { diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index b92814c4d..f51b8ef52 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -412,6 +412,9 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) // //do nothing //} } + if (OB_SUCC(ret) && OB_FAIL(check_fulltext_dict_table(table_schema))) { + SQL_RESV_LOG(WARN, "invalid fulltext dictionary table", K(ret), K(table_schema)); + } } } @@ -546,6 +549,19 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) } return ret; } + +int ObCreateTableResolver::check_fulltext_dict_table(const ObTableSchema &table_schema) const +{ + int ret = OB_SUCCESS; + if (!table_schema.is_fulltext_dict()) { + } else if (!has_explicit_organization_index()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT table must specify ORGANIZATION INDEX"); + } else if (OB_FAIL(table_schema.check_fulltext_dict_structure())) { + SQL_RESV_LOG(WARN, "invalid fulltext dictionary structure", K(ret), K(table_schema)); + } + return ret; +} // Generate array of primary key column names // Generate an array composed of the indices of uk in index_arg_list // Check if the primary key and unique index are built on exactly the same (including order) columns or column families @@ -1690,6 +1706,8 @@ int ObCreateTableResolver::set_index_option_to_arg() SQL_RESV_LOG(WARN, "allocator is null.", K(ret)); } else { index_arg_.index_option_.block_size_ = block_size_; + index_arg_.database_name_ = database_name_; + index_arg_.table_name_ = table_name_; if (OB_FAIL(ob_write_string(*allocator_, compress_method_, index_arg_.index_option_.compress_method_))) { SQL_RESV_LOG(WARN, "set compress func name failed", K(ret)); diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.h b/src/sql/resolver/ddl/ob_create_table_resolver.h index ee99f9d61..04d099416 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver.h @@ -124,6 +124,7 @@ class ObCreateTableResolver: public ObCreateTableResolverBase uint64_t gen_column_group_id(); int add_inner_index_for_heap_gtt(); int check_max_row_data_length(const ObTableSchema &table_schema); + int check_fulltext_dict_table(const ObTableSchema &table_schema) const; int set_default_micro_index_clustered_(share::schema::ObTableSchema &table_schema); int resolve_primary_key_node_in_heap_table(const ParseNode *element, common::ObArray &stats, ObSEArray &resolved_cols); diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index 50fd7c851..54d10b13d 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -362,6 +362,7 @@ int ObCreateTableResolverBase::resolve_column_group(const ParseNode *cg_node) int ObCreateTableResolverBase::resolve_table_organization(common::ObServerConfig *tenant_config, ParseNode *node) { int ret = OB_SUCCESS; + has_explicit_organization_index_ = false; // get the table organization from the tenant config { const char *ptr = NULL; @@ -401,6 +402,7 @@ int ObCreateTableResolverBase::resolve_table_organization(common::ObServerConfig table_organization_ = ObTableOrganizationType::OB_HEAP_ORGANIZATION; } else if (T_ORGANIZATION_INDEX == option_node->children_[0]->type_) { table_organization_ = ObTableOrganizationType::OB_INDEX_ORGANIZATION; + has_explicit_organization_index_ = true; } } } else if (stmt_->get_stmt_type() == stmt::T_ALTER_TABLE) { diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 9ff8ace97..02ebdcacc 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -113,6 +113,7 @@ ObDDLResolver::ObDDLResolver(ObResolverParams ¶ms) mocked_external_table_column_ids_(), index_params_(), table_organization_(ObTableOrganizationType::OB_ORGANIZATION_INVALID), + has_explicit_organization_index_(false), mv_refresh_dop_(0), vec_column_name_(), vec_index_type_(INDEX_TYPE_MAX), @@ -1643,6 +1644,25 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } break; } + case T_FULLTEXT_DICT: { + if (OB_ISNULL(option_node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "FULLTEXT_DICT value is null", K(ret)); + } else if (stmt::T_CREATE_TABLE != stmt_->get_stmt_type() || is_index_option) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "FULLTEXT_DICT outside CREATE TABLE"); + } else { + const ObString value(static_cast(option_node->children_[0]->str_len_), + option_node->children_[0]->str_value_); + if (0 != value.case_compare("Y")) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "FULLTEXT_DICT must be 'Y'"); + } else { + static_cast(stmt_)->get_create_table_arg().schema_.set_fulltext_dict(); + } + } + break; + } case T_TABLEGROUP: { if (!is_index_option) { if (OB_ISNULL(option_node->children_[0])) { diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 99c75aa9d..70773b5e9 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -987,6 +987,7 @@ class ObDDLResolver : public ObStmtResolver bool &is_prefix); bool is_support_split_index_key(const INDEX_KEYNAME index_keyname); bool is_organization_set_to_heap() { return table_organization_ == ObTableOrganizationType::OB_HEAP_ORGANIZATION; } + bool has_explicit_organization_index() const { return has_explicit_organization_index_; } int64_t block_size_; int64_t consistency_level_; INDEX_TYPE index_scope_; @@ -1056,6 +1057,7 @@ class ObDDLResolver : public ObStmtResolver common::ObBitSet<> mocked_external_table_column_ids_; common::ObString index_params_; ObTableOrganizationType table_organization_; + bool has_explicit_organization_index_; int64_t mv_refresh_dop_; common::ObString vec_column_name_; ObIndexType vec_index_type_; diff --git a/src/sql/resolver/ddl/ob_fts_index_builder_util.cpp b/src/sql/resolver/ddl/ob_fts_index_builder_util.cpp index bf952b346..1bb1d1690 100644 --- a/src/sql/resolver/ddl/ob_fts_index_builder_util.cpp +++ b/src/sql/resolver/ddl/ob_fts_index_builder_util.cpp @@ -25,6 +25,10 @@ #include "rootserver/ob_root_service.h" #include "plugin/sys/ob_plugin_helper.h" #include "share/schema/ob_schema_utils.h" // relocated-definition owner +#include "share/schema/ob_dependency_info.h" +#include "storage/fts/ob_fts_plugin_helper.h" +#include "storage/fts/dict/ob_ft_dict_hub.h" +#include "storage/ddl/ob_ddl_lock.h" namespace oceanbase { @@ -36,6 +40,83 @@ namespace share { namespace { +int make_qualified_table_name(ObIAllocator &allocator, + const ObString &database_name, + const ObString &table_name, + ObString &qualified_name) +{ + int ret = OB_SUCCESS; + const int64_t length = database_name.length() + 1 + table_name.length(); + char *buffer = nullptr; + if (OB_ISNULL(buffer = static_cast(allocator.alloc(length)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + MEMCPY(buffer, database_name.ptr(), database_name.length()); + buffer[database_name.length()] = '.'; + MEMCPY(buffer + database_name.length() + 1, table_name.ptr(), table_name.length()); + qualified_name.assign_ptr(buffer, length); + } + return ret; +} + +int qualify_ik_dictionary_table(storage::ObFTParserJsonProps &properties, + const ObString &property_name, + const ObString &database_name, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + ObString table_name; + if (property_name == storage::ObFTSLiteral::CONFIG_NAME_DICT_TABLE) { + ret = properties.config_get_dict_table(table_name); + } else if (property_name == storage::ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE) { + ret = properties.config_get_stopword_table(table_name); + } else if (property_name == storage::ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE) { + ret = properties.config_get_quantifier_table(table_name); + } else { + ret = OB_INVALID_ARGUMENT; + } + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else if (OB_SUCC(ret) && nullptr == table_name.find('.')) { + ObString qualified_name; + if (database_name.empty()) { + ret = OB_ERR_NO_DB_SELECTED; + LOG_WARN("cannot qualify dictionary table without a database", K(ret), K(property_name), K(table_name)); + } else if (OB_FAIL(make_qualified_table_name( + allocator, database_name, table_name, qualified_name))) { + LOG_WARN("fail to qualify dictionary table", K(ret), K(property_name), K(table_name)); + } else if (property_name == storage::ObFTSLiteral::CONFIG_NAME_DICT_TABLE) { + ret = properties.config_set_dict_table(qualified_name); + } else if (property_name == storage::ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE) { + ret = properties.config_set_stopword_table(qualified_name); + } else { + ret = properties.config_set_quantifier_table(qualified_name); + } + } + return ret; +} + +int qualify_ik_dictionary_tables(storage::ObFTParserJsonProps &properties, + const ObString &database_name, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(qualify_ik_dictionary_table(properties, + storage::ObFTSLiteral::CONFIG_NAME_DICT_TABLE, + database_name, + allocator))) { + } else if (OB_FAIL(qualify_ik_dictionary_table(properties, + storage::ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE, + database_name, + allocator))) { + } else if (OB_FAIL(qualify_ik_dictionary_table(properties, + storage::ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE, + database_name, + allocator))) { + } + return ret; +} + bool get_function_args(const ObString &expr_string, const ObString &func_name, ObString &args) @@ -2118,6 +2199,8 @@ int ObFtsIndexBuilderUtil::generate_fts_parser_property( LOG_WARN("fail to init json props", K(ret)); } else if (OB_FAIL(json_props.parse_from_valid_str(arg.index_option_.parser_properties_))) { LOG_WARN("fail to parse json props", K(ret), K(arg.index_option_.parser_properties_)); + } else if (OB_FAIL(qualify_ik_dictionary_tables(json_props, arg.database_name_, allocator))) { + LOG_WARN("fail to qualify IK dictionary tables", K(ret), K(arg.database_name_)); } else if (OB_FAIL(json_props.rebuild_props_for_ddl(arg.index_option_.parser_name_, collation_type, true))) { @@ -2157,15 +2240,15 @@ int ObFtsIndexBuilderUtil::try_load_and_lock_dictionary_tables( { int ret = OB_SUCCESS; if (index_schema.is_fts_index_aux() || index_schema.is_fts_doc_word_aux()) { - bool need_to_load_dic = false; - ObCharsetType charset_type = ObCharsetType::CHARSET_INVALID; - const ObString &parser_name = index_schema.get_parser_name(); + const ObString &parser_name = index_schema.get_parser_name_str(); + storage::ObFTParser parser; ObTableSchema::const_column_iterator tmp_begin = index_schema.column_begin(); ObTableSchema::const_column_iterator tmp_end = index_schema.column_end(); - if (OB_FAIL(check_need_to_load_dic(parser_name, need_to_load_dic))) { - LOG_WARN("fail to check need to load dic", K(ret), K(parser_name), K(need_to_load_dic)); - } else if (need_to_load_dic) { + if (OB_FAIL(parser.parse_from_str(parser_name.ptr(), parser_name.length()))) { + LOG_WARN("fail to parse fulltext parser name", K(ret), K(parser_name)); + } else if (!is_need_dictionary(ObString::make_string(parser.get_parser_name().str()))) { + } else { for (; OB_SUCC(ret) && tmp_begin != tmp_end; tmp_begin++) { ObColumnSchemaV2 *col = (*tmp_begin); if (OB_ISNULL(col)) { @@ -2199,6 +2282,99 @@ int ObFtsIndexBuilderUtil::try_load_and_lock_dictionary_tables( LOG_WARN("fail to lock all dictionaries", K(ret), K(1UL), K(dic_loader_handle)); } } + + if (OB_SUCC(ret) && !index_schema.get_parser_property_str().empty()) { + storage::ObFTParserJsonProps properties; + ObSchemaGetterGuard schema_guard; + storage::ObFTDictHub *dict_hub = nullptr; + if (OB_FAIL(properties.init())) { + LOG_WARN("fail to initialize parser properties", K(ret)); + } else if (OB_FAIL(properties.parse_from_valid_str(index_schema.get_parser_property_str()))) { + LOG_WARN("fail to parse parser properties", K(ret), K(index_schema)); + } else if (OB_ISNULL(GCTX.root_service_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(GCTX.root_service_->get_schema_service().get_tenant_schema_guard(schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(storage::ObFTParsePluginData::instance().get_dict_hub(dict_hub))) { + LOG_WARN("fail to get dictionary hub", K(ret)); + } + + const char *property_names[] = {storage::ObFTSLiteral::CONFIG_NAME_DICT_TABLE, + storage::ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE, + storage::ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE}; + const char *builtin_names[] = {storage::ObFTSLiteral::FT_DEFAULT_IK_DICT_UTF8_TABLE, + storage::ObFTSLiteral::FT_DEFAULT_IK_STOPWORD_UTF8_TABLE, + storage::ObFTSLiteral::FT_DEFAULT_IK_QUANTIFIER_UTF8_TABLE}; + const char *reasons[] = {"IK_MAIN_DICTIONARY", "IK_STOPWORD_DICTIONARY", "IK_QUANTIFIER_DICTIONARY"}; + for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(property_names); ++i) { + ObString full_table_name; + if (0 == ObString(property_names[i]).compare(storage::ObFTSLiteral::CONFIG_NAME_DICT_TABLE)) { + ret = properties.config_get_dict_table(full_table_name); + } else if (0 == ObString(property_names[i]).compare(storage::ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE)) { + ret = properties.config_get_stopword_table(full_table_name); + } else { + ret = properties.config_get_quantifier_table(full_table_name); + } + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else if (OB_SUCC(ret) && 0 != full_table_name.case_compare(builtin_names[i])) { + const char *dot = full_table_name.find('.'); + const ObTableSchema *dict_table_schema = nullptr; + if (OB_ISNULL(dot)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("dictionary table must be qualified", K(ret), K(full_table_name)); + } else { + const ObString database_name(dot - full_table_name.ptr(), full_table_name.ptr()); + const ObString table_name(full_table_name.length() - (dot - full_table_name.ptr()) - 1, + dot + 1); + if (OB_FAIL(schema_guard.get_table_schema(database_name, + table_name, + false, + dict_table_schema))) { + LOG_WARN("fail to resolve dictionary table", K(ret), K(full_table_name)); + } else if (OB_ISNULL(dict_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + } else if (!dict_table_schema->is_fulltext_dict()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "referenced table is not a FULLTEXT_DICT table"); + } else if (OB_FAIL(dict_table_schema->check_fulltext_dict_structure())) { + LOG_WARN("invalid fulltext dictionary structure", K(ret), K(full_table_name)); + } else if (OB_ISNULL(dict_hub)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(storage::ObDDLLock::lock_table_in_trans( + *dict_table_schema, + transaction::tablelock::SHARE, + trans))) { + LOG_WARN("fail to lock user dictionary table", K(ret), K(full_table_name)); + } else if (OB_FAIL(dict_hub->refresh_user_dict(database_name, + table_name, + dict_table_schema->get_table_id()))) { + LOG_WARN("fail to reload user dictionary for fulltext index", + K(ret), K(full_table_name)); + } else { + ObDependencyInfo dependency; + dependency.set_dep_obj_id(index_schema.get_table_id()); + dependency.set_dep_obj_type(ObObjectType::INDEX); + dependency.set_order(i); + dependency.set_dep_timestamp(-1); + dependency.set_ref_obj_id(dict_table_schema->get_table_id()); + dependency.set_ref_obj_type(ObObjectType::TABLE); + dependency.set_ref_timestamp(dict_table_schema->get_schema_version()); + dependency.set_dep_obj_owner_id(index_schema.get_data_table_id()); + dependency.set_property(i + 1); + dependency.set_schema_version(index_schema.get_schema_version()); + if (OB_FAIL(dependency.set_dep_reason(ObString::make_string(reasons[i])))) { + LOG_WARN("fail to set dependency reason", K(ret)); + } else if (OB_FAIL(dependency.set_ref_obj_name(table_name))) { + LOG_WARN("fail to set dependency object name", K(ret)); + } else if (OB_FAIL(dependency.insert_schema_object_dependency(trans))) { + LOG_WARN("fail to persist dictionary dependency", K(ret), K(dependency)); + } + } + } + } + } + } } } return ret; diff --git a/src/sql/resolver/ddl/ob_rename_table_resolver.cpp b/src/sql/resolver/ddl/ob_rename_table_resolver.cpp index 4def71360..1b78ac113 100644 --- a/src/sql/resolver/ddl/ob_rename_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_rename_table_resolver.cpp @@ -16,6 +16,8 @@ #define USING_LOG_PREFIX SQL_RESV #include "sql/resolver/ddl/ob_rename_table_resolver.h" +#include "share/schema/ob_dependency_info.h" +#include "share/ob_server_struct.h" namespace oceanbase { @@ -126,7 +128,24 @@ int ObRenameTableResolver::resolve_rename_action(const ParseNode &rename_action_ if (OB_FAIL(rename_table_stmt->add_rename_table_item(rename_table_item))) { LOG_WARN("failed to add rename table item", K(rename_table_item), K(ret)); } else if (OB_NOT_NULL(table_schema)) { - if (table_schema->is_mlog_table()) { + bool has_dictionary_dependency = false; + if (table_schema->is_fulltext_dict()) { + ObArray dependencies; + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(ObDependencyInfo::collect_dep_infos( + table_schema->get_table_id(), *GCTX.sql_proxy_, dependencies))) { + LOG_WARN("fail to collect dictionary dependencies", K(ret), KPC(table_schema)); + } else { + for (int64_t i = 0; !has_dictionary_dependency && i < dependencies.count(); ++i) { + has_dictionary_dependency = ObObjectType::INDEX == dependencies.at(i).get_dep_obj_type(); + } + } + } + if (OB_SUCC(ret) && has_dictionary_dependency) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "renaming a referenced FULLTEXT_DICT table is"); + } else if (table_schema->is_mlog_table()) { ret = OB_NOT_SUPPORTED; LOG_WARN("rename materialized view log is not supported", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename materialized view log is"); diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 9a50e3a11..085caa07f 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -389,6 +389,10 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(ClearMergeError); break; } + case T_REFRESH_FULLTEXT_DICT: { + REGISTER_STMT_RESOLVER(RefreshFulltextDict); + break; + } case T_CREATE_CATALOG: case T_ALTER_CATALOG: case T_DROP_CATALOG: diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 4fc07a22b..15f481093 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -114,6 +114,7 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_SET_NAMES, no_priv_needed, 113) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CLEAR_LOCATION_CACHE, get_sys_tenant_alter_system_priv, 114) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_RELOAD_GTS, get_sys_tenant_alter_system_priv, 115) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CLEAR_MERGE_ERROR, get_sys_tenant_alter_system_priv, 119) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_REFRESH_FULLTEXT_DICT, get_sys_tenant_alter_system_priv, 130) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_UPGRADE_VIRTUAL_SCHEMA, get_sys_tenant_alter_system_priv, 121) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_EMPTY_QUERY, no_priv_needed, 123) OB_STMT_TYPE_DEF(T_CREATE_OUTLINE, get_create_outline_stmt_need_privs, 124, ACTION_TYPE_CREATE_OUTLINE) diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 4bc44dad3..5c1d75205 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -315,6 +315,7 @@ ob_set_subtarget(ob_storage ik fts/dict/ob_ft_dict_hub.cpp fts/dict/ob_ft_range_dict.cpp fts/dict/ob_ft_dict_table_iter.cpp + fts/dict/ob_ft_user_dict.cpp fts/dict/ob_ft_cache.cpp ) diff --git a/src/storage/fts/dict/ob_ft_dict_hub.cpp b/src/storage/fts/dict/ob_ft_dict_hub.cpp index b8122a0ff..67880391c 100644 --- a/src/storage/fts/dict/ob_ft_dict_hub.cpp +++ b/src/storage/fts/dict/ob_ft_dict_hub.cpp @@ -36,6 +36,8 @@ int ObFTDictHub::init() LOG_WARN("init dict map failed", K(ret)); } else if (OB_FAIL(rw_dict_lock_.init(K_MAX_DICT_BUCKET))) { LOG_WARN("init dict lock failed", K(ret)); + } else if (OB_FAIL(user_dict_manager_.init())) { + LOG_WARN("init user dictionary manager failed", K(ret)); } else { is_inited_ = true; } @@ -45,6 +47,7 @@ int ObFTDictHub::init() int ObFTDictHub::destroy() { int ret = OB_SUCCESS; + user_dict_manager_.destroy(); is_inited_ = false; return ret; } diff --git a/src/storage/fts/dict/ob_ft_dict_hub.h b/src/storage/fts/dict/ob_ft_dict_hub.h index d8de9d172..dabad451d 100644 --- a/src/storage/fts/dict/ob_ft_dict_hub.h +++ b/src/storage/fts/dict/ob_ft_dict_hub.h @@ -20,6 +20,7 @@ #include "lib/charset/ob_charset.h" #include "lib/lock/ob_bucket_lock.h" #include "storage/fts/dict/ob_ft_dict_def.h" +#include "storage/fts/dict/ob_ft_user_dict.h" namespace oceanbase { @@ -96,7 +97,7 @@ class ObFTCacheRangeContainer; class ObFTDictHub { public: - ObFTDictHub() : is_inited_(false), dict_map_(), rw_dict_lock_() {} + ObFTDictHub() : is_inited_(false), dict_map_(), rw_dict_lock_(), user_dict_manager_() {} ~ObFTDictHub() {} int init(); @@ -106,6 +107,14 @@ class ObFTDictHub int build_cache(const ObFTDictDesc &desc, ObFTCacheRangeContainer &container); int load_cache(const ObFTDictDesc &desc, ObFTCacheRangeContainer &container); + int get_user_dict(const common::ObString &table_name, ObFTUserDictHandle &handle) + { return user_dict_manager_.get_dict(table_name, handle); } + int refresh_user_dict(const common::ObString &table_name) + { return user_dict_manager_.refresh(table_name); } + int refresh_user_dict(const common::ObString &database_name, + const common::ObString &table_name, + const uint64_t table_id) + { return user_dict_manager_.refresh(database_name, table_name, table_id); } private: int get_dict_info(const ObFTDictInfoKey &key, ObFTDictInfo &info); @@ -118,6 +127,7 @@ class ObFTDictHub // holds info of dict hash::ObHashMap dict_map_; ObBucketLock rw_dict_lock_; + ObFTUserDictManager user_dict_manager_; }; } // namespace storage diff --git a/src/storage/fts/dict/ob_ft_user_dict.cpp b/src/storage/fts/dict/ob_ft_user_dict.cpp new file mode 100644 index 000000000..59a24e4d7 --- /dev/null +++ b/src/storage/fts/dict/ob_ft_user_dict.cpp @@ -0,0 +1,436 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX STORAGE_FTS + +#include "storage/fts/dict/ob_ft_user_dict.h" + +#include "common/mysqlclient/ob_mysql_proxy.h" +#include "common/mysqlclient/ob_mysql_result.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log_module.h" +#include "lib/utility/ob_smart_var.h" +#include "share/ob_server_struct.h" +#include "share/schema/ob_schema_getter_guard.h" +#include "share/schema/ob_table_schema.h" +#include "storage/fts/dict/ob_ft_dat_dict.h" +#include "storage/fts/dict/ob_ft_trie.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace common; +using namespace share::schema; + +namespace +{ +int append_quoted_identifier(const ObString &identifier, ObSqlString &sql) +{ + int ret = OB_SUCCESS; + if (identifier.empty()) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(sql.append("`"))) { + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < identifier.length(); ++i) { + if ('`' == identifier[i] && OB_FAIL(sql.append("``"))) { + } else if ('`' != identifier[i] && OB_FAIL(sql.append(identifier.ptr() + i, 1))) { + } + } + if (OB_SUCC(ret) && OB_FAIL(sql.append("`"))) { + } + } + return ret; +} +} // namespace + +ObFTUserDict::ObFTUserDict() + : allocator_(ObMemAttr("FTUserDict")), + reader_(nullptr), + ref_count_(1), + table_id_(OB_INVALID_ID), + word_count_(0) +{ +} + +ObFTUserDict::~ObFTUserDict() +{ + if (nullptr != reader_) { + reader_->~ObFTDATReader(); + reader_ = nullptr; + } + allocator_.reset(); +} + +int ObFTUserDict::build_query(const ObString &database_name, + const ObString &table_name, + ObSqlString &sql) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(sql.append("SELECT word FROM "))) { + } else if (OB_FAIL(append_quoted_identifier(database_name, sql))) { + } else if (OB_FAIL(sql.append("."))) { + } else if (OB_FAIL(append_quoted_identifier(table_name, sql))) { + } else if (OB_FAIL(sql.append(" ORDER BY word"))) { + } + return ret; +} + +int ObFTUserDict::build(const ObString &database_name, + const ObString &table_name, + const uint64_t table_id) +{ + int ret = OB_SUCCESS; + ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; + ObArenaAllocator trie_allocator(ObMemAttr("FTUserTrie")); + ObFTTrie trie(trie_allocator, CS_TYPE_UTF8MB4_BIN); + ObSqlString sql; + if (database_name.empty() || table_name.empty() || OB_INVALID_ID == table_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid user dictionary identity", K(ret), K(database_name), K(table_name), K(table_id)); + } else if (OB_ISNULL(sql_proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", K(ret)); + } else if (OB_FAIL(build_query(database_name, table_name, sql))) { + LOG_WARN("fail to build dictionary query", K(ret), K(database_name), K(table_name)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, result) { + sqlclient::ObMySQLResult *mysql_result = nullptr; + if (OB_FAIL(sql_proxy->read(result, sql.ptr()))) { + LOG_WARN("fail to read user dictionary", K(ret), K(database_name), K(table_name)); + } else if (OB_ISNULL(mysql_result = result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dictionary query returned null result", K(ret)); + } else { + while (OB_SUCC(ret)) { + ObString word; + if (OB_FAIL(mysql_result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to read dictionary row", K(ret)); + } + break; + } else if (OB_FAIL(mysql_result->get_varchar("word", word))) { + LOG_WARN("fail to read dictionary word", K(ret)); + } else if (word.empty()) { + LOG_DEBUG("ignore empty dictionary word", K(table_id)); + } else if (OB_FAIL(trie.insert(word, {}))) { + LOG_WARN("fail to insert dictionary word", K(ret), K(word)); + } else { + ++word_count_; + } + } + } + } + } + + if (OB_SUCC(ret) && word_count_ > 0) { + ObFTDATBuilder builder(allocator_); + ObFTDAT *dat = nullptr; + size_t dat_size = 0; + if (OB_FAIL(builder.init(trie))) { + LOG_WARN("fail to initialize user dictionary DAT", K(ret)); + } else if (OB_FAIL(builder.build_from_trie(trie))) { + LOG_WARN("fail to build user dictionary DAT", K(ret)); + } else if (OB_FAIL(builder.get_mem_block(dat, dat_size))) { + LOG_WARN("fail to get user dictionary DAT", K(ret)); + } else if (OB_ISNULL(dat) || 0 == dat_size) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("empty DAT for nonempty dictionary", K(ret), K(word_count_)); + } else if (OB_ISNULL(reader_ = OB_NEWx(ObFTDATReader, &allocator_, dat))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate user dictionary reader", K(ret)); + } + } + if (OB_SUCC(ret)) { + table_id_ = table_id; + LOG_INFO("loaded user fulltext dictionary", K(database_name), K(table_name), K(table_id_), K(word_count_)); + } + return ret; +} + +int ObFTUserDict::match(const ObString &single_word, ObDATrieHit &hit) const +{ + return match_with_hit(single_word, hit, hit); +} + +int ObFTUserDict::match(const ObString &words, bool &is_match) const +{ + int ret = OB_SUCCESS; + int64_t char_len = 0; + is_match = false; + ObDATrieHit hit(this, 0); + if (nullptr != reader_) { + for (int64_t offset = 0; OB_SUCC(ret) && offset < words.length(); offset += char_len) { + if (OB_FAIL(ObCharset::first_valid_char(CS_TYPE_UTF8MB4_BIN, + words.ptr() + offset, + words.length() - offset, + char_len))) { + LOG_WARN("invalid dictionary lookup string", K(ret), K(words)); + } else if (OB_FAIL(match_with_hit(ObString(char_len, words.ptr() + offset), hit, hit))) { + LOG_WARN("fail to match dictionary word", K(ret)); + } else if (hit.is_match() && offset + char_len == words.length()) { + is_match = true; + break; + } else if (hit.is_unmatch()) { + break; + } + } + } + return ret; +} + +int ObFTUserDict::match_with_hit(const ObString &single_word, + const ObDATrieHit &last_hit, + ObDATrieHit &hit) const +{ + int ret = OB_SUCCESS; + if (nullptr == reader_) { + if (&last_hit != &hit) { + hit = last_hit; + } + hit.set_unmatch(); + } else if (OB_UNLIKELY(last_hit.dict_ != this)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("dictionary hit belongs to another dictionary", K(ret), KP(last_hit.dict_), KP(this)); + } else if (OB_FAIL(reader_->match_with_hit(single_word, last_hit, hit))) { + LOG_WARN("fail to match user dictionary", K(ret)); + } + return ret; +} + +ObFTUserDictHandle &ObFTUserDictHandle::operator=(const ObFTUserDictHandle &other) +{ + if (this != &other) { + reset(); + if (nullptr != other.dict_) { + dict_ = other.dict_; + dict_->inc_ref(); + } + } + return *this; +} + +int ObFTUserDictHandle::set_dict(ObFTUserDict *dict) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(dict)) { + ret = OB_INVALID_ARGUMENT; + } else { + reset(); + dict_ = dict; + dict_->inc_ref(); + } + return ret; +} + +void ObFTUserDictHandle::reset() +{ + if (nullptr != dict_) { + ObFTUserDict *dict = dict_; + dict_ = nullptr; + if (0 == dict->dec_ref()) { + OB_DELETE(ObFTUserDict, ObMemAttr("FTUserDict"), dict); + } + } +} + +int ObFTUserDictManager::init() +{ + int ret = OB_SUCCESS; + static constexpr int64_t BUCKET_COUNT = 128; + if (is_inited_) { + ret = OB_INIT_TWICE; + } else if (OB_FAIL(dict_map_.create(BUCKET_COUNT, "FTUserMap"))) { + LOG_WARN("fail to create user dictionary map", K(ret)); + } else if (OB_FAIL(dict_lock_.init(BUCKET_COUNT))) { + LOG_WARN("fail to initialize user dictionary lock", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +void ObFTUserDictManager::release_dict(ObFTUserDict *dict) +{ + if (nullptr != dict && 0 == dict->dec_ref()) { + OB_DELETE(ObFTUserDict, ObMemAttr("FTUserDict"), dict); + } +} + +void ObFTUserDictManager::destroy() +{ + if (dict_map_.created()) { + for (auto iter = dict_map_.begin(); iter != dict_map_.end(); ++iter) { + release_dict(iter->second); + } + dict_map_.destroy(); + } + is_inited_ = false; +} + +int ObFTUserDictManager::resolve_table(const ObString &full_table_name, + ObString &database_name, + ObString &table_name, + uint64_t &table_id) const +{ + int ret = OB_SUCCESS; + const char *dot = full_table_name.find('.'); + const ObDatabaseSchema *database_schema = nullptr; + const ObTableSchema *table_schema = nullptr; + ObSchemaGetterGuard schema_guard; + table_id = OB_INVALID_ID; + if (OB_ISNULL(dot) || dot == full_table_name.ptr() + || dot == full_table_name.ptr() + full_table_name.length() - 1 + || nullptr != ObString(full_table_name.length() - (dot - full_table_name.ptr()) - 1, + dot + 1).find('.')) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("dictionary table name must be database.table", K(ret), K(full_table_name)); + } else { + database_name.assign_ptr(full_table_name.ptr(), dot - full_table_name.ptr()); + table_name.assign_ptr(dot + 1, full_table_name.length() - (dot - full_table_name.ptr()) - 1); + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.get_database_schema(database_name, database_schema))) { + LOG_WARN("fail to get dictionary database", K(ret), K(database_name)); + } else if (OB_ISNULL(database_schema)) { + ret = OB_ERR_BAD_DATABASE; + } else if (OB_FAIL(schema_guard.get_table_schema(database_schema->get_database_id(), + table_name, + false, + table_schema))) { + LOG_WARN("fail to get dictionary table", K(ret), K(database_name), K(table_name)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + } else if (!table_schema->is_fulltext_dict()) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "referenced table is not a FULLTEXT_DICT table"); + } else if (OB_FAIL(table_schema->check_fulltext_dict_structure())) { + LOG_WARN("invalid fulltext dictionary structure", K(ret), KPC(table_schema)); + } else { + table_id = table_schema->get_table_id(); + } + } + return ret; +} + +int ObFTUserDictManager::build_dict(const ObString &database_name, + const ObString &table_name, + const uint64_t table_id, + ObFTUserDict *&dict) const +{ + int ret = OB_SUCCESS; + dict = OB_NEW(ObFTUserDict, ObMemAttr("FTUserDict")); + if (OB_ISNULL(dict)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else if (OB_FAIL(dict->build(database_name, table_name, table_id))) { + LOG_WARN("fail to build user dictionary", K(ret), K(database_name), K(table_name), K(table_id)); + release_dict(dict); + dict = nullptr; + } + return ret; +} + +int ObFTUserDictManager::get_dict(const ObString &full_table_name, ObFTUserDictHandle &handle) +{ + int ret = OB_SUCCESS; + ObString database_name; + ObString table_name; + uint64_t table_id = OB_INVALID_ID; + ObFTUserDict *dict = nullptr; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(resolve_table(full_table_name, database_name, table_name, table_id))) { + LOG_WARN("fail to resolve user dictionary", K(ret), K(full_table_name)); + } else { + { + ObBucketHashRLockGuard guard(dict_lock_, table_id); + ret = dict_map_.get_refactored(table_id, dict); + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else if (OB_SUCC(ret) && OB_FAIL(handle.set_dict(dict))) { + LOG_WARN("fail to pin user dictionary", K(ret), K(table_id)); + } + } + if (OB_SUCC(ret) && !handle.is_valid()) { + ObFTUserDict *new_dict = nullptr; + if (OB_FAIL(build_dict(database_name, table_name, table_id, new_dict))) { + } else { + ObBucketHashWLockGuard guard(dict_lock_, table_id); + if (OB_FAIL(dict_map_.get_refactored(table_id, dict))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = dict_map_.set_refactored(table_id, new_dict); + dict = new_dict; + new_dict = nullptr; + } + } + if (OB_SUCC(ret) && OB_FAIL(handle.set_dict(dict))) { + LOG_WARN("fail to pin loaded user dictionary", K(ret), K(table_id)); + } + } + release_dict(new_dict); + } + } + return ret; +} + +int ObFTUserDictManager::refresh(const ObString &full_table_name) +{ + int ret = OB_SUCCESS; + ObString database_name; + ObString table_name; + uint64_t table_id = OB_INVALID_ID; + if (OB_FAIL(resolve_table(full_table_name, database_name, table_name, table_id))) { + LOG_WARN("fail to resolve dictionary for refresh", K(ret), K(full_table_name)); + } else if (OB_FAIL(refresh(database_name, table_name, table_id))) { + LOG_WARN("fail to refresh dictionary", K(ret), K(full_table_name), K(table_id)); + } + return ret; +} + +int ObFTUserDictManager::refresh(const ObString &database_name, + const ObString &table_name, + const uint64_t table_id) +{ + int ret = OB_SUCCESS; + ObFTUserDict *new_dict = nullptr; + ObFTUserDict *old_dict = nullptr; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(build_dict(database_name, table_name, table_id, new_dict))) { + } else { + ObBucketHashWLockGuard guard(dict_lock_, table_id); + int get_ret = dict_map_.get_refactored(table_id, old_dict); + if (OB_SUCCESS != get_ret && OB_HASH_NOT_EXIST != get_ret) { + ret = get_ret; + old_dict = nullptr; + } else if (OB_FAIL(dict_map_.set_refactored(table_id, new_dict, 1))) { + LOG_WARN("fail to publish refreshed dictionary", K(ret), K(table_id)); + old_dict = nullptr; + } else { + new_dict = nullptr; + } + } + release_dict(old_dict); + release_dict(new_dict); + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/fts/dict/ob_ft_user_dict.h b/src/storage/fts/dict/ob_ft_user_dict.h new file mode 100644 index 000000000..c82779ea1 --- /dev/null +++ b/src/storage/fts/dict/ob_ft_user_dict.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OCEANBASE_STORAGE_FTS_DICT_OB_FT_USER_DICT_H_ +#define OCEANBASE_STORAGE_FTS_DICT_OB_FT_USER_DICT_H_ + +#include "lib/allocator/page_arena.h" +#include "lib/atomic/ob_atomic.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/lock/ob_bucket_lock.h" +#include "lib/string/ob_sql_string.h" +#include "storage/fts/dict/ob_ft_dict.h" + +namespace oceanbase +{ +namespace storage +{ + +template class ObFTDATReader; + +class ObFTUserDict final : public ObIFTDict +{ +public: + ObFTUserDict(); + ~ObFTUserDict() override; + + int build(const common::ObString &database_name, + const common::ObString &table_name, + const uint64_t table_id); + int init() override { return OB_SUCCESS; } + int match(const common::ObString &single_word, ObDATrieHit &hit) const override; + int match(const common::ObString &words, bool &is_match) const override; + int match_with_hit(const common::ObString &single_word, + const ObDATrieHit &last_hit, + ObDATrieHit &hit) const override; + + int64_t inc_ref() { return ATOMIC_AAF(&ref_count_, 1); } + int64_t dec_ref() { return ATOMIC_SAF(&ref_count_, 1); } + uint64_t get_table_id() const { return table_id_; } + int64_t get_word_count() const { return word_count_; } + +private: + int build_query(const common::ObString &database_name, + const common::ObString &table_name, + common::ObSqlString &sql) const; + +private: + common::ObArenaAllocator allocator_; + ObFTDATReader *reader_; + int64_t ref_count_; + uint64_t table_id_; + int64_t word_count_; + DISALLOW_COPY_AND_ASSIGN(ObFTUserDict); +}; + +class ObFTUserDictHandle final +{ +public: + ObFTUserDictHandle() : dict_(nullptr) {} + ObFTUserDictHandle(const ObFTUserDictHandle &other) : dict_(nullptr) { *this = other; } + ~ObFTUserDictHandle() { reset(); } + ObFTUserDictHandle &operator=(const ObFTUserDictHandle &other); + + int set_dict(ObFTUserDict *dict); + void reset(); + bool is_valid() const { return nullptr != dict_; } + ObIFTDict *get_dict() const { return dict_; } + +private: + ObFTUserDict *dict_; +}; + +class ObFTUserDictManager final +{ +public: + ObFTUserDictManager() : is_inited_(false), dict_map_(), dict_lock_() {} + ~ObFTUserDictManager() { destroy(); } + + int init(); + void destroy(); + int get_dict(const common::ObString &full_table_name, ObFTUserDictHandle &handle); + int refresh(const common::ObString &full_table_name); + int refresh(const common::ObString &database_name, + const common::ObString &table_name, + const uint64_t table_id); + +private: + int resolve_table(const common::ObString &full_table_name, + common::ObString &database_name, + common::ObString &table_name, + uint64_t &table_id) const; + int build_dict(const common::ObString &database_name, + const common::ObString &table_name, + const uint64_t table_id, + ObFTUserDict *&dict) const; + static void release_dict(ObFTUserDict *dict); + +private: + bool is_inited_; + common::hash::ObHashMap dict_map_; + common::ObBucketLock dict_lock_; + DISALLOW_COPY_AND_ASSIGN(ObFTUserDictManager); +}; + +} // namespace storage +} // namespace oceanbase + +#endif // OCEANBASE_STORAGE_FTS_DICT_OB_FT_USER_DICT_H_ diff --git a/src/storage/fts/ik/ob_fast_list.h b/src/storage/fts/ik/ob_fast_list.h new file mode 100644 index 000000000..c381e44ee --- /dev/null +++ b/src/storage/fts/ik/ob_fast_list.h @@ -0,0 +1,246 @@ +/** + * Copyright (c) 2024 OceanBase + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ +#define _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ + +#include "storage/fts/ik/ob_fast_segment_array.h" + +namespace oceanbase +{ +namespace storage +{ + +// A list whose POD-like nodes come from reusable segmented storage. Individual +// element destructors are intentionally not invoked; the pool is reclaimed in +// bulk when the parser is reset. +template +class ObFastList +{ + struct Node + { + Node() : next(nullptr), prev(nullptr), value() {} + explicit Node(const T &v) : next(nullptr), prev(nullptr), value(v) {} + + Node *next; + Node *prev; + T value; + }; + + struct NodeHolder + { + operator Node *() { return reinterpret_cast(this); } + operator const Node *() const { return reinterpret_cast(this); } + + Node *next; + Node *prev; + }; + +public: + class iterator + { + public: + iterator() : node_(nullptr) {} + explicit iterator(Node *node) : node_(node) {} + + T &operator*() const { return node_->value; } + T *operator->() const { return &node_->value; } + iterator &operator++() + { + node_ = nullptr == node_ ? nullptr : node_->next; + return *this; + } + iterator operator++(int) + { + iterator tmp(*this); + ++(*this); + return tmp; + } + iterator &operator--() + { + node_ = nullptr == node_ ? nullptr : node_->prev; + return *this; + } + iterator operator--(int) + { + iterator tmp(*this); + --(*this); + return tmp; + } + bool operator==(const iterator &other) const { return node_ == other.node_; } + bool operator!=(const iterator &other) const { return node_ != other.node_; } + + private: + friend class ObFastList; + Node *node_; + }; + + class const_iterator + { + public: + const_iterator() : node_(nullptr) {} + explicit const_iterator(const Node *node) : node_(const_cast(node)) {} + const_iterator(const iterator &it) : node_(it.node_) {} + + const T &operator*() const { return node_->value; } + const T *operator->() const { return &node_->value; } + const_iterator &operator++() + { + node_ = nullptr == node_ ? nullptr : node_->next; + return *this; + } + const_iterator operator++(int) + { + const_iterator tmp(*this); + ++(*this); + return tmp; + } + const_iterator &operator--() + { + node_ = nullptr == node_ ? nullptr : node_->prev; + return *this; + } + const_iterator operator--(int) + { + const_iterator tmp(*this); + --(*this); + return tmp; + } + bool operator==(const const_iterator &other) const { return node_ == other.node_; } + bool operator!=(const const_iterator &other) const { return node_ != other.node_; } + + private: + friend class ObFastList; + Node *node_; + }; + +public: + explicit ObFastList(ObIAllocator &allocator) : pool_(allocator), size_(0) + { + root_.next = root_; + root_.prev = root_; + } + + ~ObFastList() { reset(); } + + void reset() + { + clear_(); + pool_.reset(); + } + + void reuse() + { + root_.next = root_; + root_.prev = root_; + size_ = 0; + pool_.reuse(); + } + + bool empty() const { return 0 == size_; } + int64_t size() const { return size_; } + + T &get_first() { return root_.next->value; } + const T &get_first() const { return root_.next->value; } + T &get_last() { return root_.prev->value; } + const T &get_last() const { return root_.prev->value; } + + iterator begin() { return iterator(root_.next); } + iterator end() { return iterator(root_); } + const_iterator begin() const { return const_iterator(root_.next); } + const_iterator end() const { return const_iterator(root_); } + iterator last() { return iterator(root_.prev); } + const_iterator last() const { return const_iterator(root_.prev); } + + int push_front(const T &value) { return insert_before_(root_.next, value); } + int push_back(const T &value) { return insert_before_(root_, value); } + int insert(iterator pos, const T &value) { return insert_before_(pos.node_, value); } + int insert(const_iterator pos, const T &value) { return insert_before_(pos.node_, value); } + + int pop_front() + { + int ret = OB_SUCCESS; + if (empty()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + remove_node_(root_.next); + } + return ret; + } + + int pop_back() + { + int ret = OB_SUCCESS; + if (empty()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + remove_node_(root_.prev); + } + return ret; + } + +private: + void clear_() + { + Node *cur = root_.next; + while (cur != root_) { + Node *next = cur->next; + cur->next = nullptr; + cur->prev = nullptr; + cur = next; + } + root_.next = root_; + root_.prev = root_; + size_ = 0; + } + + int insert_before_(Node *pos, const T &value) + { + int ret = OB_SUCCESS; + Node *node = nullptr; + if (OB_FAIL(alloc_node_(node, value))) { + } else { + Node *anchor = nullptr == pos ? root_ : pos; + node->next = anchor; + node->prev = anchor->prev; + anchor->prev->next = node; + anchor->prev = node; + ++size_; + } + return ret; + } + + int alloc_node_(Node *&node, const T &value) + { + int ret = OB_SUCCESS; + Node dummy(value); + if (OB_FAIL(pool_.push_back(dummy))) { + } else { + node = &pool_.at(pool_.count() - 1); + } + return ret; + } + + void remove_node_(Node *node) + { + if (nullptr != node && node != root_) { + node->prev->next = node->next; + node->next->prev = node->prev; + node->next = nullptr; + node->prev = nullptr; + --size_; + } + } + +private: + ObFastSegmentArray pool_; + NodeHolder root_; + int64_t size_; +}; + +} // namespace storage +} // namespace oceanbase + +#endif // _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ diff --git a/src/storage/fts/ik/ob_fast_segment_array.h b/src/storage/fts/ik/ob_fast_segment_array.h new file mode 100644 index 000000000..ae85729f2 --- /dev/null +++ b/src/storage/fts/ik/ob_fast_segment_array.h @@ -0,0 +1,187 @@ +/** + * Copyright (c) 2024 OceanBase + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ +#define _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ + +#define USING_LOG_PREFIX STORAGE_FTS + +#include "lib/allocator/ob_allocator.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log_module.h" + +namespace oceanbase +{ +namespace storage +{ + +// Reusable segmented storage for POD-like objects on tokenization hot paths. +// The real block size is rounded up to a power of two so locating an element +// needs only a shift and a mask. +template +class ObFastSegmentArray +{ + static_assert(block_capacity > 2 && block_capacity <= (1 << 30), + "block capacity must be greater than 2 and no greater than (1 << 30)"); + +public: + static constexpr int64_t BLOCK_POINTER_ARRAY_CAPACITY = 64; + static constexpr int64_t BLOCK_CAPACITY_POWER = + 64 - __builtin_clzll(static_cast(block_capacity) - 1); + static constexpr int64_t REAL_BLOCK_CAPACITY = static_cast(1) << BLOCK_CAPACITY_POWER; + static constexpr int64_t BLOCK_LOCATOR = REAL_BLOCK_CAPACITY - 1; + + explicit ObFastSegmentArray( + ObIAllocator &allocator, + int64_t init_block_arr_cap = BLOCK_POINTER_ARRAY_CAPACITY) + : allocator_(allocator), + block_arr_(nullptr), + block_arr_cap_(0), + block_count_(0), + size_(0), + init_block_arr_cap_(init_block_arr_cap > 0 + ? init_block_arr_cap + : BLOCK_POINTER_ARRAY_CAPACITY) + {} + + ~ObFastSegmentArray() { reset(); } + + int push_back(const T &value) + { + int ret = OB_SUCCESS; + const int64_t block_idx = size_ >> BLOCK_CAPACITY_POWER; + if (OB_FAIL(ensure_block_(block_idx))) { + } else { + const int64_t inner_idx = size_ & BLOCK_LOCATOR; + block_arr_[block_idx][inner_idx] = value; + ++size_; + } + return ret; + } + + int alloc(T *&ptr) + { + int ret = OB_SUCCESS; + const int64_t block_idx = size_ >> BLOCK_CAPACITY_POWER; + if (OB_FAIL(ensure_block_(block_idx))) { + } else { + const int64_t inner_idx = size_ & BLOCK_LOCATOR; + ptr = &block_arr_[block_idx][inner_idx]; + ++size_; + } + return ret; + } + + int free_an_obj() + { + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 == size_)) { + ret = OB_ERR_UNEXPECTED; + } else { + --size_; + } + return ret; + } + + const T &at(const int64_t idx) const + { + const int64_t block_idx = idx >> BLOCK_CAPACITY_POWER; + const int64_t inner_idx = idx & BLOCK_LOCATOR; + return block_arr_[block_idx][inner_idx]; + } + + T &at(const int64_t idx) + { + return const_cast(static_cast(*this).at(idx)); + } + + int64_t count() const { return size_; } + bool empty() const { return 0 == size_; } + + void reuse() { size_ = 0; } + + void reset() + { + for (int64_t i = 0; i < block_count_; ++i) { + if (nullptr != block_arr_[i]) { + allocator_.free(block_arr_[i]); + block_arr_[i] = nullptr; + } + } + if (nullptr != block_arr_) { + allocator_.free(block_arr_); + } + block_arr_ = nullptr; + block_arr_cap_ = 0; + block_count_ = 0; + size_ = 0; + } + +private: + int ensure_block_(const int64_t block_idx) + { + int ret = OB_SUCCESS; + if (block_idx >= block_count_) { + if (block_idx >= block_arr_cap_ && OB_FAIL(expand_block_array_(block_idx + 1))) { + LOG_WARN("failed to expand block array", K(ret), K(block_idx)); + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(nullptr == block_arr_[block_idx])) { + void *buf = allocator_.alloc(REAL_BLOCK_CAPACITY * static_cast(sizeof(T))); + if (OB_UNLIKELY(nullptr == buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate segment block", K(ret), K(block_idx)); + } else { + block_arr_[block_idx] = static_cast(buf); + ++block_count_; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected initialized segment block", K(ret), K(block_idx)); + } + } + } + return ret; + } + + int expand_block_array_(const int64_t need_cap) + { + int ret = OB_SUCCESS; + int64_t next_cap = block_arr_cap_ > 0 ? block_arr_cap_ : init_block_arr_cap_; + while (next_cap < need_cap) { + next_cap <<= 1; + } + const int64_t bytes = next_cap * static_cast(sizeof(T *)); + T **next_arr = static_cast(allocator_.alloc(bytes)); + if (OB_ISNULL(next_arr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate segment block pointer array", K(ret), K(next_cap)); + } else { + MEMSET(next_arr, 0, bytes); + if (nullptr != block_arr_) { + MEMCPY(next_arr, block_arr_, block_count_ * static_cast(sizeof(T *))); + allocator_.free(block_arr_); + } + block_arr_ = next_arr; + block_arr_cap_ = next_cap; + } + return ret; + } + +private: + ObIAllocator &allocator_; + T **block_arr_; + int64_t block_arr_cap_; + int64_t block_count_; + int64_t size_; + const int64_t init_block_arr_cap_; + + DISALLOW_COPY_AND_ASSIGN(ObFastSegmentArray); +}; + +} // namespace storage +} // namespace oceanbase + +#endif // _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ diff --git a/src/storage/fts/ik/ob_ik_arbitrator.cpp b/src/storage/fts/ik/ob_ik_arbitrator.cpp index ee4f18ed9..8a529436e 100644 --- a/src/storage/fts/ik/ob_ik_arbitrator.cpp +++ b/src/storage/fts/ik/ob_ik_arbitrator.cpp @@ -36,11 +36,10 @@ int ObIKArbitrator::process(TokenizeContext &ctx) { int ret = OB_SUCCESS; - ObList &tokens = ctx.token_list().tokens(); + ObFastList &tokens = ctx.token_list().tokens(); ObIKTokenChain *chain_need_arbitrate = nullptr; bool use_smart = ctx.is_smart(); - if (OB_FAIL(prepare(ctx))) { - } else if (OB_ISNULL(chain_need_arbitrate = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { + if (OB_ISNULL(chain_need_arbitrate = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed"); } else { @@ -52,7 +51,7 @@ int ObIKArbitrator::process(TokenizeContext &ctx) } else if (OB_FAIL(chain_need_arbitrate->add_token_if_conflict(token, is_add))) { LOG_WARN("add token if conflict failed", K(ret)); } else if (!is_add) { - ObFTSortList::CellIter iter = chain_need_arbitrate->list().tokens().begin(); + ObFTLightSortList::CellIter iter = chain_need_arbitrate->list().tokens().begin(); ObIKTokenChain *judge_result = nullptr; if (chain_need_arbitrate->list().tokens().size() == 1 || !use_smart) { if (OB_FAIL(add_chain(chain_need_arbitrate))) { @@ -121,19 +120,19 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) int64_t char_len = 0; ObIKTokenChain *chain = nullptr; - for (int64_t current = 0; OB_SUCC(ret) && current < ctx.fulltext_len();) { - ObFTCharUtil::CharType type; - // maybe not so good to keep single, check it later - if (OB_FAIL(ObCharset::first_valid_char(ctx.collation(), - ctx.fulltext() + current, - ctx.fulltext_len() - current, - char_len))) { - LOG_WARN("Failed to get next valid char", K(ret)); - } else if (OB_FAIL(ObFTCharUtil::classify_first_char(ctx.collation(), - ctx.fulltext() + current, - char_len, - type))) { - LOG_WARN("Failed to classify first char", K(ret)); + const int64_t buffer_start_cursor = ctx.get_buffer_start_cursor(); + const int64_t buffer_end_cursor = ctx.get_buffer_end_cursor(); + for (int64_t current = buffer_start_cursor; + OB_SUCC(ret) && current < buffer_end_cursor;) { + ObFTCharUtil::CharType type = ObFTCharUtil::CharType::USELESS; + if (OB_FAIL(ObFTCharUtil::classify_first_valid_char(ctx.get_cs_type(), + ctx.fulltext() + current, + ctx.fulltext_len() - current, + ctx.well_formed_len_fn(), + ctx.get_cs(), + char_len, + type))) { + LOG_WARN("failed to classify first valid character", K(ret)); } else if (ObFTCharUtil::CharType::USELESS == type) { current += char_len; // skip useless char } else if (OB_FAIL(chains_.get_refactored(current, chain)) && OB_HASH_NOT_EXIST != ret) { @@ -152,7 +151,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) bool is_ignore = false; if (ObFTCharUtil::CharType::CHINESE == type) { token.type_ = ObIKTokenType::IK_CHINESE_TOKEN; - if (OB_FAIL(ctx.result_list().push_back(token))) { + if (OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to output result to ctx", K(ret)); } } else if (ObFTCharUtil::CharType::OTHER_CJK == type) { @@ -162,7 +161,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) is_ignore))) { LOG_WARN("Failed to check ignore", K(ret)); } else if (!is_ignore && !FALSE_IT(token.type_ = ObIKTokenType::IK_OTHER_CJK_TOKEN) - && OB_FAIL(ctx.result_list().push_back(token))) { + && OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to add token to ctx result", K(ret)); } else { // ignore @@ -180,12 +179,16 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) } else { // output single word between two token while (OB_SUCC(ret) && current < token.offset_) { - if (OB_FAIL(ObCharset::first_valid_char(ctx.collation(), - ctx.fulltext() + current, - ctx.fulltext_len() - current, - char_len))) { - LOG_WARN("Failed to get next valid char, ", K(ret)); - break; + if (OB_FAIL(ObFTCharUtil::classify_first_valid_char(ctx.get_cs_type(), + ctx.fulltext() + current, + ctx.fulltext_len() - current, + ctx.well_formed_len_fn(), + ctx.get_cs(), + char_len, + type))) { + LOG_WARN("failed to classify first valid character", K(ret)); + } else if (ObFTCharUtil::CharType::USELESS == type) { + current += char_len; } else { ObIKToken token; token.offset_ = current; @@ -195,7 +198,9 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) bool is_ignore = false; if (ObFTCharUtil::CharType::CHINESE == type) { token.type_ = ObIKTokenType::IK_CHINESE_TOKEN; - ctx.result_list().push_back(token); + if (OB_FAIL(ctx.get_results().push_back(token))) { + LOG_WARN("failed to add token to result", K(ret)); + } } else if (ObFTCharUtil::CharType::OTHER_CJK == type) { if (OB_FAIL(ObFTCharUtil::is_ignore_single_cjk(ctx.collation(), ctx.fulltext() + current, @@ -204,7 +209,9 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) LOG_WARN("Failed to check ignore", K(ret)); } else if (!is_ignore) { token.type_ = ObIKTokenType::IK_OTHER_CJK_TOKEN; - ctx.result_list().push_back(token); + if (OB_FAIL(ctx.get_results().push_back(token))) { + LOG_WARN("failed to add token to result", K(ret)); + } } else { } } @@ -214,7 +221,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ctx.result_list().push_back(token))) { + } else if (OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to add token to ctx result", K(ret)); } else // output the token @@ -233,14 +240,14 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) int ObIKArbitrator::optimize(TokenizeContext &ctx, ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, int64_t fulltext_len, ObIKTokenChain *&best) { int ret = OB_SUCCESS; ObIKTokenChain *option = nullptr; - ObList conflict_stack(alloc_); + ObList conflict_stack(alloc_); if (OB_ISNULL(option = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -254,7 +261,7 @@ int ObIKArbitrator::optimize(TokenizeContext &ctx, LOG_WARN("Copy best option failed", K(ret)); } else { while (OB_SUCC(ret) && !conflict_stack.empty()) { - ObFTSortList::CellIter iter = conflict_stack.get_last(); + ObFTLightSortList::CellIter iter = conflict_stack.get_last(); conflict_stack.pop_back(); if (OB_FAIL(remove_conflict(*iter, option))) { LOG_WARN("Failed to remove conflict", K(ret)); @@ -286,10 +293,10 @@ int ObIKArbitrator::optimize(TokenizeContext &ctx, } int ObIKArbitrator::try_add_next_words(ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, ObIKTokenChain *option, bool need_conflict, - ObList &conflict_stack) + ObList &conflict_stack) { int ret = OB_SUCCESS; if (OB_ISNULL(chain) || OB_ISNULL(option)) { @@ -324,13 +331,10 @@ int ObIKArbitrator::remove_conflict(const ObIKToken &token, ObIKTokenChain *opti return ret; } -int ObIKArbitrator::prepare(TokenizeContext &ctx) +int ObIKArbitrator::prepare() { int ret = OB_SUCCESS; - - int cal_bucket_num = MAX(ctx.fulltext_len() / 100, 10); - cal_bucket_num = MIN(cal_bucket_num, 100); - if (OB_FAIL(chains_.create(cal_bucket_num, ObMemAttr("IK ARBITRATE")))) { + if (OB_FAIL(chains_.create(HANDLE_SIZE_LIMIT * 2L, ObMemAttr("IK ARBITRATE")))) { LOG_WARN("create chain map failed", K(ret)); } return ret; @@ -357,5 +361,11 @@ ObIKArbitrator::~ObIKArbitrator() chains_.destroy(); alloc_.reset(); } + +void ObIKArbitrator::reuse() +{ + chains_.reuse(); + alloc_.reset_remain_one_page(); +} } // namespace storage } // namespace oceanbase diff --git a/src/storage/fts/ik/ob_ik_arbitrator.h b/src/storage/fts/ik/ob_ik_arbitrator.h index 40837f39f..514d7b8c8 100644 --- a/src/storage/fts/ik/ob_ik_arbitrator.h +++ b/src/storage/fts/ik/ob_ik_arbitrator.h @@ -38,22 +38,23 @@ class ObIKArbitrator int output_result(TokenizeContext &ctx); -private: - int prepare(TokenizeContext &ctx); + int prepare(); + void reuse(); +private: int add_chain(ObIKTokenChain *chain); int optimize(TokenizeContext &ctx, ObIKTokenChain *option, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, int64_t fulltext_len, ObIKTokenChain *&best); int try_add_next_words(ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, ObIKTokenChain *option, bool need_conflict, - ObList &conflict_stack); + ObList &conflict_stack); int remove_conflict(const ObIKToken &token, ObIKTokenChain *option); diff --git a/src/storage/fts/ik/ob_ik_char_util.h b/src/storage/fts/ik/ob_ik_char_util.h index 56702ef9b..205100f5d 100644 --- a/src/storage/fts/ik/ob_ik_char_util.h +++ b/src/storage/fts/ik/ob_ik_char_util.h @@ -52,6 +52,19 @@ class ObFTCharUtil const uint8_t char_len, CharType &type); + static int classify_first_valid_char( + ObCharsetType cs_type, + const char *buf, + int64_t buf_size, + size_t (*well_formed_len_fn)(const ObCharsetInfo *, + const char *, + const char *, + size_t, + int *), + const ObCharsetInfo *cs, + int64_t &char_len, + CharType &type); + static int check_cn_number(ObCollationType coll_type, const char *input, const uint8_t char_len, @@ -455,36 +468,73 @@ template inline int ObFTCharUtil::do_classify(const char *input, const uint8_t char_len, CharType &type) { int ret = OB_SUCCESS; - bool checker = false; type = CharType::USELESS; - - if (OB_FAIL(is_alpha(input, char_len, checker))) { - } else if (checker) { + ob_wc_t unicode = 0; + if (OB_FAIL(decode_unicode(input, char_len, unicode))) { + STORAGE_FTS_LOG(WARN, "Failed to decode unicode", K(ret)); + } else if (ObUnicodeBlockUtils::is_chinese(unicode)) { + type = CharType::CHINESE; + } else if (ObUnicodeBlockUtils::is_alpha(unicode)) { type = CharType::ENGLISH_LETTER; - } else if (OB_FAIL(is_arabic(input, char_len, checker))) { - STORAGE_FTS_LOG(WARN, "Failed to check arabic letter", K(ret)); - } else if (checker) { + } else if (ObUnicodeBlockUtils::is_arabic(unicode)) { type = CharType::ARABIC_LETTER; - } else if (OB_FAIL(is_chinese(input, char_len, checker))) { - STORAGE_FTS_LOG(WARN, "Failed to check chinese letter", K(ret)); - } else if (checker) { - type = CharType::CHINESE; - } else if (OB_FAIL(is_other_cjk(input, char_len, checker))) { - STORAGE_FTS_LOG(WARN, "Failed to check other cjk letter", K(ret)); - } else if (checker) { + } else if (ObUnicodeBlockUtils::is_other_cjk(unicode)) { type = CharType::OTHER_CJK; - } else if (OB_FAIL(is_surrogate_high(input, char_len, checker))) { - STORAGE_FTS_LOG(WARN, "Failed to check surrogate high letter", K(ret)); - } else if (checker) { + } else if (!(CS_TYPE == CHARSET_UTF16 || CS_TYPE == CHARSET_UTF16LE)) { + type = CharType::USELESS; + } else if (ObUnicodeBlockUtils::check_high_surrogate(unicode)) { type = CharType::SURROGATE_HIGH; - } else if (OB_FAIL(is_surrogate_low(input, char_len, checker))) { - STORAGE_FTS_LOG(WARN, "Failed to check surrogate low letter", K(ret)); - } else if (checker) { + } else if (ObUnicodeBlockUtils::check_low_surrogate(unicode)) { type = CharType::SURROGATE_LOW; } return ret; } +inline int ObFTCharUtil::classify_first_valid_char( + ObCharsetType cs_type, + const char *buf, + int64_t buf_size, + size_t (*well_formed_len_fn)(const ObCharsetInfo *, + const char *, + const char *, + size_t, + int *), + const ObCharsetInfo *cs, + int64_t &char_len, + CharType &type) +{ + int ret = OB_SUCCESS; + char_len = 0; + type = CharType::USELESS; + if (OB_UNLIKELY(nullptr == buf || buf_size <= 0 || nullptr == well_formed_len_fn + || nullptr == cs)) { + ret = OB_INVALID_ARGUMENT; + } else { + int error = 0; + char_len = static_cast(well_formed_len_fn(cs, buf, buf + buf_size, 1, &error)); + if (OB_UNLIKELY(0 != error || char_len <= 0)) { + ret = OB_ITER_END; + } else { + switch (cs_type) { + case CHARSET_UTF8MB4: + ret = do_classify(buf, static_cast(char_len), type); + break; + case CHARSET_UTF16: + ret = do_classify(buf, static_cast(char_len), type); + break; + case CHARSET_UTF16LE: + ret = do_classify(buf, static_cast(char_len), type); + break; + default: + ret = OB_NOT_SUPPORTED; + STORAGE_FTS_LOG(WARN, "Not supported charset type", K(ret), K(cs_type)); + break; + } + } + } + return ret; +} + inline int ObFTCharUtil::classify_first_char(ObCollationType coll_type, const char *input, const uint8_t char_len, diff --git a/src/storage/fts/ik/ob_ik_cjk_processor.h b/src/storage/fts/ik/ob_ik_cjk_processor.h index c1df9da3a..ead83a5f3 100644 --- a/src/storage/fts/ik/ob_ik_cjk_processor.h +++ b/src/storage/fts/ik/ob_ik_cjk_processor.h @@ -41,6 +41,8 @@ class ObIKCJKProcessor : public ObIIKProcessor const uint8_t char_len, const ObFTCharUtil::CharType type) override; + void reuse() override { hits_.clear(); } + private: ObList hits_; const ObIFTDict &dict_main_; diff --git a/src/storage/fts/ik/ob_ik_letter_processor.h b/src/storage/fts/ik/ob_ik_letter_processor.h index 144d18487..b1ea39def 100644 --- a/src/storage/fts/ik/ob_ik_letter_processor.h +++ b/src/storage/fts/ik/ob_ik_letter_processor.h @@ -34,6 +34,13 @@ class ObIKLetterProcessor : public ObIIKProcessor const uint8_t char_len, const ObFTCharUtil::CharType type) override; + void reuse() override + { + reset_english_state(); + reset_arabic_state(); + reset_mix_state(); + } + private: int process_english_letter(TokenizeContext &ctx, const char *ch, diff --git a/src/storage/fts/ik/ob_ik_processor.cpp b/src/storage/fts/ik/ob_ik_processor.cpp index 24e89441d..ed29e1ef6 100644 --- a/src/storage/fts/ik/ob_ik_processor.cpp +++ b/src/storage/fts/ik/ob_ik_processor.cpp @@ -39,48 +39,96 @@ int ObIIKProcessor::process(TokenizeContext &ctx) const char *ch = nullptr; uint8_t char_len = 0; - if (OB_FAIL(ctx.current_char_type(type))) { - LOG_WARN("fail to get current char type", K(ret)); - } else if (OB_FAIL(ctx.current_char(ch, char_len))) { - LOG_WARN("Fail to get current char", K(ret)); + if (OB_FAIL(ctx.current_char_and_type(ch, char_len, type))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get current char and type", K(ret)); + } } else if (OB_FAIL(do_process(ctx, ch, char_len, type))) { LOG_WARN("Failed to do process char", K(ret)); } return ret; } -TokenizeContext::TokenizeContext(ObCollationType coll_type, - ObIAllocator &allocator, - const char *fulltext, - int64_t fulltext_len, - bool is_smart) - : coll_type_(coll_type), fulltext_(fulltext), fulltext_len_(fulltext_len), cursor_(0), - next_char_len_(0), handle_size_(0), is_smart_(is_smart), token_list_(allocator), - result_list_(allocator) +TokenizeContext::TokenizeContext(ObIAllocator &allocator) + : coll_type_(CS_TYPE_INVALID), + fulltext_(nullptr), + fulltext_len_(0), + cursor_(0), + next_char_len_(0), + next_char_type_(ObFTCharUtil::CharType::USELESS), + handle_size_(0), + is_smart_(false), + token_list_(allocator), + results_(allocator), + result_idx_(0), + cs_(nullptr), + well_formed_len_(nullptr), + cs_type_(CHARSET_INVALID), + buffer_start_cursor_(0) { } -int TokenizeContext::init() +int TokenizeContext::init(ObCollationType coll_type, + const char *fulltext, + int64_t fulltext_len, + bool is_smart) { int ret = OB_SUCCESS; - - if (OB_ISNULL(fulltext_) || fulltext_len_ <= 0) { + if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tokenization input", K(ret), K(coll_type), KP(fulltext), K(fulltext_len)); + } else if (FALSE_IT(coll_type_ = coll_type)) { + } else if (FALSE_IT(fulltext_ = fulltext)) { + } else if (FALSE_IT(fulltext_len_ = fulltext_len)) { + } else if (FALSE_IT(is_smart_ = is_smart)) { + } else if (FALSE_IT(cs_type_ = ObCharset::charset_type_by_coll(coll_type_))) { + } else if (OB_ISNULL(cs_ = ObCharset::get_charset(coll_type_))) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported charset or collation", K(ret), K(coll_type_)); + } else if (OB_ISNULL(cs_->cset) || OB_ISNULL(cs_->cset->well_formed_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("charset has no well-formed-length function", K(ret), K(coll_type_)); + } else if (FALSE_IT(well_formed_len_ = cs_->cset->well_formed_len)) { } else if (OB_FAIL(prepare_next_char())) { LOG_WARN("Failed to prepare next char", K(ret)); } return ret; } +int TokenizeContext::reuse_context(const char *fulltext, int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tokenization input", K(ret), KP(fulltext), K(fulltext_len)); + } else { + fulltext_ = fulltext; + fulltext_len_ = fulltext_len; + cursor_ = 0; + next_char_len_ = 0; + next_char_type_ = ObFTCharUtil::CharType::USELESS; + buffer_start_cursor_ = 0; + if (OB_FAIL(reset_resource())) { + LOG_WARN("failed to reset tokenize context", K(ret)); + } else if (OB_FAIL(prepare_next_char())) { + LOG_WARN("failed to prepare first character", K(ret)); + } + } + return ret; +} + int TokenizeContext::reset_resource() { handle_size_ = 0; - result_list_.reset(); - token_list_.reset(); + results_.reuse(); + result_idx_ = 0; + token_list_.reuse(); return OB_SUCCESS; } -int TokenizeContext::current_char(const char *&ch, uint8_t &char_len) +int TokenizeContext::current_char_and_type(const char *&ch, + uint8_t &char_len, + ObFTCharUtil::CharType &type) { int ret = OB_SUCCESS; if (cursor_ >= fulltext_len_) { @@ -88,16 +136,6 @@ int TokenizeContext::current_char(const char *&ch, uint8_t &char_len) } else { ch = fulltext_ + cursor_; char_len = next_char_len_; - } - return ret; -} - -int TokenizeContext::current_char_type(ObFTCharUtil::CharType &type) -{ - int ret = OB_SUCCESS; - if (cursor_ >= fulltext_len_) { - ret = OB_ITER_END; - } else { type = next_char_type_; } return ret; @@ -106,16 +144,16 @@ int TokenizeContext::current_char_type(ObFTCharUtil::CharType &type) int TokenizeContext::prepare_next_char() { int ret = OB_SUCCESS; - if (OB_FAIL(ObCharset::first_valid_char(coll_type_, - fulltext_ + cursor_, - fulltext_len_ - cursor_, - next_char_len_))) { - LOG_WARN("Failed to get first valid char, ", K(ret)); - } else if (OB_FAIL(ObFTCharUtil::classify_first_char(coll_type_, - fulltext_ + cursor_, - next_char_len_, - next_char_type_))) { - LOG_WARN("Failed to classify first char", K(ret)); + if (OB_FAIL(ObFTCharUtil::classify_first_valid_char(cs_type_, + fulltext_ + cursor_, + fulltext_len_ - cursor_, + well_formed_len_, + cs_, + next_char_len_, + next_char_type_))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to classify first valid character", K(ret)); + } } return ret; } @@ -136,9 +174,8 @@ int TokenizeContext::step_next() } else { cursor_ += next_char_len_; handle_size_++; - if (OB_FAIL(prepare_next_char())) { + if (OB_FAIL(prepare_next_char()) && OB_ITER_END != ret) { LOG_WARN("Failed to prepare next char", K(ret)); - } else { } } return ret; @@ -160,6 +197,8 @@ bool TokenizeContext::iter_end() const { return cursor_ >= fulltext_len_; } bool TokenizeContext::is_smart() const { return is_smart_; } +bool TokenizeContext::is_results_exhaust() const { return result_idx_ >= results_.count(); } + int TokenizeContext::add_token(const char *fulltext, int64_t offset, int64_t length, @@ -182,7 +221,7 @@ int TokenizeContext::add_token(const char *fulltext, TokenizeContext::~TokenizeContext() { token_list_.reset(); - result_list_.reset(); + results_.reset(); } int TokenizeContext::get_next_token(const char *&word, @@ -191,10 +230,9 @@ int TokenizeContext::get_next_token(const char *&word, int64_t &char_cnt) { int ret = OB_SUCCESS; - if (!result_list_.empty()) { - ObIKToken &token = result_list_.get_first(); - result_list_.pop_front(); - if (!result_list_.empty()) { + if (result_idx_ < results_.count()) { + ObIKToken &token = results_.at(result_idx_++); + if (result_idx_ < results_.count()) { if (OB_FAIL(compound(token))) { LOG_WARN("Failed to compound", K(ret)); } else { @@ -216,11 +254,11 @@ int TokenizeContext::get_next_token(const char *&word, int TokenizeContext::compound(ObIKToken &token) { int ret = OB_SUCCESS; - ObList &list = result_list_; + ObFastSegmentArray &list = results_; if (is_smart_) { - if (!list.empty()) { + if (result_idx_ < list.count()) { if (ObIKTokenType::IK_ARABIC_TOKEN == token.type_) { - ObIKToken &next = list.get_first(); + ObIKToken &next = list.at(result_idx_); bool append = false; if (ObIKTokenType::IK_CNNUM_TOKEN == next.type_) { @@ -243,12 +281,12 @@ int TokenizeContext::compound(ObIKToken &token) // pass } if (append) { - list.pop_front(); + ++result_idx_; } } // There may be another round of append - if (OB_SUCC(ret)) { - ObIKToken next = list.get_first(); + if (OB_SUCC(ret) && result_idx_ < list.count()) { + ObIKToken &next = list.at(result_idx_); bool append = false; if (ObIKTokenType::IK_COUNT_TOKEN == next.type_) { if (token.offset_ + token.length_ == next.offset_) { @@ -258,7 +296,7 @@ int TokenizeContext::compound(ObIKToken &token) } } if (append) { - list.pop_front(); + ++result_idx_; } } } diff --git a/src/storage/fts/ik/ob_ik_processor.h b/src/storage/fts/ik/ob_ik_processor.h index 19a073ca5..a86a77d80 100644 --- a/src/storage/fts/ik/ob_ik_processor.h +++ b/src/storage/fts/ik/ob_ik_processor.h @@ -22,6 +22,7 @@ #include "lib/utility/ob_macro_utils.h" #include "storage/fts/ik/ob_ik_char_util.h" #include "storage/fts/ik/ob_ik_token.h" +#include "storage/fts/ik/ob_fast_segment_array.h" namespace oceanbase { @@ -30,23 +31,24 @@ namespace storage class TokenizeContext { public: - TokenizeContext(ObCollationType coll_type, - ObIAllocator &allocator, - const char *fulltext, - int64_t fulltext_len, - bool is_smart); + explicit TokenizeContext(ObIAllocator &allocator); ~TokenizeContext(); - int init(); + int init(ObCollationType coll_type, + const char *fulltext, + int64_t fulltext_len, + bool is_smart); + int reuse_context(const char *fulltext, int64_t fulltext_len); int reset_resource(); int get_next_token(const char *&word, int64_t &word_len, int64_t &offset, int64_t &char_cnt); int compound(ObIKToken &result); - int current_char(const char *&ch, uint8_t &char_len); - int current_char_type(ObFTCharUtil::CharType &type); + int current_char_and_type(const char *&ch, + uint8_t &char_len, + ObFTCharUtil::CharType &type); int step_next(); @@ -59,6 +61,7 @@ class TokenizeContext bool is_last() const; bool iter_end() const; bool is_smart() const; + bool is_results_exhaust() const; int add_chain(ObIKTokenChain *chain); int add_token(const char *fulltext, @@ -67,11 +70,25 @@ class TokenizeContext int64_t char_cnt, ObIKTokenType type); - ObFTSortList &token_list() { return token_list_; } + ObFTFastSortList &token_list() { return token_list_; } - ObList &result_list() { return result_list_; } + ObFastSegmentArray &get_results() { return results_; } - int32_t handle_size() const { return handle_size_; } + uint32_t handle_size() const { return handle_size_; } + + const ObCharsetInfo *get_cs() const { return cs_; } + ObCharsetType get_cs_type() const { return cs_type_; } + size_t (*well_formed_len_fn() const)(const ObCharsetInfo *, + const char *, + const char *, + size_t, + int *) + { + return well_formed_len_; + } + void calc_buffer_start_cursor() { buffer_start_cursor_ = cursor_; } + int64_t get_buffer_start_cursor() const { return buffer_start_cursor_; } + int64_t get_buffer_end_cursor() const { return cursor_; } private: int prepare_next_char(); @@ -87,8 +104,17 @@ class TokenizeContext uint32_t handle_size_; bool is_smart_; - ObFTSortList token_list_; - ObList result_list_; + ObFTFastSortList token_list_; + ObFastSegmentArray results_; + int64_t result_idx_; + const ObCharsetInfo *cs_; + size_t (*well_formed_len_)(const ObCharsetInfo *, + const char *, + const char *, + size_t, + int *); + ObCharsetType cs_type_; + int64_t buffer_start_cursor_; private: DISALLOW_COPY_AND_ASSIGN(TokenizeContext); @@ -108,6 +134,8 @@ class ObIIKProcessor const uint8_t char_len, const ObFTCharUtil::CharType type) = 0; + + virtual void reuse() = 0; }; } // namespace storage diff --git a/src/storage/fts/ik/ob_ik_quantifier_processor.h b/src/storage/fts/ik/ob_ik_quantifier_processor.h index 570ddc2ec..c158bd948 100644 --- a/src/storage/fts/ik/ob_ik_quantifier_processor.h +++ b/src/storage/fts/ik/ob_ik_quantifier_processor.h @@ -41,6 +41,14 @@ class ObIKQuantifierProcessor : public ObIIKProcessor const uint8_t char_len, const ObFTCharUtil::CharType type); + void reuse() override + { + count_hits_.clear(); + start_ = -1; + end_ = -1; + quan_char_cnt_ = 0; + } + private: int process_CN_number(TokenizeContext &ctx, const char *ch, diff --git a/src/storage/fts/ik/ob_ik_surrogate_processor.h b/src/storage/fts/ik/ob_ik_surrogate_processor.h index 2beeb9e95..01acc67e5 100644 --- a/src/storage/fts/ik/ob_ik_surrogate_processor.h +++ b/src/storage/fts/ik/ob_ik_surrogate_processor.h @@ -35,6 +35,8 @@ class ObIKSurrogateProcessor : public ObIIKProcessor const uint8_t char_len, const ObFTCharUtil::CharType type) override; + void reuse() override { reset(); } + private: void reset() { diff --git a/src/storage/fts/ik/ob_ik_token.cpp b/src/storage/fts/ik/ob_ik_token.cpp index 6cdc4eb82..545358e6b 100644 --- a/src/storage/fts/ik/ob_ik_token.cpp +++ b/src/storage/fts/ik/ob_ik_token.cpp @@ -25,7 +25,8 @@ namespace oceanbase { namespace storage { -int ObFTSortList::add_token(const ObIKToken &token) +template +int ObFTSortList::add_token(const ObIKToken &token) { int ret = OB_SUCCESS; @@ -44,7 +45,8 @@ int ObFTSortList::add_token(const ObIKToken &token) LOG_WARN("fail to push back token", K(ret)); } } else { - for (ObFTSortList::CellIter iter = tokens_.last(); OB_SUCC(ret) && iter != tokens_.end(); + for (typename ObFTSortList::CellIter iter = tokens_.last(); + OB_SUCC(ret) && iter != tokens_.end(); --iter) { if (token < *iter) { continue; @@ -63,6 +65,18 @@ int ObFTSortList::add_token(const ObIKToken &token) return ret; } +template +int64_t ObFTSortList::max() +{ + return tokens_.empty() ? 0 : tokens_.get_last().offset_ + tokens_.get_last().length_; +} + +template +int64_t ObFTSortList::min() +{ + return tokens_.empty() ? 0 : tokens_.get_first().offset_; +} + int ObIKTokenChain::add_token_if_conflict(const ObIKToken &token, bool &added) { int ret = OB_SUCCESS; @@ -165,7 +179,7 @@ int ObIKTokenChain::copy(ObIKTokenChain *other) min_offset_ = other->min_offset_; max_offset_ = other->max_offset_; payload_ = other->payload_; - ObFTSortList::CellIter iter = other->list().tokens().begin(); + ObFTLightSortList::CellIter iter = other->list().tokens().begin(); for (; OB_SUCC(ret) && iter != other->list().tokens().end(); ++iter) { bool added = false; @@ -218,19 +232,7 @@ int ObIKTokenChain::pop_back(ObIKToken &token) return ret; } -int64_t ObFTSortList::max() -{ - if (tokens_.empty()) { - return 0; - } - return tokens_.get_last().offset_ + tokens_.get_last().length_; -} -int64_t ObFTSortList::min() -{ - if (tokens_.empty()) { - return 0; - } - return tokens_.get_first().offset_; -} +template class ObFTSortList>; +template class ObFTSortList>; } // namespace storage } // namespace oceanbase diff --git a/src/storage/fts/ik/ob_ik_token.h b/src/storage/fts/ik/ob_ik_token.h index 3788bccd8..541622ab7 100644 --- a/src/storage/fts/ik/ob_ik_token.h +++ b/src/storage/fts/ik/ob_ik_token.h @@ -19,10 +19,13 @@ #include "lib/allocator/ob_allocator.h" #include "lib/list/ob_list.h" +#include "storage/fts/ik/ob_fast_list.h" namespace oceanbase { namespace storage { +static constexpr uint32_t HANDLE_SIZE_LIMIT = 256; + enum class ObIKTokenType : int8_t { IK_CHINESE_TOKEN = 0, @@ -71,6 +74,7 @@ struct ObIKToken } }; +template class ObFTSortList { public: @@ -82,22 +86,34 @@ class ObFTSortList bool is_empty() const { return tokens_.empty(); } void reset() { tokens_.reset(); } + void reuse() { reuse_list_(tokens_); } int64_t min(); int64_t max(); - ObList &tokens() { return tokens_; } - const ObList &tokens() const { return tokens_; } + ListType &tokens() { return tokens_; } + const ListType &tokens() const { return tokens_; } public: - typedef ObList::iterator CellIter; - typedef ObList::const_iterator ConstCellIter; + typedef typename ListType::iterator CellIter; + typedef typename ListType::const_iterator ConstCellIter; private: - ObList tokens_; + static void reuse_list_(ObFastList &list) { list.reuse(); } + + template + static void reuse_list_(T &list) + { + list.reset(); + } + + ListType tokens_; }; +typedef ObFTSortList> ObFTLightSortList; +typedef ObFTSortList> ObFTFastSortList; + class ObIKTokenChain { public: @@ -113,7 +129,7 @@ class ObIKTokenChain bool check_conflict(const ObIKToken &token); - ObFTSortList &list() { return list_; } + ObFTLightSortList &list() { return list_; } bool better_than(const ObIKTokenChain &other) const; @@ -135,7 +151,7 @@ class ObIKTokenChain int min_offset_ = -1; int max_offset_ = -1; int payload_ = -1; - ObFTSortList list_; + ObFTLightSortList list_; }; } // namespace storage diff --git a/src/storage/fts/ob_beng_ft_parser.cpp b/src/storage/fts/ob_beng_ft_parser.cpp index 5acb7ff78..7a0f9843c 100644 --- a/src/storage/fts/ob_beng_ft_parser.cpp +++ b/src/storage/fts/ob_beng_ft_parser.cpp @@ -54,7 +54,7 @@ int ObBEngFTParser::get_next_token( } else if (OB_ISNULL(token.ptr_) || OB_UNLIKELY(0 >= token.len_ || 0 >= token_freq)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), KP(token.ptr_), K(token.len_), K(token_freq)); - } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(token.len_)))) { + } else if (OB_ISNULL(buf = static_cast(scratch_allocator_.alloc(token.len_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate word memory", K(ret), K(token.len_)); } else { @@ -68,6 +68,30 @@ int ObBEngFTParser::get_next_token( return ret; } +int ObBEngFTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("basic English parser has not been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext for English parser reuse", K(ret), KP(fulltext), K(fulltext_len)); + } else if (OB_UNLIKELY(UINT32_MAX < fulltext_len)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("document is too large for English analyzer", K(ret), K(fulltext_len)); + } else { + doc_.set_string(fulltext, fulltext_len); + if (OB_FAIL(segment(doc_, token_stream_))) { + LOG_WARN("failed to segment fulltext with reused English parser", K(ret)); + } else if (OB_ISNULL(token_stream_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("English token stream is null", K(ret)); + } + } + return ret; +} + int ObBEngFTParser::init(ObFTParserParam *param) { int ret = OB_SUCCESS; @@ -85,7 +109,13 @@ int ObBEngFTParser::init(ObFTParserParam *param) analysis_ctx_.cs_ = param->cs_; analysis_ctx_.filter_stopword_ = false; analysis_ctx_.need_grouping_ = false; - if (OB_FAIL(english_analyzer_.init(analysis_ctx_, *param->allocator_))) { + ObIAllocator *metadata_allocator = nullptr != param->metadata_allocator_ + ? param->metadata_allocator_ + : param->allocator_; + if (OB_ISNULL(metadata_allocator)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("English parser metadata allocator is null", K(ret)); + } else if (OB_FAIL(english_analyzer_.init(analysis_ctx_, *metadata_allocator))) { LOG_WARN("fail to init english analyzer", K(ret), KPC(param), K(analysis_ctx_)); } else if (OB_FAIL(segment(doc_, token_stream_))) { LOG_WARN("fail to segment fulltext by parser", K(ret), KP(param->fulltext_), K(param->ft_length_)); @@ -152,13 +182,22 @@ int ObBasicEnglishFTParserDesc::segment( { int ret = OB_SUCCESS; ObBEngFTParser *parser = nullptr; + ObIAllocator *metadata_allocator = nullptr == param ? nullptr + : (nullptr != param->metadata_allocator_ ? param->metadata_allocator_ : param->allocator_); + ObIAllocator *scratch_allocator = nullptr == param ? nullptr + : (nullptr != param->scratch_allocator_ ? param->scratch_allocator_ : param->allocator_); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("default ft parser desc hasn't be initialized", K(ret), K(is_inited_)); - } else if (OB_ISNULL(param) || OB_ISNULL(param->fulltext_) || OB_UNLIKELY(!param->is_valid())) { + } else if (OB_ISNULL(param) || OB_ISNULL(param->fulltext_) + || OB_ISNULL(metadata_allocator) || OB_ISNULL(scratch_allocator) + || OB_UNLIKELY(!param->is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(param)); - } else if (OB_ISNULL(parser = OB_NEWx(ObBEngFTParser, param->allocator_, *(param->allocator_)))) { + } else if (OB_ISNULL(parser = OB_NEWx(ObBEngFTParser, + metadata_allocator, + *metadata_allocator, + *scratch_allocator))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate basic english ft parser", K(ret)); } else if (OB_FAIL(parser->init(param))) { @@ -168,7 +207,7 @@ int ObBasicEnglishFTParserDesc::segment( } if (OB_FAIL(ret)) { - OB_DELETEx(ObBEngFTParser, param->allocator_, parser); + OB_DELETEx(ObBEngFTParser, metadata_allocator, parser); } return ret; @@ -180,9 +219,12 @@ void ObBasicEnglishFTParserDesc::free_token_iter( { if (OB_NOT_NULL(iter)) { abort_unless(nullptr != param); - abort_unless(nullptr != param->allocator_); + ObIAllocator *metadata_allocator = nullptr != param->metadata_allocator_ + ? param->metadata_allocator_ + : param->allocator_; + abort_unless(nullptr != metadata_allocator); iter->~ObITokenIterator(); - param->allocator_->free(iter); + metadata_allocator->free(iter); } } diff --git a/src/storage/fts/ob_beng_ft_parser.h b/src/storage/fts/ob_beng_ft_parser.h index fea7ed7ce..0df0b6eb8 100644 --- a/src/storage/fts/ob_beng_ft_parser.h +++ b/src/storage/fts/ob_beng_ft_parser.h @@ -33,8 +33,10 @@ class ObBEngFTParser final : public plugin::ObITokenIterator static const int64_t FT_MIN_WORD_LEN = 3; static const int64_t FT_MAX_WORD_LEN = 84; public: - explicit ObBEngFTParser(common::ObIAllocator &allocator) - : allocator_(allocator), + ObBEngFTParser(common::ObIAllocator &metadata_allocator, + common::ObIAllocator &scratch_allocator) + : metadata_allocator_(metadata_allocator), + scratch_allocator_(scratch_allocator), analysis_ctx_(), english_analyzer_(), doc_(), @@ -50,6 +52,7 @@ class ObBEngFTParser final : public plugin::ObITokenIterator int64_t &word_len, int64_t &char_len, int64_t &word_freq) override; + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; VIRTUAL_TO_STRING_KV(K_(analysis_ctx), K_(english_analyzer), KP_(token_stream), K_(is_inited)); private: @@ -57,7 +60,8 @@ class ObBEngFTParser final : public plugin::ObITokenIterator const common::ObDatum &doc, share::ObITokenStream *&token_stream); private: - common::ObIAllocator &allocator_; + common::ObIAllocator &metadata_allocator_; + common::ObIAllocator &scratch_allocator_; share::ObTextAnalysisCtx analysis_ctx_; share::ObEnglishTextAnalyzer english_analyzer_; common::ObDatum doc_; diff --git a/src/storage/fts/ob_fts_literal.h b/src/storage/fts/ob_fts_literal.h index 976a988a7..44e864aad 100644 --- a/src/storage/fts/ob_fts_literal.h +++ b/src/storage/fts/ob_fts_literal.h @@ -36,7 +36,9 @@ class ObFTSLiteral final static constexpr const char *CONFIG_NAME_NGRAM_TOKEN_SIZE = "ngram_token_size"; static constexpr const char *CONFIG_NAME_STOPWORD_TABLE = "stopword_table"; static constexpr const char *CONFIG_NAME_DICT_TABLE = "dict_table"; - static constexpr const char *CONFIG_NAME_QUANTIFIER_TABLE = "quanitfier_table"; + static constexpr const char *CONFIG_NAME_QUANTIFIER_TABLE = "quantifier_table"; + // Parser properties persisted before the key typo was fixed still use this spelling. + static constexpr const char *CONFIG_NAME_LEGACY_QUANTIFIER_TABLE = "quanitfier_table"; static constexpr const char *CONFIG_NAME_IK_MODE = "ik_mode"; static constexpr const char *CONFIG_NAME_MIN_NGRAM_SIZE = "min_ngram_size"; static constexpr const char *CONFIG_NAME_MAX_NGRAM_SIZE = "max_ngram_size"; diff --git a/src/storage/fts/ob_fts_parser_property.cpp b/src/storage/fts/ob_fts_parser_property.cpp index 1a1b5ac3b..6a4b04d09 100644 --- a/src/storage/fts/ob_fts_parser_property.cpp +++ b/src/storage/fts/ob_fts_parser_property.cpp @@ -292,6 +292,10 @@ int ObFTParserJsonProps::parse_from_valid_str(const ObString &str) } else if (OB_FAIL(ObJsonParser::get_tree(&allocator_, str, root))) { LOG_WARN("Fail to parse json", K(ret)); } else { + if (nullptr != root_) { + root_->reset(); + root_->~ObIJsonBase(); + } root_ = root; } return ret; @@ -455,10 +459,13 @@ int ObFTParserJsonProps::config_get_quantifier_table(ObString &str) const ret = OB_NOT_INIT; } else { ObIJsonBase *value = nullptr; - if (OB_FAIL( - root_->get_object_value(ObString(ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE), value))) { - if (OB_SEARCH_NOT_FOUND == ret) { - } else { + ret = root_->get_object_value(ObString(ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE), value); + if (OB_SEARCH_NOT_FOUND == ret) { + ret = root_->get_object_value( + ObString(ObFTSLiteral::CONFIG_NAME_LEGACY_QUANTIFIER_TABLE), value); + } + if (OB_FAIL(ret)) { + if (OB_SEARCH_NOT_FOUND != ret) { LOG_WARN("Fail to get quantifier_table", K(ret)); } } else if (OB_ISNULL(value)) { @@ -629,6 +636,7 @@ int ObFTParserJsonProps::ik_rebuild_props_for_ddl(const bool log_to_user) static const char *supported[] = { ObFTSLiteral::CONFIG_NAME_DICT_TABLE, ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE, + ObFTSLiteral::CONFIG_NAME_LEGACY_QUANTIFIER_TABLE, ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE, ObFTSLiteral::CONFIG_NAME_IK_MODE, }; @@ -660,11 +668,11 @@ int ObFTParserJsonProps::ik_rebuild_props_for_ddl(const bool log_to_user) if (OB_FAIL(ret)) { // do nothing - } else if (OB_FAIL(config_get_quantifier_table(table_name))) { + } else if (OB_FAIL(config_get_stopword_table(table_name))) { if (OB_SEARCH_NOT_FOUND == ret) { if (OB_FAIL( - config_set_quantifier_table(ObFTSLiteral::FT_DEFAULT_IK_QUANTIFIER_UTF8_TABLE))) { - LOG_WARN("Failed to set default quantifier table", K(ret)); + config_set_stopword_table(ObFTSLiteral::FT_DEFAULT_IK_STOPWORD_UTF8_TABLE))) { + LOG_WARN("Failed to set default stopword table", K(ret)); } } } else { @@ -684,6 +692,23 @@ int ObFTParserJsonProps::ik_rebuild_props_for_ddl(const bool log_to_user) // check quantifier table valid } + if (OB_SUCC(ret)) { + ObIJsonBase *legacy_value = nullptr; + int legacy_ret = root_->get_object_value( + ObString(ObFTSLiteral::CONFIG_NAME_LEGACY_QUANTIFIER_TABLE), legacy_value); + if (OB_SUCCESS == legacy_ret) { + if (OB_FAIL(config_set_quantifier_table(table_name))) { + LOG_WARN("Failed to normalize legacy quantifier table key", K(ret), K(table_name)); + } else if (OB_FAIL(root_->object_remove( + ObString(ObFTSLiteral::CONFIG_NAME_LEGACY_QUANTIFIER_TABLE)))) { + LOG_WARN("Failed to remove legacy quantifier table key", K(ret)); + } + } else if (OB_SEARCH_NOT_FOUND != legacy_ret && OB_SUCCESS != legacy_ret) { + ret = legacy_ret; + LOG_WARN("Failed to inspect legacy quantifier table key", K(ret)); + } + } + if (OB_FAIL(ret)) { // do nothing } else { @@ -1060,6 +1085,69 @@ int ObFTParserJsonProps::show_parser_properties(const ObFTParserJsonProps &prope } } + ObString dict_table; + if (FAILEDx(properties.config_get_dict_table(dict_table))) { + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("Fail to get dict_table", K(ret)); + } + } else if (0 != dict_table.case_compare(ObFTSLiteral::FT_DEFAULT_IK_DICT_UTF8_TABLE)) { + __FT_PARSER_PROPERTY_SHOW_COMMA(need_comma); + if (FAILEDx(databuff_printf(buf, + buf_len, + pos, + "%s=\"%.*s\"", + ObFTSLiteral::CONFIG_NAME_DICT_TABLE, + static_cast(dict_table.length()), + dict_table.ptr()))) { + LOG_WARN("fail to printf dict_table", K(ret), K(buf_len), K(pos), K(dict_table)); + } + } + + ObString stopword_table; + if (FAILEDx(properties.config_get_stopword_table(stopword_table))) { + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("Fail to get stopword_table", K(ret)); + } + } else if (0 != stopword_table.case_compare( + ObFTSLiteral::FT_DEFAULT_IK_STOPWORD_UTF8_TABLE)) { + __FT_PARSER_PROPERTY_SHOW_COMMA(need_comma); + if (FAILEDx(databuff_printf(buf, + buf_len, + pos, + "%s=\"%.*s\"", + ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE, + static_cast(stopword_table.length()), + stopword_table.ptr()))) { + LOG_WARN("fail to printf stopword_table", K(ret), K(buf_len), K(pos), K(stopword_table)); + } + } + + ObString quantifier_table; + if (FAILEDx(properties.config_get_quantifier_table(quantifier_table))) { + if (OB_SEARCH_NOT_FOUND == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("Fail to get quantifier_table", K(ret)); + } + } else if (0 != quantifier_table.case_compare( + ObFTSLiteral::FT_DEFAULT_IK_QUANTIFIER_UTF8_TABLE)) { + __FT_PARSER_PROPERTY_SHOW_COMMA(need_comma); + if (FAILEDx(databuff_printf(buf, + buf_len, + pos, + "%s=\"%.*s\"", + ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE, + static_cast(quantifier_table.length()), + quantifier_table.ptr()))) { + LOG_WARN("fail to printf quantifier_table", + K(ret), K(buf_len), K(pos), K(quantifier_table)); + } + } + ObString ik_mode; if (FAILEDx(properties.config_get_ik_mode(ik_mode))) { if (OB_SEARCH_NOT_FOUND == ret) { @@ -1072,8 +1160,9 @@ int ObFTParserJsonProps::show_parser_properties(const ObFTParserJsonProps &prope if (FAILEDx(databuff_printf(buf, buf_len, pos, - "%s=\"%s\"", + "%s=\"%.*s\"", ObFTSLiteral::CONFIG_NAME_IK_MODE, + static_cast(ik_mode.length()), ik_mode.ptr()))) { LOG_WARN("fail to printf ik mode", K(ret), K(buf_len), K(pos), K(ik_mode)); } @@ -1148,7 +1237,9 @@ int ObFTParserJsonProps::show_parser_properties(const ObFTParserJsonProps &prope #undef __FT_PARSER_PROPERTY_SHOW_COMMA -int ObFTParserProperty::parse_for_parser_helper(const ObFTParser &parser, const ObString &json_str) +int ObFTParserProperty::parse_for_parser_helper(const ObFTParser &parser, + const ObString &json_str, + common::ObIAllocator &allocator) { int ret = OB_SUCCESS; ObFTParserJsonProps props; @@ -1158,12 +1249,25 @@ int ObFTParserProperty::parse_for_parser_helper(const ObFTParser &parser, const LOG_WARN("fail to parse from json str", K(ret), K(json_str)); } else { if (parser.is_ik()) { - // set dict tables and copy dict name - dict_table_ = ObString(ObFTSLiteral::CONFIG_NAME_DICT_TABLE); - stopword_table_ = ObString(ObFTSLiteral::CONFIG_NAME_STOPWORD_TABLE); - quantifier_table_ = ObString(ObFTSLiteral::CONFIG_NAME_QUANTIFIER_TABLE); + ObString dict_table; + ObString stopword_table; + ObString quantifier_table; + if (OB_FAIL(props.config_get_dict_table(dict_table))) { + LOG_WARN("fail to get dict table", K(ret)); + } else if (OB_FAIL(props.config_get_stopword_table(stopword_table))) { + LOG_WARN("fail to get stopword table", K(ret)); + } else if (OB_FAIL(props.config_get_quantifier_table(quantifier_table))) { + LOG_WARN("fail to get quantifier table", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator, dict_table, dict_table_))) { + LOG_WARN("fail to copy dict table", K(ret), K(dict_table)); + } else if (OB_FAIL(ob_write_string(allocator, stopword_table, stopword_table_))) { + LOG_WARN("fail to copy stopword table", K(ret), K(stopword_table)); + } else if (OB_FAIL(ob_write_string(allocator, quantifier_table, quantifier_table_))) { + LOG_WARN("fail to copy quantifier table", K(ret), K(quantifier_table)); + } ObString ik_smart; - if (OB_FAIL(props.config_get_ik_mode(ik_smart))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(props.config_get_ik_mode(ik_smart))) { if (OB_SEARCH_NOT_FOUND == ret) { // from old version, ik_mode is not set, so use default value ik_mode_smart_ = true; diff --git a/src/storage/fts/ob_fts_parser_property.h b/src/storage/fts/ob_fts_parser_property.h index 1cd0f397d..2c749677e 100644 --- a/src/storage/fts/ob_fts_parser_property.h +++ b/src/storage/fts/ob_fts_parser_property.h @@ -137,7 +137,9 @@ struct ObFTParserProperty final public: ObFTParserProperty(); ~ObFTParserProperty() = default; - int parse_for_parser_helper(const ObFTParser &parser, const ObString &json_str); + int parse_for_parser_helper(const ObFTParser &parser, + const ObString &json_str, + common::ObIAllocator &allocator); bool is_equal(const ObFTParserProperty &other) const { diff --git a/src/storage/fts/ob_fts_plugin_helper.cpp b/src/storage/fts/ob_fts_plugin_helper.cpp index 6f59ece2a..3880e14a7 100644 --- a/src/storage/fts/ob_fts_plugin_helper.cpp +++ b/src/storage/fts/ob_fts_plugin_helper.cpp @@ -236,7 +236,6 @@ int ObFTParseHelper::segment( const ObCharsetInfo *cs, const char *ft, const int64_t ft_len, - common::ObIAllocator &allocator, ObAddWord &add_word) { int ret = OB_SUCCESS; @@ -244,34 +243,44 @@ int ObFTParseHelper::segment( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(parser_version), KP(parser_desc), KP(cs), K(ft), K(ft_len)); } else { - ObFTParserParam param; - ObITokenIterator *iter = nullptr; - param.allocator_ = &allocator; - param.cs_ = cs; - param.fulltext_ = ft; - param.ft_length_ = ft_len; - param.parser_version_ = parser_version; - param.plugin_param_ = plugin_param; - param.ngram_token_size_ = property.ngram_token_size_; - param.ik_param_.mode_ - = (property.ik_mode_smart_ ? ObFTIKParam::Mode::SMART : ObFTIKParam::Mode::MAX_WORD); - param.min_ngram_size_ = property.min_ngram_token_size_; - param.max_ngram_size_ = property.max_ngram_token_size_; - - if (OB_FAIL(parser_desc->segment(¶m, iter))) { - LOG_WARN("fail to segment", K(ret), K(param)); - } else if (OB_ISNULL(iter)) { + if (nullptr == parser_iter_ || parser_cs_ != cs) { + if (nullptr != parser_iter_) { + destroy_parser_(); + } + if (OB_FAIL(create_parser_( + property, parser_version, plugin_param, cs, ft, ft_len))) { + LOG_WARN("failed to create fulltext parser", K(ret), K(parser_version), KP(cs)); + } + } else { + parser_param_->fulltext_ = ft; + parser_param_->ft_length_ = ft_len; + if (OB_FAIL(parser_iter_->reuse_parser(ft, ft_len))) { + if (OB_NOT_SUPPORTED == ret) { + ret = OB_SUCCESS; + destroy_parser_(); + if (OB_FAIL(create_parser_( + property, parser_version, plugin_param, cs, ft, ft_len))) { + LOG_WARN("failed to recreate non-reusable fulltext parser", K(ret)); + } + } else { + LOG_WARN("failed to reuse fulltext parser", K(ret), K(parser_version), KP(cs)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(parser_iter_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, token iterator is nullptr", K(ret), KP(iter)); + LOG_WARN("unexpected null token iterator", K(ret)); } else { const char *word = nullptr; int64_t word_len = 0; int64_t char_cnt = 0; int64_t word_freq = 0; while (OB_SUCC(ret)) { - if (OB_FAIL(iter->get_next_token(word, word_len, char_cnt, word_freq))) { + if (OB_FAIL(parser_iter_->get_next_token(word, word_len, char_cnt, word_freq))) { if (OB_ITER_END != ret) { - LOG_WARN("fail to get next token", K(ret), KPC(iter)); + LOG_WARN("fail to get next token", K(ret), KPC(parser_iter_)); } } else if (OB_FAIL(add_word.process_word(word, word_len, char_cnt, word_freq))) { LOG_WARN("fail to process one word", K(ret), KP(word), K(word_len), K(char_cnt), K(word_freq)); @@ -281,21 +290,88 @@ int ObFTParseHelper::segment( ret = OB_SUCCESS; } } - if (OB_NOT_NULL(iter)) { - parser_desc->free_token_iter(¶m, iter); - iter = nullptr; + } + return ret; +} + +int ObFTParseHelper::create_parser_(const ObFTParserProperty &property, + const int64_t parser_version, + ObPluginParam *plugin_param, + const ObCharsetInfo *cs, + const char *fulltext, + const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(parser_desc_) || OB_ISNULL(cs) || OB_ISNULL(fulltext) + || fulltext_len <= 0 || parser_version < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid parser creation arguments", K(ret), KP(parser_desc_), KP(cs), + KP(fulltext), K(fulltext_len), K(parser_version)); + } else if (OB_ISNULL(parser_param_ = OB_NEWx(ObFTParserParam, &parser_metadata_allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate parser parameters", K(ret)); + } else { + parser_param_->allocator_ = &parser_metadata_allocator_; + parser_param_->metadata_allocator_ = &parser_metadata_allocator_; + parser_param_->scratch_allocator_ = &parser_scratch_allocator_; + parser_param_->cs_ = cs; + parser_param_->fulltext_ = fulltext; + parser_param_->ft_length_ = fulltext_len; + parser_param_->parser_version_ = parser_version; + parser_param_->plugin_param_ = plugin_param; + parser_param_->ngram_token_size_ = property.ngram_token_size_; + parser_param_->ik_param_.mode_ = property.ik_mode_smart_ + ? ObFTIKParam::Mode::SMART + : ObFTIKParam::Mode::MAX_WORD; + parser_param_->ik_param_.main_dict_ = property.dict_table_; + parser_param_->ik_param_.quan_dict_ = property.quantifier_table_; + parser_param_->ik_param_.stopword_dict_ = property.stopword_table_; + parser_param_->min_ngram_size_ = property.min_ngram_token_size_; + parser_param_->max_ngram_size_ = property.max_ngram_token_size_; + + if (OB_FAIL(parser_desc_->segment(parser_param_, parser_iter_))) { + LOG_WARN("failed to initialize fulltext parser", K(ret), KPC(parser_param_)); + } else if (OB_ISNULL(parser_iter_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fulltext parser descriptor returned a null iterator", K(ret)); + } else { + parser_cs_ = cs; } } + if (OB_FAIL(ret)) { + destroy_parser_(); + } return ret; } +void ObFTParseHelper::destroy_parser_() +{ + if (nullptr != parser_iter_ && nullptr != parser_desc_ && nullptr != parser_param_) { + parser_desc_->free_token_iter(parser_param_, parser_iter_); + } + parser_iter_ = nullptr; + parser_cs_ = nullptr; + if (nullptr != parser_param_) { + OB_DELETEx(ObFTParserParam, &parser_metadata_allocator_, parser_param_); + parser_param_ = nullptr; + } + parser_scratch_allocator_.reset(); + parser_metadata_allocator_.reset(); +} + ObFTParseHelper::ObFTParseHelper() : allocator_(nullptr), parser_desc_(nullptr), plugin_param_(nullptr), parser_name_(), add_word_flag_(), + property_allocator_(common::ObMemAttr("FTParserProp")), + parser_metadata_allocator_(common::ObMemAttr("FTParserMeta")), + parser_scratch_allocator_(common::ObMemAttr("FTParserScratch")), parser_property_(), + parser_param_(nullptr), + parser_iter_(nullptr), + parser_cs_(nullptr), is_inited_(false) { } @@ -319,7 +395,8 @@ int ObFTParseHelper::init( LOG_WARN("invalid argument", K(ret), KP(allocator), K(plugin_name)); } else if (OB_FAIL(parser_name_.parse_from_str(plugin_name.ptr(), plugin_name.length()))) { LOG_WARN("fail to parse name from cstring", K(ret), K(plugin_name)); - } else if (OB_FAIL(parser_property_.parse_for_parser_helper(parser_name_, plugin_properties))) { + } else if (OB_FAIL(parser_property_.parse_for_parser_helper( + parser_name_, plugin_properties, property_allocator_))) { LOG_WARN("fail to parse parser property from cstring", K(ret), K(plugin_properties), K(parser_name_)); } else if (OB_FAIL(ObPluginHelper::find_ftparser(parser_name_.get_parser_name().str(), parser_desc_, plugin_param_))) { @@ -346,10 +423,13 @@ int ObFTParseHelper::init( void ObFTParseHelper::reset() { + destroy_parser_(); parser_desc_ = nullptr; plugin_param_ = nullptr; allocator_ = nullptr; add_word_flag_.clear(); + parser_property_ = ObFTParserProperty(); + property_allocator_.reset(); is_inited_ = false; } @@ -358,11 +438,15 @@ int ObFTParseHelper::segment( const char *fulltext, const int64_t fulltext_len, int64_t &doc_length, - ObFTWordMap &words) const + ObFTWordMap &words) { int ret = OB_SUCCESS; const ObCharsetInfo *cs = nullptr; ObCollationType type = meta.get_collation_type(); + const char *parse_text = fulltext; + int64_t parse_text_len = fulltext_len; + ObString normalized_text; + ObAddWordFlag effective_flag = add_word_flag_; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("this fulltext parser helper hasn't been initialized", K(ret), K(is_inited_)); @@ -377,16 +461,40 @@ int ObFTParseHelper::segment( LOG_WARN("unexpected error, charset info is nullptr", K(ret), K(type)); } else { words.reuse(); - ObAddWord add_word(parser_property_, meta, add_word_flag_, *allocator_, words); - if (OB_FAIL(segment( + parser_scratch_allocator_.reset_remain_one_page(); + const bool builtin_main = parser_property_.dict_table_.empty() + || 0 == parser_property_.dict_table_.case_compare( + ObFTSLiteral::FT_DEFAULT_IK_DICT_UTF8_TABLE); + const bool builtin_quantifier = parser_property_.quantifier_table_.empty() + || 0 == parser_property_.quantifier_table_.case_compare( + ObFTSLiteral::FT_DEFAULT_IK_QUANTIFIER_UTF8_TABLE); + const bool builtin_stopword = parser_property_.stopword_table_.empty() + || 0 == parser_property_.stopword_table_.case_compare( + ObFTSLiteral::FT_DEFAULT_IK_STOPWORD_UTF8_TABLE); + const bool can_normalize_once = add_word_flag_.casedown() + && (!parser_name_.is_ik() || (builtin_main && builtin_quantifier && builtin_stopword)); + if (can_normalize_once + && OB_FAIL(ObCharset::tolower(type, + ObString(fulltext_len, fulltext), + normalized_text, + parser_scratch_allocator_))) { + LOG_WARN("failed to normalize fulltext before parsing", K(ret), K(type)); + } else if (can_normalize_once && !normalized_text.empty()) { + parse_text = normalized_text.ptr(); + parse_text_len = normalized_text.length(); + effective_flag.clear_casedown(); + } + + ObAddWord add_word(parser_property_, meta, effective_flag, *allocator_, words); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(segment( parser_property_, parser_name_.get_parser_version(), parser_desc_, plugin_param_, cs, - fulltext, - fulltext_len, - *allocator_, + parse_text, + parse_text_len, add_word))) { LOG_WARN("fail to segment fulltext", K(ret), K(parser_name_), KP(parser_desc_), KP(cs), KP(fulltext), K(fulltext_len), KP(allocator_), K(parser_property_)); @@ -409,12 +517,14 @@ int ObFTParseHelper::check_is_the_same( if (is_inited_) { storage::ObFTParser parser_name; ObFTParserProperty parser_property; + ObArenaAllocator property_allocator(ObMemAttr("FTPropCheck")); if (OB_UNLIKELY(plugin_name.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(plugin_name)); } else if (OB_FAIL(parser_name.parse_from_str(plugin_name.ptr(), plugin_name.length()))) { LOG_WARN("fail to parse name from cstring", K(ret), K(plugin_name)); - } else if (OB_FAIL(parser_property.parse_for_parser_helper(parser_name, plugin_properties))) { + } else if (OB_FAIL(parser_property.parse_for_parser_helper( + parser_name, plugin_properties, property_allocator))) { LOG_WARN("fail to parse parser property from cstring", K(ret), K(plugin_properties), K(parser_name_)); } else if (parser_name == parser_name_ && parser_property.is_equal(parser_property_)) { is_same = true; diff --git a/src/storage/fts/ob_fts_plugin_helper.h b/src/storage/fts/ob_fts_plugin_helper.h index 84fefd75b..31c96c1f3 100644 --- a/src/storage/fts/ob_fts_plugin_helper.h +++ b/src/storage/fts/ob_fts_plugin_helper.h @@ -18,12 +18,14 @@ #define OB_FTS_PLUGIN_HELPER_H_ #include "lib/allocator/ob_fifo_allocator.h" +#include "lib/allocator/page_arena.h" #include "lib/charset/ob_charset.h" #include "lib/string/ob_string.h" #include "object/ob_object.h" #include "share/ob_plugin_helper.h" #include "storage/fts/ob_fts_parser_property.h" #include "storage/fts/ob_fts_struct.h" +#include "plugin/interface/ob_plugin_ftparser_intf.h" namespace oceanbase { @@ -154,7 +156,7 @@ class ObFTParseHelper final * "ngram_token_size":2, * "stopword_table":"default", * "dict_table":"none", - * "quanitfier_table":"none" + * "quantifier_table":"none" * } * * @return error code @@ -177,7 +179,7 @@ class ObFTParseHelper final const char *fulltext, const int64_t fulltext_len, int64_t &doc_length, - ObFTWordMap &words) const; + ObFTWordMap &words); int check_is_the_same( const common::ObString &plugin_name, const common::ObString &plugin_properties, @@ -209,7 +211,7 @@ class ObFTParseHelper final TO_STRING_KV(KP_(allocator), K_(parser_name), KP_(parser_desc), K_(is_inited)); private: - static int segment( + int segment( const ObFTParserProperty &property, const int64_t parser_version, const plugin::ObIFTParserDesc *parser_desc, @@ -217,8 +219,14 @@ class ObFTParseHelper final const ObCharsetInfo *cs, const char *fulltext, const int64_t fulltext_len, - common::ObIAllocator &allocator, ObAddWord &add_word); + int create_parser_(const ObFTParserProperty &property, + const int64_t parser_version, + plugin::ObPluginParam *plugin_param, + const ObCharsetInfo *cs, + const char *fulltext, + const int64_t fulltext_len); + void destroy_parser_(); int set_add_word_flag(const plugin::ObIFTParserDesc &ftparser_desc); private: common::ObIAllocator *allocator_; @@ -226,7 +234,13 @@ class ObFTParseHelper final plugin::ObPluginParam *plugin_param_; ObFTParser parser_name_; ObAddWordFlag add_word_flag_; + common::ObArenaAllocator property_allocator_; + common::ObArenaAllocator parser_metadata_allocator_; + common::ObArenaAllocator parser_scratch_allocator_; ObFTParserProperty parser_property_; + plugin::ObFTParserParam *parser_param_; + plugin::ObITokenIterator *parser_iter_; + const ObCharsetInfo *parser_cs_; bool is_inited_; private: diff --git a/src/storage/fts/ob_fts_stop_word.cpp b/src/storage/fts/ob_fts_stop_word.cpp index 66f733f03..f88d8a94b 100644 --- a/src/storage/fts/ob_fts_stop_word.cpp +++ b/src/storage/fts/ob_fts_stop_word.cpp @@ -44,19 +44,27 @@ int ObStopWordChecker::init() if (inited_) { ret = OB_INIT_TWICE; - } else if (OB_FAIL(stopword_set_.create(DEFAULT_STOPWORD_BUCKET_NUM, "StopWordSet", "StopWordSet"))) { + } else if (OB_FAIL(general_ci_stopword_set_.create( + DEFAULT_STOPWORD_BUCKET_NUM, "StopWordSet", "StopWordSet"))) { LOG_WARN("fail to create stop word set", K(ret)); + } else if (OB_FAIL(binary_stopword_set_.create( + DEFAULT_STOPWORD_BUCKET_NUM, "StopWordBin", "StopWordBin"))) { + LOG_WARN("fail to create binary stop word set", K(ret)); } else { - ObObjMeta stop_meta; - stop_meta.set_varchar(); - stop_meta.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); - - stopword_type_.set_meta(stop_meta); + general_ci_stopword_type_.set_varchar(); + general_ci_stopword_type_.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); + binary_stopword_type_.set_varchar(); + binary_stopword_type_.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); const int64_t stopword_count = sizeof(ob_stop_word_list) / sizeof(ob_stop_word_list[0]); for (int64_t i = 0; OB_SUCC(ret) && i < stopword_count; ++i) { - ObFTWord stopword(STRLEN(ob_stop_word_list[i]), ob_stop_word_list[i], stopword_type_); - if (OB_FAIL(stopword_set_.set_refactored(stopword))) { - LOG_WARN("fail to set stop word", K(ret), K(stopword)); + ObFTWord general_ci_stopword( + STRLEN(ob_stop_word_list[i]), ob_stop_word_list[i], general_ci_stopword_type_); + ObFTWord binary_stopword( + STRLEN(ob_stop_word_list[i]), ob_stop_word_list[i], binary_stopword_type_); + if (OB_FAIL(general_ci_stopword_set_.set_refactored(general_ci_stopword))) { + LOG_WARN("fail to set general-ci stop word", K(ret), K(general_ci_stopword)); + } else if (OB_FAIL(binary_stopword_set_.set_refactored(binary_stopword))) { + LOG_WARN("fail to set binary stop word", K(ret), K(binary_stopword)); } } @@ -70,7 +78,8 @@ int ObStopWordChecker::init() void ObStopWordChecker::destroy() { if (inited_) { - stopword_set_.destroy(); + general_ci_stopword_set_.destroy(); + binary_stopword_set_.destroy(); inited_ = false; } } @@ -80,7 +89,6 @@ int ObStopWordChecker::check_stopword(const ObFTWord &word, bool &is_stopword) int ret = OB_SUCCESS; - common::ObArenaAllocator allocator(lib::ObMemAttr("ChkStopWord")); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObStopWordChecker hasn't been initialized", K(ret), K(inited_)); @@ -88,31 +96,44 @@ int ObStopWordChecker::check_stopword(const ObFTWord &word, bool &is_stopword) ret = OB_INVALID_ARGUMENT; LOG_WARN("word is empty", K(ret), K(word)); } else { - common::ObString cmp_str; - // do nothing set out with in if type is the same. - if (OB_FAIL(common::ObCharset::charset_convert( - allocator, - word.get_word().get_string(), - word.get_collation_type(), - stopword_type_.get_collation_type(), - cmp_str))) { - LOG_WARN("fail to convert charset", K(ret), K(word), K(stopword_type_)); + const ObCollationType coll = word.get_collation_type(); + if (CS_TYPE_UTF8MB4_GENERAL_CI == coll || CS_TYPE_UTF8MB4_BIN == coll) { + const ObObjMeta &stopword_meta = CS_TYPE_UTF8MB4_BIN == coll + ? binary_stopword_type_ + : general_ci_stopword_type_; + StopWordSet &stopword_set = CS_TYPE_UTF8MB4_BIN == coll + ? binary_stopword_set_ + : general_ci_stopword_set_; + ObFTWord lookup(word.get_word().get_string().length(), + word.get_word().get_string().ptr(), + stopword_meta); + ret = stopword_set.exist_refactored(lookup); } else { - ObFTWord converted(cmp_str.length(), cmp_str.ptr(), stopword_type_); - ret = stopword_set_.exist_refactored(converted); - if (OB_HASH_NOT_EXIST == ret) { - is_stopword = false; - ret = OB_SUCCESS; - } else if (OB_HASH_EXIST == ret) { - is_stopword = true; - ret = OB_SUCCESS; - } else if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the exist of hastset shouldn't return success", K(ret), K(word), K(converted)); + common::ObArenaAllocator allocator(lib::ObMemAttr("ChkStopWord")); + common::ObString cmp_str; + if (OB_FAIL(common::ObCharset::charset_convert(allocator, + word.get_word().get_string(), + coll, + general_ci_stopword_type_.get_collation_type(), + cmp_str))) { + LOG_WARN("fail to convert charset", K(ret), K(word), K(general_ci_stopword_type_)); } else { - LOG_WARN("fail to do exist", K(ret), K(word), K(converted)); + ObFTWord converted(cmp_str.length(), cmp_str.ptr(), general_ci_stopword_type_); + ret = general_ci_stopword_set_.exist_refactored(converted); } } + if (OB_HASH_NOT_EXIST == ret) { + is_stopword = false; + ret = OB_SUCCESS; + } else if (OB_HASH_EXIST == ret) { + is_stopword = true; + ret = OB_SUCCESS; + } else if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stop word hash set returned unexpected success", K(ret), K(word)); + } else { + LOG_WARN("fail to check stop word", K(ret), K(word)); + } } return ret; } @@ -219,7 +240,6 @@ int ObAddWord::check_stopword(const ObFTWord &ft_word, bool &is_stopword) int ObAddWord::groupby_word(const ObFTWord &word, const int64_t word_freq) { int ret = OB_SUCCESS; - int64_t word_count = 0; if (OB_UNLIKELY(word.empty() || word_freq <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(word), K(word_freq)); @@ -227,16 +247,10 @@ int ObAddWord::groupby_word(const ObFTWord &word, const int64_t word_freq) if (OB_FAIL(word_map_.set_refactored(word, 1/*word count*/))) { LOG_WARN("fail to set fulltext word and count", K(ret), K(word)); } - } else if (OB_FAIL(word_map_.get_refactored(word, word_count)) && OB_HASH_NOT_EXIST != ret) { - LOG_WARN("fail to get fulltext word", K(ret), K(word)); } else { - if (OB_HASH_NOT_EXIST == ret) { - word_count = 1; - } else { - word_count += word_freq; - } - if (OB_FAIL(word_map_.set_refactored(word, word_count, 1/*overwrite*/))) { - LOG_WARN("fail to set fulltext word and count", K(ret), K(word), K(word_count)); + UpdateWordCount update_word_count(word_freq); + if (OB_FAIL(word_map_.set_or_update(word, word_freq, update_word_count))) { + LOG_WARN("fail to aggregate fulltext word count", K(ret), K(word), K(word_freq)); } } return ret; diff --git a/src/storage/fts/ob_fts_stop_word.h b/src/storage/fts/ob_fts_stop_word.h index 1213f9091..2200c6213 100644 --- a/src/storage/fts/ob_fts_stop_word.h +++ b/src/storage/fts/ob_fts_stop_word.h @@ -84,8 +84,10 @@ class ObStopWordChecker final static const int64_t DEFAULT_STOPWORD_BUCKET_NUM = 37L; typedef common::hash::ObHashSet StopWordSet; - StopWordSet stopword_set_; - ObObjMeta stopword_type_; + StopWordSet general_ci_stopword_set_; + StopWordSet binary_stopword_set_; + ObObjMeta general_ci_stopword_type_; + ObObjMeta binary_stopword_type_; bool inited_ = false; @@ -119,6 +121,20 @@ class ObAddWord final K(word_map_.size())); private: + class UpdateWordCount final + { + public: + explicit UpdateWordCount(const int64_t word_freq) : word_freq_(word_freq) {} + int operator()(common::hash::HashMapPair &pair) + { + pair.second += word_freq_; + return OB_SUCCESS; + } + + private: + int64_t word_freq_; + }; + bool is_min_max_word(const int64_t c_len) const; int casedown_word(const ObFTWord &src, ObFTWord &dst); int check_stopword(const ObFTWord &word, bool &is_stopword); diff --git a/src/storage/fts/ob_fts_struct.cpp b/src/storage/fts/ob_fts_struct.cpp index 916ddef7c..8507985e0 100644 --- a/src/storage/fts/ob_fts_struct.cpp +++ b/src/storage/fts/ob_fts_struct.cpp @@ -25,33 +25,69 @@ namespace oceanbase namespace storage { +int ObFTWord::init(const char *ptr, const int64_t length, const ObObjMeta &meta) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == ptr || length <= 0)) { + ret = OB_INVALID_ARGUMENT; + } else { + sql::ObExprBasicFuncs *funcs = + ObDatumFuncs::get_basic_func(meta.get_type(), meta.get_collation_type()); + ObDatumCmpFuncType cmp_func = get_datum_cmp_func(meta, meta); + if (OB_UNLIKELY(nullptr == funcs || nullptr == funcs->default_hash_ || nullptr == cmp_func)) { + ret = OB_ERR_UNEXPECTED; + } else { + word_.set_string(ptr, length); + meta_ = meta; + hash_func_ = funcs->default_hash_; + cmp_func_ = cmp_func; + hash_calculated_ = false; + hash_value_ = 0; + } + } + return ret; +} + int ObFTWord::hash(uint64_t &hash_val) const { int ret = OB_SUCCESS; - sql::ObExprBasicFuncs *funcs = ObDatumFuncs::get_basic_func(meta_.get_type(), meta_.get_collation_type()); - if (OB_ISNULL(funcs)) { - ret = OB_ERR_UNEXPECTED; - } else if (funcs->default_hash_ == nullptr) { + if (OB_LIKELY(hash_calculated_)) { + hash_val = hash_value_; + } else if (OB_UNLIKELY(nullptr == hash_func_)) { ret = OB_ERR_UNEXPECTED; } else { - ret = funcs->default_hash_(word_, 0, hash_val); + if (OB_FAIL(hash_func_(word_, 0, hash_value_))) { + } else { + hash_calculated_ = true; + hash_val = hash_value_; + } } return ret; } + bool ObFTWord::operator==(const ObFTWord &other) const { + int ret = OB_SUCCESS; bool is_equal = false; + if (OB_FAIL(compare_(other, is_equal))) { + ob_abort(); + } + return is_equal; +} + +int ObFTWord::compare_(const ObFTWord &other, bool &is_equal) const +{ int ret = OB_SUCCESS; int cmp_ret = 0; - ObDatumCmpFuncType func = get_datum_cmp_func(meta_, other.meta_); - if (func == nullptr) { - ob_abort(); - } else if (OB_FAIL(func(word_, other.word_, cmp_ret))) { - ob_abort(); + is_equal = false; + if (hash_calculated_ && other.hash_calculated_ && hash_value_ != other.hash_value_) { + } else if (OB_UNLIKELY(nullptr == cmp_func_)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(cmp_func_(word_, other.word_, cmp_ret))) { } else { is_equal = (cmp_ret == 0); } - return is_equal; + return ret; } } // namespace storage } // namespace oceanbase diff --git a/src/storage/fts/ob_fts_struct.h b/src/storage/fts/ob_fts_struct.h index 488ac2e23..eb61bbcd3 100644 --- a/src/storage/fts/ob_fts_struct.h +++ b/src/storage/fts/ob_fts_struct.h @@ -31,13 +31,22 @@ namespace storage class ObFTWord final { public: - ObFTWord() : word_(), meta_() {} - ObFTWord(const int64_t length, const char *ptr, const ObObjMeta &meta) : meta_(meta) + ObFTWord() + : hash_calculated_(false), + hash_value_(0), + hash_func_(nullptr), + cmp_func_(nullptr), + word_(), + meta_() + {} + ObFTWord(const int64_t length, const char *ptr, const ObObjMeta &meta) : ObFTWord() { - word_.set_string(ptr, length); + (void)init(ptr, length, meta); } ~ObFTWord() = default; + int init(const char *ptr, const int64_t length, const ObObjMeta &meta); + OB_INLINE const ObDatum &get_word() const { return word_; } OB_INLINE ObCollationType get_collation_type() const { return meta_.get_collation_type(); } OB_INLINE bool empty() const { return word_.get_string().empty(); } @@ -48,11 +57,20 @@ class ObFTWord final TO_STRING_KV(K_(meta), K_(word)); private: + int compare_(const ObFTWord &other, bool &is_equal) const; + +private: + mutable bool hash_calculated_; + mutable uint64_t hash_value_; + ObDatumHashFuncType hash_func_; + ObDatumCmpFuncType cmp_func_; ObDatum word_; ObObjMeta meta_; }; typedef common::hash::ObHashMap ObFTWordMap; +typedef ObFTWord ObFTToken; +typedef ObFTWordMap ObFTTokenMap; class ObAddWordFlag final { diff --git a/src/storage/fts/ob_ik_ft_parser.cpp b/src/storage/fts/ob_ik_ft_parser.cpp index 1c5ca659f..e0259c0ac 100644 --- a/src/storage/fts/ob_ik_ft_parser.cpp +++ b/src/storage/fts/ob_ik_ft_parser.cpp @@ -64,6 +64,8 @@ int ObIKFTParser::init(const ObFTParserParam ¶m) LOG_WARN("Failed to init ctx", K(ret)); } else if (OB_FAIL(init_segmenter(param))) { LOG_WARN("Failed to init segmenters", K(ret)); + } else if (OB_FAIL(arb_.prepare())) { + LOG_WARN("failed to prepare IK arbitrator", K(ret)); } if (OB_FAIL(ret)) { @@ -76,6 +78,29 @@ int ObIKFTParser::init(const ObFTParserParam ¶m) return ret; } +int ObIKFTParser::reuse_parser(const char *fulltext, int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("IK parser has not been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext for parser reuse", K(ret), KP(fulltext), K(fulltext_len)); + } else if (OB_FAIL(ctx_->reuse_context(fulltext, fulltext_len))) { + LOG_WARN("failed to reuse IK tokenize context", K(ret)); + } else { + for (ObList::iterator iter = segmenters_.begin(); + iter != segmenters_.end(); + ++iter) { + (*iter)->reuse(); + } + arb_.reuse(); + scratch_alloc_.reset_remain_one_page(); + } + return ret; +} + int ObIKFTParser::get_next_token(const char *&word, int64_t &word_len, int64_t &char_cnt, @@ -103,11 +128,10 @@ int ObIKFTParser::get_next_token(const char *&word, } } else { bool is_stop = false; - // if (!OB_ISNULL(dict_stop_) - // && OB_FAIL(dict_stop_->match(ObString(len, output_word + offset), is_stop))) { - // LOG_WARN("Failed to match stopwords", K(ret)); - // } else - if (!is_stop) { + if (!OB_ISNULL(dict_stop_) + && OB_FAIL(dict_stop_->match(ObString(len, output_word + offset), is_stop))) { + LOG_WARN("Failed to match stopwords", K(ret)); + } else if (!is_stop) { word = output_word + offset; word_len = len; char_cnt = cnt; @@ -126,7 +150,7 @@ int ObIKFTParser::produce() { int ret = OB_SUCCESS; // Loop until end or has data to output - while (OB_SUCC(ret) && ctx_->result_list().empty() && !ctx_->iter_end()) { + while (OB_SUCC(ret) && ctx_->is_results_exhaust() && !ctx_->iter_end()) { if (OB_FAIL(process_next_batch())) { if (OB_ITER_END == ret) { // ok @@ -169,19 +193,21 @@ int ObIKFTParser::process_next_batch() if (ctx_->iter_end()) { ret = OB_ITER_END; } else { + ctx_->calc_buffer_start_cursor(); while (OB_SUCC(ret) && !do_seg && !ctx_->iter_end()) { const char *ch; uint8_t char_len = 0; ObFTCharUtil::CharType type = ObFTCharUtil::CharType::USELESS; - if (OB_FAIL(ctx_->current_char(ch, char_len))) { - LOG_WARN("Failed to get current char", K(ret)); - } else if (OB_FAIL(ctx_->current_char_type(type))) { - LOG_WARN("Failed to get current char type", K(ret)); + if (OB_FAIL(ctx_->current_char_and_type(ch, char_len, type))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get current character and type", K(ret)); + } } else if (OB_FAIL(process_one_char(*ctx_, ch, char_len, type))) { LOG_WARN("Failed to process one char", K(ret)); } else { // 1. check segmention - if (ctx_->handle_size() > SEGMENT_LIMIT && type == ObFTCharUtil::CharType::USELESS) { + if (ctx_->handle_size() >= HANDLE_SIZE_LIMIT + && type == ObFTCharUtil::CharType::USELESS) { do_seg = true; } @@ -196,11 +222,12 @@ int ObIKFTParser::process_next_batch() } if (OB_SUCC(ret) || OB_ITER_END == ret) { - ObIKArbitrator arb; - if (OB_FAIL(arb.process(*ctx_))) { + if (OB_FAIL(arb_.process(*ctx_))) { LOG_WARN("Failed to process arbitrator", K(ret)); - } else if (OB_FAIL(arb.output_result(*ctx_))) { + } else if (OB_FAIL(arb_.output_result(*ctx_))) { LOG_WARN("Failed to make result list"); + } else { + arb_.reuse(); } } else { // Already logged. @@ -270,14 +297,11 @@ int ObIKFTParserDesc::get_add_word_flag(ObAddWordFlag &flag) const int ObIKFTParser::init_dict(const plugin::ObFTParserParam ¶m) { int ret = OB_SUCCESS; - ObIFTDict *tmp_dict = nullptr; - if (OB_ISNULL(hub_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Dict hub is not inited", K(ret)); } - ObFTRangeDict *dict = nullptr; ObFTDictDesc main_dict_desc("main_dict", ObFTDictType::DICT_IK_MAIN, ObCharsetType::CHARSET_UTF8MB4, @@ -293,31 +317,56 @@ int ObIKFTParser::init_dict(const plugin::ObFTParserParam ¶m) ObCharsetType::CHARSET_UTF8MB4, ObCollationType::CS_TYPE_UTF8MB4_BIN); - if (should_read_newest_table()) { - // clear dict cache, always false now - } else { - if (OB_FAIL(init_single_dict(main_dict_desc, cache_main_))) { - LOG_WARN("Failed to init main dict", K(ret)); - } else if (OB_FAIL(init_single_dict(quan_dict_desc, cache_quan_))) { - LOG_WARN("Failed to init quantifier dict", K(ret)); - } else if (OB_FAIL(init_single_dict(stopword_dict_desc, cache_stop_))) { - LOG_WARN("Failed to init stopword dict", K(ret)); - } - } - if (OB_FAIL(ret)) { - // already logged. - } else if (OB_FAIL(build_dict_from_cache(main_dict_desc, cache_main_, dict_main_))) { - LOG_WARN("Failed to build dict main", K(ret)); - } else if (OB_FAIL(build_dict_from_cache(quan_dict_desc, cache_quan_, dict_quan_))) { - LOG_WARN("Failed to build dict quantifier", K(ret)); - } else if (OB_FAIL(build_dict_from_cache(stopword_dict_desc, cache_stop_, dict_stop_))) { - LOG_WARN("Failed to build dict stopword", K(ret)); + } else if (OB_FAIL(init_dict_category(param.ik_param_.main_dict_, + ObFTSLiteral::FT_DEFAULT_IK_DICT_UTF8_TABLE, + main_dict_desc, + cache_main_, + user_main_, + dict_main_))) { + LOG_WARN("Failed to initialize IK main dictionary", K(ret), K(param.ik_param_.main_dict_)); + } else if (OB_FAIL(init_dict_category(param.ik_param_.quan_dict_, + ObFTSLiteral::FT_DEFAULT_IK_QUANTIFIER_UTF8_TABLE, + quan_dict_desc, + cache_quan_, + user_quan_, + dict_quan_))) { + LOG_WARN("Failed to initialize IK quantifier dictionary", K(ret), K(param.ik_param_.quan_dict_)); + } else if (OB_FAIL(init_dict_category(param.ik_param_.stopword_dict_, + ObFTSLiteral::FT_DEFAULT_IK_STOPWORD_UTF8_TABLE, + stopword_dict_desc, + cache_stop_, + user_stop_, + dict_stop_))) { + LOG_WARN("Failed to initialize IK stopword dictionary", K(ret), K(param.ik_param_.stopword_dict_)); } return ret; } +int ObIKFTParser::init_dict_category(const ObString &configured_table, + const ObString &builtin_table, + const ObFTDictDesc &builtin_desc, + ObFTCacheRangeContainer &container, + ObFTUserDictHandle &user_handle, + ObIFTDict *&dict) +{ + int ret = OB_SUCCESS; + if (configured_table.empty() || 0 == configured_table.case_compare(builtin_table)) { + if (OB_FAIL(init_single_dict(builtin_desc, container))) { + LOG_WARN("fail to initialize built-in dictionary cache", K(ret), K(builtin_table)); + } else if (OB_FAIL(build_dict_from_cache(builtin_desc, container, dict))) { + LOG_WARN("fail to build built-in dictionary", K(ret), K(builtin_table)); + } + } else if (OB_FAIL(hub_->get_user_dict(configured_table, user_handle))) { + LOG_WARN("fail to load user dictionary", K(ret), K(configured_table)); + } else if (OB_ISNULL(dict = user_handle.get_dict())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("user dictionary handle is empty", K(ret), K(configured_table)); + } + return ret; +} + int ObIKFTParser::init_single_dict(ObFTDictDesc desc, ObFTCacheRangeContainer &container) { int ret = OB_SUCCESS; @@ -342,14 +391,13 @@ int ObIKFTParser::init_ctx(const ObFTParserParam ¶m) LOG_WARN("Illegal collation type", K(ret)); } else if (OB_ISNULL(ctx_ = OB_NEWx(TokenizeContext, &allocator_, - coll_type_, - allocator_, - param.fulltext_, - param.ft_length_, - param.ik_param_.mode_ == ObFTIKParam::Mode::SMART))) { + allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc ctx", K(ret)); - } else if (OB_FAIL(ctx_->init())) { + } else if (OB_FAIL(ctx_->init(coll_type_, + param.fulltext_, + param.ft_length_, + param.ik_param_.mode_ == ObFTIKParam::Mode::SMART))) { LOG_WARN("Failed to init ctx", K(ret)); } if (OB_FAIL(ret)) { @@ -372,13 +420,15 @@ int ObIKFTParser::init_segmenter(const ObFTParserParam ¶m) } else if (OB_ISNULL(dict_quan_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Dict quan is null.", K(ret)); - } else if (OB_ISNULL(cnqsg = OB_NEWx(ObIKQuantifierProcessor, &allocator_, *dict_quan_, allocator_))) { + } else if (OB_ISNULL(cnqsg = OB_NEWx( + ObIKQuantifierProcessor, &allocator_, *dict_quan_, scratch_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc cn quantifier segmenter", K(ret)); } else if (OB_ISNULL(dict_main_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Dict main is null.", K(ret)); - } else if (OB_ISNULL(cjksg = OB_NEWx(ObIKCJKProcessor, &allocator_, *dict_main_, allocator_))) { + } else if (OB_ISNULL(cjksg = OB_NEWx( + ObIKCJKProcessor, &allocator_, *dict_main_, scratch_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc cjk segmenter", K(ret)); } else if (OB_ISNULL(surrogate_seg = OB_NEWx(ObIKSurrogateProcessor, &allocator_))) { @@ -427,18 +477,25 @@ void ObIKFTParser::reset() cache_quan_.reset(); cache_stop_.reset(); - if (!OB_ISNULL(dict_main_)) { + if (!OB_ISNULL(dict_main_) && !user_main_.is_valid()) { dict_main_->~ObIFTDict(); allocator_.free(dict_main_); } - if (!OB_ISNULL(dict_quan_)) { + if (!OB_ISNULL(dict_quan_) && !user_quan_.is_valid()) { dict_quan_->~ObIFTDict(); allocator_.free(dict_quan_); } - if (!OB_ISNULL(dict_stop_)) { + if (!OB_ISNULL(dict_stop_) && !user_stop_.is_valid()) { dict_stop_->~ObIFTDict(); allocator_.free(dict_stop_); } + dict_main_ = nullptr; + dict_quan_ = nullptr; + dict_stop_ = nullptr; + user_main_.reset(); + user_quan_.reset(); + user_stop_.reset(); + scratch_alloc_.reset(); is_inited_ = false; } diff --git a/src/storage/fts/ob_ik_ft_parser.h b/src/storage/fts/ob_ik_ft_parser.h index d25901f52..7becd0591 100644 --- a/src/storage/fts/ob_ik_ft_parser.h +++ b/src/storage/fts/ob_ik_ft_parser.h @@ -21,6 +21,8 @@ #include "storage/fts/dict/ob_ft_cache_container.h" #include "storage/fts/dict/ob_ft_dict.h" #include "storage/fts/dict/ob_ft_dict_def.h" +#include "storage/fts/dict/ob_ft_user_dict.h" +#include "storage/fts/ik/ob_ik_arbitrator.h" #include "storage/fts/ik/ob_ik_processor.h" #include "plugin/interface/ob_plugin_ftparser_intf.h" @@ -46,7 +48,11 @@ class ObIKFTParser final : public plugin::ObITokenIterator cache_stop_(allocator), dict_main_(nullptr), dict_quan_(nullptr), - dict_stop_(nullptr) + dict_stop_(nullptr), + user_main_(), + user_quan_(), + user_stop_(), + arb_() { } @@ -59,6 +65,8 @@ class ObIKFTParser final : public plugin::ObITokenIterator int64_t &char_cnt, int64_t &word_freq) override; + int reuse_parser(const char *fulltext, int64_t fulltext_len) override; + VIRTUAL_TO_STRING_KV(K(is_inited_)); private: @@ -87,6 +95,12 @@ class ObIKFTParser final : public plugin::ObITokenIterator int build_dict_from_cache(const ObFTDictDesc &desc, ObFTCacheRangeContainer &container, ObIFTDict *&dict); + int init_dict_category(const common::ObString &configured_table, + const common::ObString &builtin_table, + const ObFTDictDesc &builtin_desc, + ObFTCacheRangeContainer &container, + ObFTUserDictHandle &user_handle, + ObIFTDict *&dict); private: static constexpr int SEGMENT_LIMIT = 1000; @@ -106,6 +120,11 @@ class ObIKFTParser final : public plugin::ObITokenIterator ObIFTDict *dict_main_; ObIFTDict *dict_quan_; ObIFTDict *dict_stop_; + ObFTUserDictHandle user_main_; + ObFTUserDictHandle user_quan_; + ObFTUserDictHandle user_stop_; + ObIKArbitrator arb_; + ObArenaAllocator scratch_alloc_; DISABLE_COPY_ASSIGN(ObIKFTParser); }; diff --git a/src/storage/fts/ob_ngram2_ft_parser.cpp b/src/storage/fts/ob_ngram2_ft_parser.cpp index b694b1fff..28448109d 100644 --- a/src/storage/fts/ob_ngram2_ft_parser.cpp +++ b/src/storage/fts/ob_ngram2_ft_parser.cpp @@ -87,6 +87,18 @@ int ObNgram2FTParser::get_next_token(const char *&word, return ret; } +int ObNgram2FTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ngram2 parser has not been initialized", K(ret)); + } else if (OB_FAIL(ngram_impl_.reuse_parser(fulltext, fulltext_len))) { + LOG_WARN("failed to reuse ngram2 implementation", K(ret)); + } + return ret; +} + ObNgram2FTParserDesc::ObNgram2FTParserDesc() : is_inited_(false) {} int ObNgram2FTParserDesc::init(ObPluginParam *param) diff --git a/src/storage/fts/ob_ngram2_ft_parser.h b/src/storage/fts/ob_ngram2_ft_parser.h index e4cb237ed..f4a3957bd 100644 --- a/src/storage/fts/ob_ngram2_ft_parser.h +++ b/src/storage/fts/ob_ngram2_ft_parser.h @@ -36,6 +36,7 @@ class ObNgram2FTParser final : public plugin::ObITokenIterator int64_t &word_len, int64_t &char_len, int64_t &word_freq) override; + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; VIRTUAL_TO_STRING_KV(K_(is_inited)); diff --git a/src/storage/fts/ob_ngram_ft_parser.cpp b/src/storage/fts/ob_ngram_ft_parser.cpp index 03e1a5b11..fbb226112 100644 --- a/src/storage/fts/ob_ngram_ft_parser.cpp +++ b/src/storage/fts/ob_ngram_ft_parser.cpp @@ -91,6 +91,18 @@ int ObNgramFTParser::get_next_token( return ret; } +int ObNgramFTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ngram parser has not been initialized", K(ret)); + } else if (OB_FAIL(ngram_impl_.reuse_parser(fulltext, fulltext_len))) { + LOG_WARN("failed to reuse ngram implementation", K(ret)); + } + return ret; +} + ObNgramFTParserDesc::ObNgramFTParserDesc() : is_inited_(false) { diff --git a/src/storage/fts/ob_ngram_ft_parser.h b/src/storage/fts/ob_ngram_ft_parser.h index 07dedbe9e..4a20a4d06 100644 --- a/src/storage/fts/ob_ngram_ft_parser.h +++ b/src/storage/fts/ob_ngram_ft_parser.h @@ -40,6 +40,7 @@ class ObNgramFTParser final : public plugin::ObITokenIterator int64_t &word_len, int64_t &char_len, int64_t &word_freq) override; + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; VIRTUAL_TO_STRING_KV(K_(is_inited)); diff --git a/src/storage/fts/ob_whitespace_ft_parser.cpp b/src/storage/fts/ob_whitespace_ft_parser.cpp index a6d86e49b..3e7a905ac 100644 --- a/src/storage/fts/ob_whitespace_ft_parser.cpp +++ b/src/storage/fts/ob_whitespace_ft_parser.cpp @@ -54,6 +54,23 @@ void ObSpaceFTParser::reset() is_inited_ = false; } +int ObSpaceFTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("space parser has not been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext for space parser reuse", K(ret), KP(fulltext), K(fulltext_len)); + } else { + start_ = fulltext; + next_ = fulltext; + end_ = fulltext + fulltext_len; + } + return ret; +} + int ObSpaceFTParser::init(ObFTParserParam *param) { int ret = OB_SUCCESS; diff --git a/src/storage/fts/ob_whitespace_ft_parser.h b/src/storage/fts/ob_whitespace_ft_parser.h index 460ae3d6a..aaa64526c 100644 --- a/src/storage/fts/ob_whitespace_ft_parser.h +++ b/src/storage/fts/ob_whitespace_ft_parser.h @@ -35,6 +35,7 @@ class ObSpaceFTParser final : public plugin::ObITokenIterator int init(plugin::ObFTParserParam *param); void reset(); + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; virtual int get_next_token( const char *&word, int64_t &word_len, diff --git a/src/storage/fts/utils/ob_ft_ngram_impl.cpp b/src/storage/fts/utils/ob_ft_ngram_impl.cpp index 4fcbd5733..5f435487a 100644 --- a/src/storage/fts/utils/ob_ft_ngram_impl.cpp +++ b/src/storage/fts/utils/ob_ft_ngram_impl.cpp @@ -73,6 +73,24 @@ void ObFTNgramImpl::reset() is_inited_ = false; } +int ObFTNgramImpl::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ngram parser has not been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext for ngram parser reuse", K(ret), KP(fulltext), K(fulltext_len)); + } else { + fulltext_start_ = fulltext; + fulltext_end_ = fulltext + fulltext_len; + cur_ = fulltext_start_; + window_.reset(); + } + return ret; +} + int ObFTNgramImpl::get_next_token(const char *&word, int64_t &word_len, diff --git a/src/storage/fts/utils/ob_ft_ngram_impl.h b/src/storage/fts/utils/ob_ft_ngram_impl.h index 3a59a62c3..560d7ea1c 100644 --- a/src/storage/fts/utils/ob_ft_ngram_impl.h +++ b/src/storage/fts/utils/ob_ft_ngram_impl.h @@ -41,6 +41,8 @@ class ObFTNgramImpl final void reset(); + int reuse_parser(const char *fulltext, const int64_t fulltext_len); + int get_next_token(const char *&word, int64_t &word_len, int64_t &char_cnt, int64_t &word_freq); private: diff --git a/tools/deploy/mysql_test/test_suite/ai_funcs/r/ik_custom_dict.result b/tools/deploy/mysql_test/test_suite/ai_funcs/r/ik_custom_dict.result index 0d502104e..e35cb2ac3 100644 --- a/tools/deploy/mysql_test/test_suite/ai_funcs/r/ik_custom_dict.result +++ b/tools/deploy/mysql_test/test_suite/ai_funcs/r/ik_custom_dict.result @@ -2,36 +2,146 @@ DROP DATABASE IF EXISTS ai_ik_test; CREATE DATABASE ai_ik_test; USE ai_ik_test; -# dictionary table (FULLTEXT_DICT='Y') loaded with custom words, then refreshed into memory +# invalid dictionary table definitions are rejected +CREATE TABLE invalid_no_iot (word varchar(100) primary key) +DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT table must specify ORGANIZATION INDEX +CREATE TABLE invalid_column_name (term varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT table column must be named word +CREATE TABLE invalid_without_pk (word varchar(100)) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT word column must be the primary key +CREATE TABLE invalid_type (word int primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT word column must be VARCHAR +CREATE TABLE invalid_extra_column (word varchar(100) primary key, extra int) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT table must contain only the word column +CREATE TABLE invalid_length (word varchar(501) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT word column length must be between 1 and 500 +CREATE TABLE invalid_flag (word varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='N'; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT must be 'Y' +# create main, stopword, and quantifier dictionary tables CREATE TABLE my_dict (word varchar(100) primary key) ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci FULLTEXT_DICT='Y'; +CREATE TABLE my_stopwords (word varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +CREATE TABLE my_quantifiers (word varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; INSERT INTO my_dict (word) VALUES ('OceanBase'),('分布式数据库'),('全文索引'); -ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_dict; -# business table + fulltext index that uses the custom dictionary +INSERT INTO my_stopwords (word) VALUES ('全文索引'); +INSERT INTO my_quantifiers (word) VALUES ('盒'); +# SHOW CREATE persists the dictionary flag +SHOW CREATE TABLE my_dict; +Table Create Table +my_dict CREATE TABLE `my_dict` ( + `word` varchar(100) NOT NULL, + PRIMARY KEY (`word`) +) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 FULLTEXT_DICT = 'Y' +# REFRESH accepts qualified and current-database table names +ALTER SYSTEM REFRESH FULLTEXT DICT my_dict; +ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_stopwords; +ALTER SYSTEM REFRESH FULLTEXT DICT my_quantifiers; +CREATE TABLE ordinary (word varchar(100) primary key) DEFAULT CHARSET=utf8mb4; +ALTER SYSTEM REFRESH FULLTEXT DICT ordinary; +ERROR HY000: Incorrect arguments to table is not a FULLTEXT_DICT table +ALTER SYSTEM REFRESH FULLTEXT DICT missing_dict; +ERROR 42S02: Table doesn't exist +# non-qualified parser properties are persisted as database-qualified names CREATE TABLE articles ( id int unsigned NOT NULL AUTO_INCREMENT, title varchar(200) NOT NULL, PRIMARY KEY (id) ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; ALTER TABLE articles ADD FULLTEXT INDEX ft_title(title) -WITH PARSER ik PARSER_PROPERTIES=(dict_table='ai_ik_test.my_dict'); +WITH PARSER ik PARSER_PROPERTIES=(dict_table='my_dict', +stopword_table='my_stopwords', +quantifier_table='my_quantifiers', +ik_mode='max_word'); +SHOW CREATE TABLE articles; +Table Create Table +articles CREATE TABLE `articles` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `title` varchar(200) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + FULLTEXT KEY `ft_title` (`title`) WITH PARSER ik PARSER_PROPERTIES=(dict_table="ai_ik_test.my_dict",stopword_table="ai_ik_test.my_stopwords",quantifier_table="ai_ik_test.my_quantifiers",ik_mode="max_word") BLOCK_SIZE 16384 +) ORGANIZATION INDEX AUTO_INCREMENT = 1 AUTO_INCREMENT_MODE = 'ORDER' DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_bin INSERT INTO articles (title) VALUES ('OceanBase数据库介绍'),('全文索引使用指南'); -# custom-dictionary words match their rows +# custom main dictionary matches, custom stopword filters SELECT id, title FROM articles WHERE MATCH(title) AGAINST('OceanBase' IN BOOLEAN MODE) ORDER BY id; id title 1 OceanBase数据库介绍 SELECT id, title FROM articles WHERE MATCH(title) AGAINST('全文索引' IN BOOLEAN MODE) ORDER BY id; id title -2 全文索引使用指南 # a custom dict_table REPLACES the built-in main dict: 数据库 is not listed, so it breaks # into single characters and matches nothing SELECT id, title FROM articles WHERE MATCH(title) AGAINST('数据库' IN BOOLEAN MODE) ORDER BY id; id title -# dynamic update: add a word -> REFRESH -> index a new row -> it matches +# TOKENIZE supports all three custom tables and max_word mode +SELECT TOKENIZE('这是全文索引的介绍', 'ik', +'[{"output":"all"},{"additional_args":[{"dict_table":"my_dict"},{"stopword_table":"my_stopwords"},{"quantifier_table":"my_quantifiers"},{"ik_mode":"max_word"}]}]') AS tokens; +tokens +{"tokens": [{"绍": 1}, {"介": 1}, {"这": 1}, {"是": 1}, {"的": 1}], "doc_len": 5} +SELECT TOKENIZE('3盒', 'ik', +'[{"output":"all"},{"additional_args":[{"dict_table":"my_dict"},{"stopword_table":"my_stopwords"},{"quantifier_table":"my_quantifiers"},{"ik_mode":"smart"}]}]') AS quantity_tokens; +quantity_tokens +{"tokens": [{"3盒": 1}], "doc_len": 1} +# dynamic update affects only rows indexed after REFRESH INSERT INTO my_dict (word) VALUES ('新词汇'); -ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_dict; +INSERT INTO articles (title) VALUES ('刷新前新词汇'); +SELECT id, title FROM articles WHERE MATCH(title) AGAINST('新词汇' IN BOOLEAN MODE) ORDER BY id; +id title +ALTER SYSTEM REFRESH FULLTEXT DICT my_dict; INSERT INTO articles (title) VALUES ('包含新词汇的标题'); SELECT id, title FROM articles WHERE MATCH(title) AGAINST('新词汇' IN BOOLEAN MODE) ORDER BY id; id title -3 包含新词汇的标题 +4 包含新词汇的标题 +# referenced dictionaries reject structural DDL and DROP +ALTER TABLE my_dict ADD COLUMN extra int; +ERROR 0A000: structural ALTER on a referenced FULLTEXT_DICT table is not supported +ALTER TABLE my_dict MODIFY COLUMN word varchar(101); +ERROR 0A000: structural ALTER on a referenced FULLTEXT_DICT table is not supported +ALTER TABLE my_dict RENAME COLUMN word TO term; +ERROR 0A000: structural ALTER on a referenced FULLTEXT_DICT table is not supported +ALTER TABLE my_dict RENAME TO my_dict_renamed; +ERROR 0A000: structural ALTER on a referenced FULLTEXT_DICT table is not supported +RENAME TABLE my_stopwords TO my_stopwords_renamed; +ERROR 0A000: renaming a referenced FULLTEXT_DICT table is not supported +DROP TABLE my_quantifiers; +ERROR HY000: dropping a referenced FULLTEXT_DICT table is not allowed +# non-structural ALTER preserves FULLTEXT_DICT and dependencies are released with the index +ALTER TABLE my_dict COMMENT='still a dictionary'; +SHOW CREATE TABLE my_dict; +Table Create Table +my_dict CREATE TABLE `my_dict` ( + `word` varchar(100) NOT NULL, + PRIMARY KEY (`word`) +) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 FULLTEXT_DICT = 'Y' +ALTER TABLE articles DROP INDEX ft_title; +DROP TABLE my_dict, my_stopwords, my_quantifiers; +# an unreferenced dictionary table can be dropped directly +CREATE TABLE unused_dict (word varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +DROP TABLE unused_dict; +# an unreferenced dictionary may be altered, but an invalid structure cannot be loaded +CREATE TABLE invalidated_dict (word varchar(100) primary key) +ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +INSERT INTO invalidated_dict VALUES ('domain_term'); +ALTER TABLE invalidated_dict ADD COLUMN extra int; +ALTER SYSTEM REFRESH FULLTEXT DICT invalidated_dict; +ERROR HY000: Incorrect arguments to FULLTEXT_DICT table must contain only the word column +SELECT TOKENIZE('domain_term', 'ik', +'[{"output":"all"},{"additional_args":[{"dict_table":"invalidated_dict"}]}]'); +ERROR HY000: Incorrect arguments to FULLTEXT_DICT table must contain only the word column +CREATE TABLE invalid_articles ( +id int primary key, +title varchar(200) NOT NULL +) DEFAULT CHARSET=utf8mb4; +ALTER TABLE invalid_articles ADD FULLTEXT INDEX ft_invalid(title) +WITH PARSER ik PARSER_PROPERTIES=(dict_table='invalidated_dict'); +ERROR HY000: Invalid argument +DROP TABLE invalid_articles, invalidated_dict; DROP DATABASE ai_ik_test; diff --git a/tools/deploy/mysql_test/test_suite/ai_funcs/t/ik_custom_dict.test b/tools/deploy/mysql_test/test_suite/ai_funcs/t/ik_custom_dict.test index 6d4b90e22..fbd214d46 100644 --- a/tools/deploy/mysql_test/test_suite/ai_funcs/t/ik_custom_dict.test +++ b/tools/deploy/mysql_test/test_suite/ai_funcs/t/ik_custom_dict.test @@ -5,33 +5,133 @@ DROP DATABASE IF EXISTS ai_ik_test; CREATE DATABASE ai_ik_test; USE ai_ik_test; ---echo # dictionary table (FULLTEXT_DICT='Y') loaded with custom words, then refreshed into memory +--echo # invalid dictionary table definitions are rejected +--error 1210 +CREATE TABLE invalid_no_iot (word varchar(100) primary key) + DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_column_name (term varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_without_pk (word varchar(100)) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_type (word int primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_extra_column (word varchar(100) primary key, extra int) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_length (word varchar(501) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +--error 1210 +CREATE TABLE invalid_flag (word varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='N'; + +--echo # create main, stopword, and quantifier dictionary tables CREATE TABLE my_dict (word varchar(100) primary key) ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci FULLTEXT_DICT='Y'; +CREATE TABLE my_stopwords (word varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +CREATE TABLE my_quantifiers (word varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; INSERT INTO my_dict (word) VALUES ('OceanBase'),('分布式数据库'),('全文索引'); -ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_dict; +INSERT INTO my_stopwords (word) VALUES ('全文索引'); +INSERT INTO my_quantifiers (word) VALUES ('盒'); + +--echo # SHOW CREATE persists the dictionary flag +--replace_regex / ROW_FORMAT = .*$// +SHOW CREATE TABLE my_dict; + +--echo # REFRESH accepts qualified and current-database table names +ALTER SYSTEM REFRESH FULLTEXT DICT my_dict; +ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_stopwords; +ALTER SYSTEM REFRESH FULLTEXT DICT my_quantifiers; +CREATE TABLE ordinary (word varchar(100) primary key) DEFAULT CHARSET=utf8mb4; +--error 1210 +ALTER SYSTEM REFRESH FULLTEXT DICT ordinary; +--error 1146 +ALTER SYSTEM REFRESH FULLTEXT DICT missing_dict; ---echo # business table + fulltext index that uses the custom dictionary +--echo # non-qualified parser properties are persisted as database-qualified names CREATE TABLE articles ( id int unsigned NOT NULL AUTO_INCREMENT, title varchar(200) NOT NULL, PRIMARY KEY (id) ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; ALTER TABLE articles ADD FULLTEXT INDEX ft_title(title) - WITH PARSER ik PARSER_PROPERTIES=(dict_table='ai_ik_test.my_dict'); + WITH PARSER ik PARSER_PROPERTIES=(dict_table='my_dict', + stopword_table='my_stopwords', + quantifier_table='my_quantifiers', + ik_mode='max_word'); +--replace_regex / ROW_FORMAT = .*$// +SHOW CREATE TABLE articles; INSERT INTO articles (title) VALUES ('OceanBase数据库介绍'),('全文索引使用指南'); ---echo # custom-dictionary words match their rows +--echo # custom main dictionary matches, custom stopword filters SELECT id, title FROM articles WHERE MATCH(title) AGAINST('OceanBase' IN BOOLEAN MODE) ORDER BY id; SELECT id, title FROM articles WHERE MATCH(title) AGAINST('全文索引' IN BOOLEAN MODE) ORDER BY id; --echo # a custom dict_table REPLACES the built-in main dict: 数据库 is not listed, so it breaks --echo # into single characters and matches nothing SELECT id, title FROM articles WHERE MATCH(title) AGAINST('数据库' IN BOOLEAN MODE) ORDER BY id; ---echo # dynamic update: add a word -> REFRESH -> index a new row -> it matches +--echo # TOKENIZE supports all three custom tables and max_word mode +SELECT TOKENIZE('这是全文索引的介绍', 'ik', + '[{"output":"all"},{"additional_args":[{"dict_table":"my_dict"},{"stopword_table":"my_stopwords"},{"quantifier_table":"my_quantifiers"},{"ik_mode":"max_word"}]}]') AS tokens; +SELECT TOKENIZE('3盒', 'ik', + '[{"output":"all"},{"additional_args":[{"dict_table":"my_dict"},{"stopword_table":"my_stopwords"},{"quantifier_table":"my_quantifiers"},{"ik_mode":"smart"}]}]') AS quantity_tokens; + +--echo # dynamic update affects only rows indexed after REFRESH INSERT INTO my_dict (word) VALUES ('新词汇'); -ALTER SYSTEM REFRESH FULLTEXT DICT ai_ik_test.my_dict; +INSERT INTO articles (title) VALUES ('刷新前新词汇'); +SELECT id, title FROM articles WHERE MATCH(title) AGAINST('新词汇' IN BOOLEAN MODE) ORDER BY id; +ALTER SYSTEM REFRESH FULLTEXT DICT my_dict; INSERT INTO articles (title) VALUES ('包含新词汇的标题'); SELECT id, title FROM articles WHERE MATCH(title) AGAINST('新词汇' IN BOOLEAN MODE) ORDER BY id; +--echo # referenced dictionaries reject structural DDL and DROP +--error 1235 +ALTER TABLE my_dict ADD COLUMN extra int; +--error 1235 +ALTER TABLE my_dict MODIFY COLUMN word varchar(101); +--error 1235 +ALTER TABLE my_dict RENAME COLUMN word TO term; +--error 1235 +ALTER TABLE my_dict RENAME TO my_dict_renamed; +--error 1235 +RENAME TABLE my_stopwords TO my_stopwords_renamed; +--error 4179 +DROP TABLE my_quantifiers; + +--echo # non-structural ALTER preserves FULLTEXT_DICT and dependencies are released with the index +ALTER TABLE my_dict COMMENT='still a dictionary'; +--replace_regex / ROW_FORMAT = .*$// +SHOW CREATE TABLE my_dict; +ALTER TABLE articles DROP INDEX ft_title; +DROP TABLE my_dict, my_stopwords, my_quantifiers; + +--echo # an unreferenced dictionary table can be dropped directly +CREATE TABLE unused_dict (word varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +DROP TABLE unused_dict; + +--echo # an unreferenced dictionary may be altered, but an invalid structure cannot be loaded +CREATE TABLE invalidated_dict (word varchar(100) primary key) + ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 FULLTEXT_DICT='Y'; +INSERT INTO invalidated_dict VALUES ('domain_term'); +ALTER TABLE invalidated_dict ADD COLUMN extra int; +--error 1210 +ALTER SYSTEM REFRESH FULLTEXT DICT invalidated_dict; +--error 1210 +SELECT TOKENIZE('domain_term', 'ik', + '[{"output":"all"},{"additional_args":[{"dict_table":"invalidated_dict"}]}]'); +CREATE TABLE invalid_articles ( + id int primary key, + title varchar(200) NOT NULL +) DEFAULT CHARSET=utf8mb4; +--error 1210 +ALTER TABLE invalid_articles ADD FULLTEXT INDEX ft_invalid(title) + WITH PARSER ik PARSER_PROPERTIES=(dict_table='invalidated_dict'); +DROP TABLE invalid_articles, invalidated_dict; + DROP DATABASE ai_ik_test;