diff --git a/.github/workflows/upload_engine.yaml b/.github/workflows/upload_engine.yaml index ddd3ed5..701b005 100644 --- a/.github/workflows/upload_engine.yaml +++ b/.github/workflows/upload_engine.yaml @@ -17,12 +17,8 @@ jobs: include: - name: document-search engine_config: document-search/notebooks/blog_engine_config.yaml - - name: fashion-catalog-semantic-search - engine_config: fashion-catalog/python/engine_configs/simple_semantic_search.yaml - name: fashion-catalog-autotune engine_config: fashion-catalog/python/engine_configs/autotune_lightgbm.yaml - - name: movie-recommendations-v1 - engine_config: movie-recommendations/model/engine_configs/search_and_recommendations_v1.yaml - name: movie-recommendations-v2 engine_config: movie-recommendations/model/engine_configs/search_and_recommendations_v2.yaml defaults: diff --git a/apps/document-search/notebooks/blog_engine_config.yaml b/apps/document-search/notebooks/blog_engine_config.yaml index cc14394..4d9cab8 100644 --- a/apps/document-search/notebooks/blog_engine_config.yaml +++ b/apps/document-search/notebooks/blog_engine_config.yaml @@ -1,22 +1,46 @@ version: v2 -name: demo___blog_semantic_search +name: shaped_blog_semantic_search_and_rag + data: item_table: type: query - query: select toDateTime(toInt64(created_at) / 1000) as created_at_datetime, item_id, author, categories, chunk_metadata, content, created_at, featured, main_image, name, popular, article_id, read_length_in_mins, release_date, roles, slug, updated_at, url from blog_post_chunked order by created_at_datetime asc + query: | + select + toDateTime(toInt64(created_at) / 1000) as created_at_datetime, + article_id, + item_id, + author, + categories, + chunk_metadata, + content, + created_at, + featured, + main_image, + name, + popular, + read_length_in_mins, + release_date, + roles, + slug, + updated_at, + url + from blog_articles_chunked + order by created_at_datetime asc + index: lexical_search: item_fields: - - name - - content - - author - - categories - - roles + - name + - content + - author + - categories + - roles + embeddings: - name: content_embedding encoder: type: hugging_face - model_name: sentence-transformers/all-MiniLM-L6-v2 + model_name: Alibaba-NLP/gte-modernbert-base item_fields: - name - content diff --git a/apps/fashion-catalog/python/engine_configs/autotune_lightgbm.yaml b/apps/fashion-catalog/python/engine_configs/autotune_lightgbm.yaml index b4bc17b..d338786 100644 --- a/apps/fashion-catalog/python/engine_configs/autotune_lightgbm.yaml +++ b/apps/fashion-catalog/python/engine_configs/autotune_lightgbm.yaml @@ -1,25 +1,61 @@ +# this model demonstrates autotuning a lightgbm model version: v2 -name: hm__autotune_lightgbm +name: hm_recommendations_and_search + data: - item_dataset: - name: hm__articles - query: select * EXCEPT (article_id), article_id as item_id from hm__articles - user_dataset: - name: hm__customers - query: select * EXCEPT (customer_id), customer_id as user_id from hm__customers - interactions_dataset: - name: hm__transactions - query: select * EXCEPT (article_id, customer_id), article_id as item_id, customer_id as user_id from hm__transactions - index: - search: - item_fields: - - prod_name - - prod_type_name - - product_group_name - - colour_group_name - - department_name -training: - models: - - policy_type: lightgbm - - policies: - - lightgbm \ No newline at end of file + item_table: + type: table + name: hm_articles + + user_table: + type: table + name: hm_customers + + interaction_table: + type: query + query: select *, '5' as label from hm_transactions + + schema_override: + item: + id: item_id + features: + - name: image_url + type: Image + +index: + embeddings: + - name: image_embedding + encoder: + type: hugging_face + model_name: openai/clip-vit-base-patch32 + item_fields: + - image_url + - name: text_content_embedding + encoder: + type: hugging_face + model_name: Alibaba-NLP/gte-modernbert-base + item_fields: + - prod_name + - index_name + - detail_desc + - section_name + - colour_group_name + - product_group_name + - graphical_appearance_name + - perceived_colour_value_name + + lexical_search: + item_fields: + - prod_name + - index_name + - detail_desc + - section_name + - colour_group_name + - product_group_name + - graphical_appearance_name + - perceived_colour_value_name + +training: + models: + - name: lightgbm + policy_type: lightgbm \ No newline at end of file diff --git a/apps/movie-recommendations/model/engine_configs/search_and_recommendations_v2.yaml b/apps/movie-recommendations/model/engine_configs/search_and_recommendations_v2.yaml index 4bda6f5..c3890c4 100644 --- a/apps/movie-recommendations/model/engine_configs/search_and_recommendations_v2.yaml +++ b/apps/movie-recommendations/model/engine_configs/search_and_recommendations_v2.yaml @@ -1,34 +1,49 @@ version: v2 -name: movielens__simple_semantic_search_2 -data: - item_dataset: - name: movies_2018_enriched - query: | - SELECT r.movie_id as item_id, r.movie_title as title, r.poster_url, - r.imdb_url_new as imdb_url, r.genres as genres, r.description as - description, r.interests as interests, r.release_date as release_date, - r.cast as cast, r.directors as directors, r.writers as writers FROM - movies_2018_enriched r - index: - embeddings: - - name: metadata_embedding - encoder: - model_name: sentence-transformers/all-MiniLM-L6-v2 - item_fields: - - title - - description - - genres - - description - - directors - - writers - - cast - type: hugging_face - search: - item_fields: - - title - - description - - genres - - description - - directors - - writers - - cast \ No newline at end of file +name: movielens_recommendations_and_search + +data: + item_table: + type: query + query: select item_id as item_id, toDateTimeOrNull(release_date) AS release_datetime, cast, genres, imdbId, writers, directors, interests, poster_url, description, movie_title from movielens_items_enriched order by release_datetime asc + + interaction_table: + type: query + query: SELECT r.user_id as user_id, r.item_id as item_id, r.timestamp AS created_at, r.label AS label FROM movielens_ratings_2018 r + + schema_override: + item: + id: item_id + features: + - name: poster_url + type: Image + +index: + embeddings: + - name: item_content_embedding + encoder: + type: hugging_face + model_name: Alibaba-NLP/gte-modernbert-base + item_fields: + - movie_title + - description + - cast + - genres + - writers + - directors + - interests + - name: poster_embedding + encoder: + type: hugging_face + model_name: openai/clip-vit-base-patch32 + item_fields: + - poster_url + + lexical_search: + item_fields: + - movie_title + - description + - cast + - genres + - writers + - directors + - interests \ No newline at end of file