Fix variable-length arrays

Author: jwijffels

R/paragraph2vec.R

@@ -284,7 +284,7 @@ summary.paragraph2vec_trained <- function(object, type = "vocabulary", which = c
 #' \item{'doc2doc', 'word2doc', 'word2word', 'sent2doc' can be chosen if \code{type} is set to 'nearest' indicating to extract respectively
 #' the closest document to a document (doc2doc), the closest document to a word (word2doc), the closest word to a word (word2word) or the closest document to sentences (sent2doc).}
 #' }
-#' @param top_n show only the top n nearest neighbours. Defaults to 10. Only used for \code{type} 'nearest'.
+#' @param top_n show only the top n nearest neighbours. Defaults to 10, with a maximum value of 100. Only used for \code{type} 'nearest'.
 #' @param normalize logical indicating to normalize the embeddings. Defaults to \code{TRUE}. Only used for \code{type} 'embedding'.
 #' @param encoding set the encoding of the text elements to the specified encoding. Defaults to 'UTF-8'. 
 #' @param ... not used
@@ -350,6 +350,7 @@ predict.paragraph2vec <- function(object, newdata,
   type  <- match.arg(type)
   which <- match.arg(which)
   top_n <- as.integer(top_n)
+  stopifnot(top_n <= 100)
   if(type == "embedding"){
     stopifnot(which %in% c("docs", "words"))
     if(is.character(newdata)){

README.md

@@ -50,24 +50,18 @@ x        <- subset(x, nwords < 1000 & nchar(text) > 0)
 -  Build the model 
 
 
-```r
-model <- paragraph2vec(x = x, type = "PV-DBOW", dim = 100, iter = 20, min_count = 5, 
-                       lr = 0.05, threads = 4)
-```
-
-
 ```r
 ## Low-dimensional model using DM, low number of iterations, for speed and display purposes
-model <- paragraph2vec(x = x, type = "PV-DM",   dim = 5,   iter = 3,  min_count = 5, 
-                       lr = 0.05, threads = 1)
+model <- paragraph2vec(x = x, type = "PV-DM", dim = 5, iter = 3,  
+                       min_count = 5, lr = 0.05, threads = 1)
 str(model)
 ```
 
 ```
 ## List of 3
 ##  $ model  :<externalptr> 
 ##  $ data   :List of 4
-##   ..$ file        : chr "C:\\Users\\Jan\\AppData\\Local\\Temp\\RtmpApjuPd\\textspace_1ef05c50176.txt"
+##   ..$ file        : chr "C:\\Users\\Jan\\AppData\\Local\\Temp\\Rtmpk9Npjg\\textspace_1c4458cb6943.txt"
 ##   ..$ n           : num 170469
 ##   ..$ n_vocabulary: num 3867
 ##   ..$ n_docs      : num 1000
@@ -84,6 +78,13 @@ str(model)
 ##  - attr(*, "class")= chr "paragraph2vec_trained"
 ```
 
+
+```r
+## More realistic model
+model <- paragraph2vec(x = x, type = "PV-DBOW", dim = 100, iter = 20, 
+                       min_count = 5, lr = 0.05, threads = 4)
+```
+
 -  Get the embedding of the documents or words and get the vocabulary
 
 
@@ -104,14 +105,22 @@ sentences <- list(
 embedding <- predict(model, newdata = sentences,                     type = "embedding")
 embedding <- predict(model, newdata = c("geld", "koning"),           type = "embedding", which = "words")
 embedding <- predict(model, newdata = c("doc_1", "doc_10", "doc_3"), type = "embedding", which = "docs")
-embedding
+ncol(embedding)
 ```
 
 ```
-##              [,1]      [,2]       [,3]        [,4]        [,5]
-## doc_1  0.09160496 0.5503142 -0.5195833 0.162630379 -0.62637627
-## doc_10 0.43539885 0.1009961 -0.8531511 0.266749799  0.03471836
-## doc_3  0.59375095 0.3877517 -0.6868675 0.002579026 -0.15910600
+## [1] 100
+```
+
+```r
+embedding[, 1:4]
+```
+
+```
+##              [,1]        [,2]        [,3]        [,4]
+## doc_1  0.08172660 -0.03679979  0.05726605 -0.06496991
+## doc_10 0.13976580  0.10821507 -0.06986591 -0.05825572
+## doc_3  0.09486584 -0.07999156  0.03448128  0.02999697
 ```
 
 -  Get similar documents or words when providing sentences, documents or words
@@ -124,20 +133,20 @@ nn
 
 ```
 ## [[1]]
-##      term1   term2 similarity rank
-## 1 proximus   neemt  0.9994797    1
-## 2 proximus plaatse  0.9994527    2
-## 3 proximus     ver  0.9993714    3
-## 4 proximus  gratis  0.9992922    4
-## 5 proximus hiermee  0.9992417    5
+##      term1              term2 similarity rank
+## 1 proximus telefoontoestellen  0.5571629    1
+## 2 proximus            belfius  0.4994604    2
+## 3 proximus         toenmalige  0.4873388    3
+## 4 proximus internetverbinding  0.4730936    4
+## 5 proximus       gefactureerd  0.4568973    5
 ## 
 ## [[2]]
-##    term1        term2 similarity rank
-## 1 koning      pleiten  0.9984228    1
-## 2 koning     ongeacht  0.9983451    2
-## 3 koning pensionering  0.9982112    3
-## 4 koning    profielen  0.9981233    4
-## 5 koning    beschermd  0.9978001    5
+##    term1          term2 similarity rank
+## 1 koning       grondwet  0.5572801    1
+## 2 koning verplaatsingen  0.5373006    2
+## 3 koning     ministerie  0.5140343    3
+## 4 koning        familie  0.4943074    4
+## 5 koning       vereiste  0.4715540    5
 ```
 
 ```r
@@ -148,19 +157,19 @@ nn
 ```
 ## [[1]]
 ##      term1   term2 similarity rank
-## 1 proximus  doc_77  0.9989672    1
-## 2 proximus doc_263  0.9989251    2
-## 3 proximus doc_260  0.9982057    3
-## 4 proximus doc_344  0.9980863    4
-## 5 proximus doc_408  0.9979483    5
+## 1 proximus doc_105  0.6922343    1
+## 2 proximus doc_863  0.5826316    2
+## 3 proximus doc_186  0.5146015    3
+## 4 proximus doc_862  0.5051525    4
+## 5 proximus doc_746  0.4467830    5
 ## 
 ## [[2]]
 ##    term1   term2 similarity rank
-## 1 koning doc_553  0.9980003    1
-## 2 koning doc_477  0.9964797    2
-## 3 koning doc_658  0.9955103    3
-## 4 koning  doc_99  0.9953933    4
-## 5 koning doc_163  0.9953347    5
+## 1 koning  doc_44  0.6228581    1
+## 2 koning doc_583  0.5643232    2
+## 3 koning  doc_45  0.5535781    3
+## 4 koning doc_797  0.4408725    4
+## 5 koning doc_943  0.4039679    5
 ```
 
 ```r
@@ -171,19 +180,19 @@ nn
 ```
 ## [[1]]
 ##     term1   term2 similarity rank
-## 1 doc_198 doc_882  0.9992993    1
-## 2 doc_198 doc_709  0.9990637    2
-## 3 doc_198 doc_122  0.9989671    3
-## 4 doc_198 doc_121  0.9988763    4
-## 5 doc_198 doc_569  0.9988336    5
+## 1 doc_198 doc_343  0.4893735    1
+## 2 doc_198 doc_569  0.4858374    2
+## 3 doc_198 doc_358  0.4831750    3
+## 4 doc_198 doc_498  0.4766597    4
+## 5 doc_198 doc_983  0.4761481    5
 ## 
 ## [[2]]
 ##     term1   term2 similarity rank
-## 1 doc_285 doc_722  0.9988106    1
-## 2 doc_285 doc_467  0.9977189    2
-## 3 doc_285 doc_250  0.9976925    3
-## 4 doc_285 doc_174  0.9975280    4
-## 5 doc_285 doc_294  0.9968556    5
+## 1 doc_285 doc_319  0.5304061    1
+## 2 doc_285 doc_286  0.5205777    2
+## 3 doc_285  doc_76  0.5086077    3
+## 4 doc_285  doc_74  0.4975725    4
+## 5 doc_285 doc_537  0.4802507    5
 ```
 
 ```r
@@ -197,19 +206,19 @@ nn
 ```
 ## $sent1
 ##   term1   term2 similarity rank
-## 1 sent1 doc_980  0.9784521    1
-## 2 sent1 doc_758  0.9678799    2
-## 3 sent1 doc_806  0.9547009    3
-## 4 sent1 doc_764  0.9544759    4
-## 5 sent1 doc_842  0.9529226    5
+## 1 sent1 doc_740  0.4637638    1
+## 2 sent1 doc_742  0.4621139    2
+## 3 sent1 doc_206  0.4315273    3
+## 4 sent1 doc_825  0.4221503    4
+## 5 sent1 doc_151  0.4183135    5
 ## 
 ## $sent2
 ##   term1   term2 similarity rank
-## 1 sent2 doc_842  0.9873239    1
-## 2 sent2 doc_764  0.9832168    2
-## 3 sent2 doc_564  0.9739662    3
-## 4 sent2 doc_980  0.9675324    4
-## 5 sent2 doc_542  0.9622889    5
+## 1 sent2 doc_105  0.5789919    1
+## 2 sent2 doc_186  0.4938067    2
+## 3 sent2 doc_862  0.4848365    3
+## 4 sent2 doc_863  0.4685720    4
+## 5 sent2 doc_620  0.4497271    5
 ```
 
 ```r

man/predict.paragraph2vec.Rd

@@ -62,28 +62,28 @@ BEGIN_RCPP
 END_RCPP
 }
 // paragraph2vec_nearest
-Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, std::size_t top_n, std::string type);
+Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, int top_n, std::string type);
 RcppExport SEXP _doc2vec_paragraph2vec_nearest(SEXP ptrSEXP, SEXP xSEXP, SEXP top_nSEXP, SEXP typeSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
     Rcpp::traits::input_parameter< SEXP >::type ptr(ptrSEXP);
     Rcpp::traits::input_parameter< std::string >::type x(xSEXP);
-    Rcpp::traits::input_parameter< std::size_t >::type top_n(top_nSEXP);
+    Rcpp::traits::input_parameter< int >::type top_n(top_nSEXP);
     Rcpp::traits::input_parameter< std::string >::type type(typeSEXP);
     rcpp_result_gen = Rcpp::wrap(paragraph2vec_nearest(ptr, x, top_n, type));
     return rcpp_result_gen;
 END_RCPP
 }
 // paragraph2vec_nearest_sentence
-Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, std::size_t top_n);
+Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, int top_n);
 RcppExport SEXP _doc2vec_paragraph2vec_nearest_sentence(SEXP ptrSEXP, SEXP xSEXP, SEXP top_nSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
     Rcpp::traits::input_parameter< SEXP >::type ptr(ptrSEXP);
     Rcpp::traits::input_parameter< Rcpp::List >::type x(xSEXP);
-    Rcpp::traits::input_parameter< std::size_t >::type top_n(top_nSEXP);
+    Rcpp::traits::input_parameter< int >::type top_n(top_nSEXP);
     rcpp_result_gen = Rcpp::wrap(paragraph2vec_nearest_sentence(ptr, x, top_n));
     return rcpp_result_gen;
 END_RCPP

src/RcppExports.cpp

@@ -13,6 +13,7 @@
 #define MAX_EXP 6
 #define MAX_SENTENCE_LENGTH 1000
 #define MAX_CODE_LENGTH 40
+#define MAX_DOC2VEC_KNN_R 100
 #define MAX_DOC2VEC_KNN 2000
 const int vocab_hash_size = 30000000;
 const int negtive_sample_table_size = 1e8;

src/doc2vec/common_define.h

@@ -92,9 +92,9 @@ std::vector<std::string> paragraph2vec_dictionary(SEXP ptr, std::string type = "
 
 
 // [[Rcpp::export]]
-Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, std::size_t top_n = 10, std::string type = "doc2doc") {
+Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, int top_n = 10, std::string type = "doc2doc") {
   Rcpp::XPtr<Doc2Vec> model(ptr);
-  knn_item_t knn_items[top_n];
+  knn_item_t knn_items[MAX_DOC2VEC_KNN_R];
   if(type == "doc2doc"){
     model->doc_knn_docs(x.c_str(), knn_items, top_n);
   }else if(type == "word2doc"){
@@ -114,6 +114,9 @@ Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, std::size_t top_n
     distance.push_back(kv.similarity);
     r = r + 1;
     rank.push_back(r);
+    if(r >= top_n || r >= MAX_DOC2VEC_KNN_R) {
+      break;
+    }
   } 
   Rcpp::DataFrame out = Rcpp::DataFrame::create(
     Rcpp::Named("term1") = x,
@@ -126,7 +129,7 @@ Rcpp::DataFrame paragraph2vec_nearest(SEXP ptr, std::string x, std::size_t top_n
 }
 
 // [[Rcpp::export]]
-Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, std::size_t top_n = 10) {
+Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, int top_n = 10) {
   Rcpp::XPtr<Doc2Vec> model(ptr);
   real * infer_vector = NULL;
   //int errnr = posix_memalign((void **)&infer_vector, 128, model->dim() * sizeof(real));
@@ -146,7 +149,7 @@ Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, std::size_t to
     }
     model->infer_doc(&doc, infer_vector);
     // Get closest docs to sentence
-    knn_item_t knn_items[top_n];
+    knn_item_t knn_items[MAX_DOC2VEC_KNN_R];
     model->sent_knn_docs(&doc, knn_items, top_n, infer_vector);
     // Collect result in data.frame
     std::vector<std::string> keys;
@@ -159,6 +162,9 @@ Rcpp::List paragraph2vec_nearest_sentence(SEXP ptr, Rcpp::List x, std::size_t to
       distance.push_back(kv.similarity);
       r = r + 1;
       rank.push_back(r);
+      if(r >= top_n || r >= MAX_DOC2VEC_KNN_R) {
+        break;
+      }
     } 
     Rcpp::DataFrame out = Rcpp::DataFrame::create(
       Rcpp::Named("term1") = Rcpp::as<std::string>(rownames_(i)),