diff --git a/packages/client/app/components/component-manuscripts/src/Manuscripts.jsx b/packages/client/app/components/component-manuscripts/src/Manuscripts.jsx index 3b4c3ac66..280eb4d0d 100644 --- a/packages/client/app/components/component-manuscripts/src/Manuscripts.jsx +++ b/packages/client/app/components/component-manuscripts/src/Manuscripts.jsx @@ -5,8 +5,6 @@ import { useContext, useState } from 'react' import { useNavigate } from 'react-router-dom' import styled from 'styled-components' import { Trans, useTranslation } from 'react-i18next' -import { ToastContainer } from 'react-toastify' -import 'react-toastify/dist/ReactToastify.css' import { grid } from '@coko/client' import Page from '../../../ui/shared/Page' @@ -59,6 +57,7 @@ const FlexRow = styled.div` display: flex; gap: ${grid(2)}; justify-content: flex-end; + margin-bottom: ${grid(2)}; ` const FlexRowWithSmallGapAbove = styled(FlexRow)` @@ -416,17 +415,6 @@ const Manuscripts = props => { )} > - {topRightControls} diff --git a/packages/client/app/components/component-manuscripts/src/ManuscriptsPage.jsx b/packages/client/app/components/component-manuscripts/src/ManuscriptsPage.jsx index 5ae9fff09..28add0aba 100644 --- a/packages/client/app/components/component-manuscripts/src/ManuscriptsPage.jsx +++ b/packages/client/app/components/component-manuscripts/src/ManuscriptsPage.jsx @@ -4,8 +4,6 @@ import { useState, useContext, useRef, useEffect } from 'react' import { useLocation } from 'react-router-dom' -import { toast } from 'react-toastify' -import 'react-toastify/dist/ReactToastify.css' import { useTranslation } from 'react-i18next' import { useQuery, @@ -16,6 +14,8 @@ import { } from '@apollo/client/react' import fnv from 'fnv-plus' import { saveAs } from 'file-saver' +import { useNotification } from '@coko/client' + import { ConfigContext } from '../../config/src' import { GET_MANUSCRIPTS_AND_FORM, @@ -46,6 +46,7 @@ const ManuscriptsPage = () => { const location = useLocation() const { t } = useTranslation() const currentUser = useCurrentUser() + const notify = useNotification() const config = useContext(ConfigContext) const { urlFrag } = config @@ -94,20 +95,14 @@ const ManuscriptsPage = () => { ) useSubscription(IMPORTED_MANUSCRIPTS, { - onSubscriptionData: data => { - const { - subscriptionData: { - data: { manuscriptsImportStatus }, - }, - } = data - + onData: ({ data }) => { setIsImporting(false) applyQueryParams({ [URI_PAGENUM_PARAM]: 1 }) + queryObject.refetch() - toast.success( - manuscriptsImportStatus && 'Manuscripts successfully imported', - { hideProgressBar: true }, - ) + if (data.data.manuscriptsImportStatus) { + notify.success({ title: 'Manuscripts successfully imported' }) + } }, }) diff --git a/packages/client/package.json b/packages/client/package.json index 184db2f64..ef8a0ef5a 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -76,7 +76,6 @@ "react-router": "^7.14.2", "react-router-dom": "^7.14.2", "react-select": "^4.2.1", - "react-toastify": "^7.0.4", "react-uid": "^2.3.3", "reactjs-popup": "^2.0.5", "recharts": "^3.8.1", diff --git a/packages/client/yarn.lock b/packages/client/yarn.lock index f034a2bba..ec301cd9d 100644 --- a/packages/client/yarn.lock +++ b/packages/client/yarn.lock @@ -3622,13 +3622,6 @@ __metadata: languageName: node linkType: hard -"clsx@npm:^1.1.1": - version: 1.2.1 - resolution: "clsx@npm:1.2.1" - checksum: 10c0/34dead8bee24f5e96f6e7937d711978380647e936a22e76380290e35486afd8634966ce300fc4b74a32f3762c7d4c0303f442c3e259f4ce02374eb0c82834f27 - languageName: node - linkType: hard - "clsx@npm:^2.1.1": version: 2.1.1 resolution: "clsx@npm:2.1.1" @@ -5434,7 +5427,6 @@ __metadata: react-router: "npm:^7.14.2" react-router-dom: "npm:^7.14.2" react-select: "npm:^4.2.1" - react-toastify: "npm:^7.0.4" react-uid: "npm:^2.3.3" reactjs-popup: "npm:^2.0.5" recharts: "npm:^3.8.1" @@ -7233,18 +7225,6 @@ __metadata: languageName: node linkType: hard -"react-toastify@npm:^7.0.4": - version: 7.0.4 - resolution: "react-toastify@npm:7.0.4" - dependencies: - clsx: "npm:^1.1.1" - peerDependencies: - react: ">=16" - react-dom: ">=16" - checksum: 10c0/2aa396f2aefe4c30498eca3e4bbb560c1b6503065808c98003711cc6273bc1328160cb2a2e844e05ff29ee8d52ad9094d0ebd6ea233d32e38f8516c859c009d6 - languageName: node - linkType: hard - "react-transition-group@npm:^4.3.0": version: 4.4.5 resolution: "react-transition-group@npm:4.4.5" diff --git a/packages/server/controllers/manuscript/importManuscripts.js b/packages/server/controllers/manuscript/importManuscripts.js index a7e1b2631..9d2928f15 100644 --- a/packages/server/controllers/manuscript/importManuscripts.js +++ b/packages/server/controllers/manuscript/importManuscripts.js @@ -18,19 +18,19 @@ const shouldRunDefaultImportsForColab = [true, 'true'].includes( ) const importManuscripts = async (groupId, ctx) => { - // eslint-disable-next-line no-console - console.log(`Importing manuscripts. Triggered by ${ctx.userId ?? 'system'}`) + logger.info(`Importing manuscripts. Triggered by ${ctx.userId ?? 'system'}`) const key = `${groupId}-imports` if (importsInProgress.has(key)) { - // eslint-disable-next-line no-console - console.log('Import already in progress. Aborting new import') + logger.info('Import already in progress. Aborting new import') return false } - try { - importsInProgress.add(key) + importsInProgress.add(key) + + let promises + try { const activeConfig = await Config.query().findOne({ groupId, active: true, @@ -42,7 +42,7 @@ const importManuscripts = async (groupId, ctx) => { ? 'evaluated' : 'accepted' - const promises = [runImports(groupId, evaluatedStatusString, ctx.userId)] + promises = [runImports(groupId, evaluatedStatusString, ctx.userId)] if (activeConfig.formData.instanceName === 'preprint2') { promises.push(importArticlesFromBiorxiv(groupId, ctx)) @@ -53,57 +53,71 @@ const importManuscripts = async (groupId, ctx) => { ) { promises.push(importArticlesFromBiorxivWithFullTextSearch(groupId, ctx)) } + } catch (error) { + importsInProgress.delete(key) + throw error + } - if (!promises.length) return false + if (!promises.length) { + importsInProgress.delete(key) + return false + } - Promise.all(promises) - .catch(error => logger.error(error)) - .finally(async () => { - subscriptionManager.publish('IMPORT_MANUSCRIPTS_STATUS', { - manuscriptsImportStatus: true, - }) + Promise.all(promises) + .catch(error => logger.error(error)) + .finally(() => { + importsInProgress.delete(key) + subscriptionManager.publish('IMPORT_MANUSCRIPTS_STATUS', { + manuscriptsImportStatus: true, }) + }) - return true - } finally { - importsInProgress.delete(key) - } + return true } const importManuscriptsFromSemanticScholar = async (groupId, ctx) => { const key = `${groupId}-SemanticScholar` if (importsInProgress.has(key)) return false - try { - importsInProgress.add(key) + importsInProgress.add(key) + + let promises + try { const activeConfig = await Config.query().findOne({ groupId, active: true, }) - const promises = [] + promises = [] if ( activeConfig.formData.integrations?.semanticScholar.enableSemanticScholar ) { promises.push(importArticlesFromSemanticScholar(groupId, ctx)) } + } catch (error) { + importsInProgress.delete(key) + throw error + } - if (!promises.length) return false + if (!promises.length) { + importsInProgress.delete(key) + return false + } - Promise.all(promises) - .catch(error => logger.error(error)) - .finally(async () => { - subscriptionManager.publish('IMPORT_MANUSCRIPTS_STATUS', { - manuscriptsImportStatus: true, - }) + // The import lock is held until this background work settles, not until + // this function returns, since the caller doesn't await it. + Promise.all(promises) + .catch(error => logger.error(error)) + .finally(() => { + importsInProgress.delete(key) + subscriptionManager.publish('IMPORT_MANUSCRIPTS_STATUS', { + manuscriptsImportStatus: true, }) + }) - return true - } finally { - importsInProgress.delete(key) - } + return true } module.exports = { diff --git a/packages/server/package.json b/packages/server/package.json index 2a08887f1..2f0fa74bc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -32,7 +32,7 @@ "citeproc-js-node": "^0.0.3", "fast-xml-parser": "^4.0.7", "fnv-plus": "^1.3.1", - "form-data": "^4.0.0", + "form-data": "^4.0.6", "fs-extra": "4.0.3", "google-spreadsheet": "^3.1.15", "graphql": "16.9.0", diff --git a/packages/server/services/importArticles/biorxiv-full-text-import.js b/packages/server/services/importArticles/biorxiv-full-text-import.js index 01a5bbe82..a3fddcde6 100644 --- a/packages/server/services/importArticles/biorxiv-full-text-import.js +++ b/packages/server/services/importArticles/biorxiv-full-text-import.js @@ -15,6 +15,7 @@ const { } = require('./importTools') const CURSOR_LIMIT = 200 // This permits up to 10,000 matches, but prevents infinite loop +const PAGES_PER_SAVE_BATCH = 20 const SAVE_CHUNK_SIZE = 50 const TIMEOUT_MS = 30000 const MAX_TRIES = 5 @@ -60,26 +61,6 @@ const formatSearchQueryWithoutCursor = (dateFrom, dateTo) => { const restrictToSubjects = (imports, subjects) => imports.filter(preprint => subjects.includes(preprint.category)) -const importAll = async (queryWithoutCursor, subjects) => { - const imports = [] - let totalRetrievedCount = 0 - - for (let cursor = 0; cursor < CURSOR_LIMIT; cursor += 1) { - const queryString = `${queryWithoutCursor}&cursor=${cursor}` - // eslint-disable-next-line no-await-in-loop - const data = await doAxiosQueryWithRetry(queryString) - if (!data || !data.collection || !data.collection.length) break - totalRetrievedCount += data.collection.length - imports.push(...restrictToSubjects(data.collection, subjects)) - console.log( - `Retrieved ${totalRetrievedCount} of ${data.total_results} total papers from biorxiv (${imports.length} in desired subjects)`, - ) - if (imports.length >= data.total_results) break - } - - return imports -} - const populateUrlAndAbstract = imports => imports.map(item => ({ ...item, @@ -106,39 +87,14 @@ const stripInternalDuplicates = imports => { return Object.values(importsByDoi) } -/** Strip any preprints that share a DOI with a manuscript already belonging to the group */ -const stripDuplicates = async (imports, groupId) => { - // TODO retrieving all manuscripts to check DOIs is inefficient! - const manuscripts = await Manuscript.query().where({ groupId }) - - const currentDois = new Set( - manuscripts.map(m => m.submission.$doi).filter(Boolean), - ) - - return imports.filter(({ doi }) => !currentDois.has(doi)) -} - -const getData = async (groupId, ctx) => { - const sourceId = await getServerId('biorxiv') - const lastImportDate = await getLastImportDate(sourceId, groupId) - const minDate = Math.max(lastImportDate, await getDate2WeeksAgo()) - const dateFrom = dateToIso8601(minDate) - const dateTo = dateToIso8601(Date.now()) - - const queryWithoutCursor = formatSearchQueryWithoutCursor(dateFrom, dateTo) - - console.log(`Requesting papers from biorxiv...`) - let imports = await importAll(queryWithoutCursor, [ - 'biophysics', - 'biochemistry', - ]) - imports = stripInternalDuplicates(imports) - imports = await stripDuplicates(imports, groupId) - imports = populateUrlAndAbstract(imports) - - const emptySubmission = getEmptySubmission(groupId) - - const newManuscripts = imports.map( +const buildNewManuscripts = ( + imports, + groupId, + ctx, + sourceId, + emptySubmission, +) => + imports.map( ({ doi, title, @@ -194,21 +150,122 @@ const getData = async (groupId, ctx) => { }), ) - try { - const result = [] +/** Saves one batch of already-subject-filtered raw bioRxiv records: dedupes internal + * version duplicates, skips DOIs already known (in the DB, or saved earlier in this + * run), and inserts the rest in chunks. Adds the saved DOIs to `knownDois` so later + * batches in the same run skip them too. + */ +const saveBatch = async ( + rawImports, + groupId, + ctx, + sourceId, + emptySubmission, + knownDois, +) => { + const deduped = stripInternalDuplicates(rawImports).filter( + ({ doi }) => !knownDois.has(doi), + ) - for (let i = 0; i < newManuscripts.length; i += SAVE_CHUNK_SIZE) { - const chunk = newManuscripts.slice(i, i + SAVE_CHUNK_SIZE) + const imports = populateUrlAndAbstract(deduped) - // eslint-disable-next-line no-await-in-loop - const inserted = await Manuscript.query().upsertGraphAndFetch(chunk, { - relate: true, - }) + const newManuscripts = buildNewManuscripts( + imports, + groupId, + ctx, + sourceId, + emptySubmission, + ) + + const result = [] + + for (let i = 0; i < newManuscripts.length; i += SAVE_CHUNK_SIZE) { + const chunk = newManuscripts.slice(i, i + SAVE_CHUNK_SIZE) + // eslint-disable-next-line no-await-in-loop + const inserted = await Manuscript.query().upsertGraphAndFetch(chunk, { + relate: true, + }) + Array.prototype.push.apply(result, inserted) + } + + imports.forEach(({ doi }) => knownDois.add(doi)) + + return result +} + +const getData = async (groupId, ctx) => { + const sourceId = await getServerId('biorxiv') + const lastImportDate = await getLastImportDate(sourceId, groupId) + const minDate = Math.max(lastImportDate, await getDate2WeeksAgo()) + const dateFrom = dateToIso8601(minDate) + const dateTo = dateToIso8601(Date.now()) + + const queryWithoutCursor = formatSearchQueryWithoutCursor(dateFrom, dateTo) + const subjects = ['biophysics', 'biochemistry'] + const emptySubmission = getEmptySubmission(groupId) + const manuscripts = await Manuscript.query().where({ groupId }) + + const knownDois = new Set( + manuscripts.map(m => m.submission.$doi).filter(Boolean), + ) + + console.log(`Requesting papers from biorxiv...`) + + const result = [] + let pageBuffer = [] + let totalRetrievedCount = 0 + let totalMatchedCount = 0 + let hadSaveError = false + + const flush = async () => { + if (!pageBuffer.length) return + const batch = pageBuffer + pageBuffer = [] + + try { + const inserted = await saveBatch( + batch, + groupId, + ctx, + sourceId, + emptySubmission, + knownDois, + ) Array.prototype.push.apply(result, inserted) + } catch (e) { + hadSaveError = true + console.error(e.message) } + } + + for (let cursor = 0; cursor < CURSOR_LIMIT; cursor += 1) { + const queryString = `${queryWithoutCursor}&cursor=${cursor}` + // eslint-disable-next-line no-await-in-loop + const data = await doAxiosQueryWithRetry(queryString) + if (!data || !data.collection || !data.collection.length) break + totalRetrievedCount += data.collection.length + + const matched = restrictToSubjects(data.collection, subjects) + totalMatchedCount += matched.length + pageBuffer.push(...matched) + + console.log( + `Retrieved ${totalRetrievedCount} of ${data.total_results} total papers from biorxiv (${totalMatchedCount} in desired subjects)`, + ) + + if ((cursor + 1) % PAGES_PER_SAVE_BATCH === 0) { + // eslint-disable-next-line no-await-in-loop + await flush() + } + } + + await flush() - if (lastImportDate > 0) { + // Only advance the import history if every batch saved cleanly, so a partial + // failure doesn't cause this date range to be skipped on the next run. + if (!hadSaveError) { + if (lastImportDate) { await ArticleImportHistory.query() .update({ date: new Date().toISOString(), @@ -221,15 +278,13 @@ const getData = async (groupId, ctx) => { groupId, }) } + } - console.log( - `Imported ${result.length} preprints from biorxiv (discarding duplicates).`, - ) + console.log( + `Imported ${result.length} preprints from biorxiv (discarding duplicates).`, + ) - return result - } catch (e) { - console.error(e.message) - } + return result } module.exports = getData diff --git a/packages/server/services/importArticles/biorxiv-import.js b/packages/server/services/importArticles/biorxiv-import.js index 450468e2b..c314da30d 100644 --- a/packages/server/services/importArticles/biorxiv-import.js +++ b/packages/server/services/importArticles/biorxiv-import.js @@ -20,6 +20,8 @@ const { const { getSubmissionForm } = require('../../controllers/review.controllers') +const TIMEOUT_MS = 30000 + const getData = async (groupId, ctx) => { const dateTwoWeeksAgo = +new Date(new Date(Date.now()).toISOString().split('T')[0]) - 12096e5 @@ -60,8 +62,11 @@ const getData = async (groupId, ctx) => { const requests = async (cursor = 0, minDate, results = []) => { const { data } = await axios.get( `https://api.biorxiv.org/covid19/${cursor}`, + { timeout: TIMEOUT_MS }, ) + if (!data.collection || !data.collection.length) return results + const isDatesOutdated = data.collection.some( ({ rel_date }) => +new Date(rel_date) < minDate, ) diff --git a/packages/server/services/importArticles/europepmc-import.js b/packages/server/services/importArticles/europepmc-import.js index 2a398c13b..9f12ac2b5 100644 --- a/packages/server/services/importArticles/europepmc-import.js +++ b/packages/server/services/importArticles/europepmc-import.js @@ -7,6 +7,8 @@ const Manuscript = require('../../models/manuscript/manuscript.model') const { getSubmissionForm } = require('../../controllers/review.controllers') +const TIMEOUT_MS = 30000 + // Not sure if this code is used anymore could not find any relevant import triggers for europepmc? const getData = async (groupId, ctx) => { const dateTwoWeeksAgo = @@ -48,7 +50,9 @@ const getData = async (groupId, ctx) => { /* eslint-disable-next-line default-param-last */ const requests = async (cursor = '', minDate, results = []) => { - const { data } = await axios.get(`${requestUrl}&cursorMark=${cursor}`) + const { data } = await axios.get(`${requestUrl}&cursorMark=${cursor}`, { + timeout: TIMEOUT_MS, + }) const briefInfoAboutArticles = data.resultList.result @@ -112,6 +116,7 @@ const getData = async (groupId, ctx) => { return axios .get( `https://api.biorxiv.org/details/${server.toLowerCase()}/${doi}/na/json`, + { timeout: TIMEOUT_MS }, ) .then(response => response.data.collection[0]) }, diff --git a/packages/server/services/importArticles/pubmed-import.js b/packages/server/services/importArticles/pubmed-import.js index 5e619108c..add27d5d4 100644 --- a/packages/server/services/importArticles/pubmed-import.js +++ b/packages/server/services/importArticles/pubmed-import.js @@ -1,4 +1,4 @@ -/* eslint-disable consistent-return, no-await-in-loop, no-nested-ternary, no-console, no-shadow */ +/* eslint-disable consistent-return, no-await-in-loop, no-nested-ternary, no-shadow */ const axios = require('axios') const xml2json = require('xml-js') const FormData = require('form-data') @@ -12,6 +12,8 @@ const flattenObj = require('../../utils/flattenObj') const { getSubmissionForm } = require('../../controllers/review.controllers') const selectVersionRegexp = /(v)(?!.*\1)/g +const TIMEOUT_MS = 30000 +const EFETCH_CHUNK_SIZE = 200 const delay = milisec => { return new Promise(resolve => { @@ -113,12 +115,13 @@ const getData = async (groupId, ctx) => { formData, { headers: formData.getHeaders(), + timeout: TIMEOUT_MS, }, ) return { topic, ids: data.esearchresult.idlist } } catch (err) { - console.error( + logger.error( `Failed to retrieve pubmed data for topic ${topic}. Query:\n${query}\n${err.message}`, ) } @@ -186,46 +189,10 @@ const getData = async (groupId, ctx) => { const topicsIdsResult = topicsIdsResponse.map( async ({ topic, ids }, index) => { if (!ids.length) { - return + return [] } - const formData = new FormData() - await delay(3000 * index) - const idList = ids.join(',') - - const eFetchUrlParameters = { - db: 'pubmed', - id: idList, - tool: 'my_tool', - email: 'my_email@example.com', - retmode: 'xml', - } - - Object.entries(eFetchUrlParameters).map(([key, value]) => - formData.append(key, value), - ) - - const url = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi` - - let idsResponse - - try { - idsResponse = await axios - .post(url, formData, { responseType: 'text' }) - .then(response => response.data) - } catch (fetchError) { - logger.error( - `[PUBMED-IMPORT]: failed to fetch from ${url}: ${fetchError.message}`, - ) - } - - const { PubmedArticleSet } = await JSON.parse( - xml2json.xml2json(idsResponse, { - compact: true, - spaces: 2, - }), - ) const currentDOIs = currentArticleURLs .map(articleUrl => { @@ -240,8 +207,8 @@ const getData = async (groupId, ctx) => { .split(selectVersionRegexp)[0] } - console.log('broken url should be here') - console.log(articleUrl) + logger.info('broken url should be here') + logger.info(articleUrl) return null } @@ -260,140 +227,193 @@ const getData = async (groupId, ctx) => { : singleElocationId(MedlineCitation.Article.ELocationID) } - const singlePubmedArticles = MedlineCitation => - !currentDOIs.includes(pubmedDOI(MedlineCitation)) - ? [PubmedArticleSet.PubmedArticle] - : [] - - const withoutDuplicates = Array.isArray(PubmedArticleSet.PubmedArticle) - ? PubmedArticleSet.PubmedArticle.filter(({ MedlineCitation }) => { - return !currentDOIs.includes(pubmedDOI(MedlineCitation)) - }) - : singlePubmedArticles(PubmedArticleSet.PubmedArticle.MedlineCitation) - - const newManuscripts = withoutDuplicates - .map(({ MedlineCitation }) => { - const { AuthorList, ArticleTitle, Abstract, Journal } = - MedlineCitation.Article - - const year = Journal.JournalIssue.PubDate.Year - ? Journal.JournalIssue.PubDate.Year._text - : null + const idChunks = [] - const month = Journal.JournalIssue.PubDate.Month - ? Journal.JournalIssue.PubDate.Month._text - : null + for (let i = 0; i < ids.length; i += EFETCH_CHUNK_SIZE) { + idChunks.push(ids.slice(i, i + EFETCH_CHUNK_SIZE)) + } - const day = Journal.JournalIssue.PubDate.Day - ? Journal.JournalIssue.PubDate.Day._text - : null + const insertedForTopic = [] - const publishedDate = [year, month, day].filter(Boolean).join('-') + for (const idChunk of idChunks) { + const formData = new FormData() + const idList = idChunk.join(',') - const topics = topic ? [topic] : [] + const eFetchUrlParameters = { + db: 'pubmed', + id: idList, + tool: 'my_tool', + email: 'my_email@example.com', + retmode: 'xml', + } - // for some titles HTML is returned, need to find _text property in nested object - const flattedArticleTitle = flattenObj(ArticleTitle) + Object.entries(eFetchUrlParameters).map(([key, value]) => + formData.append(key, value), + ) - // the name of nested property in objects is always _text - const titlePropName = Object.keys(flattedArticleTitle).find(key => - key.includes('_text'), + const url = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi` + + let idsResponse + + try { + idsResponse = await axios + .post(url, formData, { + headers: formData.getHeaders(), + responseType: 'text', + timeout: TIMEOUT_MS, + }) + .then(response => response.data) + } catch (fetchError) { + logger.error( + `[PUBMED-IMPORT]: failed to fetch from ${url}: ${fetchError.message}`, ) + continue + } + + const { PubmedArticleSet } = await JSON.parse( + xml2json.xml2json(idsResponse, { + compact: true, + spaces: 2, + }), + ) - const articleTitle = flattedArticleTitle[titlePropName] - - let abstract = '' - - if (Abstract?.AbstractText) { - if (Abstract.AbstractText.length) { - abstract = Abstract.AbstractText.map( - textWithAttributes => - `

${ - textWithAttributes._attributes - ? textWithAttributes._attributes.Label - : '' - }
${joinToStringIfArray( - textWithAttributes._text, - )}

`, - ) - .join('') - .replace(/\n/gi, '') - } else { - abstract = joinToStringIfArray(Abstract.AbstractText._text) + const singlePubmedArticles = MedlineCitation => + !currentDOIs.includes(pubmedDOI(MedlineCitation)) + ? [PubmedArticleSet.PubmedArticle] + : [] + + const withoutDuplicates = Array.isArray(PubmedArticleSet.PubmedArticle) + ? PubmedArticleSet.PubmedArticle.filter(({ MedlineCitation }) => { + return !currentDOIs.includes(pubmedDOI(MedlineCitation)) + }) + : singlePubmedArticles(PubmedArticleSet.PubmedArticle.MedlineCitation) + + const newManuscripts = withoutDuplicates + .map(({ MedlineCitation }) => { + const { AuthorList, ArticleTitle, Abstract, Journal } = + MedlineCitation.Article + + const year = Journal.JournalIssue.PubDate.Year + ? Journal.JournalIssue.PubDate.Year._text + : null + + const month = Journal.JournalIssue.PubDate.Month + ? Journal.JournalIssue.PubDate.Month._text + : null + + const day = Journal.JournalIssue.PubDate.Day + ? Journal.JournalIssue.PubDate.Day._text + : null + + const publishedDate = [year, month, day].filter(Boolean).join('-') + + const topics = topic ? [topic] : [] + + // for some titles HTML is returned, need to find _text property in nested object + const flattedArticleTitle = flattenObj(ArticleTitle) + + // the name of nested property in objects is always _text + const titlePropName = Object.keys(flattedArticleTitle).find(key => + key.includes('_text'), + ) + + const articleTitle = flattedArticleTitle[titlePropName] + + let abstract = '' + + if (Abstract?.AbstractText) { + if (Abstract.AbstractText.length) { + abstract = Abstract.AbstractText.map( + textWithAttributes => + `

${ + textWithAttributes._attributes + ? textWithAttributes._attributes.Label + : '' + }
${joinToStringIfArray( + textWithAttributes._text, + )}

`, + ) + .join('') + .replace(/\n/gi, '') + } else { + abstract = joinToStringIfArray(Abstract.AbstractText._text) + } } - } - return publishedDate - ? { - status: 'new', - isImported: true, - importSource: pubmedImportSourceId.id, - importSourceServer: 'pubmed', - submission: { - ...emptySubmission, - firstAuthor: AuthorList - ? AuthorList.Author.length - ? AuthorList.Author.map( - ({ ForeName, LastName }) => - `${ForeName ? ForeName._text : ''} ${ - LastName ? LastName._text : '' + return publishedDate + ? { + status: 'new', + isImported: true, + importSource: pubmedImportSourceId.id, + importSourceServer: 'pubmed', + submission: { + ...emptySubmission, + firstAuthor: AuthorList + ? AuthorList.Author.length + ? AuthorList.Author.map( + ({ ForeName, LastName }) => + `${ForeName ? ForeName._text : ''} ${ + LastName ? LastName._text : '' + }`, + ).join(', ') + : [ + `${ + AuthorList.Author.ForeName + ? AuthorList.Author.ForeName._text + : '' + } ${ + AuthorList.Author.LastName + ? AuthorList.Author.LastName._text + : '' }`, - ).join(', ') - : [ - `${ - AuthorList.Author.ForeName - ? AuthorList.Author.ForeName._text - : '' - } ${ - AuthorList.Author.LastName - ? AuthorList.Author.LastName._text - : '' - }`, - ] - : [], - datePublished: publishedDate, - $sourceUri: `https://doi.org/${pubmedDOI(MedlineCitation)}`, - $title: articleTitle, - $abstract: abstract, - topics, - initialTopicsOnImport: topics, - journal: Journal.Title._text, - }, - meta: {}, - submitterId: ctx.userId, - channels: [ - { - topic: 'Manuscript discussion', - type: 'all', - groupId, + ] + : [], + datePublished: publishedDate, + $sourceUri: `https://doi.org/${pubmedDOI(MedlineCitation)}`, + $title: articleTitle, + $abstract: abstract, + topics, + initialTopicsOnImport: topics, + journal: Journal.Title._text, }, - { - topic: 'Editorial discussion', - type: 'editorial', - groupId, - }, - ], - files: [], - reviews: [], - teams: [], - groupId, - } - : null - }) - .filter(Boolean) + meta: {}, + submitterId: ctx.userId, + channels: [ + { + topic: 'Manuscript discussion', + type: 'all', + groupId, + }, + { + topic: 'Editorial discussion', + type: 'editorial', + groupId, + }, + ], + files: [], + reviews: [], + teams: [], + groupId, + } + : null + }) + .filter(Boolean) - if (!newManuscripts.length) return [] + if (!newManuscripts.length) continue - try { - const inserted = await Manuscript.query().upsertGraphAndFetch( - newManuscripts, - { relate: true }, - ) + try { + const inserted = await Manuscript.query().upsertGraphAndFetch( + newManuscripts, + { relate: true }, + ) - return inserted - } catch (e) { - console.error(e.message) + insertedForTopic.push(...inserted) + } catch (e) { + logger.error(e.message) + } } + + return insertedForTopic }, ) diff --git a/packages/server/services/importArticles/semantic-scholar-papers-import.js b/packages/server/services/importArticles/semantic-scholar-papers-import.js index a33a93264..c0a5cb05f 100644 --- a/packages/server/services/importArticles/semantic-scholar-papers-import.js +++ b/packages/server/services/importArticles/semantic-scholar-papers-import.js @@ -17,6 +17,7 @@ const semanticScholarServers = require('./semanitc-scholar-servers.json') const { getUrlByDoi } = require('../../utils/crossrefCommsUtils') const SAVE_CHUNK_SIZE = 50 +const TIMEOUT_MS = 30000 const getData = async (groupId, ctx) => { const activeConfig = await Config.getCached(groupId) @@ -78,6 +79,7 @@ const getData = async (groupId, ctx) => { headers: { 'Content-Type': 'application/json', }, + timeout: TIMEOUT_MS, }, ) diff --git a/packages/server/utils/crossrefCommsUtils.js b/packages/server/utils/crossrefCommsUtils.js index 6e6b96f32..7de7c9313 100644 --- a/packages/server/utils/crossrefCommsUtils.js +++ b/packages/server/utils/crossrefCommsUtils.js @@ -11,6 +11,7 @@ const http = rateLimit(axios.create(), { const apiUrl = 'https://api.crossref.org/v1/works/' const defaultMailTo = 'unknown@unknown.com' +const TIMEOUT_MS = 30000 /** Pass the response of every CrossRef API call to this function to ensure rate limits are updated */ const updateRateLimit = response => { @@ -35,7 +36,9 @@ const updateRateLimit = response => { // imposes a hard limit of around 10 queries/sec from a single IP address. const getUrlByDoiFromDataCite = async doi => { try { - const response = await axios.get(`https://doi.org/api/handles/${doi}`) + const response = await axios.get(`https://doi.org/api/handles/${doi}`, { + timeout: TIMEOUT_MS, + }) if (response.status === 200) { const url = response.data?.values?.[0]?.data?.value @@ -178,7 +181,7 @@ const getFormattedReferencesFromCrossRef = async ( try { const response = await http.get('https://api.crossref.org/v1/works', { params, - timeout: 15000, + timeout: TIMEOUT_MS, headers: { 'User-Agent': `Kotahi (Axios 0.21${ crossrefRetrievalEmail ? `; mailto:${crossrefRetrievalEmail}` : '' @@ -236,7 +239,7 @@ const getFormattedReferencesFromCrossRefDOI = async ( try { const response = await http.get('https://api.crossref.org/v1/works', { params, - timeout: 15000, + timeout: TIMEOUT_MS, headers: { 'User-Agent': `Kotahi (Axios 0.21${ crossrefRetrievalEmail ? `; mailto:${crossrefRetrievalEmail}` : '' diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index a896a8278..29971287c 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -6212,6 +6212,19 @@ __metadata: languageName: node linkType: hard +"form-data@npm:^4.0.6": + version: 4.0.6 + resolution: "form-data@npm:4.0.6" + dependencies: + asynckit: "npm:^0.4.0" + combined-stream: "npm:^1.0.8" + es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.4" + mime-types: "npm:^2.1.35" + checksum: 10c0/43947a77bf0ff45c6ceed789778982d47a3f3e720a74b71721174ebf3310a5f1a8be1d6b38a3ee3688e8a18a2c4273073ec0844cd37efda3eaf46d41c9c318ff + languageName: node + linkType: hard + "form-data@npm:~2.3.2": version: 2.3.3 resolution: "form-data@npm:2.3.3" @@ -6794,6 +6807,15 @@ __metadata: languageName: node linkType: hard +"hasown@npm:^2.0.4": + version: 2.0.4 + resolution: "hasown@npm:2.0.4" + dependencies: + function-bind: "npm:^1.1.2" + checksum: 10c0/2d8de939e270b70618f8cebb69746620db10617dbb495bc66ddad326955ea24d3ca4af133aff3eb7c1853e0218f867bc2b050ec26fe02e3aea58f880ffc5e506 + languageName: node + linkType: hard + "he@npm:1.2.0, he@npm:^1.2.0": version: 1.2.0 resolution: "he@npm:1.2.0" @@ -7610,7 +7632,7 @@ __metadata: citeproc-js-node: "npm:^0.0.3" fast-xml-parser: "npm:^4.0.7" fnv-plus: "npm:^1.3.1" - form-data: "npm:^4.0.0" + form-data: "npm:^4.0.6" fs-extra: "npm:4.0.3" google-spreadsheet: "npm:^3.1.15" graphql: "npm:16.9.0" @@ -8173,7 +8195,7 @@ __metadata: languageName: node linkType: hard -"mime-types@npm:^2.1.12, mime-types@npm:~2.1.19, mime-types@npm:~2.1.24": +"mime-types@npm:^2.1.12, mime-types@npm:^2.1.35, mime-types@npm:~2.1.19, mime-types@npm:~2.1.24": version: 2.1.35 resolution: "mime-types@npm:2.1.35" dependencies: