Skip to content
This repository was archived by the owner on May 5, 2021. It is now read-only.

Commit 8a0dae1

Browse files
MateStrysewskeMateStrysewske
authored
Merge pull request SORMAS-Foundation#2618 from hzi-braunschweig/2539_improve_case_import
SORMAS-Foundation#2539 performance improvements on case import
2 parents a9e3680 + be62cc3 commit 8a0dae1

24 files changed

Lines changed: 1341 additions & 666 deletions

File tree

‎sormas-api/src/main/java/de/symeda/sormas/api/FacadeProvider.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,16 @@
1717
*******************************************************************************/
1818
package de.symeda.sormas.api;
1919

20+
import javax.naming.InitialContext;
21+
import javax.naming.NamingException;
22+
2023
import de.symeda.sormas.api.action.ActionFacade;
2124
import de.symeda.sormas.api.campaign.CampaignFacade;
2225
import de.symeda.sormas.api.campaign.data.CampaignFormDataFacade;
2326
import de.symeda.sormas.api.campaign.form.CampaignFormMetaFacade;
2427
import de.symeda.sormas.api.caze.CaseFacade;
2528
import de.symeda.sormas.api.caze.CaseStatisticsFacade;
29+
import de.symeda.sormas.api.caze.caseimport.CaseImportFacade;
2630
import de.symeda.sormas.api.caze.classification.CaseClassificationFacade;
2731
import de.symeda.sormas.api.caze.maternalhistory.MaternalHistoryFacade;
2832
import de.symeda.sormas.api.clinicalcourse.ClinicalCourseFacade;
@@ -64,9 +68,6 @@
6468
import de.symeda.sormas.api.visit.VisitFacade;
6569
import de.symeda.sormas.api.visualization.VisualizationFacade;
6670

67-
import javax.naming.InitialContext;
68-
import javax.naming.NamingException;
69-
7071
public class FacadeProvider {
7172

7273
private static final String JNDI_PREFIX = "java:global/sormas-ear/sormas-backend/";
@@ -99,6 +100,10 @@ public static CaseStatisticsFacade getCaseStatisticsFacade() {
99100
return get().lookupEjbRemote(CaseStatisticsFacade.class);
100101
}
101102

103+
public static CaseImportFacade getCaseImportFacade() {
104+
return get().lookupEjbRemote(CaseImportFacade.class);
105+
}
106+
102107
public static ContactFacade getContactFacade() {
103108
return get().lookupEjbRemote(ContactFacade.class);
104109
}

‎sormas-api/src/main/java/de/symeda/sormas/api/caze/caseimport/CaseImportEntities.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* SORMAS® - Surveillance Outbreak Response Management & Analysis System
3+
* Copyright © 2016-2020 Helmholtz-Zentrum für Infektionsforschung GmbH (HZI)
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* GNU General Public License for more details.
12+
* You should have received a copy of the GNU General Public License
13+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
14+
*/
15+
16+
package de.symeda.sormas.api.caze.caseimport;
17+
18+
import java.io.Serializable;
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
22+
import de.symeda.sormas.api.caze.CaseDataDto;
23+
import de.symeda.sormas.api.person.PersonDto;
24+
import de.symeda.sormas.api.sample.PathogenTestDto;
25+
import de.symeda.sormas.api.sample.SampleDto;
26+
import de.symeda.sormas.api.user.UserReferenceDto;
27+
28+
public class CaseImportEntities implements Serializable {
29+
30+
private static final long serialVersionUID = -4565794925738392508L;
31+
32+
private final PersonDto person;
33+
private final CaseDataDto caze;
34+
private final List<SampleDto> samples;
35+
private final List<PathogenTestDto> pathogenTests;
36+
37+
public CaseImportEntities(UserReferenceDto reportingUser) {
38+
person = PersonDto.build();
39+
caze = createCase(person, reportingUser);
40+
41+
samples = new ArrayList<>();
42+
pathogenTests = new ArrayList<>();
43+
}
44+
45+
public static CaseDataDto createCase(PersonDto person, UserReferenceDto reportingUser) {
46+
CaseDataDto caze = CaseDataDto.build(person.toReference(), null);
47+
caze.setReportingUser(reportingUser);
48+
49+
return caze;
50+
}
51+
52+
public CaseImportEntities(PersonDto person, CaseDataDto caze) {
53+
this.person = person;
54+
this.caze = caze;
55+
56+
samples = new ArrayList<>();
57+
pathogenTests = new ArrayList<>();
58+
}
59+
60+
public PersonDto getPerson() {
61+
return person;
62+
}
63+
64+
public CaseDataDto getCaze() {
65+
return caze;
66+
}
67+
68+
public List<SampleDto> getSamples() {
69+
return samples;
70+
}
71+
72+
public List<PathogenTestDto> getPathogenTests() {
73+
return pathogenTests;
74+
}
75+
}

‎sormas-api/src/main/java/de/symeda/sormas/api/caze/caseimport/CaseImportFacade.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* SORMAS® - Surveillance Outbreak Response Management & Analysis System
3+
* Copyright © 2016-2020 Helmholtz-Zentrum für Infektionsforschung GmbH (HZI)
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* GNU General Public License for more details.
12+
* You should have received a copy of the GNU General Public License
13+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
14+
*/
15+
16+
package de.symeda.sormas.api.caze.caseimport;
17+
18+
import javax.ejb.Remote;
19+
20+
import de.symeda.sormas.api.importexport.InvalidColumnException;
21+
22+
@Remote
23+
public interface CaseImportFacade {
24+
25+
ImportLineResultDto<CaseImportEntities> importCaseData(
26+
String[] values,
27+
String[] entityClasses,
28+
String[] entityProperties,
29+
String[][] entityPropertyPaths,
30+
boolean ignoreEmptyEntries)
31+
throws InvalidColumnException;
32+
33+
ImportLineResultDto<CaseImportEntities> updateCaseWithImportData(
34+
String personUuid,
35+
String caseUuid,
36+
String[] values,
37+
String[] entityClasses,
38+
String[][] entityPropertyPaths)
39+
throws InvalidColumnException;
40+
41+
ImportLineResultDto<CaseImportEntities> saveImportedEntities(CaseImportEntities entities);
42+
}

‎sormas-api/src/main/java/de/symeda/sormas/api/caze/caseimport/ImportLineResult.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* SORMAS® - Surveillance Outbreak Response Management & Analysis System
3+
* Copyright © 2016-2020 Helmholtz-Zentrum für Infektionsforschung GmbH (HZI)
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* GNU General Public License for more details.
12+
* You should have received a copy of the GNU General Public License
13+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
14+
*/
15+
package de.symeda.sormas.api.caze.caseimport;
16+
17+
public enum ImportLineResult {
18+
19+
SUCCESS,
20+
ERROR,
21+
DUPLICATE;
22+
}

‎sormas-api/src/main/java/de/symeda/sormas/api/caze/caseimport/ImportLineResultDto.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
* SORMAS® - Surveillance Outbreak Response Management & Analysis System
3+
* Copyright © 2016-2020 Helmholtz-Zentrum für Infektionsforschung GmbH (HZI)
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* GNU General Public License for more details.
12+
* You should have received a copy of the GNU General Public License
13+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
14+
*/
15+
16+
package de.symeda.sormas.api.caze.caseimport;
17+
18+
import java.io.Serializable;
19+
20+
public class ImportLineResultDto<E> implements Serializable {
21+
22+
private static final long serialVersionUID = -9004769653154669800L;
23+
24+
private final ImportLineResult result;
25+
private final String message;
26+
private final E importEntities;
27+
28+
private ImportLineResultDto(ImportLineResult result, String message, E importEntities) {
29+
this.result = result;
30+
this.message = message;
31+
this.importEntities = importEntities;
32+
}
33+
34+
public ImportLineResult getResult() {
35+
return result;
36+
}
37+
38+
public String getMessage() {
39+
return message;
40+
}
41+
42+
public E getImportEntities() {
43+
return importEntities;
44+
}
45+
46+
public boolean isError() {
47+
return result == ImportLineResult.ERROR;
48+
}
49+
50+
public boolean isDuplicate() {
51+
return result == ImportLineResult.DUPLICATE;
52+
}
53+
54+
public boolean isSuccess() {
55+
return result == ImportLineResult.SUCCESS;
56+
}
57+
58+
public static <E> ImportLineResultDto<E> successResult() {
59+
return new ImportLineResultDto<>(ImportLineResult.SUCCESS, null, null);
60+
}
61+
62+
public static <E> ImportLineResultDto<E> errorResult(String message) {
63+
return new ImportLineResultDto<>(ImportLineResult.ERROR, message, null);
64+
}
65+
66+
public static <E> ImportLineResultDto<E> duplicateResult(E entities) {
67+
return new ImportLineResultDto<>(ImportLineResult.DUPLICATE, null, entities);
68+
}
69+
}

‎sormas-api/src/main/java/de/symeda/sormas/api/person/PersonHelper.java‎

Lines changed: 4 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -17,64 +17,22 @@
1717
*******************************************************************************/
1818
package de.symeda.sormas.api.person;
1919

20+
import java.util.Date;
21+
22+
import org.apache.commons.lang3.StringUtils;
23+
2024
import de.symeda.sormas.api.Language;
2125
import de.symeda.sormas.api.caze.BurialInfoDto;
2226
import de.symeda.sormas.api.person.ApproximateAgeType.ApproximateAgeHelper;
2327
import de.symeda.sormas.api.utils.DataHelper;
2428
import de.symeda.sormas.api.utils.DateHelper;
25-
import org.apache.commons.lang3.StringUtils;
26-
import org.simmetrics.metrics.StringMetrics;
27-
28-
import java.text.Normalizer;
29-
import java.util.Date;
30-
import java.util.regex.Pattern;
3129

3230
public final class PersonHelper {
3331

3432
private PersonHelper() {
3533
// Hide Utility Class Constructor
3634
}
3735

38-
public static final double DEFAULT_NAME_SIMILARITY_THRESHOLD = 0.6D;
39-
40-
/**
41-
* Calculates the trigram distance between both names and returns true
42-
* if the similarity is high enough to consider them a possible match.
43-
* Uses a default of {@link PersonHelper#DEFAULT_NAME_SIMILARITY_THRESHOLD} for the threshold.
44-
*/
45-
protected static boolean areFullNamesSimilar(final String firstName, final String secondName, Double similarityThreshold) {
46-
final String name = normalizeString(firstName);
47-
final String otherName = normalizeString(secondName);
48-
return StringMetrics.qGramsDistance().compare(name, otherName)
49-
>= (similarityThreshold != null ? similarityThreshold : DEFAULT_NAME_SIMILARITY_THRESHOLD);
50-
}
51-
52-
/**
53-
* Calculates the trigram distance between firstName/lastname (also viceversa lastname/firstname) and otherFirstName/otherLastName,
54-
* returns true if the similarity is high enough to consider them a possible match.
55-
*/
56-
public static boolean areNamesSimilar(
57-
final String firstName,
58-
final String lastName,
59-
final String otherFirstName,
60-
final String otherLastName,
61-
Double similarityThreshold) {
62-
final String name = createFullName(firstName, lastName);
63-
final String nameInverted = createFullName(lastName, firstName);
64-
final String otherName = createFullName(otherFirstName, otherLastName);
65-
return areFullNamesSimilar(name, otherName, similarityThreshold) || areFullNamesSimilar(nameInverted, otherName, similarityThreshold);
66-
}
67-
68-
private static String createFullName(String firstName, String lastName) {
69-
return firstName + StringUtils.SPACE + lastName;
70-
}
71-
72-
public static String normalizeString(String str) {
73-
String nfdNormalizedString = Normalizer.normalize(str, Normalizer.Form.NFD).toLowerCase();
74-
Pattern pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
75-
return pattern.matcher(nfdNormalizedString).replaceAll("");
76-
}
77-
7836
public static String formatBirthdate(Integer birthdateDD, Integer birthdateMM, Integer birthdateYYYY, Language language) {
7937

8038
if (birthdateDD == null && birthdateMM == null && birthdateYYYY == null) {

0 commit comments

Comments
 (0)