Skip to content

Commit 64783c5

Browse files
labkey-klumlabkey-klum
committed
run and batch creation
1 parent a846a01 commit 64783c5

1 file changed

Lines changed: 191 additions & 3 deletions

File tree

‎signalData/src/org/labkey/signaldata/pipeline/SignalDataImportTask.java‎

Lines changed: 191 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,29 +3,66 @@
33
import org.apache.commons.lang3.StringUtils;
44
import org.apache.logging.log4j.Logger;
55
import org.jetbrains.annotations.NotNull;
6+
import org.jetbrains.annotations.Nullable;
7+
import org.labkey.api.assay.AssayProvider;
8+
import org.labkey.api.assay.AssayRunUploadContext;
69
import org.labkey.api.assay.AssayService;
10+
import org.labkey.api.assay.DefaultAssayRunCreator;
11+
import org.labkey.api.collections.CaseInsensitiveHashMap;
12+
import org.labkey.api.data.Container;
13+
import org.labkey.api.dataiterator.MapDataIterator;
14+
import org.labkey.api.exp.ExperimentException;
15+
import org.labkey.api.exp.api.ExpData;
716
import org.labkey.api.exp.api.ExpProtocol;
17+
import org.labkey.api.exp.api.ExperimentService;
18+
import org.labkey.api.exp.query.ExpDataTable;
19+
import org.labkey.api.files.FileContentService;
820
import org.labkey.api.pipeline.AbstractTaskFactory;
921
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
22+
import org.labkey.api.pipeline.PipeRoot;
1023
import org.labkey.api.pipeline.PipelineJob;
24+
import org.labkey.api.pipeline.PipelineService;
1125
import org.labkey.api.pipeline.RecordedActionSet;
1226
import org.labkey.api.pipeline.file.FileAnalysisJobSupport;
27+
import org.labkey.api.query.ValidationException;
1328
import org.labkey.api.reader.DataLoader;
1429
import org.labkey.api.reader.DataLoaderFactory;
1530
import org.labkey.api.util.DateUtil;
1631
import org.labkey.api.util.FileType;
1732
import org.labkey.api.util.FileUtil;
33+
import org.labkey.api.webdav.WebdavResource;
34+
import org.labkey.api.webdav.WebdavService;
35+
import org.labkey.signaldata.assay.SignalDataAssayDataHandler;
1836
import org.labkey.vfs.FileLike;
37+
import org.labkey.vfs.FileSystemLike;
1938

39+
import java.io.File;
40+
import java.io.IOException;
2041
import java.io.InputStream;
42+
import java.net.URI;
43+
import java.net.URLDecoder;
44+
import java.nio.file.Path;
45+
import java.time.LocalDateTime;
46+
import java.time.format.DateTimeFormatter;
47+
import java.util.ArrayList;
2148
import java.util.Collections;
49+
import java.util.HashMap;
2250
import java.util.List;
2351
import java.util.Map;
52+
import java.util.Objects;
53+
54+
import static org.labkey.api.files.FileContentService.UPLOADED_FILE;
2455

2556
public class SignalDataImportTask extends PipelineJob.Task<SignalDataImportTask.Factory>
2657
{
2758
public static final String PROTOCOL_NAME_PROPERTY = "protocolName";
2859

60+
// metadata file column names
61+
private static final String INPUT_NAME = "Name";
62+
private static final String INPUT_DATA_FILE = "DataFile";
63+
64+
private String _folderName;
65+
2966
private SignalDataImportTask(SignalDataImportTask.Factory factory, PipelineJob job)
3067
{
3168
super(factory, job);
@@ -40,6 +77,7 @@ public RecordedActionSet run()
4077
job.setLogFile(support.getDataDirectory().resolveChild(FileUtil.makeFileNameWithTimestamp("triggered_signaldata_import", "log")));
4178
job.setStatus("RELOADING", "Job started at: " + DateUtil.nowISO());
4279
Logger log = job.getLogger();
80+
Container container = job.getContainer();
4381

4482
// validate the protocol
4583
String protocolName = job.getParameters().get(PROTOCOL_NAME_PROPERTY);
@@ -49,7 +87,7 @@ public RecordedActionSet run()
4987
return new RecordedActionSet();
5088
}
5189

52-
ExpProtocol protocol = AssayService.get().getAssayProtocolByName(job.getContainer(), protocolName);
90+
ExpProtocol protocol = AssayService.get().getAssayProtocolByName(container, protocolName);
5391
if (protocol == null)
5492
{
5593
log.error("Could not resolve the specified protocol name : {}", protocolName);
@@ -60,8 +98,136 @@ public RecordedActionSet run()
6098
assert support.getInputFiles().size() == 1;
6199
FileLike dataFile = support.getInputFiles().getFirst();
62100

63-
log.info("Loading {}", dataFile.getName());
64-
List<Map<String, Object>> dataRows = parseMetadata(dataFile, log);
101+
try
102+
{
103+
FileLike runRoot = getTargetFolder(container, log);
104+
if (runRoot == null)
105+
return new RecordedActionSet();
106+
107+
log.info("Loading {}", dataFile.getName());
108+
List<Map<String, Object>> dataRows = parseMetadata(dataFile, log);
109+
List<Map<String, Object>> dataInputs = new ArrayList<>();
110+
111+
for (Map<String, Object> row : dataRows)
112+
{
113+
// parse out the name and datafile properties
114+
String name = Objects.toString(row.get(INPUT_NAME), "");
115+
String dataFilePath = Objects.toString(row.get(INPUT_DATA_FILE), "").trim();
116+
117+
// validate the existance of the datafile property and make a copy to the run root
118+
if (StringUtils.isBlank(dataFilePath))
119+
{
120+
log.warn("Skipping row '{}' with blank DataFile property", name);
121+
continue;
122+
}
123+
124+
// If the value is just a filename (no directory separators), resolve it relative to
125+
// the metadata file's directory; otherwise treat it as a full server-side path
126+
String dataFileName = FileUtil.getFileName(Path.of(dataFilePath));
127+
FileLike sourceFile;
128+
if (dataFilePath.equals(dataFileName))
129+
{
130+
sourceFile = dataFile.getParent().resolveChild(dataFilePath);
131+
}
132+
else
133+
{
134+
sourceFile = FileSystemLike.wrapFile(new File(dataFilePath));
135+
}
136+
137+
if (!sourceFile.exists())
138+
{
139+
log.info("Data file not found: {}", sourceFile.getPath());
140+
row.remove(INPUT_DATA_FILE);
141+
continue;
142+
}
143+
144+
// add a data input entry for the run
145+
Map<String, Object> dataInput = new CaseInsensitiveHashMap<>();
146+
dataInputs.add(dataInput);
147+
148+
log.info("Copying {} to run folder", sourceFile.getName());
149+
FileLike destFile = runRoot.resolveChild(sourceFile.getName());
150+
FileUtil.copyFile(sourceFile, destFile);
151+
152+
log.info("Ensuring input data is created for {}", destFile.getName());
153+
URI uri = FileContentService.get().getWebDavUrl(destFile, container, FileContentService.PathType.full);
154+
if (uri != null)
155+
{
156+
WebdavResource resource = WebdavService.get().lookup(uri.getPath());
157+
if (resource != null)
158+
{
159+
ExpData data = FileContentService.get().getDataObject(resource, container);
160+
if (data == null)
161+
{
162+
// create the ExpData object for the input data
163+
data = ExperimentService.get().createData(container, UPLOADED_FILE);
164+
data.setName(destFile.getName());
165+
data.setDataFileURI(destFile.toURI());
166+
data.save(job.getUser());
167+
}
168+
169+
FileLike d = FileUtil.getAbsoluteCaseSensitiveFile(destFile);
170+
String url = d.toURI().toURL().toString();
171+
172+
if (url != null)
173+
{
174+
dataInput.put(ExpDataTable.Column.Name.name(), data.getName());
175+
dataInput.put(ExpDataTable.Column.DataFileUrl.name(), data.getDataFileUrl());
176+
177+
// file data type for this run data field
178+
String dataFileUrl = URLDecoder.decode(url, "UTF-8");
179+
row.replace(INPUT_DATA_FILE, dataFileUrl.replace("file:", ""));
180+
}
181+
}
182+
}
183+
}
184+
185+
// create and save the run
186+
AssayProvider provider = AssayService.get().getProvider(protocol);
187+
if (provider != null)
188+
{
189+
AssayRunUploadContext.Factory<?,?> runFactory = provider.createRunUploadFactory(protocol, job.getUser(), container);
190+
191+
runFactory.setName(_folderName);
192+
runFactory.setLogger(log);
193+
runFactory.setRawData(MapDataIterator.of(dataRows));
194+
runFactory.setRunProperties(Map.of("RunIdentifier", _folderName));
195+
196+
Map<ExpData, String> inputDatasMap = new HashMap<>();
197+
for (Map<String, Object> inputMap : dataInputs)
198+
{
199+
String dataFileUrl = Objects.toString(inputMap.get(ExpDataTable.Column.DataFileUrl.name()), "");
200+
if (!dataFileUrl.isEmpty())
201+
{
202+
ExpData expData = ExperimentService.get().getExpDataByURL(dataFileUrl, container);
203+
if (expData != null)
204+
inputDatasMap.put(expData, Objects.toString(inputMap.get(ExpDataTable.Column.Name.name()), ""));
205+
}
206+
}
207+
if (!inputDatasMap.isEmpty())
208+
runFactory.setInputDatas(inputDatasMap);
209+
210+
// generate output data
211+
Map<Object, String> outputData = new HashMap<>();
212+
DefaultAssayRunCreator.generateResultData(job.getUser(), container, provider, dataRows, outputData, log);
213+
runFactory.setOutputDatas(outputData);
214+
215+
try
216+
{
217+
provider.getRunCreator().saveExperimentRun(runFactory.create(), null);
218+
}
219+
catch (ValidationException | ExperimentException e)
220+
{
221+
log.error("Error saving assay run: {}", e.getMessage(), e);
222+
throw new RuntimeException(e);
223+
}
224+
}
225+
}
226+
catch (Exception e)
227+
{
228+
log.error("Error importing data : {}", e.getMessage());
229+
throw new RuntimeException(e);
230+
}
65231

66232
return new RecordedActionSet();
67233
}
@@ -87,6 +253,28 @@ private List<Map<String, Object>> parseMetadata(FileLike dataFile, Logger log)
87253
}
88254
}
89255

256+
@Nullable
257+
private FileLike getTargetFolder(Container container, Logger log) throws IOException
258+
{
259+
PipeRoot root = PipelineService.get().findPipelineRoot(container);
260+
if (root != null)
261+
{
262+
_folderName = LocalDateTime.now()
263+
.format(DateTimeFormatter.ofPattern("yyyy_M_d_H_m_s"));
264+
265+
//Create folder if needed
266+
FileLike runRoot = root.getRootFileLike().resolveChild(SignalDataAssayDataHandler.NAMESPACE).resolveChild(_folderName);
267+
if (!runRoot.exists())
268+
runRoot.mkdirs();
269+
270+
return runRoot;
271+
}
272+
else
273+
log.error("Unable to find a pipeline root for container : {}", container.getPath());
274+
275+
return null;
276+
}
277+
90278
public static class Factory extends AbstractTaskFactory<AbstractTaskFactorySettings, Factory>
91279
{
92280
public Factory()

0 commit comments

Comments
 (0)