Skip to content

Commit 9f6132b

Browse files
authored
remove collectAsList call (#13)
1 parent aa113cc commit 9f6132b

1 file changed

Lines changed: 3 additions & 4 deletions

File tree

src/main/java/com/teragrep/functions/dpf_02/BatchCollect.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,14 +132,13 @@ public void collect(Dataset<Row> batchDF, Long batchId, List<AbstractStep> postB
132132
if (!skipLimiting && limit > 0) {
133133
orderedDs = orderedDs.limit(limit);
134134
}
135-
List<Row> collected = orderedDs.collectAsList();
136-
Dataset<Row> createdDsFromCollected = SparkSession.builder().getOrCreate().createDataFrame(collected, this.inputSchema);
135+
137136
Dataset<Row> current;
138137
if (this.savedDs == null) {
139-
current = createdDsFromCollected;
138+
current = orderedDs;
140139
}
141140
else {
142-
current = savedDs.union(createdDsFromCollected);
141+
current = savedDs.union(orderedDs);
143142
}
144143

145144
current = orderDataset(current);

0 commit comments

Comments
 (0)