1 | /** |
---|
2 | * Importer service |
---|
3 | * |
---|
4 | * The importer service handles the import of tabular, comma delimited and Excel format |
---|
5 | * based files. |
---|
6 | * |
---|
7 | * @package importer |
---|
8 | * @author t.w.abma@umcutrecht.nl |
---|
9 | * @since 20100126 |
---|
10 | * |
---|
11 | * Revision information: |
---|
12 | * $Rev: 1093 $ |
---|
13 | * $Author: t.w.abma@umcutrecht.nl $ |
---|
14 | * $Date: 2010-11-06 15:01:11 +0000 (za, 06 nov 2010) $ |
---|
15 | */ |
---|
16 | |
---|
17 | package dbnp.importer |
---|
18 | import org.apache.poi.ss.usermodel.* |
---|
19 | |
---|
20 | import dbnp.studycapturing.TemplateFieldType |
---|
21 | import dbnp.studycapturing.Template |
---|
22 | import dbnp.studycapturing.SamplingEvent |
---|
23 | import dbnp.studycapturing.Study |
---|
24 | import dbnp.studycapturing.Subject |
---|
25 | import dbnp.studycapturing.Event |
---|
26 | import dbnp.studycapturing.Sample |
---|
27 | |
---|
28 | class ImporterService { |
---|
29 | def AuthenticationService |
---|
30 | |
---|
31 | boolean transactional = true |
---|
32 | |
---|
33 | /** |
---|
34 | * @param is input stream representing the (workbook) resource |
---|
35 | * @return high level representation of the workbook |
---|
36 | */ |
---|
37 | Workbook getWorkbook(InputStream is) { |
---|
38 | WorkbookFactory.create(is) |
---|
39 | } |
---|
40 | |
---|
41 | /** |
---|
42 | * @param wb high level representation of the workbook |
---|
43 | * @param sheetindex sheet to use within the workbook |
---|
44 | * @return header representation as a MappingColumn hashmap |
---|
45 | */ |
---|
46 | def getHeader(Workbook wb, int sheetindex, int headerrow, int datamatrix_start, theEntity=null){ |
---|
47 | |
---|
48 | def sheet = wb.getSheetAt(sheetindex) |
---|
49 | def sheetrow = sheet.getRow(datamatrix_start) |
---|
50 | //def header = [] |
---|
51 | def header = [:] |
---|
52 | def df = new DataFormatter() |
---|
53 | def property = new String() |
---|
54 | |
---|
55 | //for (Cell c: sheet.getRow(datamatrix_start)) { |
---|
56 | |
---|
57 | (0..sheetrow.getLastCellNum() -1 ).each { columnindex -> |
---|
58 | |
---|
59 | //def index = c.getColumnIndex() |
---|
60 | def datamatrix_celltype = sheet.getRow(datamatrix_start).getCell(columnindex,Row.CREATE_NULL_AS_BLANK).getCellType() |
---|
61 | def datamatrix_celldata = df.formatCellValue(sheet.getRow(datamatrix_start).getCell(columnindex)) |
---|
62 | def datamatrix_cell = sheet.getRow(datamatrix_start).getCell(columnindex) |
---|
63 | def headercell = sheet.getRow(headerrow-1+sheet.getFirstRowNum()).getCell(columnindex) |
---|
64 | def tft = TemplateFieldType.STRING //default templatefield type |
---|
65 | |
---|
66 | // Check for every celltype, currently redundant code, but possibly this will be |
---|
67 | // a piece of custom code for every cell type like specific formatting |
---|
68 | |
---|
69 | switch (datamatrix_celltype) { |
---|
70 | case Cell.CELL_TYPE_STRING: |
---|
71 | //parse cell value as double |
---|
72 | def doubleBoolean = true |
---|
73 | def fieldtype = TemplateFieldType.STRING |
---|
74 | |
---|
75 | // is this string perhaps a double? |
---|
76 | try { |
---|
77 | formatValue(datamatrix_celldata, TemplateFieldType.DOUBLE) |
---|
78 | } catch (NumberFormatException nfe) { doubleBoolean = false } |
---|
79 | finally { |
---|
80 | if (doubleBoolean) fieldtype = TemplateFieldType.DOUBLE |
---|
81 | } |
---|
82 | |
---|
83 | header[columnindex] = new dbnp.importer.MappingColumn(name:df.formatCellValue(headercell), |
---|
84 | templatefieldtype:fieldtype, |
---|
85 | index:columnindex, |
---|
86 | entity:theEntity, |
---|
87 | property:property); |
---|
88 | |
---|
89 | break |
---|
90 | case Cell.CELL_TYPE_NUMERIC: |
---|
91 | def fieldtype = TemplateFieldType.INTEGER |
---|
92 | def doubleBoolean = true |
---|
93 | def integerBoolean = true |
---|
94 | |
---|
95 | // is this cell really an integer? |
---|
96 | try { |
---|
97 | Integer.valueOf(datamatrix_celldata) |
---|
98 | } catch (NumberFormatException nfe) { integerBoolean = false } |
---|
99 | finally { |
---|
100 | if (integerBoolean) fieldtype = TemplateFieldType.INTEGER |
---|
101 | } |
---|
102 | |
---|
103 | // it's not an integer, perhaps a double? |
---|
104 | if (!integerBoolean) |
---|
105 | try { |
---|
106 | formatValue(datamatrix_celldata, TemplateFieldType.DOUBLE) |
---|
107 | } catch (NumberFormatException nfe) { doubleBoolean = false } |
---|
108 | finally { |
---|
109 | if (doubleBoolean) fieldtype = TemplateFieldType.DOUBLE |
---|
110 | } |
---|
111 | |
---|
112 | if (DateUtil.isCellDateFormatted(datamatrix_cell)) fieldtype = TemplateFieldType.DATE |
---|
113 | |
---|
114 | header[columnindex] = new dbnp.importer.MappingColumn(name:df.formatCellValue(headercell), |
---|
115 | templatefieldtype:fieldtype, |
---|
116 | index:columnindex, |
---|
117 | entity:theEntity, |
---|
118 | property:property); |
---|
119 | break |
---|
120 | case Cell.CELL_TYPE_BLANK: |
---|
121 | header[columnindex] = new dbnp.importer.MappingColumn(name:df.formatCellValue(headercell), |
---|
122 | templatefieldtype:TemplateFieldType.STRING, |
---|
123 | index:columnindex, |
---|
124 | entity:theEntity, |
---|
125 | property:property); |
---|
126 | break |
---|
127 | default: |
---|
128 | header[columnindex] = new dbnp.importer.MappingColumn(name:df.formatCellValue(headercell), |
---|
129 | templatefieldtype:TemplateFieldType.STRING, |
---|
130 | index:columnindex, |
---|
131 | entity:theEntity, |
---|
132 | property:property); |
---|
133 | break |
---|
134 | } // end of switch |
---|
135 | } // end of cell loop |
---|
136 | return header |
---|
137 | } |
---|
138 | |
---|
139 | /** |
---|
140 | * This method is meant to return a matrix of the rows and columns |
---|
141 | * used in the preview |
---|
142 | * |
---|
143 | * @param wb workbook object |
---|
144 | * @param sheetindex sheet index used |
---|
145 | * @param rows amount of rows returned |
---|
146 | * @return two dimensional array (matrix) of Cell objects |
---|
147 | */ |
---|
148 | |
---|
149 | Cell[][] getDatamatrix(Workbook wb, header, int sheetindex, int datamatrix_start, int count) { |
---|
150 | def sheet = wb.getSheetAt(sheetindex) |
---|
151 | def rows = [] |
---|
152 | def df = new DataFormatter() |
---|
153 | |
---|
154 | // walk through all rows |
---|
155 | (count <= sheet.getLastRowNum()) ? |
---|
156 | ((datamatrix_start+sheet.getFirstRowNum())..count).each { rowindex -> |
---|
157 | def row = [] |
---|
158 | |
---|
159 | // walk through every cell |
---|
160 | /*for (Cell c: sheet.getRow(rowindex)) { |
---|
161 | row.add(c) |
---|
162 | println c.getColumnIndex() + "=" +c |
---|
163 | }*/ |
---|
164 | |
---|
165 | (0..header.size()-1).each { columnindex -> |
---|
166 | def c = sheet.getRow(rowindex).getCell(columnindex, Row.CREATE_NULL_AS_BLANK) |
---|
167 | //row.add(df.formatCellValue(c)) |
---|
168 | row.add(c) |
---|
169 | //if (c.getCellType() == c.CELL_TYPE_STRING) println "STR"+c.getStringCellValue() |
---|
170 | //if (c.getCellType() == c.CELL_TYPE_NUMERIC) println "INT" +c.getNumericCellValue() |
---|
171 | } |
---|
172 | //row.add(df.formatCellValue(c)) |
---|
173 | rows.add(row) |
---|
174 | } : 0 |
---|
175 | |
---|
176 | return rows |
---|
177 | } |
---|
178 | |
---|
179 | /** |
---|
180 | * This method will move a file to a new location. |
---|
181 | * |
---|
182 | * @param file File object to move |
---|
183 | * @param folderpath folder to move the file to |
---|
184 | * @param filename (new) filename to give |
---|
185 | * @return if file has been moved succesful, the new path and filename will be returned, otherwise an empty string will be returned |
---|
186 | */ |
---|
187 | def moveFile(File file, String folderpath, String filename) { |
---|
188 | try { |
---|
189 | def rnd = ""; //System.currentTimeMillis() |
---|
190 | file.transferTo(new File(folderpath, rnd+filename)) |
---|
191 | return folderpath + filename |
---|
192 | } catch(Exception exception) { |
---|
193 | log.error "File move error, ${exception}" |
---|
194 | return "" |
---|
195 | } |
---|
196 | } |
---|
197 | |
---|
198 | /** |
---|
199 | * @return random numeric value |
---|
200 | */ |
---|
201 | def random = { |
---|
202 | return System.currentTimeMillis() + Runtime.runtime.freeMemory() |
---|
203 | } |
---|
204 | |
---|
205 | /** |
---|
206 | * Method to read data from a workbook and to import data into a two dimensional |
---|
207 | * array |
---|
208 | * |
---|
209 | * @param template_id template identifier to use fields from |
---|
210 | * @param wb POI horrible spreadsheet formatted workbook object |
---|
211 | * @param mcmap linked hashmap (preserved order) of MappingColumns |
---|
212 | * @param sheetindex sheet to use when using multiple sheets |
---|
213 | * @param rowindex first row to start with reading the actual data (NOT the header) |
---|
214 | * @return two dimensional array containing records (with entities) |
---|
215 | * |
---|
216 | * @see dbnp.importer.MappingColumn |
---|
217 | */ |
---|
218 | def importData(template_id, Workbook wb, int sheetindex, int rowindex, mcmap) { |
---|
219 | def sheet = wb.getSheetAt(sheetindex) |
---|
220 | def table = [] |
---|
221 | def failedcells = [:] // map [recordhash, [mappingcolumnlist]] values |
---|
222 | |
---|
223 | // walk through all rows and fill the table with records |
---|
224 | (rowindex..sheet.getLastRowNum()).each { i -> |
---|
225 | //table.add(createRecord(template_id, sheet.getRow(i), mcmap)) |
---|
226 | // Create an entity record based on a row read from Excel and store the cells which failed to be mapped |
---|
227 | def (record, failed) = createRecord(template_id, sheet.getRow(i), mcmap) |
---|
228 | |
---|
229 | // Add record with entity and its values to the table |
---|
230 | table.add(record) |
---|
231 | |
---|
232 | // If failed cells have been found, add them to the failed cells map |
---|
233 | // the record hashcode is later on used to put the failed data back |
---|
234 | // in the data matrix |
---|
235 | if (failed.size()!=0) failedcells.put(record.hashCode(), failed) |
---|
236 | } |
---|
237 | |
---|
238 | failedcells.each { record -> |
---|
239 | record.each { cell -> |
---|
240 | cell.each { |
---|
241 | println it.value.dump() |
---|
242 | } |
---|
243 | } |
---|
244 | } |
---|
245 | |
---|
246 | |
---|
247 | return [table,failedcells] |
---|
248 | } |
---|
249 | |
---|
250 | /** Method to put failed cells back into the datamatrix. Failed cells are cell values |
---|
251 | * which could not be stored in an entity (e.g. Humu Supiuns in an ontology field). |
---|
252 | * Empty corrections should not be stored |
---|
253 | * |
---|
254 | * @param datamatrix two dimensional array containing entities and possibly also failed cells |
---|
255 | * @param failedcells list with maps of failed cells in [mappingcolumn, cell] format |
---|
256 | * @param correctedcells map of corrected cells in [cellhashcode, value] format |
---|
257 | **/ |
---|
258 | def saveCorrectedCells(datamatrix, failedcells, correctedcells) { |
---|
259 | // Loop through all failed cells |
---|
260 | failedcells.each { mcrecord -> |
---|
261 | mcrecord.each { mappingcolumn -> |
---|
262 | // Get the corrected value |
---|
263 | def correctedvalue = correctedcells.find { it.key.toInteger() == mappingcolumn.hashCode()}.value |
---|
264 | |
---|
265 | // Find the record in the table which the mappingcolumn belongs to |
---|
266 | def tablerecord = datamatrix.find { it.hashCode() == mcrecord.key } |
---|
267 | |
---|
268 | // Loop through all entities in the record |
---|
269 | tablerecord.each { rec -> |
---|
270 | rec.each { entity -> |
---|
271 | try { |
---|
272 | // Update the entity field |
---|
273 | entity.setFieldValue(mappingcolumn.value.property[0], correctedvalue) |
---|
274 | println "Adjusted " + mappingcolumn.value.property[0] + " to " + correctedvalue |
---|
275 | } |
---|
276 | catch (Exception e) { |
---|
277 | println "Could not map corrected ontology" |
---|
278 | } |
---|
279 | } |
---|
280 | } // end of table record |
---|
281 | } // end of mapping record |
---|
282 | } // end of failed cells loop |
---|
283 | } |
---|
284 | |
---|
285 | /** |
---|
286 | * Method to store a matrix containing the entities in a record like structure. Every row in the table |
---|
287 | * contains one or more entity objects (which contain fields with values). So actually a row represents |
---|
288 | * a record with fields from one or more different entities. |
---|
289 | * |
---|
290 | * @param study entity Study |
---|
291 | * @param datamatrix two dimensional array containing entities with values read from Excel file |
---|
292 | */ |
---|
293 | def saveDatamatrix(Study study, datamatrix) { |
---|
294 | def validatedSuccesfully = 0 |
---|
295 | def entitystored = null |
---|
296 | study.refresh() |
---|
297 | |
---|
298 | // go through the data matrix, read every record and validate the entity and try to persist it |
---|
299 | datamatrix.each { record -> |
---|
300 | record.each { entity -> |
---|
301 | switch (entity.getClass()) { |
---|
302 | case Study : print "Persisting Study `" + entity + "`: " |
---|
303 | entity.owner = AuthenticationService.getLoggedInUser() |
---|
304 | if (persistEntity(entity)) validatedSuccesfully++ |
---|
305 | break |
---|
306 | case Subject : print "Persisting Subject `" + entity + "`: " |
---|
307 | entity.parent = study |
---|
308 | |
---|
309 | // is the current entity not already in the database? |
---|
310 | entitystored = isEntityStored(entity) |
---|
311 | |
---|
312 | // this entity is new, so add it to the study |
---|
313 | if (entitystored==null) study.addToSubjects(entity) |
---|
314 | else // existing entity, so update it |
---|
315 | updateEntity(entitystored, entity) |
---|
316 | |
---|
317 | if (persistEntity(study)) validatedSuccesfully++ |
---|
318 | break |
---|
319 | case Event : print "Persisting Event `" + entity + "`: " |
---|
320 | entity.parent = study |
---|
321 | study.addToEvents(entity) |
---|
322 | if (persistEntity(entity)) validatedSuccesfully++ |
---|
323 | break |
---|
324 | case Sample : print "Persisting Sample `" + entity +"`: " |
---|
325 | entity.parent = study |
---|
326 | study.addToSamples(entity) |
---|
327 | if (persistEntity(entity)) validatedSuccesfully++ |
---|
328 | break |
---|
329 | case SamplingEvent: print "Persisting SamplingEvent `" + entity + "`: " |
---|
330 | entity.parent = study |
---|
331 | study.addToSamplingEvents(entity) |
---|
332 | if (persistEntity(entity)) validatedSuccesfully++ |
---|
333 | break |
---|
334 | default : println "Skipping persisting of `" + entity.getclass() +"`" |
---|
335 | break |
---|
336 | } // end switch |
---|
337 | } // end record |
---|
338 | } // end datamatrix |
---|
339 | return validatedSuccesfully |
---|
340 | } |
---|
341 | |
---|
342 | /** |
---|
343 | * Check whether an entity already exist. A unique field in the entity is |
---|
344 | * used to check whether the instantiated entity (read from Excel) is new. |
---|
345 | * If the entity is found in the database it will be returned as is. |
---|
346 | * |
---|
347 | * @param entity entity object like a Study, Subject, Sample et cetera |
---|
348 | * @return entity if found, otherwise null |
---|
349 | */ |
---|
350 | def isEntityStored(entity) { |
---|
351 | switch (entity.getClass()) { |
---|
352 | case Study : return Study.findByCode(entity.code) |
---|
353 | break |
---|
354 | case Subject : return Subject.findByParentAndName(entity.parent, entity.name) |
---|
355 | break |
---|
356 | case Event : break |
---|
357 | case Sample : break |
---|
358 | case SamplingEvent : break |
---|
359 | default : // unknown entity |
---|
360 | return null |
---|
361 | } |
---|
362 | } |
---|
363 | |
---|
364 | /** |
---|
365 | * Find the entity and update the fields. The entity is an instance |
---|
366 | * read from Excel. This method looks in the database for the entity |
---|
367 | * having the same identifier. If it has found the same entity |
---|
368 | * already in the database, it will update the record. |
---|
369 | * |
---|
370 | * @param entitystored existing record in the database to update |
---|
371 | * @param entity entity read from Excel |
---|
372 | */ |
---|
373 | def updateEntity(entitystored, entity) { |
---|
374 | switch (entity.getClass()) { |
---|
375 | case Study : break |
---|
376 | case Subject : entitystored.properties = entity.properties |
---|
377 | entitystored.save() |
---|
378 | break |
---|
379 | case Event : break |
---|
380 | case Sample : break |
---|
381 | case SamplingEvent : break |
---|
382 | default : // unknown entity |
---|
383 | return null |
---|
384 | } |
---|
385 | } |
---|
386 | |
---|
387 | /** |
---|
388 | * Method to persist entities into the database |
---|
389 | * Checks whether entity already exists (based on identifier column 'name') |
---|
390 | * |
---|
391 | * @param entity entity object like Study, Subject, Protocol et cetera |
---|
392 | * |
---|
393 | */ |
---|
394 | boolean persistEntity(entity) { |
---|
395 | println "persisting ${entity}" |
---|
396 | // if not validated |
---|
397 | if (entity.validate()) { |
---|
398 | if (entity.save()) { //.merge? |
---|
399 | return true |
---|
400 | } |
---|
401 | else { // if save was unsuccesful |
---|
402 | entity.errors.allErrors.each { |
---|
403 | println it |
---|
404 | } |
---|
405 | return false |
---|
406 | } |
---|
407 | } |
---|
408 | else { // if not validated |
---|
409 | entity.errors.each { |
---|
410 | println it |
---|
411 | } |
---|
412 | return false |
---|
413 | } |
---|
414 | } |
---|
415 | |
---|
416 | /** |
---|
417 | * This method creates a record (array) containing entities with values |
---|
418 | * |
---|
419 | * @param template_id template identifier |
---|
420 | * @param excelrow POI based Excel row containing the cells |
---|
421 | * @param mcmap map containing MappingColumn objects |
---|
422 | * @return list of entities and list of failed cells |
---|
423 | */ |
---|
424 | def createRecord(template_id, Row excelrow, mcmap) { |
---|
425 | def df = new DataFormatter() |
---|
426 | def template = Template.get(template_id) |
---|
427 | def record = [] // list of entities and the read values |
---|
428 | def failed = [] // list of failed columns [mappingcolumn] with the value which couldn't be mapped into the entity |
---|
429 | |
---|
430 | // Initialize all possible entities with the chosen template |
---|
431 | def study = new Study(template: template) |
---|
432 | def subject = new Subject(template: template) |
---|
433 | def samplingEvent = new SamplingEvent(template: template) |
---|
434 | def event = new Event(template: template) |
---|
435 | def sample = new Sample(template: template) |
---|
436 | |
---|
437 | // Go through the Excel row cell by cell |
---|
438 | for (Cell cell: excelrow) { |
---|
439 | // get the MappingColumn information of the current cell |
---|
440 | def mc = mcmap[cell.getColumnIndex()] |
---|
441 | def value |
---|
442 | |
---|
443 | // Check if column must be imported |
---|
444 | if (mc!=null) if (!mc.dontimport) { |
---|
445 | try { |
---|
446 | value = formatValue(df.formatCellValue(cell), mc.templatefieldtype) |
---|
447 | } catch (NumberFormatException nfe) { |
---|
448 | value = "" |
---|
449 | } |
---|
450 | |
---|
451 | try { |
---|
452 | |
---|
453 | // which entity does the current cell (field) belong to? |
---|
454 | switch (mc.entity) { |
---|
455 | case Study: // does the entity already exist in the record? If not make it so. |
---|
456 | (record.any {it.getClass() == mc.entity}) ? 0 : record.add(study) |
---|
457 | study.setFieldValue(mc.property, value) |
---|
458 | break |
---|
459 | case Subject: (record.any {it.getClass() == mc.entity}) ? 0 : record.add(subject) |
---|
460 | subject.setFieldValue(mc.property, value) |
---|
461 | break |
---|
462 | case SamplingEvent: (record.any {it.getClass() == mc.entity}) ? 0 : record.add(samplingEvent) |
---|
463 | samplingEvent.setFieldValue(mc.property, value) |
---|
464 | break |
---|
465 | case Event: (record.any {it.getClass() == mc.entity}) ? 0 : record.add(event) |
---|
466 | event.setFieldValue(mc.property, value) |
---|
467 | break |
---|
468 | case Sample: (record.any {it.getClass() == mc.entity}) ? 0 : record.add(sample) |
---|
469 | sample.setFieldValue(mc.property, value) |
---|
470 | break |
---|
471 | case Object: // don't import |
---|
472 | break |
---|
473 | } // end switch |
---|
474 | } catch (IllegalArgumentException iae) { |
---|
475 | // store the mapping column and value which failed |
---|
476 | def temp = new MappingColumn() |
---|
477 | temp.properties = mc.properties |
---|
478 | temp.value = value |
---|
479 | failed.add(temp) |
---|
480 | } |
---|
481 | } // end |
---|
482 | } // end for |
---|
483 | |
---|
484 | // a failed column means that using the entity.setFieldValue() threw an exception |
---|
485 | return [record, failed] |
---|
486 | } |
---|
487 | |
---|
488 | /** |
---|
489 | * Method to parse a value conform a specific type |
---|
490 | * @param value string containing the value |
---|
491 | * @return object corresponding to the TemplateFieldType |
---|
492 | */ |
---|
493 | def formatValue(String value, TemplateFieldType type) throws NumberFormatException { |
---|
494 | switch (type) { |
---|
495 | case TemplateFieldType.STRING : return value.trim() |
---|
496 | case TemplateFieldType.TEXT : return value.trim() |
---|
497 | case TemplateFieldType.INTEGER : return (int) Double.valueOf(value) |
---|
498 | case TemplateFieldType.FLOAT : return Float.valueOf(value.replace(",",".")); |
---|
499 | case TemplateFieldType.DOUBLE : return Double.valueOf(value.replace(",",".")); |
---|
500 | case TemplateFieldType.STRINGLIST : return value.trim() |
---|
501 | case TemplateFieldType.ONTOLOGYTERM : return value.trim() |
---|
502 | case TemplateFieldType.DATE : return value |
---|
503 | default : return value |
---|
504 | } |
---|
505 | } |
---|
506 | |
---|
507 | } |
---|