Context Navigation

source: trunk/grails-app/controllers/dbnp/importer/ImporterController.groovy @ 1588

Last change on this file since 1588 was 1588, checked in by s.h.sikkema@…, 13 years ago
Fixed tests (except webtests); cleaned up Example{Studies,Templates}.groovy; decapitalized injected services; made 'transactional' properties static
Property svn:keywords set to `Rev Author Date`
File size: 24.5 KB

Line
1	package dbnp.importer
2
3	import dbnp.studycapturing.*
4	import org.dbnp.gdt.*
5	import grails.converters.JSON
6	import org.codehaus.groovy.grails.plugins.web.taglib.ValidationTagLib
7	import grails.plugins.springsecurity.Secured
8
9	/**
10	* Wizard Controller
11	*
12	* @author Jeroen Wesbeek
13	* @since 20101206
14	*
15	* Revision information:
16	* $Rev: 1588 $
17	* $Author: s.h.sikkema@gmail.com $
18	* $Date: 2011-03-04 11:30:52 +0000 (vr, 04 mrt 2011) $
19	*/
20	@Secured(['IS_AUTHENTICATED_REMEMBERED'])
21	class ImporterController {
22	// the pluginManager is used to check if the Grom
23	// plugin is available so we can 'Grom' development
24	// notifications to the unified notifications daemon
25	// (see http://www.grails.org/plugin/grom)
26	def pluginManager
27	def authenticationService
28	def fileService
29	def importerService
30	def validationTagLib = new ValidationTagLib()
31	def gdtService
32
33	/**
34	* index method, redirect to the webflow
35	* @void
36	*/
37	def index = {
38	// Grom a development message
39	if (pluginManager.getGrailsPlugin('grom')) "redirecting into the webflow".grom()
40
41	/**
42	* Do you believe it in your head?
43	* I can go with the flow
44	* Don't say it doesn't matter (with the flow) matter anymore
45	* I can go with the flow (I can go)
46	* Do you believe it in your head?
47	*/
48	redirect(action: 'pages')
49	}
50
51	/**
52	* WebFlow definition
53	* @void
54	*/
55	def pagesFlow = {
56	// start the flow
57	onStart {
58	// Grom a development message
59	if (pluginManager.getGrailsPlugin('grom')) "entering the WebFlow".grom()
60
61	// define variables in the flow scope which is availabe
62	// throughout the complete webflow also have a look at
63	// the Flow Scopes section on http://www.grails.org/WebFlow
64	//
65	// The following flow scope variables are used to generate
66	// wizard tabs. Also see common/_tabs.gsp for more information
67	flow.page = 0
68	flow.pages = [
69	[title: 'Import file'],
70	[title: 'Assign properties'],
71	[title: 'Check imported data'],
72	//[title: 'Imported'],
73	[title: 'Done']
74	]
75	flow.cancel = true;
76	flow.quickSave = true;
77
78	success()
79	}
80
81	// render the main wizard page which immediately
82	// triggers the 'next' action (hence, the main
83	// page dynamically renders the study template
84	// and makes the flow jump to the study logic)
85	mainPage {
86	render(view: "/importer/index")
87	onRender {
88	// Grom a development message
89	if (pluginManager.getGrailsPlugin('grom')) "rendering the main Ajaxflow page (index.gsp)".grom()
90
91	// let the view know we're in page 1
92	flow.page = 1
93	success()
94	}
95	on("next").to "pageOne"
96	}
97
98	// File import and entitie template selection page
99	pageOne {
100	render(view: "_page_one")
101	onRender {
102	log.info ".entering import wizard"
103	// Grom a development message
104	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial: pages/_page_one.gsp".grom()
105
106	flow.page = 1
107	flow.studies = Study.findAllWhere(owner: authenticationService.getLoggedInUser())
108	flow.importer_fuzzymatching="false"
109
110	success()
111	}
112
113	on("refresh") {
114
115	if (params.entity) {
116	flash.importer_datatemplates = Template.findAllByEntity(gdtService.getInstanceByEntity(params.entity.decodeURL()))
117	}
118
119	flash.importer_params = params
120
121	// If the file already exists an "existing*" string is added, but we don't
122	// want that after a refresh of the first step in the import wizard, so remove
123	// that string
124	flash.importer_params.importfile = params.importfile.replace('existing*', '')
125
126	success()
127	}.to "pageOne"
128
129	on("next") {
130	flash.wizardErrors = [:]
131	flash.importer_params = params
132	flash.importer_params.importfile = params.importfile.replace('existing*', '')
133
134	if (params.entity) {
135	flash.importer_datatemplates = Template.findAllByEntity(gdtService.getInstanceByEntity(params.entity.decodeURL()))
136	}
137
138	// Study selected?
139	flow.importer_study = (params.study) ? Study.get(params.study.id.toInteger()) : null
140
141	// Trying to import data into an existing study?
142	if (flow.importer_study)
143	if (flow.importer_study.canWrite(authenticationService.getLoggedInUser()))
144	fileImportPage(flow, flash, params) ? success() : error()
145	else {
146	log.error ".importer wizard wrong permissions"
147	this.appendErrorMap(['error': "You don't have the right permissions"], flash.wizardErrors)
148	error()
149	}
150	else {
151	fileImportPage(flow, flash, params) ? success() : error()
152	}
153
154	// put your bussiness logic (if applicable) in here
155	}.to "pageTwo"
156	}
157
158	// Property to column assignment page
159	pageTwo {
160	render(view: "_page_two")
161	onRender {
162	log.info ".import wizard properties page"
163
164	def template = Template.get(flow.importer_template_id)
165
166	// Grom a development message
167	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial: pages/_page_two.gsp".grom()
168
169	flow.importer_importmappings = ImportMapping.findAllByTemplate(template)
170
171	flow.page = 2
172	success()
173	}
174	on("refresh") {
175	def template = Template.get(flow.importer_template_id)
176	flow.importer_importmappings = ImportMapping.findAllByTemplate(template)
177
178	// a name was given to the current property mapping, try to store it
179	if (params.mappingname) {
180	flash.importer_columnproperty = params.columnproperty
181	propertiesSaveImportMappingPage(flow, flash, params)
182	} else // trying to load an existing import mapping
183	if (params.importmapping_id) {
184	propertiesLoadImportMappingPage(flow, flash, params)
185	}
186
187	if (params.fuzzymatching == "true")
188	flow.importer_fuzzymatching="true" else
189	flow.importer_fuzzymatching="false"
190
191	success()
192	}.to "pageTwo"
193
194	on("next") {
195	flow.importer_fuzzymatching="false"
196	if (propertiesPage(flow, flash, params)) {
197	success()
198	} else {
199	log.error ".import wizard, properties are set wrong"
200	error()
201	}
202	}.to "pageThree"
203	on("previous").to "pageOne"
204	}
205
206	// Mapping page
207	pageThree {
208	render(view: "_page_three")
209	onRender {
210	log.info ".import wizard mapping page"
211	// Grom a development message
212	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial pages/_page_three.gsp".grom()
213
214	flow.page = 3
215	success()
216	}
217	on("refresh") {
218	success()
219	}.to "pageThree"
220	on("next") {
221	if (mappingsPage(flow, flash, params)) {
222	flow.page = 4
223	success()
224	} else {
225	log.error ".import wizard mapping error, could not validate all entities"
226	error()
227	}
228	}.to "save"
229	on("previous").to "pageTwo"
230	}
231
232	// Imported data overview page
233	pageFour {
234	render(view: "_page_four")
235	onRender {
236	// Grom a development message
237	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial pages/_page_four.gsp".grom()
238
239	flow.page = 4
240	success()
241	}
242	on("next") {
243	if (importedPage(flow, params)) {
244	flow.page = 4
245	success()
246	} else {
247	log.error ".import wizard imported error, something went wrong showing the imported entities"
248	error()
249	}
250	}.to "save"
251	on("previous").to "pageThree"
252	}
253
254	// Save the imported data
255	save {
256	action {
257	// here you can validate and save the
258	// instances you have created in the
259	// ajax flow.
260	// Grom a development message
261	if (pluginManager.getGrailsPlugin('grom')) ".persisting instances to the database...".grom()
262
263	// Always delete the uploaded file in the save step to be sure it doesn't reside there anymore
264	fileService.delete(flow.importer_importedfile)
265
266	// Save all entities
267	if (saveEntities(flow, params)) {
268	success()
269	} else {
270	log.error ".import wizard, could not save entities:\n" + e.dump()
271	flow.page = 4
272	error()
273	}
274	}
275	on("error").to "error"
276	on(Exception).to "error"
277	on("success").to "finalPage"
278	}
279
280	// render errors
281	error {
282	render(view: "_error")
283	onRender {
284	// Grom a development message
285	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial pages/_error.gsp".grom()
286
287	// set page to 4 so that the navigation
288	// works (it is disabled on the final page)
289	flow.page = 4
290	}
291	on("next").to "save"
292	on("previous").to "pageFour"
293	}
294
295	// last wizard page
296	finalPage {
297	render(view: "_final_page")
298	onRender {
299	// Grom a development message
300	if (pluginManager.getGrailsPlugin('grom')) ".rendering the partial pages/_final_page.gsp".grom()
301	success()
302	}
303	onEnd {
304	// clean flow scope
305	flow.clear()
306	}
307	}
308	}
309
310	def propertiesManager = {
311	render (view:"common/_propertiesManager")
312	}
313
314	/**
315	* Return templates which belong to a certain entity type
316	*
317	* @param entity entity name string (Sample, Subject, Study et cetera)
318	* @return JSON object containing the found templates
319	*/
320	def ajaxGetTemplatesByEntity = {
321	// fetch all templates for a specific entity
322	def templates = Template.findAllByEntity(gdtService.getInstanceByEntity(params.entity.decodeURL()))
323
324
325	// render as JSON
326	render templates as JSON
327	}
328
329	/**
330	* Handle the file import page.
331	*
332	* @param Map LocalAttributeMap (the flow scope)
333	* @param Map GrailsParameterMap (the flow parameters = form data)
334	* @returns boolean true if correctly validated, otherwise false
335	*/
336	boolean fileImportPage(flow, flash, params) {
337	def importedfile = fileService.get(params['importfile'])
338	flow.importer_importedfile = params['importfile']
339
340	if (importedfile.exists()) {
341	try {
342	session.importer_workbook = importerService.getWorkbook(new FileInputStream(importedfile))
343	} catch (Exception e) {
344	log.error ".importer wizard could not load file: " + e
345	this.appendErrorMap(['error': "Wrong file (format), the importer requires an Excel file as input"], flash.wizardErrors)
346	return false
347	}
348	}
349
350	if (params.entity && params.template_id) {
351
352	try {
353	session.importer_workbook = importerService.getWorkbook(new FileInputStream(importedfile))
354	} catch (Exception e) {
355	log.error ".importer wizard could not load file: " + e
356	this.appendErrorMap(['error': "Excel file required as input"], flash.wizardErrors)
357	return false
358	}
359
360	def selectedentities = []
361
362	def entityName = gdtService.decryptEntity(params.entity.decodeURL())
363	def entityClass = gdtService.getInstanceByEntityName(entityName)
364
365	// Initialize some session variables
366	//flow.importer_workbook = wb // workbook object must be serialized for this to work
367
368	flow.importer_template_id = params.template_id
369	flow.importer_sheetindex = params.sheetindex.toInteger() - 1 // 0 == first sheet
370	flow.importer_datamatrix_start = params.datamatrix_start.toInteger() - 1 // 0 == first row
371	flow.importer_headerrow = params.headerrow.toInteger()
372	flow.importer_entityclass = entityClass
373	flow.importer_entity = gdtService.cachedEntities.find{ it.entity==entityName }
374
375	// Get the header from the Excel file using the arguments given in the first step of the wizard
376	flow.importer_header = importerService.getHeader(session.importer_workbook,
377	flow.importer_sheetindex,
378	flow.importer_headerrow,
379	flow.importer_datamatrix_start,
380	entityClass)
381
382	session.importer_datamatrix = importerService.getDatamatrix(
383	session.importer_workbook, flow.importer_header,
384	flow.importer_sheetindex,
385	flow.importer_datamatrix_start,
386	5)
387
388	flow.importer_templates = Template.get(flow.importer_template_id)
389	flow.importer_allfieldtypes = "true"
390
391	return true
392	}
393
394
395	log.error ".importer wizard not all fields are filled in"
396	this.appendErrorMap(['error': "Not all fields are filled in, please fill in or select all fields"], flash.wizardErrors)
397	return false
398	}
399
400	/**
401	* Load an existing import mapping
402	*
403	* @param Map LocalAttributeMap (the flow scope)
404	* @param Map GrailsParameterMap (the flow parameters = form data)
405	* @returns boolean true if correctly validated, otherwise false
406	*/
407	boolean propertiesLoadImportMappingPage(flow, flash, params) {
408	def im = ImportMapping.get(params.importmapping_id.toInteger())
409	im.refresh()
410
411	im.mappingcolumns.each { mappingcolumn ->
412	//def mc = new MappingColumn()
413	//mc.properties = mappingcolumn.properties
414
415	flow.importer_header[mappingcolumn.index.toInteger()] = mappingcolumn
416	}
417	}
418
419	/**
420	* Save the properties as an import mapping.
421	*
422	* @param Map LocalAttributeMap (the flow scope)
423	* @param Map GrailsParameterMap (the flow parameters = form data)
424	* @returns boolean true if correctly validated, otherwise false
425	*/
426	boolean propertiesSaveImportMappingPage(flow, flash, params) {
427	flash.wizardErrors = [:]
428	def isPreferredIdentifier = false
429
430	// Find actual Template object from the chosen template name
431	def template = Template.get(flow.importer_template_id)
432
433	// Create new ImportMapping instance and persist it
434	def im = new ImportMapping(name:params.mappingname, entity: flow.importer_entityclass, template:template).save()
435
436	params.columnproperty.index.each { columnindex, property ->
437	// Create an actual class instance of the selected entity with the selected template
438	// This should be inside the closure because in some cases in the advanced importer, the fields can have different target entities
439	//def entityClass = gdtService.getInstanceByEntityName(flow.importer_header[columnindex.toInteger()].entity.getName())
440	def entityObj = flow.importer_entityclass.newInstance(template:template)
441
442	def dontimport = (property == "dontimport") ? true : false
443
444	// Loop through all fields and find the preferred identifier
445	entityObj.giveFields().each {
446	isPreferredIdentifier = (it.preferredIdentifier && (it.name == property)) ? true : false
447	}
448
449	// Create new MappingColumn instance
450	def mc = new MappingColumn (importmapping:im,
451	name: flow.importer_header[columnindex.toInteger()].name,
452	property:property,
453	index:columnindex,
454	entityclass:flow.importer_entityclass,
455	templatefieldtype:entityObj.giveFieldType(property),
456	dontimport: dontimport,
457	identifier:isPreferredIdentifier)
458
459	// Save mappingcolumn
460	if (mc.validate()) {
461	im.addToMappingcolumns(mc)
462	}
463	else {
464	mc.errors.allErrors.each {
465	println it
466	}
467	}
468
469	// Save importmapping
470	if (im.validate()) {
471	try {
472	im.save(flush:true)
473	} catch (Exception e) {
474	//getNextException
475	log.error "importer wizard save importmapping error: " + e
476	}
477	}
478	else {
479	im.errors.allErrors.each {
480	println it
481	}
482	}
483
484	}
485	}
486
487	/**
488	* Handle the property mapping page.
489	*
490	* @param Map LocalAttributeMap (the flow scope)
491	* @param Map GrailsParameterMap (the flow parameters = form data)
492	* @returns boolean true if correctly validated, otherwise false
493	*/
494	boolean propertiesPage(flow, flash, params) {
495	flash.wizardErrors = [:]
496
497	// Find actual Template object from the chosen template name
498	def template = Template.get(flow.importer_template_id)
499
500	params.columnproperty.index.each { columnindex, property ->
501	// Create an actual class instance of the selected entity with the selected template
502	// This should be inside the closure because in some cases in the advanced importer, the fields can have different target entities
503	def entityClass = Class.forName(flow.importer_header[columnindex.toInteger()].entityclass.getName(), true, this.getClass().getClassLoader())
504	def entityObj = entityClass.newInstance(template: template)
505
506	// Store the selected property for this column into the column map for the ImporterService
507	flow.importer_header[columnindex.toInteger()].property = property
508
509	// Look up the template field type of the target TemplateField and store it also in the map
510	flow.importer_header[columnindex.toInteger()].templatefieldtype = entityObj.giveFieldType(property)
511
512	// Is a "Don't import" property assigned to the column?
513	flow.importer_header[columnindex.toInteger()].dontimport = (property == "dontimport") ? true : false
514
515	//if it's an identifier set the mapping column true or false
516	entityObj.giveFields().each {
517	(it.preferredIdentifier && (it.name == property)) ? flow.importer_header[columnindex.toInteger()].identifier = true : false
518	}
519	}
520
521	// Import the workbook and store the table with entity records and store the failed cells
522	def (table, failedcells) = importerService.importData(flow.importer_template_id,
523	session.importer_workbook,
524	flow.importer_sheetindex,
525	flow.importer_datamatrix_start,
526	flow.importer_header)
527
528	flow.importer_importeddata = table
529
530	// loop through all entities to validate them and add them to wizardErrors flash when invalid
531	/*table.each { record ->
532	record.each { entity ->
533	if (!entity.validate()) {
534	this.appendErrors(entity, flash.wizardErrors, 'entity_' + entity.getIdentifier() + '_')
535	}
536	}
537	}*/
538
539	flow.importer_failedcells = failedcells
540
541	return true
542	}
543
544	/**
545	* Handle the mapping page.
546	*
547	* @param Map LocalAttributeMap (the flow scope)
548	* @param Map GrailsParameterMap (the flow parameters = form data)
549	* @returns boolean true if correctly validated, otherwise false
550	*/
551	boolean mappingsPage(flow, flash, params) {
552	flash.wizardErrors = [:]
553	flow.importer_invalidentities = 0
554
555	flow.importer_importeddata.each { table ->
556	table.each { entity ->
557	def invalidfields = 0
558
559	// Set the fields for this entity by retrieving values from the params
560	entity.giveFields().each { field ->
561
562	// field is a date field, try to set it with the value, if someone enters a non-date value it throws
563	// an error, this should be caught to prevent a complete breakdown
564	if (field.type == org.dbnp.gdt.TemplateFieldType.DATE) {
565	try {
566	entity.setFieldValue(field.toString(), params["entity_" + entity.getIdentifier() + "_" + field.escapedName()])
567	} catch (Exception e) { log.error ".importer wizard could not set date field with value: " +
568	params["entity_" + entity.getIdentifier() + "_" + field.escapedName()]
569	}
570	} else
571
572	// field of type ontology and value "#invalidterm"?
573	if (field.type == org.dbnp.gdt.TemplateFieldType.ONTOLOGYTERM &&
574	params["entity_" + entity.getIdentifier() + "_" + field.escapedName()] == "#invalidterm"
575	) {
576	invalidfields++
577	} else
578	if (field.type == org.dbnp.gdt.TemplateFieldType.ONTOLOGYTERM &&
579	params["entity_" + entity.getIdentifier() + "_" + field.escapedName()] != "#invalidterm") {
580	if (entity) removeFailedCell(flow.importer_failedcells, entity, field)
581	entity.setFieldValue(field.toString(), params["entity_" + entity.getIdentifier() + "_" + field.escapedName()])
582	}
583	else
584
585	if (field.type == org.dbnp.gdt.TemplateFieldType.STRINGLIST &&
586	params["entity_" + entity.getIdentifier() + "_" + field.escapedName()] != "#invalidterm") {
587	if (entity) removeFailedCell(flow.importer_failedcells, entity, field)
588	entity.setFieldValue(field.toString(), params["entity_" + entity.getIdentifier() + "_" + field.escapedName()])
589	} else
590	if (field.type == org.dbnp.gdt.TemplateFieldType.STRINGLIST &&
591	params["entity_" + entity.getIdentifier() + "_" + field.escapedName()] == "#invalidterm"
592	) {
593	invalidfields++
594	} else
595
596	entity.setFieldValue(field.toString(), params["entity_" + entity.getIdentifier() + "_" + field.escapedName()])
597	}
598
599	// Determine entity class and add a parent (defined as Study in first step of wizard)
600	switch (entity.getClass()) {
601	case [Subject, Sample, Event]: entity.parent = flow.importer_study
602	}
603
604	// Try to validate the entity now all fields have been set
605	if (!entity.validate() \|\| invalidfields) {
606	flow.importer_invalidentities++
607
608	// add errors to map
609	this.appendErrors(entity, flash.wizardErrors, "entity_" + entity.getIdentifier() + "_")
610
611	entity.errors.getAllErrors().each() {
612	log.error ".import wizard imported validation error:" + it
613	}
614	} else {
615	//removeFailedCell(flow.importer_failedcells, entity)
616	} // end else if
617
618	} // end of record
619	} // end of table
620
621	return (flow.importer_invalidentities == 0) ? true : false
622	} // end of method
623
624	/**
625	* @param failedcell failed ontology cells
626	* @param entity entity to remove from the failedcells list
627	*/
628	def removeFailedCell(failedcells, entity, field) {
629	// Valid entity, remove it from failedcells
630	def entityidfield = "entity_" + entity.getIdentifier() + "_" + field.name.toLowerCase()
631
632	failedcells.each { record ->
633	def tempimportcells = []
634
635
636	record.importcells.each { cell ->
637	// remove the cell from the failed cells session
638	if (cell.entityidentifier != entityidfield) {
639	//record.removeFromImportcells(cell)
640	tempimportcells.add(cell)
641	}
642	}
643
644	record.importcells = tempimportcells
645	// } // end of importcells
646	} // end of failedcells
647	}
648
649	/**
650	* Handle the imported entities page.
651	*
652	* @param Map LocalAttributeMap (the flow scope)
653	* @param Map GrailsParameterMap (the flow parameters = form data)
654	* @returns boolean true if correctly validated, otherwise false
655	*/
656	boolean importedPage(flow, params) {
657	return true
658	}
659
660	boolean saveEntities(flow, params) {
661	//def (validatedSuccesfully, updatedEntities, failedToPersist) =
662	//try {
663	importerService.saveDatamatrix(flow.importer_study, flow.importer_importeddata, authenticationService, log)
664
665	//}
666	//catch (Exception e) {
667	// log.error ".import wizard saveEntities error\n" + e.dump()
668	// }
669
670	//flow.importer_validatedsuccesfully = validatedSuccesfully
671	//flow.importer_failedtopersist = failedToPersist
672	//flow.imported_updatedentities = updatedEntities
673	//flow.importer_totalrows = flow.importer_importeddata.size
674	//flow.importer_referer = ""
675
676	return true
677	}
678
679	/**
680	* append errors of a particular object to a map
681	* @param object
682	* @param map linkedHashMap
683	* @void
684	*/
685	def appendErrors(object, map) {
686	this.appendErrorMap(getHumanReadableErrors(object), map)
687	}
688
689	def appendErrors(object, map, prepend) {
690	this.appendErrorMap(getHumanReadableErrors(object), map, prepend)
691	}
692
693	/**
694	* append errors of one map to another map
695	* @param map linkedHashMap
696	* @param map linkedHashMap
697	* @void
698	*/
699	def appendErrorMap(map, mapToExtend) {
700	map.each() {key, value ->
701	mapToExtend[key] = ['key': key, 'value': value, 'dynamic': false]
702	}
703	}
704
705	def appendErrorMap(map, mapToExtend, prepend) {
706	map.each() {key, value ->
707	mapToExtend[prepend + key] = ['key': key, 'value': value, 'dynamic': true]
708	}
709	}
710
711	/**
712	* transform domain class validation errors into a human readable
713	* linked hash map
714	* @param object validated domain class
715	* @return object linkedHashMap
716	*/
717	def getHumanReadableErrors(object) {
718	def errors = [:]
719	object.errors.getAllErrors().each() { error ->
720	// error.codes.each() { code -> println code }
721
722	// generally speaking g.message(...) should work,
723	// however it fails in some steps of the wizard
724	// (add event, add assay, etc) so g is not always
725	// availably. Using our own instance of the
726	// validationTagLib instead so it is always
727	// available to us
728	errors[error.getArguments()[0]] = validationTagLib.message(error: error)
729	}
730
731	return errors
732	}
733
734	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: