Changeset 1591

Timestamp:

Mar 7, 2011, 12:01:52 PM (11 years ago)

Author:

robert@…

Message:

Improved simple wizard so that is can edit studies

Location:

trunk/grails-app

Files:

• controllers/dbnp/studycapturing/SimpleWizardController.groovy (modified) (34 diffs)
• services/dbnp/importer/ImporterService.groovy (modified) (4 diffs)
• views/simpleWizard/assays.gsp (modified) (2 diffs)
• views/simpleWizard/columns.gsp (modified) (1 diff)
• views/simpleWizard/complexStudy.gsp (added)
• views/simpleWizard/existingSamples.gsp (added)
• views/simpleWizard/missingFields.gsp (modified) (1 diff)
• views/simpleWizard/overview.gsp (modified) (1 diff)
• views/simpleWizard/samples.gsp (modified) (2 diffs)
• views/simpleWizard/save.gsp (modified) (1 diff)
• views/studyWizard/pages/_samples.gsp (modified) (6 diffs)

trunk/grails-app/controllers/dbnp/studycapturing/SimpleWizardController.groovy

@@ -68 +68 @@
                                 // If the user clicks next, the study should be validated
                                 if( event == "next" && validateObject( study ) ) {
-                                        toPage( "samples" );
+                                        if( study.samples?.size() ) {
+                                                // This wizard can only handle simple studies. If the
+                                                // study is too complex, this wizard doesn't provide the
+                                                // possibility to edit samples/subjects etc.
+                                                if( checkStudySimplicity( study ) ) {
+                                                        toPage( "existingSamples" );
+                                                } else {
+                                                        toPage( "complexStudy" );
+                                                }
+                                        } else {
+                                                toPage( "samples" );
+                                        }
+                                        
                                         return;
                                 }
@@ -76 +88 @@
                 // Give the study to the user
                 [ study: study ]
+        }
+        
+        /**
+         * Shows a page to mention that the study being edited is too complex
+         */
+        def complexStudy = {
+                // Retrieve the correct study
+                study = getStudyInWizard( params );
+                if( !study ) {
+                        redirect( controller: 'simpleWizard', action: 'study' );
+                        return;
+                }
+
+                def event = getEvent(params);
+
+                // If any event on this page is triggered
+                if( event ) {
+                        // Now determine what action to perform
+                        if( event == "save" ) {
+                                toPage( "save" );
+                                return;
+                        } else if( event == "previous" ) {
+                                toPage( "study" );
+                                return;
+                        }
+                }
+
+                // Give the study and other data to the user
+                [ study: study ]
+        }
+
+        /**
+         * Shows the samples page
+         */
+        def existingSamples = {
+                // Retrieve the correct study
+                study = getStudyInWizard( params );
+                if( !study ) {
+                        redirect( controller: 'simpleWizard', action: 'study' );
+                        return;
+                }
+
+                def event = getEvent(params);
+
+                // If any event on this page is triggered, we should save the entered data.
+                // If no event is triggered, the user gets here from another page. In that case,
+                // we don't set the values
+                if( event ) {
+                        // Now determine what action to perform
+                        if( event == "next" && handleExistingSamples( study, params )) {
+                                // Only continue to the next page if the information entered is correct
+                                toPage( "assays" );
+                                return;
+                        } else if( event == "previous" ) {
+                                // The user may go to the previous page, even if none of the data entered is OK.
+                                toPage( "study" );
+                                return;
+                        } else if( event == "update" && handleExistingSamples( study, params ) ) {
+                                // The user may update the samples using excel. Before that, the sample date should be saved
+                                toPage( "samples" );
+                                return;
+                        } else if( event == "skip" ) {
+                                // The user may skip the complete samples page
+                                toPage( "assays" );
+                                return;
+                        }
+                }
+
+                // Give the study and other data to the user
+                def records = importerService.getRecords( study );
+
+                [ study: study,
+                        records: records,
+                        templateCombinations: records.templateCombination.unique(),
+                        templates: [ 
+                                'Sample': Template.findAllByEntity( Sample.class ),
+                                'Subject': Template.findAllByEntity( Subject.class ),
+                                'Event': Template.findAllByEntity( Event.class ),
+                                'SamplingEvent': Template.findAllByEntity( SamplingEvent.class )
+                        ],
+                        encodedEntity: [ 
+                                'Sample': gdtService.encryptEntity( Sample.class.name ), 
+                                'Subject': gdtService.encryptEntity( Subject.class.name ), 
+                                'Event': gdtService.encryptEntity( Event.class.name ), 
+                                'SamplingEvent': gdtService.encryptEntity( SamplingEvent.class.name )
+                        ],
+                        existingSampleForm: session.simpleWizard.existingSampleForm 
+                ]
         }
 
@@ -102 +202 @@
                         } else if( event == "previous" ) {
                                 // The user may go to the previous page, even if none of the data entered is OK.
-                                toPage( "study" );
+                                if( study.samples?.size() )
+                                        toPage( "existingSamples" );
+                                else
+                                        toPage( "study" );
+
                                 return;
                         } else if( event == "skip" ) {
@@ -112 +216 @@
 
                 // Give the study and other data to the user
-                [ study: study, sampleTemplates: Template.findAllByEntity( Sample.class ), encodedEntity: gdtService.encryptEntity( Sample.class.name ), sampleForm: session.simpleWizard.sampleForm ]
+                [ study: study, 
+                        templates: [ 
+                                'Sample': Template.findAllByEntity( Sample.class ),
+                                'Subject': Template.findAllByEntity( Subject.class ),
+                                'Event': Template.findAllByEntity( Event.class ),
+                                'SamplingEvent': Template.findAllByEntity( SamplingEvent.class )
+                        ],
+                        encodedEntity: [ 
+                                'Sample': gdtService.encryptEntity( Sample.class.name ), 
+                                'Subject': gdtService.encryptEntity( Subject.class.name ), 
+                                'Event': gdtService.encryptEntity( Event.class.name ), 
+                                'SamplingEvent': gdtService.encryptEntity( SamplingEvent.class.name )
+                        ],
+                        sampleForm: session.simpleWizard.sampleForm ]
         }
 
@@ -134 +251 @@
                         // Now determine what action to perform
                         if( event == "next" && handleColumns( study, params ) ) {
+                                
                                 // Only continue to the next page if the information entered is correct
                                 if( session.simpleWizard.imported.numInvalidEntities > 0 ) {
@@ -142 +260 @@
                                                 fileService.delete( session.simpleWizard.sampleForm.importFile );
 
+                                        session.simpleWizard.sampleForm = null
+
                                         toPage( "assays" );
                                 }
                                 return;
                         } else if( event == "previous" ) {
-                                // THe user may go to the previous page, even if the data is not correct
+                                // The user may go to the previous page, even if the data is not correct
                                 toPage( "samples" );
                                 return;
                         }
                 }
-
+                
+                def templates = [:]; 
+                def domainFields = [:]
+
+                session.simpleWizard.sampleForm.templateId.each { 
+                        templates[ it.key ] = it.value ? Template.get( it.value ) : null;
+                        if( it.value ) {
+                                domainFields[ it.key ] = templates[ it.key ].entity.giveDomainFields();
+                        }
+                } 
+                
                 // Give the study and other data to the user
                 [ study: study,
                                         filename: session.simpleWizard.sampleForm.importFile,
-                                        template: Template.get( session.simpleWizard.sampleForm.templateId ),
+                                        templates: templates,
+                                        domainFields: domainFields,
                                         excel: session.simpleWizard.excel]
         }
@@ -180 +311 @@
                                 if( session.simpleWizard.imported.numInvalidEntities == 0 ) {
                                         // Only continue to the next page if the information entered is correct
-                                        
+
                                         // The import of the excel file has finished. Now delete the excelfile
-                                        if( session.simpleWizard.sampleForm.importFile ) 
+                                        if( session.simpleWizard.sampleForm.importFile ) {
                                                 fileService.delete( session.simpleWizard.sampleForm.importFile );
-                                                
+                                        }
+                                        session.simpleWizard.sampleForm = null
+
                                         toPage( "assays" );
                                         return;
@@ -203 +336 @@
                 }
 
+                // Create the right format for showing the data
+                def records = [];
+                session.simpleWizard.imported.data.each { row ->
+                        def record = [:];
+                        row.each { object ->
+                                // Attach template to session
+                                if( object.template )
+                                        attachTemplate( object.template );
+                                
+                                def entityName = object.class.name[ object.class.name.lastIndexOf( '.' ) + 1 .. -1 ]
+                                record[ entityName ] = object;
+                        }
+                        records << record;
+                }
+                
                 // Give the study and other data to the user
-                [ study: study, imported: session.simpleWizard.imported, rules: rules ]
+                println "Imported: " + session.simpleWizard.imported.failedCells;
+                
+                [ study: study, imported: session.simpleWizard.imported, records: records, rules: rules ]
         }
 
@@ -217 +367 @@
                         return;
                 }
-                
-                Assay assay
-                if( study.assays?.size() ) {
-                        assay = study.assays[0];
-                        study.removeFromAssays( assay );
-                } else {
-                        assay = new Assay();
-                }
-                        
+
+                Assay assay = getAssayInWizard( study );
+
                 def event = getEvent(params);
 
@@ -235 +379 @@
                         if( event == "skip" ) {
                                 // The user may skip the complete assay page
+
+                                // In case the user has created an assay before, it should only be kept if it
+                                // existed before this step
+                                if( session.simpleWizard.assay != null && !session.simpleWizard.assay.id ) {
+                                        session.simpleWizard.remove( "assay" )
+                                }
+
                                 toPage( "overview" );
                                 return;
                         } else if( handleAssays( assay, params ) ) {
-                                study.addToAssays( assay );
-                                
                                 // Now determine what action to perform
-                                if( event == "next" && validateObject( study ) ) {
+                                if( event == "next" && validateObject( assay ) ) {
                                         toPage( "overview" );
                                         return;
                                 } else if( event == "previous" ) {
-                                        toPage( "samples" )
+                                        if( study.samples?.size() )
+                                                toPage( "existingSamples" )
+                                        else
+                                                toPage( "samples" );
+
                                         return;
                                 }
@@ -252 +405 @@
 
                 // Give the study to the user
-                [ study: study, assay: assay ]
-        }
-        
+                [ study: study, wizardAssay: assay ]
+        }
+
         /**
          * Shows an overview of the entered study
@@ -265 +418 @@
                         return;
                 }
-                        
+
+                Assay assay = getAssayInWizard();
+
                 def event = getEvent(params);
 
@@ -277 +432 @@
                                 return;
                         } else if( event == "previous" ) {
-                                toPage( "assay" )
+                                toPage( "assays" )
                                 return;
                         }
@@ -283 +438 @@
 
                 // Give the study to the user
-                [ study: study ]
-        }
-        
+                [ study: study, wizardAssay: assay ]
+        }
+
         def save = {
                 // Retrieve the correct study
@@ -293 +448 @@
                         return;
                 }
-                
-                // Make sure all samples are attached to all assays
-                study.assays.each { assay ->
-                        assay.samples?.clear();
-                        study.samples.each { sample ->
-                                assay.addToSamples( sample )
-                        }
-                }
-                
-                // Save the study
-                if( study.save( flush: true ) ) {
+
+                def event = getEvent(params);
+
+                if( event && event == "previous" ) {
+                        toPage( "assays" );
+                        return;
+                }
+                
+                Assay newAssay = getAssayInWizard();
+
+                if( newAssay && !study.assays?.contains( newAssay ) ) {
+                        study.addToAssays( newAssay );
+                }
+
+                //attachAndValidateEntities( study );
+                
+                // Save the study. The study must be merged if it is loaded from
+                // the database before, since otherwise it will raise 'lazy initialization' errors
+                // The validation is done in the other steps, so it is skipped here.
+                def success
+                
+                if(
+                study.id ?
+                study.merge( validate: false, flush: true ) :
+                study.save( flush: true )
+                ) {
+                        // Make sure all samples are attached to all assays
+                        study.assays.each { assay ->
+                                def l = []+ assay.samples;
+                                l.each { sample ->
+                                        if( sample )
+                                                assay.removeFromSamples( sample );
+                                }
+                                assay.samples?.clear();
+
+                                study.samples.each { sample ->
+                                        assay.addToSamples( sample )
+                                }
+                        }
+
                         // Clear session
                         session.simpleWizard = null;
-                        
+
                         flash.message = "Your study is succesfully saved.";
+                        success = true
                 } else {
                         flash.error = "An error occurred while saving your study";
+
+                        study.getErrors().each { println it }
+
+                        // Remove the assay from the study again, since it is still available
+                        // in the session
+                        if( newAssay ) {
+                                study.removeFromAssays( newAssay );
+                                newAssay.parent = study;
+                        }
+
                         //validateObject( study );
+                        success = false
                 }
 
                 // Give the study to the user
-                [ study: study ]
+                [ study: study, success: success ]
         }
 
@@ -353 +549 @@
                 handleStudyUsers(study, params, 'readers')
                 handleStudyUsers(study, params, 'writers')
-                
+
                 return true
+        }
+
+        /**
+         * Handles the editing of existing samples
+         * @param study         Study to update
+         * @param params                Request parameter map
+         * @return                      True if everything went OK, false otherwise. An error message is put in flash.error
+         */
+        def handleExistingSamples( study, params ) {
+                session.simpleWizard.existingSampleForm = params
+                flash.validationErrors = [:];
+
+                def errors = false;
+                
+                // iterate through objects; set field values and validate the object
+                def eventgroups = study.samples.parentEventGroup.findAll { it }
+                def events;
+                if( !eventgroups )
+                        events = []
+                else
+                        events = eventgroups.events?.getAt(0);
+                
+                def objects = [ 
+                        'Sample': study.samples, 
+                        'Subject': study.samples.parentSubject.findAll { it }, 
+                        'SamplingEvent': study.samples.parentEvent.findAll { it }, 
+                        'Event': events.flatten().findAll { it }
+                ];      
+                objects.each {
+                        def type = it.key;
+                        def entities = it.value;
+                        
+                        entities.each { entity ->
+                                // iterate through entity fields
+                                entity.giveFields().each() { field ->
+                                        def value = params.get( type.toLowerCase() + '_' + entity.getIdentifier() + '_' + field.escapedName())
+
+                                        // set field value; name cannot be set to an empty value
+                                        if (field.name != 'name' || value) {
+                                                log.info "setting "+field.name+" to "+value
+                                                entity.setFieldValue(field.name, value)
+                                        }
+                                }
+                                
+                                // has the template changed? 
+                                def templateName = params.get(type.toLowerCase() + '_' + entity.getIdentifier() + '_template')
+                                if (templateName && entity.template?.name != templateName) {
+                                        entity.template = Template.findByName(templateName)
+                                }
+        
+                                // validate sample
+                                if (!entity.validate()) {
+                                        errors = true;
+                                        flash.validationErrors << getHumanReadableErrors( entity )
+                                }
+                        }
+                }
+
+                return !errors
         }
 
@@ -370 +625 @@
                 // still being in the temporary directory.
                 // This import step doesn't have to make that distinction, since all files remain in the temporary directory.
-                if( filename[0..8] == 'existing*' )
+                if( filename == 'existing*' )
+                        filename = '';
+                else if( filename[0..8] == 'existing*' )
                         filename = filename[9..-1]
-
-                def templateId  = params.long( 'template_id' )
+                
+                def sampleTemplateId  = params.long( 'sample_template_id' )
+                def subjectTemplateId  = params.long( 'subject_template_id' )
+                def eventTemplateId  = params.long( 'event_template_id' )
+                def samplingEventTemplateId  = params.long( 'samplingEvent_template_id' )
+                
+                // These fields have been removed from the form, so will always contain
+                // their default value. The code however remains like this for future use.
                 int sheetIndex = (params.int( 'sheetindex' ) ?: 1 )
                 int dataMatrixStart = (params.int( 'datamatrix_start' ) ?: 2 )
@@ -381 +644 @@
                 session.simpleWizard.sampleForm = [
                                         importFile: filename,
-                                        templateId: templateId,
+                                        templateId: [ 
+                                                'Sample': sampleTemplateId,
+                                                'Subject': subjectTemplateId,
+                                                'Event': eventTemplateId,
+                                                'SampingEvent': samplingEventTemplateId
+                                        ],
                                         sheetIndex: sheetIndex,
                                         dataMatrixStart: dataMatrixStart,
@@ -388 +656 @@
 
                 // Check whether the template exists
-                if (!templateId || !Template.get( templateId ) ){
-                        log.error ".simple study wizard not all fields are filled in"
+                if (!sampleTemplateId || !Template.get( sampleTemplateId ) ){
+                        log.error ".simple study wizard not all fields are filled in: " + sampleTemplateId
                         flash.error = "No template was chosen. Please choose a template for the samples you provided."
                         return false
@@ -474 +742 @@
         def handleColumns( study, params ) {
                 // Find actual Template object from the chosen template name
-                def template = Template.get(session.simpleWizard.sampleForm.templateId)
+                def templates = [:];
+                session.simpleWizard.sampleForm.templateId.each {
+                        templates[ it.key ] = it.value ? Template.get( it.value ) : null;
+                }
+                
                 def headers = session.simpleWizard.excel.data.header;
 
@@ -485 +757 @@
                 // Retrieve the chosen matches from the request parameters and put them into
                 // the headers-structure, for later reference
-                params.matches.index.each { columnindex, property ->
+                params.matches.index.each { columnindex, value ->
+                        // Determine the entity and property by splitting it
+                        def parts = value.toString().tokenize( "||" );
+                        
+                        def property
+                        def entityName
+                        if( parts.size() > 1 ) {
+                                property = parts[ 1 ];
+                                entityName = "dbnp.studycapturing." + parts[ 0 ];
+                        } else if( parts.size() == 1 ) {
+                                property = parts[ 0 ];
+                                entityName = headers[columnindex.toInteger()].entityclass.getName();
+                        }
+                        
                         // Create an actual class instance of the selected entity with the selected template
                         // This should be inside the closure because in some cases in the advanced importer, the fields can have different target entities
-                        def entityClass = Class.forName( headers[columnindex.toInteger()].entityclass.getName(), true, this.getClass().getClassLoader())
-                        def entityObj = entityClass.newInstance(template: template)
-
+                        def entityClass = Class.forName( entityName, true, this.getClass().getClassLoader())
+                        def entityObj = entityClass.newInstance(template: templates[ entityName[entityName.lastIndexOf( '.' ) + 1..-1] ])
+
+                        headers[ columnindex.toInteger() ].entityclass = entityClass
+                        
                         // Store the selected property for this column into the column map for the ImporterService
                         headers[columnindex.toInteger()].property = property
@@ -501 +788 @@
 
                         //if it's an identifier set the mapping column true or false
-                        entityObj.giveFields().each {
+                        entityClass.giveDomainFields().each {
                                 headers[columnindex.toInteger()].identifier = ( it.preferredIdentifier && (it.name == property) )
                         }
@@ -507 +794 @@
 
                 // Import the workbook and store the table with entity records and store the failed cells
-                def (table, failedcells) = importerService.importData(session.simpleWizard.sampleForm.templateId,
+                println "Importing samples for study " + study + " (" + study.id + ")";
+                
+                def imported = importerService.importOrUpdateDataBySampleIdentifier(templates,
                                 session.simpleWizard.excel.workbook,
                                 session.simpleWizard.excel.sheetIndex - 1,
                                 session.simpleWizard.excel.dataMatrixStart - 1,
-                                session.simpleWizard.excel.data.header)
+                                session.simpleWizard.excel.data.header,
+                                study,
+                                true                    // Also create entities for which no data is imported but where templates were chosen 
+                );
+
+                def table = imported.table
+                def failedcells = imported.failedCells
 
                 session.simpleWizard.imported = [
-                                        data: table,
-                                        failedCells: failedcells
-                                ];
+                        data: table,
+                        failedCells: failedcells
+                ];
 
                 // loop through all entities to validate them and add them to failedcells if an error occurs
@@ -522 +817 @@
                 def errors = [];
 
-                // Remove all samples
-                study.samples?.clear();
-
+                // Add all samples
                 table.each { record ->
                         record.each { entity ->
+                                if( entity ) {
+                                        // Determine entity class and add a parent
+                                        def preferredIdentifier = importerService.givePreferredIdentifier( entity.class );
+                                        def equalClosure = { it.getFieldValue( preferredIdentifier.name ) == entity.getFieldValue( preferredIdentifier.name ) }
+        
+                                        entity.parent = study
+                                        
+                                        switch( entity.class ) {
+                                                case Sample:
+                                                        if( preferredIdentifier && !study.samples?.find( equalClosure ) ) {
+                                                                study.addToSamples( entity );
+                                                        }
+                                                        break;
+                                                case Subject:
+                                                        if( preferredIdentifier && !study.subjects?.find( equalClosure ) ) {
+                                                                study.addToSubjects( entity );
+                                                        }
+                                                        break;
+                                                case Event:
+                                                        if( preferredIdentifier && !study.events?.find( equalClosure ) ) {
+                                                                study.addToEvents( entity );
+                                                        }
+                                                        break;
+                                                case SamplingEvent:
+                                                        if( preferredIdentifier && !study.samplingEvents?.find( equalClosure ) ) {
+                                                                study.addToSamplingEvents( entity );
+                                                        }
+                                                        break;
+                                        }
+                                        
+                                        if (!entity.validate()) {
+                                                numInvalidEntities++;
+                                                
+                                                // Add this field to the list of failed cells, in order to give the user feedback
+                                                failedcells = addNonValidatingCells( failedcells, entity )
+        
+                                                // Also create a full list of errors
+                                                errors += getHumanReadableErrors( entity );
+                                        }
+                                }
+                        }
+                }
+
+                session.simpleWizard.imported.numInvalidEntities = numInvalidEntities + failedcells?.size();
+                session.simpleWizard.imported.errors = errors;
+
+                return true
+        }
+
+        /**
+         * Handles the update of the edited fields by the user
+         * @param study         Study to update
+         * @param params                Request parameter map
+         * @return                      True if everything went OK, false otherwise. An error message is put in flash.error.
+         *                                      The field session.simpleWizard.imported.numInvalidEntities reflects the number of
+         *                                      entities that still have errors, and should be fixed before saving. The errors for those entities
+         *                                      are saved into session.simpleWizard.imported.errors
+         */
+        def handleMissingFields( study, params ) {
+                def numInvalidEntities = 0;
+                def errors = [];
+
+                // Check which fields failed previously
+                def failedCells = session.simpleWizard.imported.failedCells
+
+                session.simpleWizard.imported.data.each { table ->
+                        table.each { entity ->
+                                def invalidFields = 0
+
+                                // Set the fields for this entity by retrieving values from the params
+                                entity.giveFields().each { field ->
+                                        def fieldName = importerService.getFieldNameInTableEditor( entity, field );
+
+                                        if( params[ fieldName ] == "#invalidterm" ) {
+                                                // If the value '#invalidterm' is chosen, the user hasn't fixed anything, so this field is still incorrect
+                                                invalidFields++;
+                                        } else {
+                                                if( field.type == org.dbnp.gdt.TemplateFieldType.ONTOLOGYTERM || field.type == org.dbnp.gdt.TemplateFieldType.STRINGLIST ) {
+                                                        // If this field is an ontologyterm field or a stringlist field, the value has changed, so remove the field from
+                                                        // the failedCells list
+                                                        importerService.removeFailedCell( failedCells, entity, field )
+                                                }
+
+                                                // Update the field, regardless of the type of field
+                                                entity.setFieldValue(field.name, params[ fieldName ] )
+                                        }
+                                }
+
                                 // Determine entity class and add a parent
-                                entity.parent = study
-                                study.addToSamples( entity );
-
-                                if (!entity.validate()) {
+                                entity.parent = study;
+
+                                // Try to validate the entity now all fields have been set. If it fails, return an error
+                                if (!entity.validate() || invalidFields) {
                                         numInvalidEntities++;
 
                                         // Add this field to the list of failed cells, in order to give the user feedback
-                                        failedcells = addNonValidatingCells( failedcells, entity )
+                                        failedCells = addNonValidatingCells( failedCells, entity )
 
                                         // Also create a full list of errors
                                         errors += getHumanReadableErrors( entity );
-                                }
-                        }
-                }
+                                } else {
+                                        importerService.removeFailedCell( failedCells, entity )
+                                }
+                        } // end of record
+                } // end of table
 
                 session.simpleWizard.imported.numInvalidEntities = numInvalidEntities;
@@ -548 +931 @@
                 return true
         }
+
+        /**
+         * Handles assay input
+         * @param study         Study to update
+         * @param params                Request parameter map
+         * @return                      True if everything went OK, false otherwise. An error message is put in flash.error
+         */
+        def handleAssays( assay, params ) {
+                // did the study template change?
+                if (params.get('template') && assay.template?.name != params.get('template')) {
+                        // set the template
+                        assay.template = Template.findByName(params.remove('template'))
+                }
+
+                // does the study have a template set?
+                if (assay.template && assay.template instanceof Template) {
+                        // yes, iterate through template fields
+                        assay.giveFields().each() {
+                                // and set their values
+                                assay.setFieldValue(it.name, params.get(it.escapedName()))
+                        }
+                }
+
+                // Save the assay in session
+                session.simpleWizard.assay = assay;
+
+                return true
+        }
         
         /**
-        * Handles the update of the edited fields by the user
-        * @param study          Study to update
-        * @param params         Request parameter map
-        * @return                       True if everything went OK, false otherwise. An error message is put in flash.error.
-        *                                       The field session.simpleWizard.imported.numInvalidEntities reflects the number of
-        *                                       entities that still have errors, and should be fixed before saving. The errors for those entities
-        *                                       are saved into session.simpleWizard.imported.errors
-        */
-   def handleMissingFields( study, params ) {
-           def numInvalidEntities = 0;
-           def errors = [];
-
-           // Remove all samples before adding them again
-           study.samples?.clear();
-
-           // Check which fields failed previously
-           def failedCells = session.simpleWizard.imported.failedCells
-
-           session.simpleWizard.imported.data.each { table ->
-                   table.each { entity ->
-                           def invalidFields = 0
-
-                           // Set the fields for this entity by retrieving values from the params
-                           entity.giveFields().each { field ->
-                                   def fieldName = importerService.getFieldNameInTableEditor( entity, field );
-
-                                   if( params[ fieldName ] == "#invalidterm" ) {
-                                           // If the value '#invalidterm' is chosen, the user hasn't fixed anything, so this field is still incorrect
-                                           invalidFields++;
-                                   } else {
-                                           if( field.type == org.dbnp.gdt.TemplateFieldType.ONTOLOGYTERM || field.type == org.dbnp.gdt.TemplateFieldType.STRINGLIST ) {
-                                                   // If this field is an ontologyterm field or a stringlist field, the value has changed, so remove the field from
-                                                   // the failedCells list
-                                                   importerService.removeFailedCell( failedCells, entity, field )
-                                           }
-
-                                           // Update the field, regardless of the type of field
-                                           entity.setFieldValue(field.name, params[ fieldName ] )
-                                   }
-                           }
-
-                           // Determine entity class and add a parent
-                           entity.parent = study;
-                           study.addToSamples( entity );
-
-                           // Try to validate the entity now all fields have been set. If it fails, return an error
-                           if (!entity.validate() || invalidFields) {
-                                   numInvalidEntities++;
-
-                                   // Add this field to the list of failed cells, in order to give the user feedback
-                                   failedCells = addNonValidatingCells( failedCells, entity )
-
-                                   // Also create a full list of errors
-                                   errors += getHumanReadableErrors( entity );
-                           } else {
-                                   importerService.removeFailedCell( failedCells, entity )
-                           }
-                   } // end of record
-           } // end of table
-
-           session.simpleWizard.imported.numInvalidEntities = numInvalidEntities;
-           session.simpleWizard.imported.errors = errors;
-
-           return true
-   }
-        
-        /**
-        * Handles assay input
-        * @param study          Study to update
-        * @param params         Request parameter map
-        * @return                       True if everything went OK, false otherwise. An error message is put in flash.error
-        */
-   def handleAssays( assay, params ) {
-           // did the study template change?
-           if (params.get('template') && assay.template?.name != params.get('template')) {
-                   // set the template
-                   assay.template = Template.findByName(params.remove('template'))
-           }
-
-           // does the study have a template set?
-           if (assay.template && assay.template instanceof Template) {
-                   // yes, iterate through template fields
-                   assay.giveFields().each() {
-                           // and set their values
-                           assay.setFieldValue(it.name, params.get(it.escapedName()))
-                   }
-           }
-
-           return true
-   }
-
-   /**
-    * Validates an object and puts human readable errors in validationErrors variable
-    * @param entity             Entity to validate
-    * @return                   True iff the entity validates, false otherwise
-    */
-   protected boolean validateObject( def entity ) {
-           if( !entity.validate() ) {
-                   flash.validationErrors = getHumanReadableErrors( entity )
-                   return false;
-           }
-           return true;
-   }
-   
-   
+         * Checks whether the given study is simple enough to be edited using this controller.
+         * 
+         * The study is simple enough if the samples, subjects, events and samplingEvents can be 
+         * edited as a flat table. That is: 
+         *              - Every subject belongs to 0 or 1 eventgroup
+         *              - Every eventgroup belongs to 0 or 1 sample
+         *              - Every eventgroup has 0 or 1 subjects, 0 or 1 event and 0 or 1 samplingEvents
+         *              - If a sample belongs to an eventgroup:
+         *                      - If that eventgroup has a samplingEvent, that same samplingEvent must also be 
+         *                              the sampling event that generated this sample
+         *                      - If that eventgroup has a subject, that same subject must also be the subject
+         *                              from whom the sample was taken
+         * 
+         * @param study         Study to check
+         * @return                      True if the study can be edited by this controller, false otherwise
+         */
+        def checkStudySimplicity( study ) {
+                def simplicity = true;
+                
+                if( !study )
+                        return false
+
+                if( study.eventGroups ) {
+                        study.eventGroups.each { eventGroup ->
+                                // Check for simplicity of eventgroups: only 0 or 1 subject, 0 or 1 event and 0 or 1 samplingEvent
+                                if( eventGroup.subjects?.size() > 1 || eventGroup.events?.size() > 1 || eventGroup.samplingEvents?.size() > 1 ) {
+                                        flash.message = "One or more eventgroups contain multiple subjects or events."
+                                        simplicity = false;
+                                }
+                                
+                                // Check whether this eventgroup only belongs to (max) 1 sample
+                                def numSamples = 0;
+                                study.samples.each { sample ->
+                                        // If no id is given for the eventGroup, it has been entered in this wizard, but 
+                                        // not yet saved. In that case, it is always OK
+                                        if( eventGroup.id && sample.parentEventGroup?.id == eventGroup.id )
+                                                numSamples++;
+                                }
+                                
+                                if( numSamples > 1 ) { 
+                                        flash.message = "One or more eventgroups belong to multiple samples."
+                                        simplicity = false;
+                                }
+                        }
+                        
+                        if( !simplicity ) return false;
+
+                        // Check whether subject only belong to zero or one event group
+                        if( study.subjects ) {
+                                study.subjects.each { subject ->
+                                        def numEventGroups = 0
+                                        study.eventGroups.each { eventGroup ->
+                                                // If no id is given for the subject, it has been entered in this wizard, but
+                                                // not yet saved. In that case, it is always OK
+                                                if( subject.id && eventGroup.subjects[0]?.id == subject.id )
+                                                        numEventGroups++
+                                        }
+                                        
+                                        if( numEventGroups > 1 ) {
+                                                flash.message = "One or more subjects belong to multiple eventgroups."
+                                                simplicity = false;
+                                        }
+                                }
+                        }
+
+                        if( !simplicity ) return false;
+                        
+                        // Check whether the samples that belong to an eventgroup have the right parentObjects
+                        study.samples.each { sample ->
+                                if( sample.parentEventGroup ) {
+                                        // If no id is given for the subject, it has been entered in this wizard, but
+                                        // not yet saved. In that case, it is always OK
+                                        if( sample.parentSubject && sample.parentSubject.id) {
+                                                if( !sample.parentEventGroup.subjects || sample.parentEventGroup.subjects[0]?.id != sample.parentSubject.id ) {
+                                                        flash.message = "The structure of the eventgroups of one or more samples is too complex"
+                                                        simplicity = false;
+                                                }
+                                        }
+                                        
+                                        // If no id is given for the sampling event, it has been entered in this wizard, but
+                                        // not yet saved. In that case, it is always OK
+                                        if( sample.parentEvent && sample.parentEvent.id) {
+                                                if( !sample.parentEventGroup.samplingEvents || sample.parentEventGroup.samplingEvents[0]?.id != sample.parentEvent.id ) {
+                                                        flash.message = "The structure of the eventgroups of one or more samples is too complex"
+                                                        simplicity = false;
+                                                }
+                                        }
+                                }
+                        }
+                        
+                        if( !simplicity ) return false;
+                }
+                
+                return simplicity;
+        }
+
+        def attachAndValidateEntities( def study ) {
+                if( !session.simpleWizard?.imported?.data ) 
+                        return
+                         
+                def table = session.simpleWizard.imported.data
+                def numInvalidEntities = 0;
+                
+                // Add all samples
+                table.each { record ->
+                        println record*.class
+                        record.each { entity ->
+                                if( entity ) {
+                                        if( entity.validate() ) {
+                                                println "Saving: " + entity + " (" + entity.class.name + ")"
+                                                println study.samples;
+                                                println study.subjects;
+                                                
+                                                // Determine entity class and add a parent
+                                                def preferredIdentifier = importerService.givePreferredIdentifier( entity.class );
+                                                def equalClosure = { it.getFieldValue( preferredIdentifier.name ) == entity.getFieldValue( preferredIdentifier.name ) }
+                
+                                                //entity.parent = study
+                                                switch( entity.class ) {
+                                                        case Sample:
+                                                                if( preferredIdentifier && !study.samples?.find( equalClosure ) ) {
+                                                                        study.addToSamples( entity );
+                                                                }
+                                                                break;
+                                                        case Subject:
+                                                                if( preferredIdentifier && !study.subjects?.find( equalClosure ) ) {
+                                                                        study.addToSubjects( entity );
+                                                                }
+                                                                break;
+                                                        case Event:
+                                                                if( preferredIdentifier && !study.events?.find( equalClosure ) ) {
+                                                                        study.addToEvents( entity );
+                                                                }
+                                                                break;
+                                                        case SamplingEvent:
+                                                                if( preferredIdentifier && !study.samplingEvents?.find( equalClosure ) ) {
+                                                                        study.addToSamplingEvents( entity );
+                                                                }
+                                                                break;
+                                                }
+                                                
+                                                entity.save();
+                                                
+                                        } else {
+                                                numInvalidEntities++;
+                                        }
+                                }
+                        }
+                }
+
+                return numInvalidEntities == 0;
+        }
+                
+        /**
+         * Validates an object and puts human readable errors in validationErrors variable
+         * @param entity                Entity to validate
+         * @return                      True iff the entity validates, false otherwise
+         */
+        protected boolean validateObject( def entity ) {
+                if( !entity.validate() ) {
+                        flash.validationErrors = getHumanReadableErrors( entity )
+                        return false;
+                }
+                return true;
+        }
+
         /**
          * Adds all fields of this entity that have given an error when validating to the failedcells list
@@ -667 +1139 @@
                 entity.getErrors().getFieldErrors().each { error ->
                         String field = error.getField();
+                        
                         def mc = importerService.findMappingColumn( session.simpleWizard.excel.data.header, field );
                         def mcInstance = new MappingColumn( name: field, entityClass: Sample.class, index: -1, property: field.toLowerCase(), templateFieldType: entity.giveFieldType( field ) );
@@ -681 +1154 @@
                 return failedcells
         }
-
-
 
 
@@ -698 +1169 @@
                 if( sheetIndex > workbook.getNumberOfSheets() ) {
                         log.error ".simple study wizard Sheet index is too high: " + sheetIndex + " / " + workbook.getNumberOfSheets();
-                        flash.error = "The sheet number you provided is too high. The provided excel sheet has only " + workbook.getNumberOfSheets() + " sheet(s).";
+                        flash.error = "Your excel sheet contains too few excel sheets. The provided excel sheet has only " + workbook.getNumberOfSheets() + " sheet(s).";
                         return false
                 }
@@ -709 +1180 @@
                 if( headerRow > numRows  ) {
                         log.error ".simple study wizard Header row number is incorrect: " + headerRow + " / " + numRows;
-                        flash.error = "The header row number you provided is too high. Please provide a number equal to or below " + numRows;
+                        flash.error = "Your excel sheet doesn't contain enough rows (" + numRows + "). Please provide an excel sheet with one header row and data below";
                         return false
                 }
@@ -715 +1186 @@
                 if( dataMatrixStart > numRows  ) {
                         log.error ".simple study wizard Data row number is incorrect: " + dataMatrixStart + " / " + numRows;
-                        flash.error = "The data row number you provided is too high. Please provide a number equal to or below " + numRows;
+                        flash.error = "Your excel sheet doesn't contain enough rows (" + numRows + "). Please provide an excel sheet with one header row and data below";
                         return false
                 }
@@ -756 +1227 @@
                 Study s = Study.get( id );
 
+                if( !s ) {
+                        flash.error = "No study found with given id";
+                        return null;
+                }
                 if( !s.canWrite( authenticationService.getLoggedInUser() ) ) {
                         flash.error = "No authorization to edit this study."
@@ -764 +1239 @@
         }
 
+        /**
+         * Attach a template to the session
+         * @param t
+         * @return
+         */
+        protected attachTemplate( Template t ) {
+                if( t && !t.isAttached() ) {
+                        t.attach();
+
+                        t.fields.each { field ->
+                                if( field && !field.isAttached() )
+                                        field.attach();
+        
+                                field.listEntries?.each { entry ->
+                                        if( entry && !entry.isAttached() )
+                                                entry.attach();
+                                }
+                                field.ontologies?.each { entry ->
+                                        if( entry && !entry.isAttached() )
+                                                entry.attach();
+                                }
+                        }
+                }
+        }
+        
         /**
          * Retrieves the study that is saved in the wizard, 
@@ -773 +1273 @@
                 if( params.wizard && session.simpleWizard && session.simpleWizard.study ) {
                         // The user came here by clicking previous or a link on another page. Use the existing study
-                        return session.simpleWizard.study;
+                        Study s = session.simpleWizard.study;
+
+                        if( s.id && !s.isAttached() ) {
+                                s.attach();
+                        }
+                        
+                        s.samples?.each {
+                                if( it && it.id && !it.isAttached() )
+                                        it.attach();
+                                         
+                                attachTemplate( it.template )
+                        }
+                        s.subjects?.each { 
+                                if( it && it.id && !it.isAttached() )
+                                        it.attach();
+                                         
+                                attachTemplate( it.template )
+                        }
+                        s.events?.each { 
+                                if( it && it.id && !it.isAttached() )
+                                        it.attach();
+                                         
+                                attachTemplate( it.template )
+                        }
+
+                        s.samplingEvents?.each {
+                                if( it && it.id && !it.isAttached() )
+                                        it.attach();
+                                         
+                                attachTemplate( it.template )
+                        }
+                        s.assays?.each { 
+                                if( it && it.id && !it.isAttached() )
+                                        it.attach();
+                                         
+                                attachTemplate( it.template )
+                        }
+
+                        return s;
                 } else {
                         // The user didn't get here from the wizard or no study is found
+                        return null;
+                }
+        }
+
+        /**
+         * Retrieves the assay to edit in the wizard
+         * @param s             Study that the assay will be in
+         * @return              Assay object (may be empty)
+         */
+        protected Assay getAssayInWizard( Study study = null ) {
+                if( session.simpleWizard && session.simpleWizard.assay ) {
+                        Assay a = session.simpleWizard.assay;
+
+                        if( a.id && !a.isAttached() ) {
+                                a.attach();
+
+                                attachTemplate( a.template );
+                        }
+
+                        return a;
+                } else if( study ) {
+                        // The user came on the assay page for the first time
+                        if( study.assays?.size() ) {
+                                def assay = study.assays[0];
+
+                                return assay;
+                        } else {
+                                return new Assay( parent: study );
+                        }
+                } else {
                         return null;
                 }
@@ -802 +1370 @@
                 return errors
         }
-
 }

trunk/grails-app/services/dbnp/importer/ImporterService.groovy

@@ -15 +15 @@
  */
 package dbnp.importer
+
 import org.dbnp.gdt.*
 import org.apache.poi.ss.usermodel.*
@@ -186 +187 @@
         }
 
+        
+        /**
+         * Retrieves records with sample, subject, samplingevent etc. from a study
+         * @param s             Study to retrieve records from
+         * @return              A list with hashmaps [ 'objects': [ 'Sample': .., 'Subject': .., 'SamplingEvent': .., 'Event': '.. ], 'templates': [], 'templateCombination': .. ]
+         */
+        protected def getRecords( Study s ) {
+                def records = [];
+                
+                s.samples?.each {
+                        def record = [ 'objects': retrieveEntitiesBySample( it ) ];
+                
+                        def templates = [:]
+                        def templateCombination = [];
+                        record.objects.each { entity ->
+                                templates[ entity.key ] = entity.value?.template
+                                if( entity.value?.template )
+                                        templateCombination << entity.key + ": " + entity.value?.template?.name;
+                        }
+                        
+                        record.templates = templates;
+                        record.templateCombination = templateCombination.join( ', ' )
+                        
+                        records << record
+                }
+                
+                return records;
+        }
+        
+        /**
+         * Returns a subject, event and samplingEvent that belong to this sample
+         * @param s             Sample to find the information for
+         * @return
+         */
+        protected retrieveEntitiesBySample( Sample s ) {
+                return [
+                        'Sample': s,
+                        'Subject': s?.parentSubject,
+                        'SamplingEvent': s?.parentEvent,
+                        'Event': s?.parentEventGroup?.events?.getAt(0)
+                ]
+        }
+        
+        /**
+         * Imports data from a workbook into a list of ImportRecords. If some entities are already in the database,
+         * these records are updated.
+         * 
+         * This method is capable of importing Subject, Samples, SamplingEvents and Events
+         * 
+         * @param       templates       Map of templates, identified by their entity as a key. For example: [ Subject: Template x, Sample: Template y ]
+         * @param       wb                      Excel workbook to import
+         * @param       sheetindex      Number of the sheet to import data from
+         * @param       rowindex        Row to start importing from.
+         * @param       mcmap           Hashmap of mappingcolumns, with the first entry in the hashmap containing information about the first column, etc.
+         * @param       parent          Study to import all data into. Is used for determining which sample/event/subject/assay to update
+         * @param       createAllEntities       If set to true, the system will also create objects for entities that have no data imported, but do have
+         *                                                              a template assigned
+         * @return      List            List with two entries:
+         *                      0                       List with ImportRecords, one for each row in the excelsheet
+         *                      1                       List with ImportCell objects, mentioning the cells that could not be correctly imported
+         *                                              (because the value in the excelsheet can't be entered into the template field)
+         */
+        def importOrUpdateDataBySampleIdentifier( def templates, Workbook wb, int sheetindex, int rowindex, def mcmap, Study parent = null, boolean createAllEntities = true ) {
+                if( !mcmap )
+                        return;
+                        
+                // Check whether the rows should be imported in one or more entities
+                def entities
+                if( createAllEntities ) {
+                        entities = templates.entrySet().value.findAll { it }.entity;
+                } else {
+                        entities = mcmap.findAll{ !it.dontimport }.entityclass.unique();
+                }
+                
+                def sheet = wb.getSheetAt(sheetindex)
+                def table = []
+                def failedcells = [] // list of cells that have failed to import
+
+                // First check for each record whether an entity in the database should be updated,
+                // or a new entity should be added. This is done before any new object is created, since
+                // searching after new objects have been created (but not yet saved) will result in
+                //      org.hibernate.AssertionFailure: collection [...] was not processed by flush()
+                // errors
+                def existingEntities = [:]
+                for( int i = rowindex; i <= sheet.getLastRowNum(); i++ ) {
+                        existingEntities[i] = findExistingEntities( entities, sheet.getRow(i), mcmap, parent );
+                }
+                                
+                // walk through all rows and fill the table with records
+                for( int i = rowindex; i <= sheet.getLastRowNum(); i++ ) {
+                        // Create an entity record based on a row read from Excel and store the cells which failed to be mapped
+                        def (record, failed) = importOrUpdateRecord( templates, entities, sheet.getRow(i), mcmap, parent, table, existingEntities[i] );
+                        
+                        // Setup the relationships between the imported entities
+                        relateEntities( record );
+                        
+                        // Add record with entities and its values to the table
+                        table.add(record)
+
+                        // If failed cells have been found, add them to the failed cells list
+                        if (failed?.importcells?.size() > 0) failedcells.add(failed)
+                }
+                
+                return [ "table": table, "failedCells": failedcells ]
+        }
+
+        /**
+        * Checks whether entities in the given row already exist in the database
+        * they are updated.
+        *
+        * @param        entities        Entities that have to be imported for this row
+        * @param        excelRow        Excel row to import into this record
+        * @param        mcmap           Hashmap of mappingcolumns, with the first entry in the hashmap containing information about the first column, etc.
+        * @return       Map                     Map with entities that have been found for this row. The key for the entities is the entity name (e.g.: [Sample: null, Subject: <subject object>]
+        */
+   def findExistingEntities(def entities, Row excelRow, mcmap, parent ) {
+           DataFormatter df = new DataFormatter();
+
+           // Find entities based on sample identifier
+           def sample = findEntityByRow( dbnp.studycapturing.Sample, excelRow, mcmap, parent, [], df );
+           return retrieveEntitiesBySample( sample );
+   }
+   
+        /**
+         * Imports a records from the excelsheet into the database. If the entities are already in the database
+         * they are updated.
+         * 
+         * This method is capable of importing Subject, Samples, SamplingEvents and Events
+         * 
+         * @param       templates       Map of templates, identified by their entity as a key. For example: [ Sample: Template y ]
+         * @param       entities        Entities that have to be imported for this row
+         * @param       excelRow        Excel row to import into this record
+         * @param       mcmap           Hashmap of mappingcolumns, with the first entry in the hashmap containing information about the first column, etc.
+         * @param       parent          Study to import all data into. Is used for determining which sample/event/subject/assay to update
+         * @param       importedRows    Rows that have been imported before this row. These rows might contain the same entities as are
+         *                                                      imported in this row. These entities should be used again, to avoid importing duplicates.
+         * @return      List            List with two entries:
+         *                      0                       List with ImportRecords, one for each row in the excelsheet
+         *                      1                       List with ImportCell objects, mentioning the cells that could not be correctly imported
+         *                                              (because the value in the excelsheet can't be entered into the template field)
+         */
+        def importOrUpdateRecord(def templates, def entities, Row excelRow, mcmap, Study parent = null, List importedRows, Map existingEntities ) {
+                DataFormatter df = new DataFormatter();
+                def record = [] // list of entities and the read values
+                def failed = new ImportRecord() // map with entity identifier and failed mappingcolumn
+        
+                // Check whether this record mentions a sample that has been imported before. In that case,
+                // we update that record, in order to prevent importing the same sample multiple times
+                def importedEntities = [];
+                if( importedRows )
+                        importedEntities = importedRows.flatten().findAll { it.class == dbnp.studycapturing.Sample }.unique();
+
+                def importedSample = findEntityInImportedEntities( dbnp.studycapturing.Sample, excelRow, mcmap, importedEntities, df )
+                def imported = retrieveEntitiesBySample( importedSample );
+                
+                for( entity in entities ) {
+                        // Check whether this entity should be added or updated
+                        // The entity is updated is an entity with the same 'identifier' (field
+                        // specified to be the identifying field) is found in the database
+                        def entityName = entity.name[ entity.name.lastIndexOf( '.' ) + 1..-1];
+                        def template = templates[ entityName ];
+                        
+                        // If no template is specified for this entity, continue with the next
+                        if( !template )
+                                continue;
+                        
+                        // Check whether the object exists in the list of already imported entities
+                        def entityObject = imported[ entityName ]
+                        
+                        // If it doesn't, search for the entity in the database
+                        if( !entityObject && existingEntities )
+                                entityObject = existingEntities[ entityName ];
+                        
+                        // Otherwise, create a new object
+                        if( !entityObject )
+                                entityObject = entity.newInstance();
+                        
+                        // Update the template
+                        entityObject.template = template;
+                        
+                        // Go through the Excel row cell by cell
+                        for (Cell cell: excelRow) {
+                                // get the MappingColumn information of the current cell
+                                def mc = mcmap[cell.getColumnIndex()]
+                                def value
+         
+                                // Check if column must be imported
+                                if (mc != null && !mc.dontimport && mc.entityclass == entity) {
+                                        try {
+                                                value = formatValue(df.formatCellValue(cell), mc.templatefieldtype)
+                                        } catch (NumberFormatException nfe) {
+                                                value = ""
+                                        }
+         
+                                        try {
+                                                entityObject.setFieldValue(mc.property, value)
+                                        } catch (Exception iae) {
+                                                log.error ".import wizard error could not set property `" + mc.property + "` to value `" + value + "`"
+
+                                                // store the mapping column and value which failed
+                                                def identifier = entityName.toLowerCase() + "_" + entityObject.getIdentifier() + "_" + mc.property
+         
+                                                def mcInstance = new MappingColumn()
+                                                mcInstance.properties = mc.properties
+                                                failed.addToImportcells(new ImportCell(mappingcolumn: mcInstance, value: value, entityidentifier: identifier))
+                                        }
+                                } // end if
+                        } // end for
+                        
+                        // If a Study is entered, use it as a 'parent' for other entities
+                        if( entity == Study )
+                                parent = entityObject;
+                        
+                        record << entityObject;
+                }
+                
+                // a failed column means that using the entity.setFieldValue() threw an exception
+                return [record, failed]
+        }
+        
+        /**
+         * Looks into the database to find an object of the given entity that should be updated, given the excel row.
+         * This is done by looking at the 'preferredIdentifier' field of the object. If it exists in the row, and the
+         * value is already in the database for that field, an existing object is returned. Otherwise, null is returned
+         * 
+         * @param       entity          Entity to search
+         * @param       excelRow        Excelrow to search for
+         * @param       mcmap           Map with MappingColumns
+         * @param       parent          Parent study for the entity (if applicable). The returned entity will also have this parent
+         * @param       importedRows    List of entities that have been imported before. The function will first look through this list to find
+         *                                                      a matching entity.
+         * @return      An entity that has the same identifier as entered in the excelRow. The entity is first sought in the importedRows. If it
+         *                      is not found there, the database is queried. If no entity is found at all, null is returned.
+         */
+        def findEntityByRow( Class entity, Row excelRow, def mcmap, Study parent = null, List importedEntities = [], DataFormatter df = null ) {
+                if( df == null )
+                        df = new DataFormatter();
+                        
+                def identifierField = givePreferredIdentifier( entity );
+                
+                if( identifierField ) {
+                        // Check whether the identifierField is chosen in the column matching
+                        def identifierColumn = mcmap.find { it.entityclass == entity && it.property == identifierField.name };
+                        
+                        // If it is, find the identifier and look it up in the database
+                        if( identifierColumn ) {
+                                def identifierCell = excelRow.getCell( identifierColumn.index );
+                                def identifier;
+                                try {
+                                        identifier = formatValue(df.formatCellValue(identifierCell), identifierColumn.templatefieldtype)
+                                } catch (NumberFormatException nfe) {
+                                        identifier = null
+                                }
+                                
+                                // Search for an existing object with the same identifier.
+                                if( identifier ) {
+                                        // First search the already imported rows
+                                        if( importedEntities ) {
+                                                def imported = importedEntities.find { it.getFieldValue( identifierField.name ) == identifier };
+                                                if( imported )
+                                                        return imported;
+                                        }
+                                        
+                                        def c = entity.createCriteria();
+                                        
+                                        // If the entity has a field 'parent', the search should be limited to
+                                        // objects with the same parent. The method entity.hasProperty( "parent" ) doesn't
+                                        // work, since the java.lang.Class entity doesn't know of the parent property.
+                                        if( entity.belongsTo?.containsKey( "parent" ) ) {
+                                                // If the entity requires a parent, but none is given, no
+                                                // results are given from the database. This prevents the user
+                                                // of changing data in another study 
+                                                if( parent && parent.id ) {
+                                                        println "Searching (with parent ) for " + entity.name + " with " + identifierField.name + " = " + identifier
+                                                        return c.get {
+                                                                eq( identifierField.name, identifier )
+                                                                eq( "parent", parent )
+                                                        }
+                                                }
+                                        } else  {
+                                                println "Searching (without parent ) for " + entity.name + " with " + identifierField.name + " = " + identifier
+                                                return c.get {
+                                                        eq( identifierField.name, identifier )
+                                                }
+                                        }
+                                }
+                        }
+                }
+                
+                // No object is found
+                return null;
+        }
+        
+        /**
+        * Looks into the list of already imported entities to find an object of the given entity that should be 
+        * updated, given the excel row. This is done by looking at the 'preferredIdentifier' field of the object. 
+        * If it exists in the row, and the list of imported entities contains an object with the same
+        * identifier, the existing object is returned. Otherwise, null is returned
+        *
+        * @param        entity          Entity to search
+        * @param        excelRow        Excelrow to search for
+        * @param        mcmap           Map with MappingColumns
+        * @param        importedRows    List of entities that have been imported before. The function will first look through this list to find
+        *                                                       a matching entity.
+        * @return       An entity that has the same identifier as entered in the excelRow. The entity is first sought in the importedRows. If it
+        *                       is not found there, the database is queried. If no entity is found at all, null is returned.
+        */
+   def findEntityInImportedEntities( Class entity, Row excelRow, def mcmap, List importedEntities = [], DataFormatter df = null ) {
+           if( df == null )
+                   df = new DataFormatter();
+                   
+           def allFields = entity.giveDomainFields();
+           def identifierField = allFields.find { it.preferredIdentifier }
+           
+           if( identifierField ) {
+                   // Check whether the identifierField is chosen in the column matching
+                   def identifierColumn = mcmap.find { it.entityclass == entity && it.property == identifierField.name };
+                   
+                   // If it is, find the identifier and look it up in the database
+                   if( identifierColumn ) {
+                           def identifierCell = excelRow.getCell( identifierColumn.index );
+                           def identifier;
+                           try {
+                                   identifier = formatValue(df.formatCellValue(identifierCell), identifierColumn.templatefieldtype)
+                           } catch (NumberFormatException nfe) {
+                                   identifier = null
+                           }
+                           
+                           // Search for an existing object with the same identifier.
+                           if( identifier ) {
+                                        // First search the already imported rows
+                                        if( importedEntities ) {
+                                                def imported = importedEntities.find {
+                                                        def fieldValue = it.getFieldValue( identifierField.name )
+
+                                                        if( fieldValue instanceof String )
+                                                                return fieldValue.toLowerCase() == identifier.toLowerCase();
+                                                        else
+                                                                return fieldValue == identifier
+
+                                                };
+                                           if( imported )
+                                                   return imported;
+                                   }
+                           }
+                   }
+           }
+           
+           // No object is found
+           return null;
+   }
+
+        
+        /**
+         * Creates relation between multiple entities that have been imported. The entities are
+         * all created from one row in the excel sheet.
+         */
+        def relateEntities( List entities) {
+                def study = entities.find { it instanceof Study }
+                def subject = entities.find { it instanceof Subject }
+                def sample = entities.find { it instanceof Sample }
+                def event = entities.find { it instanceof Event }
+                def samplingEvent = entities.find { it instanceof SamplingEvent }
+                def assay = entities.find { it instanceof Assay }
+                
+                // A study object is found in the entity list
+                if( study ) {
+                        if( subject ) {
+                                subject.parent = study;
+                                study.addToSubjects( subject );
+                        }
+                        if( sample ) {
+                                sample.parent = study
+                                study.addToSamples( sample );
+                        }
+                        if( event ) {
+                                event.parent = study 
+                                study.addToEvents( event );
+                        }
+                        if( samplingEvent ) {
+                                samplingEvent.parent = study
+                                study.addToSamplingEvents( samplingEvent );
+                        }
+                        if( assay ) {
+                                assay.parent = study;
+                                study.addToAssays( assay );
+                        }
+                }
+
+                if( sample ) {
+                        if( subject ) sample.parentSubject = subject
+                        if( samplingEvent ) sample.parentEvent = samplingEvent;
+                        if( event ) {
+                                def evGroup = new EventGroup();
+                                evGroup.addToEvents( event );
+                                if( subject ) evGroup.addToSubjects( subject );
+                                if( samplingEvent ) evGroup.addToSamplingEvents( samplingEvent );
+                                
+                                sample.parentEventGroup = evGroup;
+                        }
+                        
+                        if( assay ) assay.addToSamples( sample );
+                }
+        }
+
         /**
          * Method to read data from a workbook and to import data into a two dimensional
@@ -258 +664 @@
          */
         def getFieldNameInTableEditor(entity, field) {
+                def entityName = entity?.class.name[ entity?.class.name.lastIndexOf(".") + 1..-1]
+                
                 if( field instanceof TemplateField )
                         field = field.escapedName();
 
-                return "entity_" + entity.getIdentifier() + "_" + field
+                return entityName.toLowerCase() + "_" + entity.getIdentifier() + "_" + field
         }
 
@@ -562 +970 @@
                 }
         }
+        
+        /**
+         * Returns the preferred identifier field for a given entity or 
+         * null if no preferred identifier is given
+         * @param entity        TemplateEntity class
+         * @return      The preferred identifier field or NULL if no preferred identifier is given
+         */
+        public TemplateField givePreferredIdentifier( Class entity ) {
+                def allFields = entity.giveDomainFields();
+                return allFields.find { it.preferredIdentifier }
+        }
 
         // classes for fuzzy string matching

trunk/grails-app/views/simpleWizard/assays.gsp

@@ -38 +38 @@
                         <input type="hidden" name="wizard" value="true" />
                         <input type="hidden" name="event" value="refresh" />
-                
                         <af:templateElement name="template" description="Template"
-                                value="${assay?.template}" entity="${dbnp.studycapturing.Assay}"
+                                value="${wizardAssay?.template}" entity="${dbnp.studycapturing.Assay}"
                                 addDummy="true" onChange="if(\$( this ).val() != '') { submitForm( 'assays' ); }">
                                 Choose the type of assay you would like to perform.
@@ -46 +45 @@
                         </af:templateElement>
                 
-                        <g:if test="${assay}">
-                                <af:templateElements ignore="externalassayid" entity="${assay}" />
+                        <g:if test="${wizardAssay}">
+                                <af:templateElements ignore="externalassayid" entity="${wizardAssay}" />
                         </g:if>
                 

trunk/grails-app/views/simpleWizard/columns.gsp

@@ -43 +43 @@
                                         <tr class="matchWith">
                                                 <g:each in="${excel.data.header}" var="mappingcolumn" status="i">
+                                                        <%
+                                                                def selectedValue;
+                                                                if( mappingcolumn.entityclass?.name && mappingcolumn.property )
+                                                                        selectedValue = mappingcolumn.entityclass.name[ mappingcolumn.entityclass.name.lastIndexOf( "." ) + 1 .. -1 ] + mappingcolumn.property;
+                                                        %>
                                                         <td>
                                                                 <g:set var="selected" value="${mappingcolumn.property}"/>
-                                                                <importer:propertyChooser name="matches" mappingcolumn="${mappingcolumn}" matchvalue="${mappingcolumn.name}" 
-                                                                        selected="${selected}" fuzzymatching="true" treshold="0.8" template_id="${template.id}" "allfieldtypes="true"/>
+                                                                <% /* Put a select box with template fields of multiple entities */ %>
+                                                                <select name="matches.index.${mappingcolumn.index}" style="font-size: 10px;">
+                                                                        <option value="dontimport">Don't import</option>
+                                                                        <g:each in="${templates}" var="entityTemplates">
+                                                                                <g:if test="${entityTemplates.value}">
+                                                                                        <optgroup label="${entityTemplates.key}">
+                                                                                                <%
+                                                                                                        def allFields = domainFields[ entityTemplates.key ] + entityTemplates.value?.fields;
+                                                                                                %>
+                                                                                                <g:each in="${allFields}" var="field">
+                                                                                                        <% 
+                                                                                                                def value = entityTemplates.key + "||" + field.name
+                                                                                                                def selected = ( value == selectedValue );
+                                                                                                        %>
+                                                                                                        <option value="${value}" <g:if test="${selected}">selected="selected"</g:if>>
+                                                                                                                ${field.name} <g:if test="${field.preferredIdentifier}">[identifier]</g:if>
+                                                                                                        
+                                                                                                </g:each>
+                                                                                        </optgroup>
+                                                                                </g:if>
+                                                                        </g:each>
+                                                                </select>
                                                         </td>
                                                 </g:each>

trunk/grails-app/views/simpleWizard/missingFields.gsp

@@ -47 +47 @@
                         
                         <div class="wizard" id="wizard">
-                            <div class="tableEditor">
-                                <g:set var="showHeader" value="${true}"/>
-                                    <g:each status="index" var="table" in="${imported.data}">
-                                                <g:each status="i" var="entity" in="${table}">
-                                                    <g:if test="${showHeader}">
-                                                                <g:set var="showHeader" value="${false}"/>
-                                                                <div class="header">
-                                                                        <div class="firstColumn"></div>
-                                                                        <af:templateColumnHeaders entity="${entity}" class="column" />
-                                                                </div>
-                                                                <input type="hidden" name="entity" value="${entity.getClass().getName()}">
-                                                    </g:if>
-                                                    <div class="row">
+                                <g:set var="showHeader" value="${true}" />
+                                <g:set var="previousTemplate" value=""/>                        
+                                <div class="tableEditor">
+                                        <g:each var="record" in="${records}">
+                                                <g:if test="${showHeader}">
+                                                        <g:set var="showHeader" value="${false}" />
+                                                        <div class="header">
                                                                 <div class="firstColumn"></div>
-                                                                <af:templateColumns id="${entity.hashCode()}" entity="${entity}" template="${entity.template}" name="entity_${entity.getIdentifier()}" class="column" subject="${entity.hashCode()}" addDummy="true" />
-                                                    </div>
-                                                </g:each>
-                                    </g:each>
-                            </div>
-                            <div class="sliderContainer">
-                              <div class="slider"></div>
-                            </div>
+                                                                <g:each var="entity" in="${record}">
+                                                                        <g:if test="${entity.value}">
+                                                                                <af:templateColumnHeaders entity="${entity.value}" class="column" columnWidths="[Name:100]"/>
+                                                                        </g:if>
+                                                                </g:each>
+                                                        </div>
+                                                </g:if>
+                                                <div class="row">
+                                                        <div class="firstColumn"></div>
+                                                        <g:each var="entity" in="${record}">
+                                                                <g:if test="${entity.value}">
+                                                                        <af:templateColumns  id="${entity.value.hashCode()}" name="${entity.key.toLowerCase()}_${entity.value.getIdentifier()}" template="${entity.value.template}" class="column" id="1" entity="${entity.value}" addDummy="true" subject="${entity.hashCode()}" />
+                                                                </g:if>
+                                                        </g:each>
+                                                </div>
+                                        </g:each>
+                                </div>
+                                <div class="sliderContainer">
+                                        <div class="slider" ></div>
+                                </div>
                     <div>
                 </g:form>

trunk/grails-app/views/simpleWizard/overview.gsp

@@ -89 +89 @@
                 
                 <div class="element">
-                        <div class="description">Sample template</div>${study.samples?.getAt(0)?.template?.name}
+                        <div class="description">Samples</div>
+                        
+                        <g:if test="${study.samples?.size()}">
+                                <g:each in="${study.samples.template.unique()}" var="currentSampleTemplate" status="j">
+                                        <g:if test="${currentSampleTemplate}">
+                                                <g:if test="${j > 0}">,</g:if>
+                                                <%=study.samples.findAll { return it.template == currentSampleTemplate; }.size()%>
+                                                ${currentSampleTemplate.name}
+                                        </g:if>
+                                </g:each>
+                        </g:if>
+                        <g:else>
+                                -
+                        </g:else>
                 </div>
-                <div class="element">
-                        <div class="description"># samples</div>${study.samples?.size()}
-                </div>
-                <div class="element">
-                        <div class="description">Assay template</div>${study.assays?.getAt(0)?.template?.name}
-                </div>
+                
+                <br />
+                <g:if test="${wizardAssay}">
+                        <div class="element">
+                                <div class="description">Assay template</div>${wizardAssay.template?.name}
+                        </div>
+                </g:if>
 
                 <br clear="all" />

trunk/grails-app/views/simpleWizard/samples.gsp

@@ -27 +27 @@
                         <input type="hidden" name="wizard" value="true" />
                         <input type="hidden" name="event" value="refresh" />
-                
-                        <g:if test="${study.samples?.size()}">
-                                <p>Current samples</p>
-                                <g:each in="${study.samples}" var="sample">
-                                        ${sample.name}<br />
-                                </g:each>
-                                
-                                <p class="options">
-                                        <a href="#" onClick="$('#samplesDialog').dialog('open'); return false;" class="add">Add samples</a>
-                                        <a href="#" onClick="$('#samplesDialog').dialog('open'); return false;" class="update">Update samples</a>
-                                </p>
-                                
-                                <% /* If samples are already present, the dialog should not be opened by default */ %>
-                                <script type="text/javascript">
-                                        $( '#samplesDialog' ).dialog({
-                                                title   : "Add/update samples",
-                                                autoOpen: false,
-                                                width   : 800,
-                                                height  : 400,
-                                                modal   : true,
-                                                position: "center",
-                                                buttons : {
-                                                        Add  : function() {
-                                                                //addUser(element_id);
-                                                                $(this).dialog("close");
-                                                        },
-                                                        Update  : function() {
-                                                                //addUser(element_id);
-                                                                $(this).dialog("close");
-                                                        },
-                                                        Close  : function() {
-                                                                $(this).dialog("close");
-                                                        }
-                                                },
-                                                close   : function() {
-                                                        /* closeFunc(this); */
-                                                }
-                                        });             
-                                </script>
-                        </g:if>
+
                         <div id="samplesDialog">
                         <span class="info"> 
                                         <span class="title">Import sample data</span> 
-                                        You can import your Excel data to the server by choosing a file from your local harddisk in the form below.
+                                        You can import your Excel data to the server by choosing a file from your local harddisk in the form below. The excel sheet should contain
+                                        data on the first sheet, and the sheet should contain one row with headers.
                                 </span> 
                     
@@ -83 +45 @@
                                         </tr>
                                         <tr>
-                                            <td width="100px">
-                                                Use data from sheet:
-                                            </td>
-                                            <td width="100px">
-                                                <g:select name="sheetindex" from="${1..25}" value="${sampleForm?.sheetIndex}"/>
-                                            </td>
-                                        </tr>
-                                        <tr>
-                                            <td width="100px">
-                                                Columnheader starts at row:
-                                            </td>
-                                            <td width="100px">
-                                                <g:select name="headerrow" from="${1..10}" value="${sampleForm?.headerRow}"/>
-                                            </td>
-                                        </tr>
-                                        <tr>
-                                            <td width="100px">
-                                                Data starts at row:
-                                            </td>
-                                            <td width="100px">
-                                                <g:select name="datamatrix_start" from="${2..10}" value="${sampleForm?.dataMatrixStart}"/>
-                                            </td>
-                                        </tr>   
-                                        <tr>
                                             <td>
                                                 <div id="datatemplate">Choose type of sample template:</div>
                                             </td>
                                             <td>
-                                                        <g:select rel="template" entity="${encodedEntity}" name="template_id" optionKey="id" optionValue="name" from="${sampleTemplates}" value="${sampleForm?.templateId}"/>
+                                                        <g:select rel="template" entity="${encodedEntity.Sample}" name="sample_template_id" optionKey="id" optionValue="name" from="${templates.Sample}"/>
                                             </td>
                                         </tr>
                                 </table>        
-                        
                         </div>
                 

trunk/grails-app/views/simpleWizard/save.gsp

@@ -22 +22 @@
                 </g:if>         
 
-                ${study}
-                
-                <g:form class="simpleWizard" name="saved" action="saved" controller="simpleWizard">
+                <g:form class="simpleWizard" name="saved" action="save" controller="simpleWizard">
                         <input type="hidden" name="wizard" value="true" />
                         <input type="hidden" name="event" value="refresh" />
                 
                         <p class="options">
-                                <g:link class="view" controller="study" action="show" id="${study.id}">View study</g:link>
-                                <g:link class="edit" controller="simpleWizard" action="study" id="${study.id}">Edit study</g:link>
-                                <g:link class="restart" controller="simpleWizard" action="study">Add another study</g:link>
+                                <g:if test="${success}">
+                                        <g:link class="view" controller="study" action="show" id="${study.id}">View study</g:link>
+                                        <g:link class="edit" controller="simpleWizard" action="study" id="${study.id}">Edit study</g:link>
+                                        <g:link class="restart" controller="simpleWizard" action="study">Add another study</g:link>
+                                </g:if>
+                                <g:else>
+                                        <a href="#" onClick="submitForm( 'saved', 'previous' ); return false;" class="previous">Previous</a>
+                                </g:else>
                         </p>
                         

trunk/grails-app/views/studyWizard/pages/_samples.gsp

@@ -36 +36 @@
 
         <g:if test="${study.samples}">
-                <g:if test="${study.samples.findAll{!it.template}.size()}">
+                <g:if test="${study.samples.findAll{!it?.template}.size()}">
                 <h1>Samples that still need to have a template assigned</h1>
                 <div class="tableEditor">
@@ -48 +48 @@
                 <g:set var="previousTemplate" value=""/>
                 <g:each var="sample" in="${study.samples}">
-                        <g:if test="${!sample.template}">
+                        <g:if test="${!sample?.template}">
                                 <div class="row">
                                         <div class="firstColumn">&nbsp;</div>
                                         <div class="column">
-                                                <g:if test="${previousTemplate != sample.parentEvent.template.name}">
-                                                        <g:set var="previousTemplate" value="${sample.parentEvent.template.name}"/>
-                                                        ${sample.parentEvent.template.name}
+                                                <g:if test="${previousTemplate != sample.parentEvent?.template?.name}">
+                                                        <g:set var="previousTemplate" value="${sample.parentEvent?.template?.name}"/>
+                                                        ${sample.parentEvent?.template?.name}
                                                         <div class="helpIcon"></div>
                                                         <div class="helpContent">
-                                                                <h1>${sample.parentEvent.template.name}</h1>
+                                                                <h1>${sample.parentEvent?.template?.name}</h1>
                                                                 <h2>Template Fields:</h2>
-                                                                <g:each var="field" in="${sample.parentEvent.giveFields()}">
+                                                                <g:each var="field" in="${sample.parentEvent?.giveFields()}">
                                                                         ${field.name[0].toUpperCase() + field.name.substring(1)}<br/>
                                                                 </g:each>
@@ -66 +66 @@
                                         </div>
                                         <div class="column">
-                                                ${sample.parentSubject.name}
+                                                ${sample.parentSubject?.name}
                                                 <div class="helpIcon"></div>
                                                 <div class="helpContent">
-                                                        <h1>${sample.parentSubject.template.name}</h1>
+                                                        <h1>${sample.parentSubject?.template?.name}</h1>
                                                         <h2>Template Fields:</h2>
-                                                        <g:each var="field" in="${sample.parentSubject.giveFields()}">
+                                                        <g:each var="field" in="${sample.parentSubject?.giveFields()}">
                                                                 ${field.name[0].toUpperCase() + field.name.substring(1)}<br/>
                                                         </g:each>
@@ -109 +109 @@
                                         <div class="firstColumn"></div>
                                         <div class="column">
-                                                <g:if test="${previousTemplate != sample.parentEvent.template.name}">
-                                                        <g:set var="previousTemplate" value="${sample.parentEvent.template.name}"/>
-                                                        ${sample.parentEvent.template.name}
+                                                <g:if test="${previousTemplate != sample.parentEvent?.template?.name}">
+                                                        <g:set var="previousTemplate" value="${sample.parentEvent?.template?.name}"/>
+                                                        ${sample.parentEvent?.template?.name}
                                                         <div class="helpIcon"></div>
                                                         <div class="helpContent">
-                                                                <h1>${sample.parentEvent.template.name}</h1>
+                                                                <h1>${sample.parentEvent?.template?.name}</h1>
                                                                 <h2>Template Fields:</h2>
-                                                                <g:each var="field" in="${sample.parentEvent.giveFields()}">
+                                                                <g:each var="field" in="${sample.parentEvent?.giveFields()}">
                                                                         ${field.name[0].toUpperCase() + field.name.substring(1)}<br/>
                                                                 </g:each>
@@ -123 +123 @@
                                         </div>
                                         <div class="column">
-                                                ${sample.parentSubject.name}
+                                                ${sample.parentSubject?.name}
                                                 <div class="helpIcon"></div>
                                                 <div class="helpContent">
-                                                        <h1>${sample.parentSubject.template.name}</h1>
+                                                        <h1>${sample.parentSubject?.template?.name}</h1>
                                                         <h2>Template Fields:</h2>
-                                                        <g:each var="field" in="${sample.parentSubject.giveFields()}">
+                                                        <g:each var="field" in="${sample.parentSubject?.giveFields()}">
                                                                 ${field.name[0].toUpperCase() + field.name.substring(1)}<br/>
                                                         </g:each>
@@ -134 +134 @@
                                         </div>
                                         <div class="column">
-                                                <af:templateSelect name="template_${sample.getIdentifier()}" entity="${dbnp.studycapturing.Sample}" value="${sample.template}" addDummy="true" tableEditorChangeEvent="switchTemplate(element);" />
+                                                <af:templateSelect name="template_${sample.getIdentifier()}" entity="${dbnp.studycapturing.Sample}" value="${sample?.template}" addDummy="true" tableEditorChangeEvent="switchTemplate(element);" />
                                         </div>
                                         <af:templateColumns name="sample_${sample.getIdentifier()}" class="column" id="1" entity="${sample}"/>