Changeset 58

Timestamp:

May 17, 2011, 1:44:07 PM (12 years ago)

Author:

robert@…

Message:

Implemented importing of classifications

Location:

trunk

Files:

• application.properties (modified) (1 diff)
• grails-app/conf/DataSource.groovy (modified) (1 diff)
• grails-app/controllers/masssequencing (added)
• grails-app/controllers/masssequencing/SandboxController.groovy (added)
• grails-app/controllers/nl/tno/massSequencing/FastaController.groovy (modified) (1 diff)
• grails-app/controllers/nl/tno/massSequencing/files/ImportController.groovy (copied) (copied from trunk/grails-app/controllers/nl/tno/massSequencing/FastaController.groovy) (10 diffs)
• grails-app/domain/nl/tno/massSequencing/AssaySample.groovy (modified) (5 diffs)
• grails-app/domain/nl/tno/massSequencing/Sequence.groovy (added)
• grails-app/domain/nl/tno/massSequencing/Study.groovy (modified) (1 diff)
• grails-app/domain/nl/tno/massSequencing/classification (added)
• grails-app/domain/nl/tno/massSequencing/classification/Classification.groovy (added)
• grails-app/domain/nl/tno/massSequencing/classification/Taxon.groovy (added)
• grails-app/services/nl/tno/massSequencing/ClassificationService.groovy (added)
• grails-app/services/nl/tno/massSequencing/FastaService.groovy (modified) (11 diffs)
• grails-app/services/nl/tno/massSequencing/SampleExcelService.groovy (modified) (1 diff)
• grails-app/services/nl/tno/massSequencing/files/ExcelService.groovy (modified) (3 diffs)
• grails-app/services/nl/tno/massSequencing/files/FileService.groovy (modified) (5 diffs)
• grails-app/services/nl/tno/massSequencing/files/ZipService.groovy (added)
• grails-app/services/nl/tno/massSequencing/imports/ImportService.groovy (added)
• grails-app/services/nl/tno/massSequencing/imports/Importer.java (added)
• grails-app/views/assay/_addFilesDialog.gsp (modified) (1 diff)
• grails-app/views/import (moved) (moved from trunk/grails-app/views/fasta)
• grails-app/views/import/showProcessResult.gsp (modified) (3 diffs)
• grails-app/views/import/showProcessScreen.gsp (modified) (1 diff)
• grails-app/views/run/_addFilesDialog.gsp (modified) (1 diff)
• grails-app/views/sandbox (added)
• test/unit/masssequencing/SandboxControllerTests.groovy (added)
• test/unit/nl/tno/massSequencing/SequenceTests.groovy (added)
• test/unit/nl/tno/massSequencing/bacteria (added)
• test/unit/nl/tno/massSequencing/bacteria/ClassificationTests.groovy (added)
• test/unit/nl/tno/massSequencing/bacteria/TaxonTests.groovy (added)
• test/unit/nl/tno/massSequencing/imports (added)
• test/unit/nl/tno/massSequencing/imports/ImportServiceTests.groovy (added)

trunk/application.properties

@@ -1 +1 @@
 #Grails Metadata file
-#Fri Apr 15 14:24:03 CEST 2011
+#Wed May 11 12:10:59 CEST 2011
 app.build.display.info=0
+app.grails.version=1.3.7
+app.name=massSequencing
+app.servlet.version=2.4
 app.version=0.1
+plugins.famfamfam=1.0.1
+plugins.hibernate=1.3.7
+plugins.jquery=1.4.4.1
+plugins.jquery-ui=1.8.7
 plugins.tomcat=1.3.7
-plugins.jquery=1.4.4.1
-plugins.famfamfam=1.0.1
-plugins.jquery-ui=1.8.7
-app.servlet.version=2.4
-plugins.hibernate=1.3.7
-app.name=massSequencing
-app.grails.version=1.3.7

trunk/grails-app/conf/DataSource.groovy

@@ -7 +7 @@
 }
 hibernate {
+        jdbc.batch_size = 50
     cache.use_second_level_cache = true
     cache.use_query_cache = true

trunk/grails-app/controllers/nl/tno/massSequencing/FastaController.groovy

@@ -9 +9 @@
         def fastaService
         def sessionFactory
-        
-        /**************************************************************************
-         *
-         * Methods for handling uploaded sequence and quality files
-         *
-         *************************************************************************/
-
-        /** 
-         * Shows a screen that processing is done
-         */
-        def showProcessScreen = {
-                def entityType = params.entityType
-
-                // Check whether files are given
-                def names = params.sequencefiles
-
-                if( !names ) {
-                        flash.message = "No files uploaded for processing"
-                        if( params.entityType && params.id)
-                                redirect( controller: params.entityType, action: 'show', 'id': params.id)
-                        else
-                                redirect( url: "" )
-                                
-                        return
-                }
-                
-                // If only 1 file is uploaded, it is given as String
-                ArrayList filenames = []
-                if( names instanceof String )
-                        filenames << names
-                else
-                        names.each { filenames << it }
-                        
-                // Save filenames in session
-                session.processFilenames = names;
-                
-                // Check for total size of the files in order to be able
-                // to show a progress bar
-                long filesize = 0;
-                names.each {
-                        filesize += fileService.get( it )?.length()
-                }
-                session.processProgress = [
-                        numFiles: names.size(),
-                        numBytes: filesize,
-                        filesProcessed: 0,
-                        bytesProcessed: 0
-                ]
-                        
-                [entityId: params.id, entityType: params.entityType, filenames: names, url: createLink( action: 'showProcessResult', id: params.id, params: [entityType: entityType] ) ]
-        }
-        
-        /**
-         * Processes uploaded files and tries to combine them with samples
-         */
-        def process = {
-                def entity
-                def assaySamples
-                
-                switch( params.entityType ) {
-                        case "run":
-                                entity = getRun( params.id );
-                                assaySamples = entity.assaySamples.findAll { it.assay.study.canRead( session.user ) };
-                                break;
-                        case "assay":
-                                entity = getAssay( params.id );
-                                assaySamples = entity.assaySamples;
-                                break;
-                        default:
-                                response.setStatus( 404, "No controller found" );
-                                render "";
-                                return;
-                }
-
-                if (!entity) {
-                        response.setStatus( 404, flash.error )
-                        render "";
-                        return
-                }
-
-                // Check whether files are given
-                def names = session.processFilenames
-
-                if( !names ) {
-                        response.setStatus( 500, "No files uploaded for processing" )
-                        render "";
-                        return
-                }
-
-                // If only 1 file is uploaded, it is given as String
-                ArrayList filenames = []
-                if( names instanceof String )
-                        filenames << names
-                else
-                        names.each { filenames << it }
-
-                /* Parses uploaded files, discards files we can not handle
-                 * 
-                 * [
-                 *              success: [
-                 *                      [filename: 'abc.fasta', type: FASTA, numSequences: 190]
-                 *                      [filename: 'cde.fasta', type: FASTA, numSequences: 140]
-                 *                      [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38]
-                 *                      [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
-                 *              ],
-                 *              failure: [
-                 *                      [filename: 'testing.xls', message: 'Type not recognized']
-                 *              ]
-                 * ]
-                 * 
-                 * The second parameter is a callback function to update progress indicators
-                 */
-                def httpSession = session;
-                def parsedFiles = fastaService.parseFiles( filenames, { files, bytes, totalFiles, totalBytes -> 
-                        httpSession.processProgress.numFiles += totalFiles;
-                        httpSession.processProgress.numBytes += totalBytes;
-                        httpSession.processProgress.filesProcessed = files; 
-                        httpSession.processProgress.bytesProcessed = bytes; 
-                } );
-                
-                // Check which assaySamples to use (only the ones visible to the user)
-                assaySamples = assaySamples.findAll { it.assay.study.canWrite( session.user ) }
-
-                // Match files with samples in the database
-                def matchedFiles = fastaService.matchFiles( parsedFiles.success, assaySamples );
-
-                // Sort files on filename
-                matchedFiles.sort { a,b -> a.fasta?.originalfilename <=> b.fasta?.originalfilename }
-
-                // Saved file matches in session to use them later on
-                session.processedFiles = [ parsed: parsedFiles, matched: matchedFiles ];
-
-                render ""
-        }
-        
-        def getProgress = {
-                if( !session.processProgress ) {
-                        response.setStatus( 500, "No progress information found" );
-                        render ""
-                        return
-                }
-                
-                render session.processProgress as JSON
-        }
-        
-        /**
-         * Show result of processing
-         */
-        def showProcessResult = {
-                // load study with id specified by param.id
-                def entity
-                
-                switch( params.entityType ) {
-                        case "run":
-                                entity = getRun( params.id )
-                                break;
-                        case "assay":
-                                entity = getAssay( params.id )
-                                break;
-                        default:
-                                response.setStatus( 404, "No entity found" );
-                                render "";
-                                return;
-                }
-
-                if (!entity) {
-                        response.setStatus( 404, flash.error )
-                        render "";
-                        return
-                }
-                
-                if( !session.processedFiles ) {
-                        flash.error = "Processing of files failed. Maybe the session timed out."
-                        redirect( controller: 'assay', action: 'show', 'id': params.id)
-                        return
-                }
-                
-                [entityType: params.entityType, entity: entity, id: params.id, parsedFiles: session.processedFiles.parsed, matchedFiles: session.processedFiles.matched, selectedRun: params.selectedRun ]
-        }
-
-        /**
-         * Returns from the upload wizard without saving the data. The uploaded files are removed
-         */
-        def returnWithoutSaving = {
-                // Delete all uploaded files from disk
-                session.processedFiles?.parsed?.success?.each {
-                        fileService.delete( it.filename );
-                }
-
-                // Redirect to the correct controller           
-                switch( params.entityType ) {
-                        case "run":
-                        case "assay":
-                                redirect( controller: params.entityType, action: "show", id: params.id );
-                                return;
-                        default:
-                                response.setStatus( 404, "No entity found" );
-                                render "";
-                                return;
-                }
-                
-                
-        }
-        
-        /**
-         * Saves processed files to the database, based on the selections made by the user
-         */
-        def saveProcessedFiles = {
-                // load entity with id specified by param.id
-                def entity
-                
-                switch( params.entityType ) {
-                        case "run":
-                                entity = getRun( params.id );
-                                break;
-                        case "assay":
-                                entity = getAssay( params.id );
-                                break;
-                        default:
-                                response.setStatus( 404, "No entity found" );
-                                render "";
-                                return;
-                }
-
-                if (!entity) {
-                        response.setStatus( 404, flash.error )
-                        render "";
-                        return
-                }
-
-                // Check whether files are given
-                def files = params.file
-
-                if( !files ) {
-                        flash.message = "No files were selected."
-                        redirect( controller: params.entityType, action: 'show', 'id': params.id)
-                        return
-                }
-
-                File permanentDir = fileService.absolutePath( ConfigurationHolder.config.massSequencing.fileDir )
-                int numSuccesful = 0;
-                def errors = [];
-                
-                // Loop through all files Those are the numeric elements in the 'files' array
-                def digitRE = ~/^\d+$/;
-                files.findAll { it.key.matches( digitRE ) }.each { file ->
-                        def filevalue = file.value;
-                        
-                        // Check if the file is selected
-                        if( filevalue.include == "on" ) {
-                                if( fileService.fileExists( filevalue.fasta ) ) {
-                                        try {
-                                                def permanent = fastaService.savePermanent( filevalue.fasta, filevalue.qual, session.processedFiles );
-                                                
-                                                // Save the data into the database
-                                                SequenceData sd = new SequenceData();
-                                                
-                                                sd.sequenceFile = permanent.fasta
-                                                sd.qualityFile = permanent.qual
-                                                sd.numSequences = permanent.numSequences
-                                                sd.averageQuality = permanent.avgQuality
-                                                        
-                                                def sample = AssaySample.get( filevalue.assaySample );
-                                                if( sample )
-                                                        sample.addToSequenceData( sd );
-                                                
-                                                if( !sd.validate() ) {
-                                                        errors << "an error occurred while saving " + filevalue.fasta + ": validation of SequenceData failed.";
-                                                } else {
-                                                        sd.save(flush:true);
-                                                }
-                                                
-                                                numSuccesful++;
-                                        } catch( Exception e ) {
-                                                errors << "an error occurred while saving " + filevalue.fasta + ": " + e.getMessage()
-                                        }
-                                }
-                        } else {
-                                // File doesn't need to be included in the system. Delete it also from disk
-                                fileService.delete( filevalue.fasta );
-                        }
-                }
-
-                // Return all files that have not been moved
-                session.processedFiles?.parsed?.success?.each {
-                        fileService.delete( it.filename );
-                }
-                
-                // Return a message to the user
-                if( numSuccesful == 0 ) {
-                        flash.error = "None of the files were imported, because "
-                        
-                        if( errors.size() > 0 ) {
-                                errors.each {
-                                        flash.error += "<br />- " + it
-                                }
-                        } else {
-                                flash.error = "none of the files were selected for import."
-                        }
-                } else {
-                        flash.message = numSuccesful + " files have been added to the system. "
-
-                        if( errors.size() > 0 ) {
-                                flash.error += errors.size() + " errors occurred during import: "
-                                errors.each {
-                                        flash.error += "<br />- " + it
-                                }
-                        }
-                }
-                
-                redirect( controller: params.entityType, action: "show", id: params.id )
-        }
         
         def deleteData = { 

trunk/grails-app/controllers/nl/tno/massSequencing/files/ImportController.groovy

@@ -1 @@
-package nl.tno.massSequencing
+package nl.tno.massSequencing.files
 
 import org.codehaus.groovy.grails.commons.ConfigurationHolder
 import org.hibernate.SessionFactory
 import grails.converters.*;
-
-class FastaController {
+import nl.tno.massSequencing.*
+
+class ImportController {
         def fileService
         def fastaService
+        def importService
+        def classificationService
         def sessionFactory
         
         /**************************************************************************
          *
-         * Methods for handling uploaded sequence and quality files
+         * Methods for handling uploaded sequence, quality and classification files
          *
          *************************************************************************/
@@ -23 +26 @@
 
                 // Check whether files are given
-                def names = params.sequencefiles
+                def names = params.list( 'sequencefiles' )
 
                 if( !names ) {
@@ -34 +37 @@
                         return
                 }
-                
-                // If only 1 file is uploaded, it is given as String
-                ArrayList filenames = []
-                if( names instanceof String )
-                        filenames << names
-                else
-                        names.each { filenames << it }
                         
                 // Save filenames in session
@@ -51 +47 @@
                         filesize += fileService.get( it )?.length()
                 }
+
                 session.processProgress = [
-                        numFiles: names.size(),
-                        numBytes: filesize,
-                        filesProcessed: 0,
-                        bytesProcessed: 0
+                        stepNum: 1,
+                        numSteps: 2,
+                        stepDescription: 'Parsing files',       // Second step is Store classification
+                        
+                        stepProgress: 0,
+                        stepTotal: filesize
                 ]
                         
@@ -71 +70 @@
                         case "run":
                                 entity = getRun( params.id );
-                                assaySamples = entity.assaySamples.findAll { it.assay.study.canRead( session.user ) };
+                                assaySamples = entity.assaySamples.findAll { it.assay.study.canWrite( session.user ) };
                                 break;
                         case "assay":
                                 entity = getAssay( params.id );
-                                assaySamples = entity.assaySamples;
+                                assaySamples = entity.assaySamples.findAll { it.assay.study.canWrite( session.user ) };
                                 break;
                         default:
@@ -115 +114 @@
                  *              ],
                  *              failure: [
-                 *                      [filename: 'testing.xls', message: 'Type not recognized']
+                 *                      [filename: 'testing.doc', message: 'Type not recognized']
                  *              ]
                  * ]
@@ -122 +121 @@
                  */
                 def httpSession = session;
-                def parsedFiles = fastaService.parseFiles( filenames, { files, bytes, totalFiles, totalBytes -> 
-                        httpSession.processProgress.numFiles += totalFiles;
-                        httpSession.processProgress.numBytes += totalBytes;
-                        httpSession.processProgress.filesProcessed = files; 
-                        httpSession.processProgress.bytesProcessed = bytes; 
-                } );
-                
-                // Check which assaySamples to use (only the ones visible to the user)
-                assaySamples = assaySamples.findAll { it.assay.study.canWrite( session.user ) }
-
+                def onProgress = { progress, total ->
+                        // Update progress 
+                        httpSession.processProgress.stepTotal = total;
+                        httpSession.processProgress.stepProgress = progress; 
+                }
+                def newStep = { total, description ->
+                        // Start a new step 
+                        httpSession.processProgress.stepTotal = total;
+                        httpSession.processProgress.stepProgress = 0;
+                        
+                        httpSession.processProgress.stepDescription = description;
+                        httpSession.processProgress.stepNum++;
+                }
+
+                def parsedFiles = importService.parseFiles( filenames, onProgress, [progress: 0, total: httpSession.processProgress.stepTotal ], newStep );
+                
+                println "Parsed files success: " + parsedFiles.success
+                
+                // Determine excel matches from the uploaded files
+                parsedFiles.success = fastaService.inferExcelMatches( parsedFiles.success );
+                
+                // Now check whether a taxonomy and groups file are uploaded. If so send them to the classificationService for
+                // parsing
+                def classificationFiles = parsedFiles.success.findAll { it.type == 'groups' || it.type == 'taxonomy' };
+                def excelFiles = parsedFiles.success.findAll { it.type == 'excel' }
+                def excelMatches = [:]
+                
+                // Find a match between input (mothur) samples and the samples in the system
+                excelFiles.each { excelFile ->
+                        if( excelFile.matches && excelFile.matches[ 'mothursamples' ] ) {
+                                excelFile.matches[ 'mothursamples' ].each { excelMatch ->
+                                        def foundSample = assaySamples.find { it.sample.name == excelMatch.samplename };
+                                        if( foundSample )
+                                                excelMatches[ excelMatch.mothurname ] = foundSample 
+                                }
+                        }
+                }
+                
+                if( classificationFiles ) {
+                        parsedFiles.success -= classificationFiles;
+
+                        // If no excel matches are found, no classifications can be stored
+                        if( !excelMatches ) {
+                                classificationFiles.each { 
+                                        it.success = false
+                                        it.message = "An excel file which maps the input samples with samples in the system must be provided."
+                                        
+                                        parsedFiles.failure << it;
+                                }
+                        } else {
+                                // Set progress to a new step, so the user knows that it will take a while
+                                long totalSize = classificationFiles.collect { it.filesize }.sum();
+                                newStep( totalSize, "Storing classification" )
+                                println "Total size: " + totalSize;
+                                
+                                classificationFiles = classificationService.storeClassification( classificationFiles, excelMatches, { bytesProcessed -> httpSession.processProgress.stepProgress += bytesProcessed } );
+                                
+                                classificationFiles.each {
+                                        if( it.success ) {
+                                                parsedFiles.success << it;
+                                        } else {
+                                                parsedFiles.failure << it;
+                                        }
+                                }
+                        }
+                }
+                
                 // Match files with samples in the database
                 def matchedFiles = fastaService.matchFiles( parsedFiles.success, assaySamples );
@@ -139 +195 @@
 
                 // Saved file matches in session to use them later on
-                session.processedFiles = [ parsed: parsedFiles, matched: matchedFiles ];
+                session.processedFiles = [ parsed: parsedFiles,  matched: matchedFiles ];
 
                 render ""
@@ -186 +242 @@
                 }
                 
-                [entityType: params.entityType, entity: entity, id: params.id, parsedFiles: session.processedFiles.parsed, matchedFiles: session.processedFiles.matched, selectedRun: params.selectedRun ]
+                [entityType: params.entityType, entity: entity, id: params.id, parsedFiles: session.processedFiles.parsed, classificationFiles: session.processedFiles.parsed.success.findAll { it.type == 'taxonomy' }, matchedFiles: session.processedFiles.matched, selectedRun: params.selectedRun ]
         }
 
@@ -252 +308 @@
                 def errors = [];
                 
-                // Loop through all files Those are the numeric elements in the 'files' array
+                // Loop through all files. Those are the numeric elements in the 'files' array
                 def digitRE = ~/^\d+$/;
                 files.findAll { it.key.matches( digitRE ) }.each { file ->

trunk/grails-app/domain/nl/tno/massSequencing/AssaySample.groovy

@@ -70 +70 @@
         }
 
+        /**
+         * Returns the number of sequences that have been classified for this sample
+         * @return
+         */
+        public int numClassifiedSequences() {
+                return Sequence.countByAssaySample( this );
+        }
+        
         /**
          * Returns the number of sequence files in the system, belonging to this
@@ -260 +268 @@
                 return  fwOligo || fwMidName || fwTotalSeq || fwMidSeq || fwPrimerSeq ||
                                 revOligo || revMidName || revTotalSeq || revMidSeq || revPrimerSeq ||
-                                numFiles() > 0;
+                                numFiles() > 0 || numClassifiedSequences() > 0;
         }
         
@@ -306 +314 @@
                         }
                 }
-
+                
+                // Copy all sequence classifications to the new assaysample
+                Sequence.executeUpdate( "UPDATE Sequence s SET s.assaySample = :new WHERE s.assaySample = :old ", [ 'old': this, 'new': otherAssaySample ] )
+        
                 // Copy run properties
                 if( otherAssaySample.run ) {
@@ -340 +351 @@
                 }
 
+                // Remove all sequence objects referencing this sequenceData object
+                Sequence.executeUpdate( "DELETE FROM Sequence s WHERE s.assaySample = ?", [this])
+                
                 resetStats();
                 save();
@@ -346 +360 @@
         }
 
+        
+        /**
+         * Remove data from associations before delete the assaySample itself
+         */
+        def beforeDelete = {
+                deleteSequenceData();
+                
+                Classification.executeUpdate( "DELETE FROM Classification c WHERE c.assaySample = ?", [this])
+        }
 }

trunk/grails-app/domain/nl/tno/massSequencing/Study.groovy

@@ -1 @@
-import java.io.StringWriter;
-
 package nl.tno.massSequencing
 

trunk/grails-app/services/nl/tno/massSequencing/FastaService.groovy

@@ -9 +9 @@
 import java.util.zip.*
 
-class FastaService {
+class FastaService implements nl.tno.massSequencing.imports.Importer {
         def fileService
         def fuzzySearchService
@@ -15 +15 @@
         def excelService
 
-        static transactional = true
-
-        /**
-         * Parses uploaded files and checks them for FASTA and QUAL files
-         * @param filenames             List of filenames currently existing in the upload directory
-         * @param onProgress    Closure to execute when progress indicators should be updated. 
-         *                                              Has 4 parameters: numFilesProcessed, numBytesProcessed that indicate the number
-         *                                              of files and bytes that have been processed in total. totalFiles, totalBytes indicate the change 
-         *                                              in total number of files and bytes (e.g. should take 1 if a new file is added to the list)
-         * @param directory             Directory to move the files to
-         * @return                              Structure with information about the parsed files. The 'success' files are
-         *                                              moved to the given directory
-         * 
-         * [
-         *              success: [
-         *                      [filename: 'abc.fasta', type: FASTA, numSequences: 190]
-         *                      [filename: 'cde.fasta', type: FASTA, numSequences: 140]
-         *                      [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38]
-         *                      [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
-         *              ],
-         *              failure: [
-         *                      [filename: 'testing.xls', type: 'unknown', message: 'Type not recognized']
-         *              ]
-         * ]
-         * 
-         */
-        def parseFiles( ArrayList filenames, Closure onProgress, File directory = null ) {
-                if( filenames.size() == 0 ) {
-                        return [ success: [], failure: [] ];
-                }
-
-                if( !directory ) {
-                        directory = fileService.absolutePath( ConfigurationHolder.config.massSequencing.fileDir )
-                }
-
-                def success = [];
-                def failure = [];
-
-                long filesProcessed = 0;
-                long bytesProcessed = 0;
-
-                // Loop through all filenames
-                for( int i = 0; i < filenames.size(); i++ ) {
-                        def filename = filenames[ i ];
-                        
-                        if( fileService.isZipFile( filename ) ) {
-                                // ZIP files are extracted and appended to the filenames list.
-                                def newfiles = fileService.extractZipFile( filename, { files, bytes, totalFiles, totalBytes ->
-                                        filesProcessed += files;
-                                        bytesProcessed += bytes;
-
-                                        onProgress( filesProcessed, bytesProcessed, totalFiles, totalBytes );
-                                } );
-                                if( newfiles ) {
-                                        newfiles.each {
-                                                filenames.add( it );
-                                        }
-                                        
-                                        // Delete the zip file itself
-                                        fileService.delete( filename );
-                                } else {
-                                        failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: "zip", message: 'Zip file could not be extracted' ];
-                                }
-                        } else {
-                                def file = fileService.get( filename );
-                                String filetype = fileService.determineFileType( file );
-
-                                if( !fileTypeValid( filetype ) ) {
-                                        // If files are not valid for parsing, delete them and return a message to the user
-                                        fileService.delete(filename);
-                                        failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: 'File type not accepted' ];
-                                } else {
-                                        try {
-                                                def contents = parseFile( file, filetype, { files, bytes ->
-                                                        filesProcessed += files;
-                                                        bytesProcessed += bytes;
-
-                                                        onProgress( filesProcessed, bytesProcessed, 0, 0 );
-                                                } );
-                                                
-                                                contents.filename = file.getName();
-                                                contents.originalfilename = fileService.originalFilename( contents.filename )
-
-                                                if( contents.success ) {
-                                                        success << contents;
-                                                } else {
-                                                        fileService.delete(filename);
-                                                        failure << contents;
-                                                }
-                                        } catch( Exception e ) {
-                                                // If anything fails during parsing, return an error message
-                                                fileService.delete(filename);
-                                                failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: e.getMessage() ];
-                                        }
-                                }
-                        }
-                }
-
-                return [ 'success': success, 'failure': failure ];
-        }
-
-        /**
-         * Matches uploaded fasta and qual files and combines them with the samples they probably belong to 
-         * @param parsedFiles   Parsed files 
-         * [
-         *              [filename: 'abc.fasta', type: FASTA, numSequences: 190]
-         *              [filename: 'cde.fasta', type: FASTA, numSequences: 140]
-         *              [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38]
-         *              [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
-         *              [filename: 'match.xls', type: EXCEL, matches: [ [ filename: 'abc.fasta', basename: 'abc', sample: 's1' ] ]
-         * ]
-         * @param samples               AssaySample objects to which the files should be matched.
-         * @return                              Structure with information about the matching. 
-         * [
-         *              [
-         *                      fasta:  [filename: 'abc.fasta', type: FASTA, numSequences: 190],
-         *                      qual:   [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38],
-         *                      feasibleQuals: [ 
-         *                              [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38],
-         *                              [filename: 'def.qual', type: QUAL, numSequences: 190, avgQuality: 21]
-         *                      ]
-         *                      sample: AssaySample object      
-         */
-        def matchFiles( def parsedFiles, def samples ) {
-                def fastas = parsedFiles.findAll { it.type == "fasta" }
-                def quals = parsedFiles.findAll { it.type == "qual" }
-                def excels = parsedFiles.findAll { it.type == "excel" }
-                
-                samples = samples.toList()
-                
-                // Collect matches from all files
-                def matches = [];
-                excels.each { m ->
-                        if( m.matches ) {
-                                m.matches.each { matches << it }
-                        }
-                }
-
-                def files = [];
-
-                fastas.each { fastaFile ->
-                        // Remove extension
-                        def matchWith = fastaFile.originalfilename.substring( 0, fastaFile.originalfilename.lastIndexOf( '.' ) )
-
-                        // Determine feasible quals (based on number of sequences )
-                        def feasibleQuals = quals.findAll { it.numSequences == fastaFile.numSequences }
-
-                        // Best matching qual file
-                        def qualIdx = fuzzySearchService.mostSimilarWithIndex( matchWith + '.qual', feasibleQuals.originalfilename );
-
-                        def qual = null
-                        if( qualIdx != null )
-                                qual = feasibleQuals[ qualIdx ];
-
-                        // Best matching sample
-                        def assaySample = null
-                        def checked = true;
-                        if( matches ) {
-                                // Match with files from excelsheet
-                                
-                                // First find the best matching filename in the list of matches.
-                                def sampleNameIdx = fuzzySearchService.mostSimilarWithIndex( matchWith, matches*.basename, 0.8 );
-                                
-                                // If one is found, use the sample name associated with it to do the matching with samples
-                                if( sampleNameIdx != null ) {
-                                        matchWith = matches[ sampleNameIdx ].sample;
-                                } else {
-                                        // If no match is found in the excel sheet, this sample is unchecked by default
-                                        checked = false;
-                                }
-                        }
-                        
-                        // Match on filenames
-                        def sampleIdx = fuzzySearchService.mostSimilarWithIndex( matchWith, samples.sample.name );
-                        if( sampleIdx != null ) {
-                                assaySample = samples[ sampleIdx ];
-                        }
-
-                        files << [
-                                                fasta: fastaFile,
-                                                feasibleQuals: feasibleQuals,
-                                                qual: qual,
-                                                assaySample: assaySample,
-                                                checked: checked
-                                        ]
-                }
-
-                return files;
-
-
-        }
+        static transactional = false
 
         /**
@@ -213 +23 @@
          * @return
          */
-        protected boolean fileTypeValid( String filetype ) {
+        public boolean canParseFileType( String filetype ) {
                 switch( filetype ) {
                         case "fasta":
                         case "qual":
-                        case "excel":
                                 return true;
                         default:
@@ -225 +34 @@
 
         /**
-         * Parses the given file
+         * Parses the given FASTA or QUAL file
+         * 
          * @param file                  File to parse
          * @param filetype              Type of the given file
          * @param onProgress    Closure to execute when progress indicators should be updated. 
-         *                                              Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number
-         *                                              of files and bytes that have been processed in this file (so the first parameter should 
-         *                                              only be 1 when the file is finished)
+         *                                              Has 2 parameters: progress that indicates the number of bytes that have been processed. The second parameter determines
+         *                                              the number of bytes that has to be processed extra after this action (e.g. when a zip file is extracted, the extracted
+         *                                              files should be added to the total number of bytes to be processed)
+         * @return                              Structure with information about the parsed files. The 'success' files are
+         *                                              moved to the given directory
          * 
-         * @return                              List structure. Examples:
+         * Examples:
+         *                      [filename: 'abc.fasta', type: FASTA, numSequences: 190]
+         *                      [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
          * 
-         *   [ success: true, filename: 'abc.fasta', type: 'fasta', numSequences: 200 ]
-         *   [ success: true, filename: 'abc.qual', type: 'qual', numSequences: 200, avgQuality: 36 ]
-         *   [ success: false, filename: 'abc.txt', type: 'txt', message: 'Filetype could not be parsed.' ]
-         */
-        protected def parseFile( File file, String filetype, Closure onProgress ) {
+         *                      [filename: 'test.zip', type: ZIP, extraFiles: [newfile1.xls, newfile2.xls, newfile3.xls]]
+         * 
+         *                      [filename: 'testing.xls', type: 'unknown', message: 'Type not recognized']
+         * 
+         */
+
+        public Map parseFile( File file, String filetype, Closure onProgress ) {
                 switch( filetype ) {
                         case "fasta":
@@ -245 +61 @@
                         case "qual":
                                 return parseQual( file, onProgress );
-                        case "excel":
-                                return parseExcelMatch( file, onProgress );
                         default:
-                                onProgress( 1, file.length() );
                                 return [ success: false, type: filetype, message: 'Filetype could not be parsed.' ]
                 }
@@ -290 +103 @@
                                 bytesProcessed += line.size();
                                 if( bytesProcessed > 1000000 ) {
-                                        onProgress( 0, bytesProcessed );
+                                        onProgress( bytesProcessed, 0 );
                                         bytesProcessed = 0;
                                 }
@@ -298 +111 @@
 
                 // Update progress and say we're finished
-                onProgress( 1, bytesProcessed );
+                onProgress( bytesProcessed, 0 );
 
                 log.trace "Finished parsing FASTA " + file.getName() + ": " + ( System.nanoTime() - startTime ) / 1000000L
@@ -388 +201 @@
                                 bytesProcessed += line.size();
                                 if( bytesProcessed > 1000000 ) {
-                                        onProgress( 0, bytesProcessed );
+                                        onProgress( bytesProcessed, 0 );
                                         bytesProcessed = 0;
                                 }
@@ -396 +209 @@
 
                 // Update progress and say we're finished
-                onProgress( 1, bytesProcessed );
+                onProgress( bytesProcessed, 0 );
 
                 log.trace "Finished parsing QUAL " + file.getName() + ": " + ( System.nanoTime() - startTime ) / 1000000L
@@ -403 +216 @@
         }
 
+        
+        /**
+        * Matches uploaded fasta and qual files and combines them with the samples they probably belong to
+        * @param parsedFiles    Parsed files
+        * [
+        *               [filename: 'abc.fasta', type: FASTA, numSequences: 190]
+        *               [filename: 'cde.fasta', type: FASTA, numSequences: 140]
+        *               [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38]
+        *               [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
+        *               [filename: 'match.xls', type: EXCEL, matches: [ [ filename: 'abc.fasta', basename: 'abc', sample: 's1' ] ]
+        * ]
+        * @param samples                AssaySample objects to which the files should be matched.
+        * @return                               Structure with information about the matching.
+        * [
+        *               [
+        *                       fasta:  [filename: 'abc.fasta', type: FASTA, numSequences: 190],
+        *                       qual:   [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38],
+        *                       feasibleQuals: [
+        *                               [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38],
+        *                               [filename: 'def.qual', type: QUAL, numSequences: 190, avgQuality: 21]
+        *                       ]
+        *                       sample: AssaySample object
+        */
+   def matchFiles( def parsedFiles, def samples ) {
+           def fastas = parsedFiles.findAll { it.type == "fasta" }
+           def quals = parsedFiles.findAll { it.type == "qual" }
+           def excels = parsedFiles.findAll { it.type == "excel" }
+           
+           samples = samples.toList()
+           
+           // Collect (filename) matches from all excel files
+           def matches = [];
+           excels.each { m ->
+                   if( m.matches && m.matches.filenames ) {
+                           // [ filename: filename, samplename: GSCFSample, basename: basename ]
+                           m.matches.filenames.each { matches << it }
+                   }
+           }
+
+           def files = [];
+
+           fastas.each { fastaFile ->
+                   // Remove extension
+                   def matchWith = fastaFile.originalfilename.substring( 0, fastaFile.originalfilename.lastIndexOf( '.' ) )
+
+                   // Determine feasible quals (based on number of sequences )
+                   def feasibleQuals = quals.findAll { it.numSequences == fastaFile.numSequences }
+
+                   // Best matching qual file
+                   def qualIdx = fuzzySearchService.mostSimilarWithIndex( matchWith + '.qual', feasibleQuals.originalfilename );
+
+                   def qual = null
+                   if( qualIdx != null )
+                           qual = feasibleQuals[ qualIdx ];
+
+                   // Best matching sample
+                   def assaySample = null
+                   def checked = true;
+                   if( matches ) {
+                           // Match with files from excelsheet
+                           
+                           // First find the best matching filename in the list of matches.
+                           def sampleNameIdx = fuzzySearchService.mostSimilarWithIndex( matchWith, matches*.basename, 0.8 );
+                           
+                           // If one is found, use the sample name associated with it to do the matching with samples
+                           if( sampleNameIdx != null ) {
+                                   matchWith = matches[ sampleNameIdx ].samplename;
+                           } else {
+                                   // If no match is found in the excel sheet, this sample is unchecked by default
+                                   checked = false;
+                           }
+                   }
+                   
+                   // Match on filenames
+                   def sampleIdx = fuzzySearchService.mostSimilarWithIndex( matchWith, samples.sample.name );
+                   if( sampleIdx != null ) {
+                           assaySample = samples[ sampleIdx ];
+                   }
+
+                   files << [
+                                           fasta: fastaFile,
+                                           feasibleQuals: feasibleQuals,
+                                           qual: qual,
+                                           assaySample: assaySample,
+                                           checked: checked
+                                   ]
+           }
+
+           return files;
+
+
+   }
+
+        
         /**
          * Parses a given excel file with a match between filenames and samples
@@ -417 +324 @@
          *   [ success: false, filename: 'def.xls', type: 'excel', message: 'File is not a valid XLS file' ]
          */
-        protected def parseExcelMatch( File file, Closure onProgress ) {
-                long startTime = System.nanoTime();
-                log.trace "Start parsing XLS " + file.getName()
-
-                def matches = []
-
-                // Read excel file
-                def wb;
-                try {
-                        wb = excelService.open( file );
-                } catch( Exception e ) {
-                        // If an exception occurs, the file can't be opened. Return the error message
-                        return [ success: false, type: "excel", filename: file.getName(), message: "Excel file could not be opened or parsed."]
-                }
-
-                // Read all data into an array, and the header in a separate array
-                def header = excelService.readRow( wb, 0, 0 );
-                def data = excelService.readData( wb, 0, 1 );
-
-                // Check whether (at least) 2 columns are present
-                if( header.size() < 2 ) {
-                        return [ success: false, type: "excel", filename: file.getName(), message: "Excel file must contain at least 2 columns, one with filenames and one with sample names"]
+        public def inferExcelMatches( parsedFiles ) {
+                // Loop through all excel files, and determine which (usable) columns are in the file
+                for( parsedFile in parsedFiles ) {
+                        if( parsedFile.type == "excel" ) {
+                                
+                                def header = parsedFile.header;
+                                def data = parsedFile.data
+                                
+                                // Check whether (at least) 2 columns are present
+                                if( header.size() < 2 ) {
+                                        parsedFile.success = false
+                                        parsedFile.exceldata = []
+                                        parsedFile.message = "Excel file must contain at least 2 columns, one with filenames and one with sample names"
+                                        continue;
+                                }
+                                                
+                                // Check the headers to see whether any columns are found
+                                def columns = [:]
+                                def matches = [ "filenames": [], "mothursamples": [] ]
+                                
+                                header.eachWithIndex { cell, i ->
+                                        switch( cell?.toLowerCase() ) {
+                                                case "gscfsample":
+                                                case "sample":
+                                                        columns[ "GSCFSample" ] = i; break;
+                                                case "file":
+                                                case "filename":
+                                                        columns[ "filename" ] = i; break;
+                                                case "mothursample":
+                                                        columns[ "MothurSample" ] = i; break;
+                                        }
+                                }
+                                                
+                                // Loop through the data and find matches
+                                data.each { row ->
+                                        def filename = columns[ "filename" ] != null ? row[ columns[ "filename" ] ] : null;
+                                        def GSCFSample = columns[ "GSCFSample" ] != null ? row[ columns[ "GSCFSample" ] ] : null;
+                                        def mothurSample = columns[ "MothurSample" ] != null ? row[ columns[ "MothurSample" ] ] : null;
+                                        
+                                        // Matches only exist if GSCF sample is entered in this row
+                                        if( GSCFSample && GSCFSample != "null" ) {
+                                                if( filename && filename != "null" ) {
+                                                        // Remove extension from the filename, but only if it is
+                                                        // .fna, .fasta, .qual, .fqa, .xls or .xlsx. Otherwise useful parts of the filename would be removed.
+                                                        def basename = ( filename =~ /\.(fna|fasta|qual|fqa|xls|xlsx)$/ ).replaceAll( "" );
+
+                                                        matches[ "filenames" ] << [ filename: filename, samplename: GSCFSample, basename: basename ];
+                                                }
+                                                
+                                                if( mothurSample && mothurSample != "null" ) {
+                                                        matches[ "mothursamples" ] << [ samplename: GSCFSample, mothurname: mothurSample ];
+                                                }
+                                        }
+                                }
+                                
+                                parsedFile[ 'columns' ] = columns;
+                                parsedFile[ 'matches' ] = matches;
+                        }
                 }
                 
-                // Check the headers to see whether the default columns are switched
-                def filenameColumn = 0;
-                def sampleColumn = 1;
-                
-                header.eachWithIndex { cell, i ->
-                        switch( cell?.toLowerCase() ) {
-                                case "sample":
-                                        sampleColumn = i; break;
-                                case "file":
-                                case "filename":
-                                        filenameColumn = i; break;
-                        }
-                }
-                
-                // If both filenames and samples are found in the same column (happens if only one of the headers is given)
-                // an error is given
-                if( filenameColumn == sampleColumn )
-                        return [ success: false, type: "excel", filename: file.getName(), message: "Excel file must contain 'Sample' and 'Filename' headers."]
-                
-                // Loop through the data and create a match
-                def maxColumn = Math.max( filenameColumn, sampleColumn );
-                data.each { row ->
-                        if( row.size() >= maxColumn ) {
-                                def filename = row[ filenameColumn ];
-                                def sample = row[ sampleColumn ];
-                                
-                                if( sample && sample != "null"  && filename && filename != "null" ) {
-                                        // Remove extension from the filename, but only if it is
-                                        // .fna, .fasta, .qual, .fqa, .xls or .xlsx. Otherwise useful parts of the filename would be removed.
-                                        def basename = ( filename =~ /\.(fna|fasta|qual|fqa|xls|xlsx)$/ ).replaceAll( "" );
-                                        
-                                        matches << [ filename: filename, sample: sample, basename: basename ];
-                                }
-                        }
-                }
-
-                // Update progress and say we're finished
-                onProgress( 1, file.size() );
-
-                log.trace "Finished parsing XLS " + file.getName() + ": " + ( System.nanoTime() - startTime ) / 1000000L
-
-                return [ success: true, type: "excel", filename: file.getName(), matches: matches ];
+                return parsedFiles
         }
 

trunk/grails-app/services/nl/tno/massSequencing/SampleExcelService.groovy

@@ -93 +93 @@
 
            // Put the headers on the first row
-           excelService.writeHeader( wb, [ "Filename", "Sample" ], sheetIndex );
+           excelService.writeHeader( wb, [ "Filename", "MothurSample", "GSCFSample" ], sheetIndex );
 
            // Adding the next lines
            ArrayList data = [];
            sortedSamples.each { assaySample ->
-                   def rowData = [ "", assaySample.sample.name ];
+                   def rowData = [ "", "", assaySample.sample.name ];
                    
                    data << rowData;

trunk/grails-app/services/nl/tno/massSequencing/files/ExcelService.groovy

@@ -1 +1 @@
 package nl.tno.massSequencing.files
 
+import groovy.lang.Closure;
+
+import java.io.File;
 import java.text.DecimalFormat
 import java.text.Format
 import java.text.NumberFormat
+import java.util.Map;
+
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@@ -14 +19 @@
  *
  */
-class ExcelService {
-    static transactional = true
+class ExcelService implements nl.tno.massSequencing.imports.Importer {
+    static transactional = false
         
         /**
@@ -254 +259 @@
 
         }
+        
+        
+        /**
+         * Determines whether a file can be processed.
+         * @param filetype      Filetype of the file
+         * @see determineFileType()
+         * @return
+         */
+        public boolean canParseFileType( String filetype ) {
+                switch( filetype ) {
+                        case "excel":
+                                return true;
+                        default:
+                                return false;
+                }
+        }
+
+        /**
+         * Parses the given excel file
+         * @param file                  File to parse
+         * @param filetype              Type of the given file (only excel is accepted)
+         * @param onProgress    Closure to execute when progress indicators should be updated. 
+         *                                              Has 2 parameters: progress that indicates the number of bytes that have been processed. The second parameter determines
+         *                                              the number of bytes that has to be processed extra after this action (e.g. when a zip file is extracted, the extracted
+         *                                              files should be added to the total number of bytes to be processed)
+         * @return                              Structure with information about the parsed files. The 'success' files are
+         *                                              moved to the given directory
+         * 
+         * Examples:
+         *                      [filename: 'abc.fasta', type: FASTA, numSequences: 190]
+         *                      [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
+         * 
+         *                      [filename: 'test.zip', type: ZIP, extraFiles: [newfile1.xls, newfile2.xls, newfile3.xls]]
+         * 
+         *                      [filename: 'testing.xls', type: 'unknown', message: 'Type not recognized']
+         * 
+         */
+        public Map parseFile( File file, String filetype, Closure onProgress ) {
+                switch( filetype ) {
+                        case "excel":
+                                return parseExcel( file, onProgress );
+                        default:
+                                return [ success: false, type: filetype, message: 'Filetype could not be parsed.' ]
+                }
+        }
+        
+        /**
+        * Parses a given excel file with a match between filenames and samples
+        * @param file                   File to parse
+        * @param onProgress             Closure to execute when progress indicators should be updated.
+        *                                               Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number
+        *                                               of files and bytes that have been processed in this file (so the first parameter should
+        *                                               only be 1 when the file is finished)
+        * @return                               List structure. The matches array contains an array of matches between filenames and sample(name)s.
+        *                                               The extension for all files are removed in the 'basename' parameter, in order to improve matching.
+        *                                               Examples:
+        *
+        *   [ success: true, filename: 'abc.xls', type: 'excel', matches: [ [ filename: 's1.qual', basename: 's1', sample: 'sample a' ], [ filename: 's9.fna', basename: 's9', sample: 'sample b' ] ]
+        *   [ success: false, filename: 'def.xls', type: 'excel', message: 'File is not a valid XLS file' ]
+        */
+   protected def parseExcel( File file, Closure onProgress ) {
+           long startTime = System.nanoTime();
+           log.trace "Start parsing XLS " + file.getName()
+
+           def matches = []
+
+           // Read excel file
+           def wb;
+           try {
+                   wb = open( file );
+           } catch( Exception e ) {
+                   // If an exception occurs, the file can't be opened. Return the error message
+                   return [ success: false, type: "excel", filename: file.getName(), message: "Excel file could not be opened or parsed."]
+           }
+
+           // Read all data into an array, and the header in a separate array
+           def header = readRow( wb, 0, 0 );
+           def data = readData( wb, 0, 1 );
+
+           // Update progress and return
+           onProgress( file.size(), 0 );
+           return [ success: true, type: "excel", filename: file.getName(), header: header, data: data ];
+   }
+
+        
 }

trunk/grails-app/services/nl/tno/massSequencing/files/FileService.groovy

@@ -363 +363 @@
                         case "fasta":
                         case "fna":
-                                return "fasta";
+                                return "fasta";         // FASTA format files (see http://en.wikipedia.org/wiki/FASTA_format)
                         case "qual":
-                        case "fqa":
-                                return "qual";
+                        case "fqa":     
+                                return "qual";          // QUAL format files for FASTA (see http://sequence.otago.ac.nz/download/ManualPartC.pdf)
                         case "xls":
                         case "xlsx":
-                                return "excel";
-                        case "zip":
-                                return "zip";
+                                return "excel";         // Excel files for metadata
+                        case "taxonomy":        
+                                return "taxonomy";      // Taxonomy files as created by Mothur (www.mothur.org) for classification of sequences
+                        case "groups":  
+                                return "groups";        // Mothur file to associate a specific sequence to a sample (www.mothur.org)
+                        case "zip":     
+                                return "zip";           // Zip files combining all other files
                         case "gz":
-                                return "gzip";
+                                return "gzip";          // GZip files combining all other files 
                         default:
                                 return "";              // meaning 'unknown'
@@ -395 +399 @@
                         case "zip":
                         case "gzip":
-                        return true;
+                                return true;
                 }
 
@@ -419 +423 @@
                 switch( this.determineFileType( f ) ) {
                         case "zip":
-                        return extractZip( f, onProgress );
+                                return extractZip( f, onProgress );
                         case "gzip":
-                        return extractGzip( f, onProgress );
+                                return extractGzip( f, onProgress );
                         default:
-                        log.error "File given for extracting files is not a valid zip file."
+                                log.error "File given for extracting files is not a valid zip file."
 
                         return [];
@@ -498 +502 @@
                                 
                                 // Update progress
-                                onProgress( 0, entry.getCompressedSize(), 1, outpath.length())
+                                onProgress( entry.getCompressedSize(), outpath.length() )
                         }
                 }
@@ -555 +559 @@
 
                         // Update progress
-                        onProgress( 0, f.length(), 1, outpath.length())
+                        onProgress( f.length(), outpath.length())
                 } catch( Exception e ) {
                         // Delete the file if it exists

trunk/grails-app/views/assay/_addFilesDialog.gsp

@@ -2 +2 @@
         <h2>Upload sequence files</h2>
         
-        <g:form name="addFiles" controller="fasta" action="showProcessScreen" id="${assay.id}">
+        <g:form name="addFiles" controller="import" action="showProcessScreen" id="${assay.id}">
                 <input type="hidden" name="entityType" value="assay" />
                 <p>

trunk/grails-app/views/import/showProcessResult.gsp

@@ -25 +25 @@
                 </g:else>
         </h1>
+        
+        <h2>Sequence files</h2>
         <g:if test="${matchedFiles.size() > 0}">
                 <p>
@@ -96 +98 @@
         </g:if>
         <g:else>
-                <h2>Uploaded files</h2>
                 <p>
                         ${parsedFiles.success?.size()} files were successfully uploaded, but no FASTA file could be determined. Please 
-                        upload at least one FASTA file.
+                        upload at least one FASTA file if sequences should be added.
                 </p>
         </g:else>
+        
+        <h2>Classification files</h2>
+        <g:if test="${classificationFiles.size() > 0}">
+                <p>
+                        The following files containing sequence classification have been imported.
+                </p>
+                <table>
+                        <thead>
+                                <tr>
+                                        <th>File</th>
+                                        <th># sequences imported</th>
+                                        <th># sequences discarded</th>
+                                </tr>
+                        </thead>
+                        <g:each in="${classificationFiles}" var="file" status="i">
+                                <tr>
+                                        <td>${file.filename}</td>
+                                        <td>${file.importedSequences}</td>
+                                        <td>${file.discardedSequences}</td>
+                                </tr>
+                        </g:each>
+                </table>
+        </g:if>
+        <g:else>
+                <p>
+                        No classification files found in uploaded data.
+                </p>
+        </g:else>
+        
+        
         
         <g:if test="${parsedFiles.failure?.size()}">
                 <h2>Failure</h2>
                 <p>
-                        The following files could not be correctly imported. The system can only interpret FASTA and QUAL files.<br />
+                        The following files could not be correctly imported. The system can only interpret FASTA, QUAL, GROUPS and TAXONOMY files.<br />
                         If these files were intended to be included into the system, please check the messages below.
                  </p>
@@ -120 +151 @@
         
         <p>
-                <g:link controller="fasta" action="returnWithoutSaving" params="[entityType: entityType, id: entity.id]">Return to ${entityType}</g:link>       
+                <g:link controller="import" action="returnWithoutSaving" params="[entityType: entityType, id: entity.id]">Return to ${entityType}</g:link>      
         </p>
 </body>

trunk/grails-app/views/import/showProcessScreen.gsp

@@ -1 +1 @@
 <html>
 <head>
-<meta name="layout" content="main" />
-<title>Processing files | Mass Sequencing | dbNP</title>
+        <meta name="layout" content="main" />
+        <title>Processing files | Mass Sequencing | dbNP</title>
 
-<script type="text/javascript">
-                        $(function() {
-                                $( "#wait_dialog" ).dialog({
-                                        height: 140,
-                                        width: 400,
-                                        title: 'Please wait while processing files',
-                                        modal: true,
-                                        autoOpen: true,
-                                        buttons: {},
-                                        closeOnEscape: false,
-                                        open: function(event, ui) { $(".ui-dialog-titlebar-close" ).hide(); },
-                                        resizable: false        
-                                });
-
-                                // Show the progressbar and update the value every second 
-                                $( "#progressbar" ).progressbar({
-                                        value: 0
-                                });
-
-                                var progressInterval = setInterval(updateProgress, 250);
-                                                                
-                                // Call processing URL
-                                $.ajax({
-                                  type: 'POST',
-                                  url: "<g:createLink controller="fasta" action="process" id="${entityId}" />",
-                                  data: "entityType=${entityType}",
-                                  success: function(data) {
-                                          
-                                          // Stop update progress bar
-                                          clearInterval( progressInterval );
-                                           
-                                          window.location.replace( "${url.encodeAsJavaScript()}" ); 
-                                  },
-                                  error: function(xhr, textStatus, errorThrown) { 
-
-                                          // Stop update progress bar (but update it for the last time)
-                                          updateProgress()
-                                          clearInterval( progressInterval );
-
-                                          alert( "Error " + xhr.getStatus() + ": " + xhr.getStatusText() ); 
-                                          //window.location.replace( "<g:createLink controller="${entityType}" action="show" id="${entityId}" />" ); 
-                                }
-                                });
+        <script type="text/javascript">
+                $(function() {
+                        $( "#wait_dialog" ).dialog({
+                                height: 200,
+                                width: 400,
+                                title: 'Please wait while processing files',
+                                modal: true,
+                                autoOpen: true,
+                                buttons: {},
+                                closeOnEscape: false,
+                                open: function(event, ui) { $(".ui-dialog-titlebar-close" ).hide(); },
+                                resizable: false        
                         });
 
-                        function updateProgress() {
-                              $.ajax({
-                                    type: "GET",
-                                    url: "<g:createLink controller="fasta" action="getProgress" />",
-                                    dataType: "json",
-                                    success: function(progress) {
-                                          $("#progressbar").progressbar("value", progress.bytesProcessed / progress.numBytes * 100 );
-                                                }
-                              });
+                        // Show the progressbar and update the value every second 
+                        $( "#progressbar" ).progressbar({
+                                value: 0
+                        });
+
+                        var progressInterval = setInterval(updateProgress, 250);
+                                                        
+                        // Call processing URL
+                        $.ajax({
+                          type: 'POST',
+                          url: "<g:createLink controller="import" action="process" id="${entityId}" />",
+                          data: "entityType=${entityType}",
+                          success: function(data) {
+                                  
+                                  // Stop update progress bar
+                                  clearInterval( progressInterval );
+                                   
+                                  window.location.replace( "${url.encodeAsJavaScript()}" ); 
+                          },
+                          error: function(xhr, textStatus, errorThrown) { 
+
+                                  // Stop update progress bar (but update it for the last time)
+                                  updateProgress()
+                                  clearInterval( progressInterval );
+
+                                  alert( "Error " + xhr.getStatus() + ": " + xhr.getStatusText() ); 
+                                  //window.location.replace( "<g:createLink controller="${entityType}" action="show" id="${entityId}" />" ); 
                         }
-                </script>
-<style type="text/css">
-        #wait_dialog {
-                padding: 20px;
-                text-align: center;
-        }
-        
-        #progressbar { margin-top: 10px; }
-</style>
+                        });
+                });
+
+                function updateProgress() {
+                        var stepDiv = $( '#step' );
+                        $.ajax({
+                        type: "GET",
+                                url: "<g:createLink controller="import" action="getProgress" />",
+                                dataType: "json",
+                                success: function(progress) {
+                                        $( '#stepNum', stepDiv ).text( progress.stepNum );
+                                        $( '#numSteps', stepDiv ).text( progress.numSteps );
+                                        $( '#stepDescription', stepDiv ).text( progress.stepDescription );
+
+                                        stepDiv.show();
+
+                                        $("#progressbar").progressbar("value", progress.stepProgress / progress.stepTotal * 100 );
+                                }
+                        });
+                }
+        </script>
+        <style type="text/css">
+                #wait_dialog {
+                        padding: 20px;
+                        text-align: center;
+                }
+                
+                #progressbar {
+                        margin-top: 10px;
+                }
+        </style>
 </head>
 <body>
-<div id="wait_dialog">
-
-        <span class="spinner" style="display: inline-block; zoom: 1; *display: inline;"></span>
-        Please wait while processing your files.
+        <div id="wait_dialog">
+                <span class="spinner" style="display: inline-block; zoom: 1; *display: inline;"></span>
+                <p>Please wait while processing your files.</p>
+                <p id="step" style="display: none;">
+                        Step <span id="stepNum"></span> / <span id="numSteps"></span>: <span id="stepDescription"></span>
+                </p>
         
-        <div id="progressbar"></div> 
+                <div id="progressbar"></div>
         
-</div>
+        </div>
 </body>
 </html>

trunk/grails-app/views/run/_addFilesDialog.gsp

@@ -2 +2 @@
         <h2>Upload sequence files</h2>
         
-        <g:form name="addFiles" controller="fasta" action="showProcessScreen" id="${run.id}">
+        <g:form name="addFiles" controller="import" action="showProcessScreen" id="${run.id}">
                 <input type="hidden" name="entityType" value="run" />
                 <p>
-                        Select sequence (.fna, .fasta) and quality (.fqa, .qual) files to upload. It is possible to zip the files before upload. You can add multiple files, if needed.
+                        Select sequence (.fna, .fasta), quality (.fqa, .qual) and classification (.groups, .taxonomy) files to upload. It is possible to zip the files before upload. You can add multiple files, if needed.
                 </p>
                 <p>
-                        The filenames should match the sample names of the samples they belong to. It is also possible to provide an
+                        The filenames should match the sample names of the samples they belong to. You should also provide an
                         excel sheet to describe which file belongs to which sample. The format should be like this <g:link controller="run" action="downloadMatchExcel" id="${run.id}">example</g:link>, with each fasta file appearing once.
                 </p>