Changeset 12 for trunk/grails-app/services/nl

Timestamp:

Feb 9, 2011, 9:04:29 AM (12 years ago)

Author:

robert@…

Message:

Implemented basic exporting functionality

Location:

trunk/grails-app/services/nl/tno/metagenomics

Files:

• FastaService.groovy (modified) (6 diffs)
• files/FileService.groovy (modified) (4 diffs)
• integration/SynchronizationService.groovy (modified) (3 diffs)

trunk/grails-app/services/nl/tno/metagenomics/FastaService.groovy

@@ -1 +1 @@
 package nl.tno.metagenomics
 
+import java.io.BufferedWriter;
 import java.io.File;
 import java.util.ArrayList;
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
 import org.codehaus.groovy.grails.commons.ConfigurationHolder
 
@@ -53 +56 @@
                 for( int i = 0; i < filenames.size(); i++ ) {
                         def filename = filenames[ i ];
-                        
+
                         if( fileService.isZipFile( filename ) ) {
                                 // ZIP files are extracted and appended to the filenames list.
                                 def newfiles = fileService.extractZipFile( filename, { files, bytes, totalFiles, totalBytes ->
-                                                        filesProcessed += files;
-                                                        bytesProcessed += bytes;
-        
-                                                        onProgress( filesProcessed, bytesProcessed, totalFiles, totalBytes );
+                                        filesProcessed += files;
+                                        bytesProcessed += bytes;
+
+                                        onProgress( filesProcessed, bytesProcessed, totalFiles, totalBytes );
                                 } );
                                 if( newfiles ) {
@@ -70 +73 @@
                                 def file = fileService.get( filename );
                                 String filetype = fileService.determineFileType( file );
-        
+
                                 if( !fileTypeValid( filetype ) ) {
                                         // If files are not valid for parsing, delete them and return a message to the user
@@ -80 +83 @@
                                                         filesProcessed += files;
                                                         bytesProcessed += bytes;
-        
+
                                                         onProgress( filesProcessed, bytesProcessed, 0, 0 );
                                                 } );
-        
+
                                                 contents.filename = file.getName();
                                                 contents.originalfilename = fileService.originalFilename( contents.filename )
-        
+
                                                 if( contents.success ) {
                                                         success << contents;
@@ -253 +256 @@
                         }
 
-
-
                 }
 
@@ -396 +397 @@
         }
 
-
+        /**
+         * Exports the fasta data of a list of assaysamples
+         * @param assaySamples  Assaysamples to export
+         * @param outStream             Outputstream to send the data to
+         * @return
+         */
+        public def export( List assaySamples, OutputStream outStream, String name ) {
+                if( !assaySamples || assaySamples.size() == 0 )
+                        return false;
+
+                // Determine the directory the uploaded files are stored in
+                File permanentDirectory = fileService.absolutePath( ConfigurationHolder.config.metagenomics.fileDir );
+
+                // First check whether qual files should be exported or not
+                // It is only exported if qual scores are available for all sequences
+                def exportQual = ( assaySamples*.numSequences().sum() == assaySamples*.numQualScores().sum() );
+
+                // First create tags for every sample
+                def tags = [];
+
+                // Determine new tag length. Since we can use 4 characters per
+                // tag position, we only have to use 4log( #samples)
+                int tagLength = Math.ceil( Math.log( assaySamples.size() ) / Math.log( 4 ) )
+                int tagNumber = 0;
+
+                assaySamples.each { assaySample ->
+                        if( assaySample.numSequences() > 0 ) {
+                                // Create a new tag for this assaysample
+                                def tag = createTag( tagLength, tagNumber++);
+
+                                // Save the tag for exporting
+                                tags << [assaySampleId: assaySample.id, sampleName: assaySample.sample.name, assayName: assaySample.assay.name, tag: tag]
+                        }
+                }
+
+                // Now create zip file for fasta and qual files
+                ZipOutputStream zipFile = new ZipOutputStream( new BufferedOutputStream( outStream ) );
+                BufferedWriter zipWriter = new BufferedWriter( new OutputStreamWriter( zipFile ) );
+                
+                // We have to loop twice through the sequenceData, since we can't write part of the sequence
+                // file and part of the qual files mixed. We have to write the full sequence file first.
+                try {
+                        zipFile.putNextEntry( new ZipEntry( name + ".fasta" ) );
+                        
+                        assaySamples.each { assaySample ->
+                                if( assaySample.numSequences() > 0 ) {
+                                        def currentTag = tags.find { it.assaySampleId == assaySample.id };
+
+                                        assaySample.sequenceData.each { sequenceData ->
+                                                copyFastaFileForExport( fileService.get( sequenceData.sequenceFile, permanentDirectory ), currentTag.tag, zipWriter)
+                                        }
+                                }
+                        }
+                        zipWriter.flush();
+                        zipFile.closeEntry();
+
+                        if( exportQual ) {
+                                zipFile.putNextEntry( new ZipEntry( name + ".qual" ) );
+
+                                assaySamples.each { assaySample ->
+                                        if( assaySample.numSequences() > 0 ) {
+                                                def currentTag = tags.find { it.assaySampleId == assaySample.id };
+
+                                                assaySample.sequenceData.each { sequenceData ->
+                                                        copyQualFileForExport( fileService.get( sequenceData.qualityFile, permanentDirectory ), currentTag.tag, zipWriter)
+                                                }
+                                        }
+                                }
+                                
+                                zipWriter.flush();
+                                zipFile.closeEntry();
+                        }
+
+                } catch( Exception e ) {
+                        log.error "Error while writing to fastafile or qualfile: " + e.getMessage();
+                } finally {
+                        // Always close zip entry
+                        try {
+                                zipFile.closeEntry();
+                        } catch( Exception e ) {
+                                log.error "Error while closing zip entry for fasta and qual: " + e.getMessage();
+                        }
+                }
+                
+                zipFile.close();
+
+                // Export a tab delimited file with tags
+                //exportSampleTagFile( tags );
+        }
+
+        /**
+         * Creates a unique tag for the given number
+         * @param length
+         * @param tagNumber
+         * @return
+         */
+        public String createTag( int length, int tagNumber ) {
+                def chars = ["C", "A", "G", "T"];
+                def numChars = chars.size();
+
+                if( tagNumber > numChars ** length )
+                        throw new Exception( "Given tag number (" + tagNumber + ") is too large for the specified length (" + length + ")")
+
+                String tag = "";
+
+                for( def i = 0; i < length; i++ ) {
+                        int currentChar = tagNumber % numChars
+
+                        tag = chars[ currentChar ] + tag;
+
+                        tagNumber = Math.floor( tagNumber / numChars );
+                }
+
+                return tag
+        }
+
+        /**
+         * Copies the contents of the given sequence file to the output file and prepends the tag to every sequences
+         * @param inFile        Filename of the file to be read
+         * @param tag           
+         * @param outWriter
+         * @return
+         */
+        protected boolean copyFastaFileForExport( File inFile, String tag, BufferedWriter outWriter ) {
+                // Walk through the lines in the file, starting with '>'
+                // (and where the following line contains a character other than '>')
+
+                try {
+                        BufferedReader inReader = new BufferedReader( new FileReader( inFile ) );
+
+                        String line = null
+                        String newLine = null
+                        String sequence = "";
+
+                        def lengthPattern = ~/length=(\d+)/
+                        def lengthMatches
+                        int length = 0;
+                        int tagLength = tag.size();
+
+                        while( ( line = inReader.readLine()) != null) {
+                                if( line.size() == 0 ) {
+                                        // Print the sequence we collected, before writing the empty line
+                                        printSequence( outWriter, sequence, tag );
+                                        sequence = "";
+
+                                        // Empty line
+                                        outWriter.newLine();
+                                } else if( line[ 0 ] == '>' ) {
+                                        // Print the sequence we collected, before writing the new comments tag
+                                        printSequence( outWriter, sequence, tag );
+                                        sequence = "";
+
+                                        // Comments line: replace length=### with the
+                                        // updated length, and put the line in the
+                                        lengthMatches = ( line =~ lengthPattern );
+                                        if( lengthMatches ) {
+                                                length = Integer.valueOf( lengthMatches[0][1] ) + tagLength;
+                                                newLine = lengthMatches.replaceAll( "length=" + length );
+                                        }
+
+                                        outWriter.write(newLine);
+                                        outWriter.newLine();
+                                } else {
+                                        // This is part of the sequence. We collect the whole sequence and
+                                        // determine in the end how to write it to the file
+                                        sequence += line;
+                                }
+                        }
+
+                        // Print the sequence we collected, before ending the file
+                        printSequence( outWriter, sequence, tag );
+                        sequence = "";
+
+                } catch( Exception e ) {
+                        log.error( "An error occurred while copying contents from " + inFile.getName() + ": " + e.getMessage() );
+                        return false;
+                }
+        }
+
+        /**
+         * Prints a sequence to the output file
+         * @param outWriter
+         * @param sequence
+         * @param tag
+         */
+        private void printSequence( BufferedWriter outWriter, String sequence, String tag, int maxWidth = 60 ) {
+                // If no sequence is given, also don't prepend it with the tag
+                if( sequence.size() == 0 )
+                        return
+
+                // Prepend the tag to the sequence
+                sequence = tag + sequence;
+
+                // Write the sequence with a width of maxWidth characters per line
+                while( sequence ) {
+                        if( sequence.size() > maxWidth ) {
+                                outWriter.write( sequence[0..maxWidth-1] );
+                                sequence = sequence[maxWidth..-1]
+                        } else {
+                                outWriter.write( sequence );
+                                sequence = null;
+                        }
+                        outWriter.newLine();
+                }
+        }
+
+        /**
+         * Copies the contents of the given qual file to the output file and prepends the tag quality score to every sequence.
+         * For every tag character '40' is prepended to the qual scores
+         * 
+         * @param inFile        Filename of the file to be read
+         * @param tag
+         * @param outWriter
+         * @return
+         */
+        protected boolean copyQualFileForExport( File inFile, String tag, BufferedWriter outWriter ) {
+                // Walk through the lines in the file, starting with '>'
+                // (and where the following line contains a character other than '>')
+                try {
+                        BufferedReader inReader = new BufferedReader( new FileReader( inFile ) );
+
+                        String line = null
+                        String newLine = null
+                        List<Integer> qualScores = []
+
+                        def lengthPattern = ~/length=(\d+)/
+                        def lengthMatches
+                        int length = 0;
+                        int tagLength = tag.size();
+
+                        while( ( line = inReader.readLine()) != null) {
+                                if( line.size() == 0 ) {
+                                        // Print the quality scores we collected, before writing the empty line
+                                        printQualScore( outWriter, qualScores, tagLength );
+                                        qualScores = [];
+
+                                        // Empty line
+                                        outWriter.newLine();
+                                } else if( line[ 0 ] == '>' ) {
+                                        // Print the quality scores we collected, before writing the empty line
+                                        printQualScore( outWriter, qualScores, tagLength );
+                                        qualScores = [];
+
+                                        // Comments line: replace length=### with the
+                                        // updated length, and put the line in the
+                                        lengthMatches = ( line =~ lengthPattern );
+                                        if( lengthMatches ) {
+                                                length = Integer.valueOf( lengthMatches[0][1] ) + tagLength;
+                                                newLine = lengthMatches.replaceAll( "length=" + length );
+                                        }
+
+                                        outWriter.write(newLine);
+                                        outWriter.newLine();
+                                } else {
+                                        // This is part of the quality score. We collect the whole set of quality
+                                        // scores and determine in the end how to write it to the file
+                                        qualScores += line.split( " " ).collect {
+                                                if( !it.isInteger() )
+                                                        return 0;
+                                                else
+                                                        return Integer.parseInt( it );
+                                        };
+                                }
+                        }
+
+                        // Print the quality scores we collected, before ending the file
+                        printQualScore( outWriter, qualScores, tagLength );
+                        qualScores = [];
+
+                } catch( Exception e ) {
+                        log.error( "An error occurred while copying contents from " + inFile.getName() + ": " + e.getMessage() );
+                        return false;
+                }
+        }
+
+        /**
+         * Prints a sequence to the output file
+         * @param outWriter
+         * @param sequence
+         * @param tag
+         */
+        private void printQualScore( BufferedWriter outWriter, List<Integer> qualScores, int tagLength, int maxWidth = 60 ) {
+                // If no qualScores are given, also don't prepend it with the tag
+                if( qualScores.size() == 0 )
+                        return
+
+                // Prepend the tag to the sequence
+                qualScores = Collections.nCopies( tagLength, 40 ) + qualScores;
+
+                // Write the sequence with a width of maxWidth characters per line
+                while( qualScores ) {
+                        if( qualScores.size() > maxWidth ) {
+                                outWriter.write( qualScores[0..maxWidth-1].join( " " ) );
+                                qualScores = qualScores[maxWidth..-1]
+                        } else {
+                                outWriter.write( qualScores.join( " " ) );
+                                qualScores = null;
+                        }
+                        outWriter.newLine();
+                }
+        }
 
 }

trunk/grails-app/services/nl/tno/metagenomics/files/FileService.groovy

@@ -147 +147 @@
                 directory = getUploadDir()
 
-                println "Moving " + file + " with originalname " + originalFilename + " to " + directory
-
                 try {
                         if( file.exists() ) {
@@ -238 +236 @@
                 }
         }
-
 
         /**
@@ -274 +271 @@
 
         /**
-         * Returns the absolute path for the given pathname. It the pathname is relative, it is taken relative to the web-app directory
+         * Returns the absolute path for the given pathname. If the pathname is relative, it is taken relative to the web-app directory
          * @param pathname
          * @return
@@ -359 +356 @@
         }
 
-
-
+        /**********************************************************************************
+         * 
+         * Methods for handling zip files
+         * 
+         **********************************************************************************/
+        
         /**
          * Determines whether a given file is a parsable zip file

trunk/grails-app/services/nl/tno/metagenomics/integration/SynchronizationService.groovy

@@ -91 +91 @@
          * @return      ArrayList       List of studies or null if the synchronization has failed
          */
-        public ArrayList<Study> synchronizeStudies() {
+        public ArrayList<Study> synchronizeStudies() throws NotAuthenticatedException, Exception {
                 if( !performSynchronization() )
                         return Study.findAllWhereTrashcan(false)
@@ -467 +467 @@
 
                 // Copy properties from gscf object
-                println "GSCF auth: " + gscfAuthorization
-                
                 if( gscfAuthorization.canRead instanceof Boolean  )
                         a.canRead = gscfAuthorization.canRead.booleanValue()
@@ -478 +476 @@
                         a.isOwner = gscfAuthorization.isOwner.booleanValue()
                         
-                println "Saved auth: " + a.canRead.toString() + " - " + a.canWrite.toString() + " - " + a.isOwner.toString()
-
                 a.save()