Changeset 75
- Timestamp:
- Nov 28, 2011, 9:18:01 AM (11 years ago)
- Location:
- trunk/grails-app
- Files:
-
- 11 edited
-
controllers/nl/tno/massSequencing/files/ImportController.groovy (modified) (5 diffs)
-
controllers/nl/tno/massSequencing/integration/RestController.groovy (modified) (1 diff)
-
controllers/nl/tno/massSequencing/integration/SynchronizeController.groovy (modified) (1 diff)
-
domain/nl/tno/massSequencing/AssaySample.groovy (modified) (7 diffs)
-
domain/nl/tno/massSequencing/Sequence.groovy (modified) (2 diffs)
-
domain/nl/tno/massSequencing/classification/Taxon.groovy (modified) (4 diffs)
-
services/nl/tno/massSequencing/ClassificationService.groovy (modified) (8 diffs)
-
services/nl/tno/massSequencing/integration/GscfService.groovy (modified) (2 diffs)
-
services/nl/tno/massSequencing/integration/SynchronizationService.groovy (modified) (1 diff)
-
services/nl/tno/massSequencing/integration/TrashService.groovy (modified) (10 diffs)
-
views/worker/process.gsp (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/grails-app/controllers/nl/tno/massSequencing/files/ImportController.groovy
r74 r75 5 5 import grails.converters.*; 6 6 import nl.tno.massSequencing.* 7 import nl.tno.massSequencing.auth.User 7 8 8 9 class ImportController { … … 96 97 97 98 // Perform the actual computations asynchronously 99 def loggedInUser = httpSession.user 98 100 runAsync { 99 101 def entity … … 105 107 break; 106 108 case "assay": 107 entity = getAssay( httpSession.process[ processId ].entityId );109 entity = getAssay( httpSession.process[ processId ].entityId, loggedInUser ); 108 110 break; 109 111 default: … … 564 566 } 565 567 566 protected Assay getAssay(def assayId ) {568 protected Assay getAssay(def assayId, User loggedInUser = null) { 567 569 // load assay with id specified by param.id 568 570 def assay … … 579 581 } 580 582 581 if (!assay.study.canRead( session.user ) ) {583 if (!assay.study.canRead( loggedInUser ?: session.user ) ) { 582 584 flash.error = "You don't have the right authorizaton to access assay " + assay.name 583 585 return null -
trunk/grails-app/controllers/nl/tno/massSequencing/integration/RestController.groovy
r63 r75 351 351 } 352 352 353 354 355 353 private def checkAssayToken( def assayToken ) { 356 354 if( !assayToken || assayToken == null ) { -
trunk/grails-app/controllers/nl/tno/massSequencing/integration/SynchronizeController.groovy
r51 r75 62 62 synchronizationService.sessionToken = session.sessionToken 63 63 synchronizationService.user = session.user 64 64 65 def previousEager = synchronizationService.eager; 66 synchronizationService.eager = true; 67 65 68 Study.findAllByTrashcan( false ).each { study -> 69 log.debug "Synchronize authrorization: " + study.id + ": " + study.name + " (" + study.token() + ")" 66 70 synchronizationService.synchronizeAuthorization( study ); 67 71 } 72 73 synchronizationService.eager = previousEager 68 74 69 75 if( params.redirectUrl ) -
trunk/grails-app/domain/nl/tno/massSequencing/AssaySample.groovy
r64 r75 1 1 package nl.tno.massSequencing 2 import nl.tno.massSequencing.classification.* 2 3 3 4 import org.codehaus.groovy.grails.commons.ApplicationHolder as AH … … 12 13 // Grails datasource is used for executing custom SQL statement 13 14 def dataSource 15 def sessionFactory 14 16 15 17 // To be computed at run time (and saved in cache) … … 76 78 */ 77 79 public int numClassifiedSequences() { 78 return Sequence.countByAssaySample( this ); 80 int numSeqs = 0; 81 if( this.sequenceData ) { 82 sequenceData.each { 83 def sequenceDataNum = Sequence.countBySequenceData( it ); 84 numSeqs += sequenceDataNum ?: 0; 85 } 86 } 87 88 return numSeqs 79 89 } 80 90 … … 280 290 otherAssaySample.numSequences = numSequences; 281 291 292 otherAssaySample.save( flush: true ); 293 282 294 // Move attached data 283 295 def dataList = [] + sequenceData?.toList() … … 293 305 // Clone the sequencedata object 294 306 def sd = dataList[ j ]?.clone(); 295 307 296 308 if( sd ) 297 309 otherAssaySample.addToSequenceData( sd ); 298 310 299 311 // Copy all sequence classifications to the new sequenceData 300 Sequence.executeUpdate( "UPDATE Sequence s SET s.sequenceData = :new WHERE s.sequenceData = :old ", [ 'old': dataList[ j ], 'new': sd ] ) 312 //log.debug "Removing sequences from sequencedata " + dataList[ j ]?.id + " to " + sd?.id 313 //Sequence.executeUpdate( "UPDATE Sequence s SET s.sequenceData = :new WHERE s.sequenceData = :old ", [ 'old': dataList[ j ], 'new': sd ] ) 314 //Sequence.executeUpdate( "DELETE FROM Sequence s WHERE s.sequenceData = :old ", [ 'old': dataList[ j ] ] ) 301 315 302 316 // Remove the old sequencedata object … … 310 324 otherAssaySample.run.removeFromAssaySamples( otherAssaySample ); 311 325 } 312 326 313 327 // Remove this sample from the run. 314 328 if( run ) { … … 319 333 otherAssaySample.run = null; 320 334 } 335 321 336 } 322 337 -
trunk/grails-app/domain/nl/tno/massSequencing/Sequence.groovy
r59 r75 3 3 import nl.tno.massSequencing.classification.Taxon 4 4 5 class Sequence {5 class Sequence implements Serializable { 6 6 // Unique name of this sequence 7 7 String name … … 22 22 23 23 static mapping = { 24 id composite: ['name'] 24 25 name index: 'name_idx' 25 26 classification index: 'taxon_idx' 26 sequenceData index: 'sequenceData_idx' 27 sequenceData index: 'sequenceData_idx', cascade: "all-delete-orphan" 28 27 29 } 28 30 } -
trunk/grails-app/domain/nl/tno/massSequencing/classification/Taxon.groovy
r70 r75 169 169 * @return First taxon that is found and matches the criteria or null if nothing is found 170 170 */ 171 public static Taxon findTaxonByPath( def path, int startLevel = 0 ) {171 public static Taxon findTaxonByPath( def path, int startLevel = 0, taxonCache ) { 172 172 if( path.size() == 0 ) 173 173 return null; 174 175 // Check taxon cache 176 def cacheTaxon = findTaxonInCache( taxonCache, path ); 177 if( cacheTaxon ) 178 return cacheTaxon; 174 179 175 180 def leafLevel = path.size() - 1 + startLevel; … … 203 208 } 204 209 210 storeTaxonInCache( taxonCache, path, leaf ); 211 205 212 return leaf; 206 213 } 207 214 208 215 return null; 216 } 217 218 public static emptyTaxonCache() { 219 return [ "top": [:], "sub": [:] ]; 220 } 221 222 protected static Taxon findTaxonInCache( cache, path ) { 223 if( path.size() <= 3 ) { 224 return cache[ "top" ][ cacheKey( path ) ]; 225 } else { 226 def topCacheKey = cacheKey( path[0..2] ); 227 228 if( !cache[ "sub" ][ topCacheKey ] ) 229 return null; 230 231 return findTaxonInCache( cache[ "sub" ][ topCacheKey ], path[ 3..-1] ); 232 } 233 } 234 235 protected static void storeTaxonInCache( cache, path, taxon ) { 236 // Keep a 'layered' cache: the first three levels are kept in this cache. The 237 // next levels are kept in a separate cache, one map for each subtree, so 238 // each map will have a size that we can still deal with 239 // [ 240 // "top": [ "bacteria": x, "bacteria;firmicutes": y ], 241 // "sub": [ "bacteria;firmicutes;abc": 242 // [ "def;ghi" : z, "def;gkl": m ] 243 // ] 244 if( path.size() <= 3 ) { 245 cache[ "top" ][ cacheKey( path ) ] = taxon; 246 } else { 247 def topCacheKey = cacheKey( path[0..2] ); 248 def restPath = path[3..-1] 249 250 if( cache[ "sub" ][ topCacheKey ] == null ) { 251 cache[ "sub" ][ topCacheKey ] = emptyTaxonCache(); 252 } 253 254 storeTaxonInCache( cache[ "sub" ][ topCacheKey ], restPath, taxon ) 255 } 256 257 } 258 259 protected static cacheKey( path ) { 260 return path.join( ";" ); 209 261 } 210 262 … … 215 267 * @param path An array or list with the names of the taxa in the path, ordered by level 216 268 * @param startLevel Which level is the first entry in the list. Defaults to zero. Can be used in order to find taxa 217 * without the whole tree being specified (e.g. without the root element) 269 * without the whole tree being specified (e.g. without the root element) 270 * @param taxonCache Hashmap with cached data about taxa found 218 271 * @return First taxon that is found and matches the criteria or a new taxon if it didn't exist 219 272 */ 220 static Taxon findOrCreateTaxonByPath( def path, int startLevel = 0 ) {221 def taxon = findTaxonByPath( path, startLevel );273 static Taxon findOrCreateTaxonByPath( def path, int startLevel = 0, def taxonCache = null ) { 274 def taxon = findTaxonByPath( path, startLevel, taxonCache ); 222 275 223 276 if( taxon ) … … 233 286 // However, we don't have to check the highest level, so we start at depth - 2 234 287 for( def level = depth - 2; level >= 0 && !found ; level-- ) { 235 parentNode = findTaxonByPath( path[ 0 .. level ], startLevel );288 parentNode = findTaxonByPath( path[ 0 .. level ], startLevel, taxonCache ); 236 289 237 290 // If this taxon is found, it is the highest level -
trunk/grails-app/services/nl/tno/massSequencing/ClassificationService.groovy
r74 r75 148 148 def i = 0; 149 149 def start = System.currentTimeMillis(); 150 def lapTime = start;151 150 152 151 // Create a stateless session in order to speed up inserts 153 152 StatelessSession statelessSession = sessionFactory.openStatelessSession(); 153 def connection = statelessSession.connection() 154 def statement 154 155 155 156 // For each line in the taxonomy file, read the sequence and save it … … 158 159 long bytesProcessed = 0; 159 160 161 def lapTime = System.currentTimeMillis(); 162 163 // Store a cache for taxa 164 def taxonCache = Taxon.emptyTaxonCache(); 165 160 166 try { 161 167 while( ( line = taxonomyReader.readLine() ) ) { … … 169 175 continue; 170 176 } 171 177 172 178 sequenceName = parts[ 0 ]; 173 179 classification = parts[ 1 ]; … … 177 183 178 184 // Find taxon or create one if needed 179 taxon = Taxon.findOrCreateTaxonByPath( taxa, startLevel );185 taxon = Taxon.findOrCreateTaxonByPath( taxa, startLevel, taxonCache ); 180 186 181 187 if( taxon ) { … … 187 193 188 194 statelessSession.insert(s); 195 189 196 } else { 190 197 log.trace( "" + i + " Sequence " + sequenceName + " not imported because it is unclassified." ) 191 198 } 192 199 193 200 if( i % 80 == 0 ) { 194 201 onProgress( bytesProcessed ); 195 202 bytesProcessed = 0; 196 203 } 197 204 198 205 importedSequences++; 199 206 … … 213 220 taxonomyReader.close(); 214 221 } 215 222 223 224 log.trace "Processing taxonomy file " + taxonomyFilename + " took: " + ( System.currentTimeMillis() - lapTime ) + "ms for " + importedSequences + " sequences. On average, that is " + ( ( System.currentTimeMillis() - lapTime ) / importedSequences ) + "ms per sequence." 225 226 // Update progress 216 227 onProgress( bytesProcessed ); 217 228 … … 236 247 Classification.executeUpdate( "DELETE FROM Classification WHERE assaySample = :assaySample", [ 'assaySample': assaySample ] ); 237 248 238 // Update classification table with data from sequences 239 240 // Unfortunately, Hibernate (through HQL) can't handle bulk insertion if a table has a 241 // composite id (Classification). See http://opensource.atlassian.com/projects/hibernate/browse/HHH-3434 242 // For that reason we execute plain SQL 243 def connection = sessionFactory.currentSession.connection() 244 def statement = connection.createStatement(); 245 246 // This statement searches within the sequences for every taxon that is mentioned (as a leaf-node in the tree) 247 // and inserts statistics about those taxons into the classification table. 249 // The process below might seem a bit complicated. If we just search the sequence table for COUNT(*) for each taxon, 250 // we would only insert statistics for the leaf nodes in the database. 248 251 // 249 // However, if we have the following sequences:252 // For example, if we have the following sequences: 250 253 // sequence1 Bacteria -> Firmicutes -> Clostridia -> (unclassified) 251 254 // sequence2 Bacteria -> (unclassified) … … 257 260 // and nothing for firmicutes. Of course, these values can be computed at any time, since 258 261 // the taxonomy tree is known. Since these values are used for searching and exporting (e.g. show all samples 259 // where firmicutes are present), it is essential that the extra values are also added. This is done in the second statement 260 261 // Insert statistics for all sequences and (leaf) taxa 262 statement.executeUpdate( connection.nativeSQL( """ 263 INSERT INTO classification (assay_sample_id, taxon_id, unclassified, total) 264 SELECT sa.id, s.classification_id, count( * ), 265 ( 266 SELECT count( * ) 267 FROM sequence s2 268 LEFT JOIN taxonomy t2 ON s2.classification_id = t2.id 269 LEFT JOIN sequence_data sd2 ON s2.sequence_data_id = sd2.id 270 LEFT JOIN assay_sample sa2 ON sd2.sample_id = sa2.id 271 WHERE sa2.id = sa.id AND t2.lft >= t.lft AND t2.rgt <= t.rgt 272 ) 273 FROM sequence s 274 LEFT JOIN taxonomy t ON s.classification_id = t.id 275 LEFT JOIN sequence_data sd ON s.sequence_data_id = sd.id 276 LEFT JOIN assay_sample sa ON sd.sample_id = sa.id 277 WHERE sa.id = """ + assaySample.id + """ 278 GROUP BY sa.id, s.classification_id, t.lft, t.rgt 279 """ ) ); 280 281 // Update the table with missing values 282 statement.executeUpdate( connection.nativeSQL( """ 283 INSERT INTO classification (assay_sample_id, taxon_id, unclassified, total) 284 SELECT * FROM ( 285 SELECT a.id as assay_sample_id, t.id as taxon_id, 0 as unclassified, 286 ( 287 SELECT sum( c.unclassified ) 288 FROM classification c 289 LEFT JOIN taxonomy t3 ON c.taxon_id = t3.id 290 WHERE c.assay_sample_id = a.id 291 AND t3.lft >= t.lft 292 AND t3.rgt <= t.rgt 293 ) AS total 294 FROM taxonomy t, assay_sample a 295 WHERE a.id = """ + assaySample.id + """ 296 AND t.id NOT IN ( 297 SELECT t2.id 298 FROM classification c 299 LEFT JOIN taxonomy t2 ON c.taxon_id = t2.id 300 WHERE c.assay_sample_id = a.id 301 ) 302 ) missingClassifications WHERE missingClassifications.total IS NOT NULL 303 """ ) ); 304 262 // where firmicutes are present), it is essential that the extra values are also added. 263 264 // First find the number of sequences that have been classified as a specific taxon or a subtaxon of that taxon 265 def numTotals = Taxon.executeQuery( "SELECT t.id, count(*) FROM Taxon t, Sequence s LEFT JOIN s.classification t2 WHERE t2.lft >= t.lft AND t2.rgt <= t.rgt AND s.sequenceData.sample = :assaySample GROUP BY t.id ORDER BY t.id", [ "assaySample": assaySample ] ); 266 267 // Also find the number of sequences that have been classified as that specific taxon, but not further down (numUnClassified) 268 def numUnclassifieds = Sequence.executeQuery( "SELECT s.classification.id, count(*) FROM Sequence s WHERE s.sequenceData.sample = :assaySample GROUP BY s.classification.id ORDER BY s.classification.id", [ "assaySample": assaySample ]); 269 270 def unClassifiedPointer = 0; 271 def unClassifiedsSize = numUnclassifieds.size() 272 273 // Combine the two lists. The numTotals list is the full list, and the numUnclassifieds list should be entered in it 274 def insertQuadriples = [] // assay_sample_id, taxon_id, unclassified, total 275 numTotals.each { lineTotal -> 276 def line = [ assaySample.id, lineTotal[ 0 ], 0, lineTotal[ 1 ] ]; 277 278 // Check whether we have an entry in numUnclassifieds. If not, the number of unclassified sequences for this taxon is zero 279 if( unClassifiedPointer < unClassifiedsSize && numUnclassifieds[ unClassifiedPointer ][ 0 ] == lineTotal[ 0 ] ) { 280 line[ 2 ] = numUnclassifieds[ unClassifiedPointer ][ 1 ]; 281 unClassifiedPointer++; 282 } 283 284 insertQuadriples << line 285 } 286 287 // Insert the quadriples into the database 288 if( insertQuadriples.size() > 0 ) { 289 // Unfortunately, Hibernate (through HQL) can't handle bulk insertion if a table has a 290 // composite id (Classification). See http://opensource.atlassian.com/projects/hibernate/browse/HHH-3434 291 // For that reason we execute plain SQL 292 def connection = sessionFactory.currentSession.connection() 293 def statement = connection.createStatement(); 294 295 // Insert statistics for all sequences and taxa. All values in the quadriples list are numbers, 296 // so no escaping has to be done. 297 def sql = "INSERT INTO classification (assay_sample_id, taxon_id, unclassified, total) VALUES"; 298 sql += insertQuadriples.collect { "( " + it.join( ", " ) + " )" }.join( ", " ); 299 300 statement.executeUpdate( connection.nativeSQL( sql ) ); 301 } 305 302 } 306 303 -
trunk/grails-app/services/nl/tno/massSequencing/integration/GscfService.groovy
r70 r75 26 26 public String urlAuthRemote( def params, def token, appendParameters = true ) { 27 27 def redirectURL = "${config.gscf.baseURL}/login/auth_remote?moduleURL=${this.moduleURL()}&consumer=${this.consumerID()}&token=${token}&" 28 def returnUrl 28 def returnUrl = config.grails.serverURL 29 29 30 30 // Append other parameters (but only if this request is a GET request) … … 38 38 } 39 39 40 returnUrl = config.grails.serverURL41 40 if (params.controller != null){ 42 41 returnUrl += "/${params.controller}" -
trunk/grails-app/services/nl/tno/massSequencing/integration/SynchronizationService.groovy
r74 r75 464 464 return Auth.findByUserAndStudy( user, study ) 465 465 466 if( !performSynchronization() )467 return Auth.findByUserAndStudy( user, study )468 469 466 def gscfAuthorization 470 467 try { -
trunk/grails-app/services/nl/tno/massSequencing/integration/TrashService.groovy
r65 r75 2 2 3 3 import nl.tno.massSequencing.* 4 import nl.tno.massSequencing.classification.* 4 5 import nl.tno.massSequencing.auth.Auth; 5 6 … … 18 19 return 19 20 20 saveDataInTrash( study );21 //saveDataInTrash( study ); 21 22 22 23 def l = [] … … 32 33 // Remove sequence data first, since it gives problems when keeping it 33 34 if( study.assays ) { 34 l = [] + study.assays*.assaySamples*.sequenceData?.flatten() 35 36 l.each { 37 if( it ) 38 it.sample.removeFromSequenceData( it ); 39 } 35 // l = [] + study.assays*.assaySamples*.sequenceData?.flatten() 36 // 37 // l.each { 38 // if( it ) 39 // it.sample.removeFromSequenceData( it ); 40 // } 41 42 l = [] + study.assays 43 l.each { moveToTrash( it ); } 40 44 } 41 45 study.delete(flush:true); … … 47 51 */ 48 52 def moveToTrash( Assay assay ) { 49 saveDataInTrash( assay );53 //saveDataInTrash( assay ); 50 54 51 55 // Remove associations … … 93 97 */ 94 98 def moveToTrash( Sample sample ) { 95 saveDataInTrash( sample );99 //saveDataInTrash( sample ); 96 100 97 101 // Remove associations … … 187 191 188 192 def assaySamples = assay.assaySamples.findAll { it.containsData() } 189 193 190 194 // For every assay sample that contains data, save that data in the trashcan 191 195 if( assaySamples.size() > 0 ) { … … 195 199 Assay dummyAssay = new Assay( assayToken: newAssayToken, name: assay.name, study: trashcan ); 196 200 trashcan.addToAssays( dummyAssay ); 197 dummyAssay.save() 198 201 dummyAssay.save( flush: true ) 202 203 // First delete all uploaded sequences for the given assaySamples 204 def sequences = Sequence.executeQuery( "FROM Sequence s WHERE s.sequenceData.sample IN (:assaySamples)", [ "assaySamples": assaySamples ] ) 205 if( sequences ) 206 Sequence.executeUpdate( "DELETE FROM Sequence s WHERE s IN (:sequences)", [ "sequences": sequences ] ) 207 199 208 assaySamples.each { assaySample -> 209 log.debug "Create copy of " + assaySample 200 210 Sample sample = assaySample.sample 201 211 … … 204 214 Sample dummySample = Sample.cloneSample( sample, newSampleToken, trashcan ); 205 215 trashcan.addToSamples( dummySample ); 206 dummySample.save() 207 216 dummySample.save( flush: true ) 217 218 log.debug "copy of sample has been saved"; 219 208 220 // Create dummy assay sample 209 221 AssaySample dummyAssaySample = new AssaySample( assay: dummyAssay, sample: dummySample, numSequences: assaySample.numSequences ); … … 211 223 dummyAssay.addToAssaySamples( dummyAssaySample ); 212 224 dummySample.addToAssaySamples( dummyAssaySample ); 213 dummyAssaySample.save(); 225 dummyAssaySample.save( flush: true ); 226 227 log.debug "copy of assaysample has been saved" 214 228 215 229 // Move data from this assay sample to the trash version of it 216 230 assaySample.moveValuableDataTo( dummyAssaySample ); 231 232 log.debug "valuable data has been copied" 217 233 218 234 // Remove the assaySample from its run, since otherwise the statistics of the run (#sequences for example) … … 220 236 dummyAssaySample.run?.removeFromAssaySamples( dummyAssaySample ); 221 237 222 dummyAssaySample.save(); 238 log.debug "old assaySample has been removed from run" 239 240 dummyAssaySample.save( flush: true ); 223 241 } 224 242 } 225 243 226 244 // Remove all assaysamples from this assay from their run 245 log.debug "removing all assaysamples from their run" 227 246 assay.assaySamples?.each { assaySample -> 228 247 assaySample.run?.removeFromAssaySamples( assaySample ); 229 248 } 249 250 251 log.debug "assaysamples removed"; 230 252 231 253 // Remove this assay from the runs, since otherwise the samples will be resaved upon delete -
trunk/grails-app/views/worker/process.gsp
r70 r75 25 25 }); 26 26 27 progressInterval = setTimeout(updateProgress, 250);27 progressInterval = setTimeout(updateProgress, 500); 28 28 29 29 }); … … 67 67 } 68 68 } else { 69 // Make sure the next progress will be retrieved in 250 ms70 progressInterval = setTimeout(updateProgress, 250);69 // Make sure the next progress will be retrieved in 500 ms 70 progressInterval = setTimeout(updateProgress, 500); 71 71 } 72 72 },
Note: See TracChangeset
for help on using the changeset viewer.
