Changeset 3 for trunk/grails-app/services/nl
Legend:
- Unmodified
- Added
- Removed
-
trunk
- Property svn:ignore
-
old new 4 4 .classpath 5 5 .project 6 fileuploads
-
- Property svn:ignore
-
trunk/grails-app/services/nl/tno/metagenomics/FastaService.groovy
r2 r3 1 1 package nl.tno.metagenomics 2 3 2 4 3 import java.io.File; … … 9 8 def fileService 10 9 def fuzzySearchService 11 12 10 11 static transactional = true 13 12 14 13 /** 15 14 * Parses uploaded files and checks them for FASTA and QUAL files 16 15 * @param filenames List of filenames currently existing in the upload directory 16 * @param onProgress Closure to execute when progress indicators should be updated. 17 * Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number 18 * of files and bytes that have been processed in total 17 19 * @param directory Directory to move the files to 18 20 * @return Structure with information about the parsed files. The 'success' files are … … 32 34 * 33 35 */ 34 def parseFiles( ArrayList filenames, File directory = null ) {36 def parseFiles( ArrayList filenames, Closure onProgress, File directory = null ) { 35 37 if( filenames.size() == 0 ) { 36 38 return [ success: [], failure: [] ]; … … 43 45 def success = []; 44 46 def failure = []; 47 48 long filesProcessed = 0; 49 long bytesProcessed = 0; 45 50 46 51 // Loop through all filenames … … 55 60 } else { 56 61 try { 57 def contents = parseFile( file, filetype ); 58 62 def contents = parseFile( file, filetype, { files, bytes -> 63 filesProcessed += files; 64 bytesProcessed += bytes; 65 66 onProgress( filesProcessed, bytesProcessed ); 67 } ); 68 59 69 contents.filename = file.getName(); 60 contents.originalfilename = fileService.originalFilename( contents.filename ) 61 70 contents.originalfilename = fileService.originalFilename( contents.filename ) 71 62 72 if( contents.success ) { 63 73 success << contents; … … 71 81 failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: e.getMessage() ]; 72 82 } 73 83 74 84 } 75 85 } … … 103 113 def quals = parsedFiles.findAll { it.type == "qual" } 104 114 samples = samples.toList() 105 115 106 116 def files = []; 107 117 108 118 fastas.each { fastaFile -> 109 119 // Remove extension 110 def matchWith = fastaFile.originalfilename.substring( 0, fastaFile. filename.lastIndexOf( '.' ) )111 120 def matchWith = fastaFile.originalfilename.substring( 0, fastaFile.originalfilename.lastIndexOf( '.' ) ) 121 112 122 // Determine feasible quals (based on number of sequences ) 113 123 def feasibleQuals = quals.findAll { it.numSequences == fastaFile.numSequences } 114 124 115 125 // Best matching qual file 116 126 def qualIdx = fuzzySearchService.mostSimilarWithIndex( matchWith + '.qual', feasibleQuals.originalfilename ); … … 119 129 if( qualIdx != null ) 120 130 qual = feasibleQuals[ qualIdx ]; 121 131 122 132 // Best matching sample 123 133 def sampleIdx = fuzzySearchService.mostSimilarWithIndex( matchWith, samples.sample.name ); 124 134 def assaySample = null 125 if( sampleIdx != null ) { 135 if( sampleIdx != null ) { 126 136 assaySample = samples[ sampleIdx ]; 127 137 } 128 138 129 139 files << [ 130 fasta: fastaFile,131 feasibleQuals: feasibleQuals,132 qual: qual,133 assaySample: assaySample134 ]135 } 136 140 fasta: fastaFile, 141 feasibleQuals: feasibleQuals, 142 qual: qual, 143 assaySample: assaySample 144 ] 145 } 146 137 147 return files; 138 139 140 } 141 142 148 149 150 } 151 152 143 153 /** 144 154 * Determines the file type of a given file, based on the extension. … … 195 205 /** 196 206 * Parses the given file 197 * @param file File to parse 198 * @param filetype Type of the given file 199 * @return List structure. Examples: 207 * @param file File to parse 208 * @param filetype Type of the given file 209 * @param onProgress Closure to execute when progress indicators should be updated. 210 * Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number 211 * of files and bytes that have been processed in this file (so the first parameter should 212 * only be 1 when the file is finished) 213 * 214 * @return List structure. Examples: 200 215 * 201 216 * [ success: true, filename: 'abc.fasta', type: 'fasta', numSequences: 200 ] … … 203 218 * [ success: false, filename: 'abc.txt', type: 'txt', message: 'Filetype could not be parsed.' ] 204 219 */ 205 protected def parseFile( File file, String filetype ) {220 protected def parseFile( File file, String filetype, Closure onProgress ) { 206 221 switch( filetype ) { 207 222 case "fasta": 208 return parseFasta( file );223 return parseFasta( file, onProgress ); 209 224 case "qual": 210 return parseQual( file );225 return parseQual( file, onProgress ); 211 226 default: 227 onProgress( 1, file.length() ); 212 228 return [ success: false, type: filetype, message: 'Filetype could not be parsed.' ] 213 229 } … … 216 232 /** 217 233 * Parses the given FASTA file 218 * @param file File to parse 219 * @return List structure. Examples: 234 * @param file File to parse 235 * @param onProgress Closure to execute when progress indicators should be updated. 236 * Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number 237 * of files and bytes that have been processed in this file (so the first parameter should 238 * only be 1 when the file is finished) 239 * @return List structure. Examples: 220 240 * 221 241 * [ success: true, filename: 'abc.fasta', type: 'fasta', numSequences: 200 ] 222 242 * [ success: false, filename: 'def.fasta', type: 'fasta', message: 'File is not a valid FASTA file' ] 223 243 */ 224 protected def parseFasta( File file ) {225 244 protected def parseFasta( File file, Closure onProgress ) { 245 226 246 long startTime = System.nanoTime(); 227 log.trace "Start parsing FASTA " + file.getName() 228 247 log.trace "Start parsing FASTA " + file.getName() 248 229 249 // Count the number of lines, starting with '>' (and where the following line contains a character other than '>') 230 250 long numSequences = 0; 251 long bytesProcessed = 0; 231 252 boolean lookingForSequence = false; 232 253 … … 241 262 } 242 263 } 243 } 244 } 264 265 266 // Update progress every time 1MB is processed 267 bytesProcessed += line.size(); 268 if( bytesProcessed > 1000000 ) { 269 onProgress( 0, bytesProcessed ); 270 bytesProcessed = 0; 271 } 272 } 273 274 275 276 } 277 278 // Update progress and say we're finished 279 onProgress( 1, bytesProcessed ); 245 280 246 281 log.trace "Finished parsing FASTA " + file.getName() + ": " + ( System.nanoTime() - startTime ) / 1000000L … … 251 286 /** 252 287 * Parses the given QUAL file 253 * @param file File to parse 254 * @return List structure. Examples: 288 * @param file File to parse 289 * @param onProgress Closure to execute when progress indicators should be updated. 290 * Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number 291 * of files and bytes that have been processed in this file (so the first parameter should 292 * only be 1 when the file is finished) 293 * @return List structure. Examples: 255 294 * 256 295 * [ success: true, filename: 'abc.qual', type: 'qual', numSequences: 200, avgQuality: 31 ] 257 296 * [ success: false, filename: 'def.qual', type: 'qual', message: 'File is not a valid QUAL file' ] 258 297 */ 259 protected def parseQual( File file ) {298 protected def parseQual( File file, Closure onProgress ) { 260 299 long startTime = System.nanoTime(); 261 300 log.trace "Start parsing QUAL " + file.getName() 262 301 263 // Count the number of lines, starting with '>'. After we've found such a character, we continue looking for 302 // Count the number of lines, starting with '>'. After we've found such a character, we continue looking for 264 303 // quality scores 265 304 long numSequences = 0; 305 long bytesProcessed = 0; 266 306 def quality = [ quality: 0.0, number: 0L ] 267 307 268 308 boolean lookingForFirstQualityScores = false; 269 309 file.eachLine { line -> … … 275 315 numSequences++; 276 316 lookingForFirstQualityScores = false; 277 317 278 318 quality = updateQuality( quality, line ); 279 319 } … … 281 321 quality = updateQuality( quality, line ); 282 322 } 283 } 284 } 323 324 // Update progress every time 1MB is processed 325 bytesProcessed += line.size(); 326 if( bytesProcessed > 1000000 ) { 327 onProgress( 0, bytesProcessed ); 328 bytesProcessed = 0; 329 } 330 } 331 332 } 333 334 // Update progress and say we're finished 335 onProgress( 1, bytesProcessed ); 285 336 286 337 log.trace "Finished parsing QUAL " + file.getName() + ": " + ( System.nanoTime() - startTime ) / 1000000L … … 288 339 return [ success: true, type: "qual", numSequences: numSequences, avgQuality: quality.quality ]; 289 340 } 290 341 291 342 /** 292 343 * Parses the given line and updates the average quality based on the scores … … 298 349 // Determine current average 299 350 List tokens = line.tokenize(); 300 Long total = 0; 301 351 Long total = 0; 352 302 353 tokens.each { 303 354 total += Integer.parseInt( it ); 304 355 } 305 356 306 357 int numTokens = tokens.size(); 307 358 308 359 // Update the given average 309 360 if( numTokens > 0 ) { … … 311 362 quality.quality = quality.quality + ( ( total / numTokens as double ) - quality.quality ) / quality.number * numTokens; 312 363 } 313 364 314 365 return quality 315 366 } 316 367 317 368 /** 318 369 * Moves a fasta and qual file to their permanent location, and returns information about these files … … 326 377 File permanentDirectory = fileService.absolutePath( ConfigurationHolder.config.metagenomics.fileDir ); 327 378 def returnStructure = [:]; 328 379 329 380 if( fileService.fileExists( fastaFile ) ) { 330 381 // Lookup the original filename … … 340 391 throw new Exception( "Fasta file to save doesn't exist on disk" ); 341 392 } 342 393 343 394 if( qualFile && fileService.fileExists( qualFile ) ) { 344 395 // Lookup the original filename 345 396 def qualData = processedFiles.parsed.success.find { it.filename == qualFile }; 346 397 347 398 if( qualData ) { 348 399 returnStructure.qual = fileService.moveFileToUploadDir( fileService.get(qualFile ), qualData.originalfilename, permanentDirectory ); … … 358 409 returnStructure.avgQuality = 0; 359 410 } 360 411 361 412 return returnStructure; 362 413 } -
trunk/grails-app/services/nl/tno/metagenomics/integration/GscfService.groovy
r2 r3 27 27 */ 28 28 public String urlAuthRemote( def params, def token ) { 29 def redirectURL = "${config.gscf.baseURL}/login/auth_remote?moduleURL=${this.moduleURL()}&consumer=${this.consumerID()}&token=${token}&returnUrl=${config. metagenomics.baseURL}"29 def redirectURL = "${config.gscf.baseURL}/login/auth_remote?moduleURL=${this.moduleURL()}&consumer=${this.consumerID()}&token=${token}&returnUrl=${config.grails.serverURL}" 30 30 31 31 if (params.controller != null){ … … 330 330 */ 331 331 private String moduleURL() { 332 return config. metagenomics.baseURL332 return config.grails.serverURL 333 333 } 334 334 -
trunk/grails-app/services/nl/tno/metagenomics/integration/SynchronizationService.groovy
r2 r3 7 7 class SynchronizationService { 8 8 def gscfService 9 9 10 10 String sessionToken = "" // Session token to use for communication 11 11 User user = null // Currently logged in user. Must be set when synchronizing authorization 12 12 boolean eager = false // When set to true, this method fetches data about all studies from GSCF. Otherwise, it will only look at the 13 // studies marked as dirty in the database. Defaults to false. 13 // studies marked as dirty in the database. Defaults to false. 14 15 // Keeps track of the last time this module performed a full synchronization. 16 static Date lastFullSynchronization = null; 14 17 15 18 static transactional = true 16 19 17 20 /** 18 21 * Determines whether the synchronization should be performed or not. This can be entered … … 22 25 protected performSynchronization() { 23 26 def conf = ConfigurationHolder.config.metagenomics.synchronization; 24 27 25 28 // If nothing is entered in configuration, return true (default value) 26 29 if( conf == null ) 27 30 return true 31 32 return conf 33 } 34 35 /** 36 * Returns true iff a full synchronization should be performed 37 * @return 38 */ 39 public boolean timeForFullSynchronization() { 40 if( SynchronizationService.lastFullSynchronization == null ) 41 return true 28 42 29 return conf 43 // Compute the time since the last full synchronization in milliseconds 44 Date today = new Date(); 45 long difference = SynchronizationService.lastFullSynchronization.getTime() - today.getTime() 46 47 if( difference / 1000 > ConfigurationHolder.config.metagenomics.fullSynchronization ) 48 return true 49 else 50 return false 51 } 52 53 /** 54 * Redirects to a temporary page to give the user a 'waiting' page while synchronizing 55 * @return 56 */ 57 public String urlForFullSynchronization( def params ) { 58 def returnUrl = ConfigurationHolder.config.grails.serverURL 59 if (params.controller != null){ 60 returnUrl += "/${params.controller}" 61 if (params.action != null){ 62 returnUrl += "/${params.action}" 63 if (params.id != null){ 64 returnUrl += "/${params.id}" 65 } 66 } 67 } 68 if( timeForFullSynchronization() ) { 69 return ConfigurationHolder.config.grails.serverURL + "/synchronize/full?redirect=" + returnUrl.encodeAsURL() 70 } else { 71 return returnUrl 72 } 73 } 74 75 /** 76 * Performs a full synchronization in order to retrieve all studies 77 * @return 78 */ 79 public void fullSynchronization() { 80 if( !timeForFullSynchronization() ) 81 return 82 83 def previousEager = eager 84 eager = true 85 synchronizeStudies(); 86 eager = previousEager 87 88 SynchronizationService.lastFullSynchronization = new Date(); 30 89 } 31 90 … … 37 96 if( !performSynchronization() ) 38 97 return Study.findAll() 39 98 40 99 // When eager fetching is enabled, ask for all studies, otherwise only ask for studies marked dirty 41 100 // Synchronization is performed on all studies, not only the studies the user has access to. Otherwise … … 47 106 studies = Study.findAllWhere( [isDirty: true] ); 48 107 } 49 108 50 109 // Perform no synchronization if no studies have to be synchronized 51 110 // Perform synchronization on only one study directly, because otherwise … … 57 116 println "Study: " + studies[ 0] 58 117 def newStudy = synchronizeStudy( studies[0] ); 59 if( newStudy ) 118 if( newStudy ) 60 119 return [ newStudy ]; 61 else 120 else 62 121 return [] 63 122 } 64 123 65 124 // Fetch all studies from GSCF 66 125 def newStudies … … 68 127 if( !eager ) { 69 128 def studyTokens = studies.studyToken; 70 129 71 130 if( studyTokens instanceof String ) { 72 131 studyTokens = [studyTokens]; … … 86 145 synchronizeStudies( newStudies ); 87 146 studies = handleDeletedStudies( studies, newStudies ); 88 147 89 148 log.trace( "Returning " + studies.size() + " studies after synchronization" ) 90 149 91 150 return studies 92 151 } 93 152 94 153 /** 95 154 * Synchronizes all studies given by 'newStudies' with existing studies in the database, and adds them … … 107 166 108 167 Study studyFound = Study.findByStudyToken( gscfStudy.studyToken as String ) 109 168 110 169 if(studyFound) { 111 170 log.trace( "Study found with name " + studyFound.name ) 112 171 113 172 // Synchronize the study itself with the data retrieved 114 173 synchronizeStudy( studyFound, gscfStudy ); … … 123 182 synchronizeAuthorization(studyFound); 124 183 synchronizeStudyAssays(studyFound); 125 184 126 185 // Mark the study as clean 127 186 studyFound.isDirty = false … … 131 190 } 132 191 } 133 192 134 193 /** 135 194 * Removes studies from the database that are expected but not found in the list from GSCF … … 151 210 log.trace( "Study " + existingStudy.studyToken + " not found. Check whether it is removed or the user just can't see it." ) 152 211 153 // Study was not given to us by GSCF. This might be because the study is removed, or because the study is not visible (anymore) 212 // Study was not given to us by GSCF. This might be because the study is removed, or because the study is not visible (anymore) 154 213 // to the current user. 155 214 // Synchronize authorization and see what is the case (it returns null if the study has been deleted) … … 160 219 } 161 220 } 162 221 163 222 return studies 164 223 } 165 224 166 225 /** 167 226 * Synchronizes the given study with the data from GSCF … … 172 231 if( !performSynchronization() ) 173 232 return study 174 233 175 234 if( study == null ) 176 235 return null … … 179 238 if( !eager && !study.isDirty ) 180 239 return study; 181 240 182 241 // Retrieve the study from GSCF 183 242 def newStudy … … 218 277 if( !performSynchronization() ) 219 278 return study 220 279 221 280 if( study == null || newStudy == null) 222 281 return null … … 224 283 // If the study hasn't changed, don't update anything 225 284 if( !eager && !study.isDirty ) { 226 return study; 285 return study; 227 286 } 228 287 … … 231 290 return null; 232 291 } 233 292 234 293 // Mark study dirty to enable synchronization 235 294 study.isDirty = true; 236 295 synchronizeAuthorization( study ); 237 296 synchronizeStudyAssays( study ); 238 297 239 298 // Update properties and mark as clean 240 299 study.name = newStudy.title … … 254 313 return study.assays.toList() 255 314 256 if( !eager && !study.isDirty ) 257 return study.assays as List 258 315 if( !eager && !study.isDirty ) 316 return study.assays as List 317 259 318 // Also update all assays, belonging to this study 260 319 // Retrieve the assays from GSCF … … 287 346 return handleDeletedAssays( study, newAssays ); 288 347 } 289 348 290 349 /** 291 350 * Synchronizes the assays of a study with the given data from GSCF … … 323 382 } 324 383 } 325 384 326 385 /** 327 386 * Removes assays from the system that have been deleted from GSCF … … 354 413 } 355 414 } 356 415 357 416 return study.assays.toList() 358 417 } 359 418 360 419 /** 361 420 * Retrieves the authorization for the currently logged in user … … 374 433 throw new Exception( "Property user of SynchronizationService must be set to the currently logged in user" ); 375 434 } 376 435 377 436 // Only perform synchronization if needed 378 437 if( !eager && !study.isDirty ) 379 438 return Auth.findByUserAndStudy( user, study ) 380 439 381 440 if( !performSynchronization() ) 382 441 return Auth.findByUserAndStudy( user, study ) 383 442 384 443 def gscfAuthorization 385 444 try { … … 389 448 // TODO: handle deletion 390 449 log.trace( "Study " + study.studyToken + " has been deleted. Remove all authorization on that study") 391 450 392 451 Auth.findAllByStudy( study ).each { 393 452 it.delete() 394 453 } 395 396 return null 397 } 398 454 455 return null 456 } 457 399 458 // Update the authorization object, or create a new one 400 459 Auth a = Auth.authorization( study, user ) 401 460 402 461 if( !a ) { 403 462 log.trace( "Authorization not found for " + study.studyToken + " and " + user.username + ". Creating a new object" ); 404 463 405 464 a = Auth.createAuth( study, user ); 406 465 } 407 466 408 467 // Copy properties from gscf object 409 468 a.canRead = gscfAuthorization.canRead 410 469 a.canWrite = gscfAuthorization.canWrite 411 470 a.isOwner = gscfAuthorization.isOwner 412 471 413 472 a.save() 414 473 415 474 return a 416 475 } … … 431 490 if( !eager && !assay.study.isDirty ) 432 491 return assay 433 492 434 493 // Retrieve the assay from GSCF 435 494 def newAssay … … 477 536 if( !eager && !assay.study.isDirty ) 478 537 return assay 479 538 480 539 // If new assay is empty, something went wrong 481 540 if( newAssay.size() == 0 ) { … … 535 594 } 536 595 537 596 538 597 synchronizeAssaySamples( assay, newSamples ); 539 598 return handleDeletedSamples( assay, newSamples ); 540 599 } 541 600 542 601 /** 543 602 * Synchronize all samples for a given assay with the data from GSCF … … 597 656 } 598 657 } 599 658 600 659 /** 601 660 * Removes samples from the system that have been removed from an assay in GSCF … … 612 671 for( int i = numAssaySamples - 1; i >= 0; i-- ) { 613 672 def existingSample = assaySamples[i]; 614 673 615 674 AssaySample sampleFound = newSamples.find { it.name == existingSample.sample.sampleToken } 616 675 617 676 if( !sampleFound ) { 618 677 log.trace( "Sample " + existingSample.sample.sampleToken + " not found. Removing it." ) 619 678 620 679 // The sample has been removed 621 680 // TODO: What to do with the data associated with this AssaySample (e.g. sequences)? See also GSCF ticket #255
Note: See TracChangeset
for help on using the changeset viewer.
