Changeset 5 for trunk/grails-app
- Timestamp:
- Jan 17, 2011, 5:08:16 PM (13 years ago)
- Location:
- trunk/grails-app
- Files:
-
- 3 edited
-
controllers/nl/tno/metagenomics/FastaController.groovy (modified) (1 diff)
-
services/nl/tno/metagenomics/FastaService.groovy (modified) (5 diffs)
-
services/nl/tno/metagenomics/files/FileService.groovy (modified) (14 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/grails-app/controllers/nl/tno/metagenomics/FastaController.groovy
r3 r5 107 107 * The second parameter is a callback function to update progress indicators 108 108 */ 109 def parsedFiles = fastaService.parseFiles( filenames, { files, bytes -> 109 def parsedFiles = fastaService.parseFiles( filenames, { files, bytes, totalFiles, totalBytes -> 110 session.processProgress.numFiles += totalFiles; 111 session.processProgress.numBytes += totalBytes; 110 112 session.processProgress.filesProcessed = files; 111 113 session.processProgress.bytesProcessed = bytes; -
trunk/grails-app/services/nl/tno/metagenomics/FastaService.groovy
r4 r5 15 15 * @param filenames List of filenames currently existing in the upload directory 16 16 * @param onProgress Closure to execute when progress indicators should be updated. 17 * Has 2 parameters: numFilesProcessed and numBytesProcessed that indicate the number 18 * of files and bytes that have been processed in total 17 * Has 4 parameters: numFilesProcessed, numBytesProcessed that indicate the number 18 * of files and bytes that have been processed in total. totalFiles, totalBytes indicate the change 19 * in total number of files and bytes (e.g. should take 1 if a new file is added to the list) 19 20 * @param directory Directory to move the files to 20 21 * @return Structure with information about the parsed files. The 'success' files are … … 50 51 51 52 // Loop through all filenames 52 filenames.each { filename -> 53 // TODO: Handle zipped files 54 def file = fileService.get( filename ); 55 String filetype = determineFileType( file ); 56 57 if( !fileTypeValid( filetype ) ) { 58 fileService.delete(filename); 59 failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: 'File type not accepted' ]; 53 for( int i = 0; i < filenames.size(); i++ ) { 54 def filename = filenames[ i ]; 55 56 if( fileService.isZipFile( filename ) ) { 57 // ZIP files are extracted and appended to the filenames list. 58 def newfiles = fileService.extractZipFile( filename, { files, bytes, totalFiles, totalBytes -> 59 filesProcessed += files; 60 bytesProcessed += bytes; 61 62 onProgress( filesProcessed, bytesProcessed, totalFiles, totalBytes ); 63 } ); 64 if( newfiles ) { 65 newfiles.each { 66 filenames.add( it ); 67 } 68 } 60 69 } else { 61 try { 62 def contents = parseFile( file, filetype, { files, bytes -> 63 filesProcessed += files; 64 bytesProcessed += bytes; 65 66 onProgress( filesProcessed, bytesProcessed ); 67 } ); 68 69 contents.filename = file.getName(); 70 contents.originalfilename = fileService.originalFilename( contents.filename ) 71 72 if( contents.success ) { 73 success << contents; 74 } else { 70 def file = fileService.get( filename ); 71 String filetype = fileService.determineFileType( file ); 72 73 if( !fileTypeValid( filetype ) ) { 74 // If files are not valid for parsing, delete them and return a message to the user 75 fileService.delete(filename); 76 failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: 'File type not accepted' ]; 77 } else { 78 try { 79 def contents = parseFile( file, filetype, { files, bytes -> 80 filesProcessed += files; 81 bytesProcessed += bytes; 82 83 onProgress( filesProcessed, bytesProcessed, 0, 0 ); 84 } ); 85 86 contents.filename = file.getName(); 87 contents.originalfilename = fileService.originalFilename( contents.filename ) 88 89 if( contents.success ) { 90 success << contents; 91 } else { 92 fileService.delete(filename); 93 failure << contents; 94 } 95 } catch( Exception e ) { 96 // If anything fails during parsing, return an error message 75 97 fileService.delete(filename); 76 failure << contents;98 failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: e.getMessage() ]; 77 99 } 78 } catch( Exception e ) { 79 // If anything fails during parsing, return an error message 80 fileService.delete(filename); 81 failure << [ filename: filename, originalfilename: fileService.originalFilename( filename ), type: filetype, message: e.getMessage() ]; 82 } 83 100 } 84 101 } 85 102 } … … 151 168 } 152 169 153 154 /**155 * Determines the file type of a given file, based on the extension.156 * @param file File to assess157 * @return158 */159 protected String determineFileType( File file ) {160 if( !file )161 return null162 163 // Determine the extension of the file164 String name = file.getName();165 if( name.lastIndexOf( '.' ) == -1 ) {166 // Without an extension the file type is unknown167 return ""168 }169 170 String extension = name.substring( name.lastIndexOf( '.' ) + 1 ).toLowerCase();171 172 switch( extension ) {173 case "fasta":174 case "fna":175 return "fasta";176 case "qual":177 case "fqa":178 return "qual";179 case "xls":180 return "excel";181 case "zip":182 return "zip";183 case "gz":184 return "gzip";185 default:186 return ""; // meaning 'unknown'187 }188 }189 190 170 /** 191 171 * Determines whether a file can be processed. … … 324 304 //quality = updateQuality( quality, line ); 325 305 } 326 306 327 307 // Update progress every time 1MB is processed 328 308 bytesProcessed += line.size(); … … 415 395 return returnStructure; 416 396 } 397 398 399 417 400 } -
trunk/grails-app/services/nl/tno/metagenomics/files/FileService.groovy
r2 r5 17 17 import org.codehaus.groovy.grails.commons.ApplicationHolder 18 18 import org.codehaus.groovy.grails.commons.ConfigurationHolder 19 import java.io.File; 20 import java.util.zip.* 19 21 20 22 /** … … 40 42 def String handleUploadedFile( def value, String currentFile, File directory = null) { 41 43 if( directory == null ) 42 44 directory = getUploadDir(); 43 45 44 46 // If NULL is given or "*deleted*", the field value is emptied and the old file is removed … … 98 100 99 101 if( !uploadDir ) 100 102 uploadDir = "fileuploads" 101 103 102 104 return absolutePath( uploadDir ); … … 108 110 def File get( String filename, File directory = null ) { 109 111 if( directory == null ) 110 112 directory = getUploadDir() 111 113 112 114 return new File( directory, filename ); … … 118 120 def boolean fileExists( String filename, File directory = null ) { 119 121 if( directory == null ) 120 122 directory = getUploadDir() 121 123 122 124 return new File( directory, filename ).exists(); … … 128 130 def boolean delete( String filename, File directory = null ) { 129 131 if( directory == null ) 130 132 directory = getUploadDir() 131 133 132 134 def f = new File( directory, filename ); … … 143 145 def String moveFileToUploadDir( File file, String originalFilename, File directory = null ) { 144 146 if( directory == null ) 145 147 directory = getUploadDir() 146 148 147 149 println "Moving " + file + " with originalname " + originalFilename + " to " + directory … … 169 171 def String moveFileToUploadDir( org.springframework.web.multipart.MultipartFile file, String originalFilename, File directory = null ) { 170 172 if( directory == null ) 171 173 directory = getUploadDir() 172 174 173 175 try { … … 208 210 def String getUniqueFilename( String originalFilename, File directory = null ) { 209 211 if( directory == null ) 210 212 directory = getUploadDir() 211 213 212 214 if( fileExists( originalFilename, directory ) ) { … … 237 239 } 238 240 239 241 240 242 /** 241 243 * Returns the original filename for a file that has been uploaded, but renamed … … 270 272 } 271 273 } 272 274 273 275 /** 274 276 * Returns the absolute path for the given pathname. It the pathname is relative, it is taken relative to the web-app directory … … 278 280 private File absolutePath( String pathname ) { 279 281 if( pathname == null) 280 282 return null 281 283 282 284 // Check if this is an absolute path … … 305 307 age = ConfigurationHolder.config.metagenomics.fileUploadMaxAge 306 308 } 307 309 308 310 // Compute the minimum timestamp a file should have not to be deleted 309 311 Date today = new Date(); … … 321 323 } 322 324 325 /** 326 * Determines the file type of a given file, based on the extension. 327 * @param file File to assess 328 * @return 329 */ 330 public String determineFileType( File file ) { 331 if( !file ) 332 return null 333 334 // Determine the extension of the file 335 String name = file.getName(); 336 if( name.lastIndexOf( '.' ) == -1 ) { 337 // Without an extension the file type is unknown 338 return "" 339 } 340 341 String extension = name.substring( name.lastIndexOf( '.' ) + 1 ).toLowerCase(); 342 343 switch( extension ) { 344 case "fasta": 345 case "fna": 346 return "fasta"; 347 case "qual": 348 case "fqa": 349 return "qual"; 350 case "xls": 351 return "excel"; 352 case "zip": 353 return "zip"; 354 case "gz": 355 return "gzip"; 356 default: 357 return ""; // meaning 'unknown' 358 } 359 } 360 361 362 363 /** 364 * Determines whether a given file is a parsable zip file 365 * @param filename Filename to assess 366 * @return 367 */ 368 public boolean isZipFile( String filename ) { 369 File f = this.get( filename ) ; 370 switch( this.determineFileType( f ) ) { 371 case "zip": 372 case "gzip": 373 return true; 374 } 375 376 return false; 377 } 378 379 /** 380 * Extracts a given zip file and returns a list of the filenames of the extracted files. 381 * 382 * Files will be extracted in the temp directory, as used by the file service 383 * 384 * @param filename Filename of the zip file to assess 385 * @return List of filenames 386 */ 387 public ArrayList extractZipFile( String filename, Closure onProgress ) { 388 File f = this.get( filename ); 389 390 if( !f ) { 391 log.error "No files was given for extracting files." 392 return [] 393 } 394 395 switch( this.determineFileType( f ) ) { 396 case "zip": 397 return extractZip( f, onProgress ); 398 case "gzip": 399 return extractGzip( f, onProgress ); 400 default: 401 log.error "File given for extracting files is not a valid zip file." 402 403 return []; 404 } 405 } 406 407 /** 408 * Extracts a ZIP file 409 * @param f Zipfile to extract 410 * @return A list of extracted filenames; files can be found in the fileService temporary directory 411 */ 412 protected ArrayList extractZip( File f, Closure onProgress ) { 413 return extractZip( f, getUploadDir(), onProgress ); 414 } 415 416 /** 417 * Extracts a ZIP file 418 * @param f Zipfile to extract 419 * @param dir Directory to extract the files to 420 * @return A list of extracted filenames 421 */ 422 protected ArrayList extractZip( File f, File outdir, Closure onProgress ) { 423 // Create a list of filenames to return later 424 def filenames = []; 425 426 // create a buffer to improve copy performance later. 427 byte[] buffer = new byte[2048]; 428 429 // open the zip file stream 430 ZipInputStream stream = new ZipInputStream(new FileInputStream( f ) ); 431 432 try 433 { 434 // now iterate through each item in the stream. The get next 435 // entry call will return a ZipEntry for each file in the 436 // stream 437 ZipEntry entry; 438 while((entry = stream.getNextEntry()) != null) 439 { 440 String s = String.format("Entry: %s len %d added %TD", 441 entry.getName(), entry.getSize(), 442 new Date(entry.getTime())); 443 log.debug s; 444 445 // Determine a unique filename for this entry 446 def newfilename = this.getUniqueFilename( entry.getName(), outdir ) 447 448 // Once we get the entry from the stream, the stream is 449 // positioned read to read the raw data, and we keep 450 // reading until read returns 0 or less. 451 File outpath = new File( outdir, newfilename ); 452 FileOutputStream output = null; 453 try 454 { 455 output = new FileOutputStream(outpath); 456 int len = 0; 457 while ((len = stream.read(buffer)) > 0) 458 { 459 output.write(buffer, 0, len); 460 } 461 462 filenames << newfilename 463 } catch( Exception e ) { 464 // Delete the file if it exists 465 if( outpath?.exists() ) { 466 outpath.delete(); 467 } 468 } 469 finally 470 { 471 // we must always close the output file 472 if(output!=null) output.close(); 473 } 474 475 // Update progress 476 onProgress( 0, entry.getCompressedSize(), 1, outpath.length()) 477 } 478 } 479 finally 480 { 481 // we must always close the zip file. 482 stream.close(); 483 } 484 485 return filenames; 486 } 487 488 /** 489 * Extracts a GZip file 490 * @param f Zipfile to extract 491 * @return A list of extracted filenames; files can be found in the fileService temporary directory 492 */ 493 protected ArrayList extractGZip( File f, Closure onProgress ) { 494 return extractGZip( f, getUploadDir(), onProgress ); 495 } 496 497 /** 498 * Extracts a GZip file 499 * @param f GZipfile to extract 500 * @param dir Directory to extract the files to 501 * @return A list of extracted filenames 502 */ 503 protected ArrayList extractGZip( File f, File outdir, Closure onProgress ) { 504 def filenames = []; 505 506 // open the input (compressed) file. 507 FileInputStream stream = new FileInputStream( f ); 508 FileOutputStream output = null; 509 File outpath; 510 511 try 512 { 513 // open the gziped file to decompress. 514 GZIPInputStream gzipstream = new GZIPInputStream(stream); 515 byte[] buffer = new byte[2048]; 516 517 // create the output file without the .gz extension. 518 String outname = f.getName()[0..f.getName().size()-3]; 519 def newfilename = this.getUniqueFilename( outname, outdir ) 520 outpath = new File( outdir, newfilename ); 521 output = new FileOutputStream(outpath); 522 523 // and copy it to a new file 524 int len; 525 while((len = gzipstream.read(buffer))>0) 526 { 527 output.write(buffer, 0, len); 528 } 529 530 filenames << newfilename; 531 532 // Update progress 533 onProgress( 0, f.length(), 1, outpath.length()) 534 } catch( Exception e ) { 535 // Delete the file if it exists 536 if( outpath?.exists() ) { 537 outpath.delete(); 538 } 539 } 540 finally 541 { 542 // both streams must always be closed. 543 if(output != null) output.close(); 544 stream.close(); 545 } 546 547 return filenames; 548 } 323 549 }
Note: See TracChangeset
for help on using the changeset viewer.
