Context Navigation

source: trunk/grails-app/controllers/nl/tno/massSequencing/files/ImportController.groovy @ 58

Last change on this file since 58 was 58, checked in by robert@…, 12 years ago
Implemented importing of classifications
File size: 12.6 KB

Line
1	package nl.tno.massSequencing.files
2
3	import org.codehaus.groovy.grails.commons.ConfigurationHolder
4	import org.hibernate.SessionFactory
5	import grails.converters.*;
6	import nl.tno.massSequencing.*
7
8	class ImportController {
9	def fileService
10	def fastaService
11	def importService
12	def classificationService
13	def sessionFactory
14
15	/**************************************************************************
16	*
17	* Methods for handling uploaded sequence, quality and classification files
18	*
19	*************************************************************************/
20
21	/**
22	* Shows a screen that processing is done
23	*/
24	def showProcessScreen = {
25	def entityType = params.entityType
26
27	// Check whether files are given
28	def names = params.list( 'sequencefiles' )
29
30	if( !names ) {
31	flash.message = "No files uploaded for processing"
32	if( params.entityType && params.id)
33	redirect( controller: params.entityType, action: 'show', 'id': params.id)
34	else
35	redirect( url: "" )
36
37	return
38	}
39
40	// Save filenames in session
41	session.processFilenames = names;
42
43	// Check for total size of the files in order to be able
44	// to show a progress bar
45	long filesize = 0;
46	names.each {
47	filesize += fileService.get( it )?.length()
48	}
49
50	session.processProgress = [
51	stepNum: 1,
52	numSteps: 2,
53	stepDescription: 'Parsing files', // Second step is Store classification
54
55	stepProgress: 0,
56	stepTotal: filesize
57	]
58
59	[entityId: params.id, entityType: params.entityType, filenames: names, url: createLink( action: 'showProcessResult', id: params.id, params: [entityType: entityType] ) ]
60	}
61
62	/**
63	* Processes uploaded files and tries to combine them with samples
64	*/
65	def process = {
66	def entity
67	def assaySamples
68
69	switch( params.entityType ) {
70	case "run":
71	entity = getRun( params.id );
72	assaySamples = entity.assaySamples.findAll { it.assay.study.canWrite( session.user ) };
73	break;
74	case "assay":
75	entity = getAssay( params.id );
76	assaySamples = entity.assaySamples.findAll { it.assay.study.canWrite( session.user ) };
77	break;
78	default:
79	response.setStatus( 404, "No controller found" );
80	render "";
81	return;
82	}
83
84	if (!entity) {
85	response.setStatus( 404, flash.error )
86	render "";
87	return
88	}
89
90	// Check whether files are given
91	def names = session.processFilenames
92
93	if( !names ) {
94	response.setStatus( 500, "No files uploaded for processing" )
95	render "";
96	return
97	}
98
99	// If only 1 file is uploaded, it is given as String
100	ArrayList filenames = []
101	if( names instanceof String )
102	filenames << names
103	else
104	names.each { filenames << it }
105
106	/* Parses uploaded files, discards files we can not handle
107	*
108	* [
109	* success: [
110	* [filename: 'abc.fasta', type: FASTA, numSequences: 190]
111	* [filename: 'cde.fasta', type: FASTA, numSequences: 140]
112	* [filename: 'abc.qual', type: QUAL, numSequences: 190, avgQuality: 38]
113	* [filename: 'cde.qual', type: QUAL, numSequences: 140, avgQuality: 29]
114	* ],
115	* failure: [
116	* [filename: 'testing.doc', message: 'Type not recognized']
117	* ]
118	* ]
119	*
120	* The second parameter is a callback function to update progress indicators
121	*/
122	def httpSession = session;
123	def onProgress = { progress, total ->
124	// Update progress
125	httpSession.processProgress.stepTotal = total;
126	httpSession.processProgress.stepProgress = progress;
127	}
128	def newStep = { total, description ->
129	// Start a new step
130	httpSession.processProgress.stepTotal = total;
131	httpSession.processProgress.stepProgress = 0;
132
133	httpSession.processProgress.stepDescription = description;
134	httpSession.processProgress.stepNum++;
135	}
136
137	def parsedFiles = importService.parseFiles( filenames, onProgress, [progress: 0, total: httpSession.processProgress.stepTotal ], newStep );
138
139	println "Parsed files success: " + parsedFiles.success
140
141	// Determine excel matches from the uploaded files
142	parsedFiles.success = fastaService.inferExcelMatches( parsedFiles.success );
143
144	// Now check whether a taxonomy and groups file are uploaded. If so send them to the classificationService for
145	// parsing
146	def classificationFiles = parsedFiles.success.findAll { it.type == 'groups' \|\| it.type == 'taxonomy' };
147	def excelFiles = parsedFiles.success.findAll { it.type == 'excel' }
148	def excelMatches = [:]
149
150	// Find a match between input (mothur) samples and the samples in the system
151	excelFiles.each { excelFile ->
152	if( excelFile.matches && excelFile.matches[ 'mothursamples' ] ) {
153	excelFile.matches[ 'mothursamples' ].each { excelMatch ->
154	def foundSample = assaySamples.find { it.sample.name == excelMatch.samplename };
155	if( foundSample )
156	excelMatches[ excelMatch.mothurname ] = foundSample
157	}
158	}
159	}
160
161	if( classificationFiles ) {
162	parsedFiles.success -= classificationFiles;
163
164	// If no excel matches are found, no classifications can be stored
165	if( !excelMatches ) {
166	classificationFiles.each {
167	it.success = false
168	it.message = "An excel file which maps the input samples with samples in the system must be provided."
169
170	parsedFiles.failure << it;
171	}
172	} else {
173	// Set progress to a new step, so the user knows that it will take a while
174	long totalSize = classificationFiles.collect { it.filesize }.sum();
175	newStep( totalSize, "Storing classification" )
176	println "Total size: " + totalSize;
177
178	classificationFiles = classificationService.storeClassification( classificationFiles, excelMatches, { bytesProcessed -> httpSession.processProgress.stepProgress += bytesProcessed } );
179
180	classificationFiles.each {
181	if( it.success ) {
182	parsedFiles.success << it;
183	} else {
184	parsedFiles.failure << it;
185	}
186	}
187	}
188	}
189
190	// Match files with samples in the database
191	def matchedFiles = fastaService.matchFiles( parsedFiles.success, assaySamples );
192
193	// Sort files on filename
194	matchedFiles.sort { a,b -> a.fasta?.originalfilename <=> b.fasta?.originalfilename }
195
196	// Saved file matches in session to use them later on
197	session.processedFiles = [ parsed: parsedFiles, matched: matchedFiles ];
198
199	render ""
200	}
201
202	def getProgress = {
203	if( !session.processProgress ) {
204	response.setStatus( 500, "No progress information found" );
205	render ""
206	return
207	}
208
209	render session.processProgress as JSON
210	}
211
212	/**
213	* Show result of processing
214	*/
215	def showProcessResult = {
216	// load study with id specified by param.id
217	def entity
218
219	switch( params.entityType ) {
220	case "run":
221	entity = getRun( params.id )
222	break;
223	case "assay":
224	entity = getAssay( params.id )
225	break;
226	default:
227	response.setStatus( 404, "No entity found" );
228	render "";
229	return;
230	}
231
232	if (!entity) {
233	response.setStatus( 404, flash.error )
234	render "";
235	return
236	}
237
238	if( !session.processedFiles ) {
239	flash.error = "Processing of files failed. Maybe the session timed out."
240	redirect( controller: 'assay', action: 'show', 'id': params.id)
241	return
242	}
243
244	[entityType: params.entityType, entity: entity, id: params.id, parsedFiles: session.processedFiles.parsed, classificationFiles: session.processedFiles.parsed.success.findAll { it.type == 'taxonomy' }, matchedFiles: session.processedFiles.matched, selectedRun: params.selectedRun ]
245	}
246
247	/**
248	* Returns from the upload wizard without saving the data. The uploaded files are removed
249	*/
250	def returnWithoutSaving = {
251	// Delete all uploaded files from disk
252	session.processedFiles?.parsed?.success?.each {
253	fileService.delete( it.filename );
254	}
255
256	// Redirect to the correct controller
257	switch( params.entityType ) {
258	case "run":
259	case "assay":
260	redirect( controller: params.entityType, action: "show", id: params.id );
261	return;
262	default:
263	response.setStatus( 404, "No entity found" );
264	render "";
265	return;
266	}
267
268
269	}
270
271	/**
272	* Saves processed files to the database, based on the selections made by the user
273	*/
274	def saveProcessedFiles = {
275	// load entity with id specified by param.id
276	def entity
277
278	switch( params.entityType ) {
279	case "run":
280	entity = getRun( params.id );
281	break;
282	case "assay":
283	entity = getAssay( params.id );
284	break;
285	default:
286	response.setStatus( 404, "No entity found" );
287	render "";
288	return;
289	}
290
291	if (!entity) {
292	response.setStatus( 404, flash.error )
293	render "";
294	return
295	}
296
297	// Check whether files are given
298	def files = params.file
299
300	if( !files ) {
301	flash.message = "No files were selected."
302	redirect( controller: params.entityType, action: 'show', 'id': params.id)
303	return
304	}
305
306	File permanentDir = fileService.absolutePath( ConfigurationHolder.config.massSequencing.fileDir )
307	int numSuccesful = 0;
308	def errors = [];
309
310	// Loop through all files. Those are the numeric elements in the 'files' array
311	def digitRE = ~/^\d+$/;
312	files.findAll { it.key.matches( digitRE ) }.each { file ->
313	def filevalue = file.value;
314
315	// Check if the file is selected
316	if( filevalue.include == "on" ) {
317	if( fileService.fileExists( filevalue.fasta ) ) {
318	try {
319	def permanent = fastaService.savePermanent( filevalue.fasta, filevalue.qual, session.processedFiles );
320
321	// Save the data into the database
322	SequenceData sd = new SequenceData();
323
324	sd.sequenceFile = permanent.fasta
325	sd.qualityFile = permanent.qual
326	sd.numSequences = permanent.numSequences
327	sd.averageQuality = permanent.avgQuality
328
329	def sample = AssaySample.get( filevalue.assaySample );
330	if( sample )
331	sample.addToSequenceData( sd );
332
333	if( !sd.validate() ) {
334	errors << "an error occurred while saving " + filevalue.fasta + ": validation of SequenceData failed.";
335	} else {
336	sd.save(flush:true);
337	}
338
339	numSuccesful++;
340	} catch( Exception e ) {
341	errors << "an error occurred while saving " + filevalue.fasta + ": " + e.getMessage()
342	}
343	}
344	} else {
345	// File doesn't need to be included in the system. Delete it also from disk
346	fileService.delete( filevalue.fasta );
347	}
348	}
349
350	// Return all files that have not been moved
351	session.processedFiles?.parsed?.success?.each {
352	fileService.delete( it.filename );
353	}
354
355	// Return a message to the user
356	if( numSuccesful == 0 ) {
357	flash.error = "None of the files were imported, because "
358
359	if( errors.size() > 0 ) {
360	errors.each {
361	flash.error += "<br />- " + it
362	}
363	} else {
364	flash.error = "none of the files were selected for import."
365	}
366	} else {
367	flash.message = numSuccesful + " files have been added to the system. "
368
369	if( errors.size() > 0 ) {
370	flash.error += errors.size() + " errors occurred during import: "
371	errors.each {
372	flash.error += "<br />- " + it
373	}
374	}
375	}
376
377	redirect( controller: params.entityType, action: "show", id: params.id )
378	}
379
380	def deleteData = {
381	// load study with id specified by param.id
382	def sequenceData
383
384	try {
385	sequenceData = SequenceData.get(params.id as Long)
386	} catch( Exception e ) {}
387
388	if (!sequenceData) {
389	flash.error = "No sequencedata found with id: $params.id"
390	redirect( controller: 'study' )
391	return
392	}
393
394	def entityId
395	def entityType
396
397	switch( params.entityType ) {
398	case "run":
399	entityId = sequenceData.sample.run?.id;
400	entityType = "run"
401	break;
402	case "assay":
403	default:
404	entityType = "assay";
405	entityId = sequenceData.sample.assay.id;
406	break;
407	}
408
409	def numFiles = sequenceData.numFiles();
410	def sample = sequenceData.sample;
411
412	// Set flushmode to auto, since otherwise the sequencedata will
413	// not be removed
414	sessionFactory.getCurrentSession().setFlushMode( org.hibernate.FlushMode.AUTO );
415
416	sample.removeFromSequenceData( sequenceData );
417	sequenceData.delete(flush:true);
418	sample.resetStats();
419	sample.save();
420
421	flash.message = numFiles + " file" + (numFiles != 1 ? "s have" : " has" ) + " been deleted from this sample"
422
423	redirect( controller: entityType, action: 'show', id: entityId )
424	}
425
426	protected Assay getAssay(def assayId) {
427	// load assay with id specified by param.id
428	def assay
429	try {
430	assay = Assay.get(assayId as Long)
431	} catch( Exception e ) {
432	flash.error = "Incorrect id given: " + assayId
433	return null
434	}
435
436	if (!assay) {
437	flash.error = "No assay found with id: " + assayId
438	return null
439	}
440
441	if (!assay.study.canRead( session.user ) ) {
442	flash.error = "You don't have the right authorizaton to access assay " + assay.name
443	return null
444	}
445
446	return assay
447	}
448
449	protected Run getRun(def runId) {
450	// load run with id specified by param.id
451	def run
452	try {
453	run = Run.get(runId as Long)
454	} catch( Exception e ) {
455	flash.error = "Incorrect id given: " + runId
456	return null
457	}
458
459	if (!run) {
460	flash.error = "No run found with id: " + runId
461	return null
462	}
463
464	return run
465	}
466	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: