Context Navigation

source: trunk/grails-app/services/nl/tno/metagenomics/integration/SynchronizationService.groovy @ 6

Last change on this file since 6 was 6, checked in by robert@…, 13 years ago

Resolved two bugs in synchronization:

full synchronization didn't work when no studies were present in the database (initial synchronization)
synchronization didn't work with spaces in the end of a studyToken

File size: 22.8 KB

Line
1	package nl.tno.metagenomics.integration
2
3	import nl.tno.metagenomics.*
4	import nl.tno.metagenomics.auth.*
5	import org.codehaus.groovy.grails.commons.ConfigurationHolder
6
7	class SynchronizationService {
8	def gscfService
9	def trashService
10
11	String sessionToken = "" // Session token to use for communication
12	User user = null // Currently logged in user. Must be set when synchronizing authorization
13	boolean eager = false // When set to true, this method fetches data about all studies from GSCF. Otherwise, it will only look at the
14	// studies marked as dirty in the database. Defaults to false.
15
16	// Keeps track of the last time this module performed a full synchronization.
17	static Date lastFullSynchronization = null;
18
19	static transactional = true
20
21	/**
22	* Determines whether the synchronization should be performed or not. This can be entered
23	* in configuration, to avoid synchronization when developing.
24	* @return
25	*/
26	protected performSynchronization() {
27	def conf = ConfigurationHolder.config.metagenomics.synchronization;
28
29	// If nothing is entered in configuration, return true (default value)
30	if( conf == null )
31	return true
32
33	return conf
34	}
35
36	/**
37	* Returns true iff a full synchronization should be performed
38	* @return
39	*/
40	public boolean timeForFullSynchronization() {
41	if( SynchronizationService.lastFullSynchronization == null )
42	return true
43
44	// Compute the time since the last full synchronization in milliseconds
45	Date today = new Date();
46	long difference = SynchronizationService.lastFullSynchronization.getTime() - today.getTime()
47
48	if( difference / 1000 > ConfigurationHolder.config.metagenomics.fullSynchronization )
49	return true
50	else
51	return false
52	}
53
54	/**
55	* Redirects to a temporary page to give the user a 'waiting' page while synchronizing
56	* @return
57	*/
58	public String urlForFullSynchronization( def params ) {
59	def returnUrl = ConfigurationHolder.config.grails.serverURL
60	if (params.controller != null){
61	returnUrl += "/${params.controller}"
62	if (params.action != null){
63	returnUrl += "/${params.action}"
64	if (params.id != null){
65	returnUrl += "/${params.id}"
66	}
67	}
68	}
69	if( timeForFullSynchronization() ) {
70	return ConfigurationHolder.config.grails.serverURL + "/synchronize/full?redirect=" + returnUrl.encodeAsURL()
71	} else {
72	return returnUrl
73	}
74	}
75
76	/**
77	* Performs a full synchronization in order to retrieve all studies
78	* @return
79	*/
80	public void fullSynchronization() {
81	if( !timeForFullSynchronization() )
82	return
83
84	def previousEager = this.eager
85	this.eager = true
86	this.synchronizeStudies();
87	this.eager = previousEager
88
89	SynchronizationService.lastFullSynchronization = new Date();
90	}
91
92	/**
93	* Synchronizes all studies with the data from GSCF.
94	* @return ArrayList List of studies or null if the synchronization has failed
95	*/
96	public ArrayList<Study> synchronizeStudies() {
97	if( !performSynchronization() )
98	return Study.list()
99
100	// When eager fetching is enabled, ask for all studies, otherwise only ask for studies marked dirty
101	// Synchronization is performed on all studies, not only the studies the user has access to. Otherwise
102	// we would never notice that a user was given read-access to a study.
103	def studies
104	if( eager ) {
105	studies = []
106	log.trace "Eager synchronization";
107	} else {
108	studies = Study.findAllWhere( [isDirty: true] );
109	log.trace "Default synchronization: " + studies.size()
110
111	// Perform no synchronization if no studies have to be synchronized
112	if( studies.size() == 0 )
113	return []
114	}
115
116	// Perform synchronization on only one study directly, because otherwise
117	// the getStudies method could throw a ResourceNotFoundException or NotAuthorizedException
118	// that can better be handled by synchronizeStudy
119	if( studies.size() == 1 ) {
120	def newStudy = synchronizeStudy( studies[0] );
121	if( newStudy )
122	return [ newStudy ];
123	else
124	return []
125	}
126
127	// Fetch all studies from GSCF
128	def newStudies
129	try {
130	if( !eager ) {
131	def studyTokens = studies.studyToken;
132
133	if( studyTokens instanceof String ) {
134	studyTokens = [studyTokens];
135	}
136	println "Only updating studies with tokens " + studyTokens.join( ',' );
137	newStudies = gscfService.getStudies(sessionToken, studyTokens)
138	} else {
139	newStudies = gscfService.getStudies(sessionToken)
140	}
141	} catch( Exception e ) { // All exceptions are thrown.
142	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
143	// synchronizing and return null
144	log.error( "Exception occurred when fetching studies: " + e.getMessage() )
145	throw new Exception( "Error while fetching studies", e)
146	}
147
148	synchronizeStudies( newStudies );
149	studies = handleDeletedStudies( studies, newStudies );
150
151	log.trace( "Returning " + studies.size() + " studies after synchronization" )
152
153	return studies
154	}
155
156	/**
157	* Synchronizes all studies given by 'newStudies' with existing studies in the database, and adds them
158	* if they don't exist
159	*
160	* @param newStudies JSON object with studies as returned by GSCF
161	* @return
162	*/
163	protected synchronizeStudies( def newStudies ) {
164	// Synchronize all studies that are returned. Studies that are not returned by GSCF might be removed
165	// but could also be invisible for the current user.
166	newStudies.each { gscfStudy ->
167	if( gscfStudy.studyToken ) {
168	log.trace( "Processing GSCF study " + gscfStudy.studyToken + ": " + gscfStudy )
169
170	Study studyFound = Study.findByStudyToken( gscfStudy.studyToken as String )
171
172	if(studyFound) {
173	log.trace( "Study found with name " + studyFound.name )
174
175	// Synchronize the study itself with the data retrieved
176	synchronizeStudy( studyFound, gscfStudy );
177	} else {
178	log.trace( "Study not found. Creating a new one" )
179
180	// If it doesn't exist, create a new object
181	studyFound = new Study( studyToken: gscfStudy.studyToken, name: gscfStudy.title, isDirty: true );
182	studyFound.save();
183
184	// Synchronize authorization and study assays (since the study itself is already synchronized)
185	synchronizeAuthorization(studyFound);
186	synchronizeStudyAssays(studyFound);
187
188	// Mark the study as clean
189	studyFound.isDirty = false
190	studyFound.save();
191	}
192	}
193	}
194	}
195
196	/**
197	* Removes studies from the database that are expected but not found in the list from GSCF
198	* @param studies List with existing studies in the database that were expected in the output of GSCF
199	* @param newStudies JSON object with studies as returned by GSCF
200	* @return List of remaining studies
201	*/
202	protected ArrayList<Study> handleDeletedStudies( def studies, def newStudies ) {
203	// If might also be that studies have been removed from the system. In that case, the studies
204	// should be deleted from this module as well. Looping backwards in order to avoid conflicts
205	// when removing elements from the list
206	def numStudies = studies.size();
207	for( int i = numStudies - 1; i >= 0; i-- ) {
208	def existingStudy = studies[i];
209
210	def studyFound = newStudies.find { it.studyToken == existingStudy.studyToken }
211
212	if( !studyFound ) {
213	log.trace( "Study " + existingStudy.studyToken + " not found. Check whether it is removed or the user just can't see it." )
214
215	// Study was not given to us by GSCF. This might be because the study is removed, or because the study is not visible (anymore)
216	// to the current user.
217	// Synchronize authorization and see what is the case (it returns null if the study has been deleted)
218	if( synchronizeAuthorization( existingStudy ) == null ) {
219	// Update studies variable to keep track of all existing studies
220	studies.remove( existingStudy )
221	}
222	}
223	}
224
225	return studies
226	}
227
228	/**
229	* Synchronizes the given study with the data from GSCF
230	* @param study Study to synchronize
231	* @return Study Synchronized study or null if the synchronization has failed
232	*/
233	public Study synchronizeStudy(Study study ) {
234	if( !performSynchronization() )
235	return study
236
237	if( study == null )
238	return null
239
240	// Trashcan should never be synchronized
241	if( study.trashcan )
242	return study
243
244	// If the study hasn't changed, don't update anything
245	if( !eager && !study.isDirty )
246	return study;
247
248	// Retrieve the study from GSCF
249	def newStudy
250	try {
251	newStudy = gscfService.getStudy(sessionToken, study.studyToken)
252	} catch( NotAuthorizedException e ) {
253	// User is not authorized to access this study. Update the authorization within the module and return
254	synchronizeAuthorization( study );
255	return null
256	} catch( ResourceNotFoundException e ) {
257	// Study can't be found within GSCF.
258	trashService.moveToTrash( study );
259	return null
260	} catch( Exception e ) { // All other exceptions
261	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
262	// synchronizing and return null
263	log.error( "Exception occurred when fetching study " + study.studyToken + ": " + e.getMessage() )
264	throw new Exception( "Error while fetching study " + study.studyToken, e)
265	}
266
267	// If no study is returned, something went wrong.
268	if( newStudy.size() == 0 ) {
269	throw new Exception( "No data returned for study " + study.studyToken + " but no error has occurred either. Please contact your system administrator" );
270	return null;
271	}
272
273	return synchronizeStudy(study, newStudy);
274	}
275
276	/**
277	* Synchronizes the given study with the data from GSCF
278	* @param study Study to synchronize
279	* @param newStudy Data to synchronize the study with
280	* @return Study Synchronized study or null if the synchronization has failed
281	*/
282	protected Study synchronizeStudy(Study study, def newStudy) {
283	if( !performSynchronization() )
284	return study
285
286	if( study == null \|\| newStudy == null)
287	return null
288
289	// If the study hasn't changed, don't update anything
290	if( !eager && !study.isDirty ) {
291	return study;
292	}
293
294	// If no study is returned, something went wrong.
295	if( newStudy.size() == 0 ) {
296	return null;
297	}
298
299	// Mark study dirty to enable synchronization
300	study.isDirty = true;
301	synchronizeAuthorization( study );
302	synchronizeStudyAssays( study );
303
304	// Update properties and mark as clean
305	study.name = newStudy.title
306	study.isDirty = false;
307	study.save()
308
309	return study
310	}
311
312	/**
313	* Synchronizes the assays of the given study with the data from GSCF
314	* @param study Study of which the assays should be synchronized
315	* @return ArrayList List of assays or null if the synchronization has failed
316	*/
317	protected ArrayList<Assay> synchronizeStudyAssays( Study study ) {
318	if( !performSynchronization() )
319	return study.assays.toList()
320
321	if( !eager && !study.isDirty )
322	return study.assays as List
323
324	// Also update all assays, belonging to this study
325	// Retrieve the assays from GSCF
326	def newAssays
327	try {
328	newAssays = gscfService.getAssays(sessionToken, study.studyToken)
329	} catch( Exception e ) { // All exceptions are thrown. If we get a NotAuthorized or NotFound Exception, something has changed in between the two requests. This will result in an error
330	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
331	// synchronizing and return null
332	log.error( "Exception occurred when fetching assays for study " + study.studyToken + ": " + e.getMessage() )
333	throw new Exception( "Error while fetching samples for assay " + study.studyToken, e)
334	}
335
336	// If no assay is returned, we remove all assays
337	// from this study and return an empty list
338	if( newAssays.size() == 0 && study.assays != null ) {
339	def studyAssays = study.assays.toArray();
340	def numStudyAssays = study.assays.size();
341	for( int i = numStudyAssays - 1; i >= 0; i-- ) {
342	def existingAssay = studyAssays[i];
343
344	// Move data to trash
345	trashService.moveToTrash( existingAssay );
346	}
347
348	return []
349	}
350
351	synchronizeStudyAssays( study, newAssays );
352	return handleDeletedAssays( study, newAssays );
353	}
354
355	/**
356	* Synchronizes the assays of a study with the given data from GSCF
357	* @param study Study to synchronize
358	* @param newAssays JSON object given by GSCF to synchronize the assays with
359	*/
360	protected void synchronizeStudyAssays( Study study, def newAssays ) {
361	// Otherwise, we search for all assays in the new list, if they
362	// already exist in the list of assays
363	newAssays.each { gscfAssay ->
364	if( gscfAssay.assayToken ) {
365	log.trace( "Processing GSCF assay " + gscfAssay.assayToken + ": " + gscfAssay )
366
367	Assay assayFound = study.assays.find { it.assayToken == gscfAssay.assayToken }
368
369	if(assayFound) {
370	log.trace( "Assay found with name " + assayFound.name )
371
372	// Synchronize the assay itself with the data retrieved
373	synchronizeAssay( assayFound, gscfAssay );
374	} else {
375	log.trace( "Assay not found in study. Creating a new one" )
376
377	// If it doesn't exist, create a new object
378	assayFound = new Assay( assayToken: gscfAssay.assayToken, name: gscfAssay.name, study: study );
379
380	log.trace( "Connecting assay to study" )
381	study.addToAssays( assayFound );
382	assayFound.save()
383
384	// Synchronize assay samples (since the assay itself is already synchronized)
385	synchronizeAssaySamples(assayFound)
386	}
387	}
388	}
389	}
390
391	/**
392	* Removes assays from the system that have been deleted from GSCF
393	* @param study Study to synchronize
394	* @param newAssays JSON object given by GSCF to synchronize the assays with
395	* @return List with all assays from the study
396	*/
397	protected ArrayList<Assay> handleDeletedAssays( Study study, def newAssays ) {
398	if( study.assays == null ) {
399	return []
400	}
401
402	// If might also be that assays have been removed from this study. In that case, the removed assays
403	// should be deleted from this study in the module as well. Looping backwards in order to avoid conflicts
404	// when removing elements from the list
405	def assays = study.assays.toArray();
406	def numAssays = assays.size();
407	for( int i = numAssays - 1; i >= 0; i-- ) {
408	def existingAssay = assays[i];
409
410	Assay assayFound = newAssays.find { it.assayToken == existingAssay.assayToken }
411
412	if( !assayFound ) {
413	log.trace( "Assay " + existingAssay.assayToken + " not found. Removing it." )
414
415	// The assay has been removed
416	trashService.moveToTrash( existingAssay );
417	}
418	}
419
420	return study.assays.toList()
421	}
422
423	/**
424	* Retrieves the authorization for the currently logged in user
425	* Since GSCF only provides authorization information about the currently
426	* logged in user, we can not guarantee that the authorization information
427	* is synchronized for all users.
428	*
429	* Make sure synchronizationService.user is set beforehand
430	*
431	* @param study Study to synchronize authorization for
432	* @return Auth object for the given study and user
433	*/
434	public Auth synchronizeAuthorization(Study study) {
435	// If the user is not set, we can't save anything to the database.
436	if( user == null ) {
437	throw new Exception( "Property user of SynchronizationService must be set to the currently logged in user" );
438	}
439
440	// Only perform synchronization if needed
441	if( !eager && !study.isDirty )
442	return Auth.findByUserAndStudy( user, study )
443
444	if( !performSynchronization() )
445	return Auth.findByUserAndStudy( user, study )
446
447	def gscfAuthorization
448	try {
449	gscfAuthorization = gscfService.getAuthorizationLevel( sessionToken, study.studyToken )
450	} catch( ResourceNotFoundException e ) {
451	// Study has been deleted, remove all authorization on that study
452	log.trace( "Study " + study.studyToken + " has been deleted. Remove all authorization on that study")
453	trashService.moveToTrash( study );
454
455	return null
456	}
457
458	// Update the authorization object, or create a new one
459	Auth a = Auth.authorization( study, user )
460
461	if( !a ) {
462	log.trace( "Authorization not found for " + study.studyToken + " and " + user.username + ". Creating a new object" );
463
464	a = Auth.createAuth( study, user );
465	}
466
467	// Copy properties from gscf object
468	a.canRead = gscfAuthorization.canRead
469	a.canWrite = gscfAuthorization.canWrite
470	a.isOwner = gscfAuthorization.isOwner
471
472	a.save()
473
474	return a
475	}
476
477	/**
478	* Synchronizes the given assay with the data from GSCF
479	* @param assay Assay to synchronize
480	* @return Assay Synchronized assay or null if the synchronization has failed
481	*/
482	public Assay synchronizeAssay(Assay assay) {
483	if( !performSynchronization() )
484	return assay
485
486	if( assay == null )
487	return null
488
489	// Only perform synchronization if needed
490	if( !eager && !assay.study.isDirty )
491	return assay
492
493	// Retrieve the assay from GSCF
494	def newAssay
495	try {
496	newAssay = gscfService.getAssay(sessionToken, assay.study.studyToken, assay.assayToken)
497	} catch( NotAuthorizedException e ) {
498	// User is not authorized to access this study. Update the authorization within the module and return
499	synchronizeAuthorization( assay.study );
500	return null
501	} catch( ResourceNotFoundException e ) {
502	// Assay can't be found within GSCF.
503	trashService.moveToTrash( assay );
504	return null
505	} catch( Exception e ) { // All other exceptions are thrown
506	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
507	// synchronizing and return null
508	log.error( "Exception occurred when fetching assay " + assay.assayToken + ": " + e.getMessage() )
509	throw new Exception( "Error while fetching assay " + assay.assayToken, e)
510	}
511
512	// If new assay is empty, this means that the assay does exist, but now belongs to another module. Remove it from our system
513	if( newAssay.size() == 0 ) {
514	log.info( "No data is returned by GSCF for assay " + assay.assayToken + "; probably the assay is connected to another module." )
515	trashService.moveToTrash( assay );
516	return null;
517	}
518
519	return synchronizeAssay( assay, newAssay );
520	}
521
522	/**
523	* Synchronizes the given assay with the data given
524	* @param assay Assay to synchronize
525	* @param newAssay New data for the assay, retrieved from GSCF
526	* @return Assay Synchronized assay or null if the synchronization has failed
527	*/
528	protected Assay synchronizeAssay(Assay assay, def newAssay) {
529	if( !performSynchronization() )
530	return assay
531
532	if( assay == null \|\| newAssay == null )
533	return null
534
535	// Only perform synchronization if needed
536	if( !eager && !assay.study.isDirty )
537	return assay
538
539	// If new assay is empty, something went wrong
540	if( newAssay.size() == 0 ) {
541	return null;
542	}
543
544	log.trace( "Assay is found in GSCF: " + assay.name + " / " + newAssay )
545	if( newAssay?.name ) {
546	assay.name = newAssay.name
547	assay.save()
548	}
549
550	// Synchronize samples
551	synchronizeAssaySamples(assay);
552
553	return assay
554	}
555
556	/**
557	* Synchronizes the samples of a given assay with the data from GSCF
558	* @param assay Assay to synchronize
559	* @return Sample List of samples or null if the synchronization failed
560	*/
561	protected ArrayList<AssaySample> synchronizeAssaySamples(Assay assay) {
562	if( !performSynchronization() )
563	return assay.assaySamples.toList()
564
565	// If no assay is given, return null
566	if( assay == null )
567	return null
568
569	// Retrieve the assay from GSCF
570	def newSamples
571	try {
572	newSamples = gscfService.getSamples(sessionToken, assay.assayToken)
573	} catch( NotAuthorizedException e ) {
574	// User is not authorized to access this study. Update the authorization within the module and return
575	synchronizeAuthorization( assay.study );
576	return null
577	} catch( ResourceNotFoundException e ) {
578	// Assay can't be found within GSCF. Samples will be removed
579	trashService.moveToTrash( assay );
580
581	return null
582	} catch( Exception e ) {
583	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
584	// synchronizing and return null
585	log.error( "Exception occurred when fetching samples for assay " + assay.assayToken + ": " + e.getMessage() )
586	throw new Exception( "Error while fetching samples for assay " + assay.assayToken, e)
587	}
588
589	// If no sample is returned, we remove all samples from the list
590	if( newSamples.size() == 0 ) {
591	assay.removeAssaySamples();
592	return []
593	}
594
595	synchronizeAssaySamples( assay, newSamples );
596	return handleDeletedSamples( assay, newSamples );
597	}
598
599	/**
600	* Synchronize all samples for a given assay with the data from GSCF
601	* @param assay Assay to synchronize samples for
602	* @param newSamples New samples in JSON object, as given by GSCF
603	*/
604	protected void synchronizeAssaySamples( Assay assay, def newSamples ) {
605	// Otherwise, we search for all samples in the new list, if they
606	// already exist in the list of samples
607	newSamples.each { gscfSample ->
608	log.trace( "Processing GSCF sample " + gscfSample.name + ": " + gscfSample )
609	if( gscfSample.name ) {
610
611	AssaySample assaySampleFound = assay.assaySamples.find { it.sample.sampleToken == gscfSample.name }
612	Sample sampleFound
613
614	if(assaySampleFound) {
615	sampleFound = assaySampleFound.sample
616	log.trace( "AssaySample found with sample " + sampleFound.name )
617
618	// Update the sample object if necessary
619	if( sampleFound.name != gscfSample.name ) {
620	sampleFound.name = gscfSample.name
621	sampleFound.save();
622	}
623	} else {
624	log.trace( "AssaySample not found in assay" )
625
626	// Check if the sample already exists in the database.
627	sampleFound = Sample.findBySampleTokenAndStudy( gscfSample.name as String, assay.study )
628
629	if( sampleFound ){
630	log.trace( "Sample " + gscfSample.name + " is found in database. Updating if necessary" )
631
632	// Update the sample object if necessary
633	if( sampleFound.name != gscfSample.name ) {
634	sampleFound.name =gscfSample.name
635	sampleFound.save();
636	}
637	} else {
638	log.trace( "Sample " + gscfSample.name + " not found in database. Creating a new object." )
639
640	// If it doesn't exist, create a new object
641	sampleFound = new Sample( sampleToken: gscfSample.name, name: gscfSample.name, study: assay.study );
642	sampleFound.save();
643	}
644
645	// Create a new assay-sample combination
646	log.trace( "Connecting sample to assay" )
647	assaySampleFound = new AssaySample();
648
649	assay.addToAssaySamples( assaySampleFound );
650	sampleFound.addToAssaySamples( assaySampleFound );
651	assaySampleFound.save();
652	}
653	}
654	}
655	}
656
657	/**
658	* Removes samples from the system that have been removed from an assay in GSCF
659	* @param assay Assay to remove samples for
660	* @param newSamples JSON object with all samples for this assay as given by GSCF
661	*/
662	protected ArrayList<AssaySample> handleDeletedSamples( Assay assay, def newSamples ) {
663	// If might also be that samples have been removed from this assay. In that case, the removed samples
664	// should be deleted from this assay. Looping backwards in order to avoid conflicts when removing elements
665	// from the list
666	if( assay.assaySamples != null ) {
667	def assaySamples = assay.assaySamples.toArray();
668	def numAssaySamples = assay.assaySamples.size();
669	for( int i = numAssaySamples - 1; i >= 0; i-- ) {
670	def existingSample = assaySamples[i];
671
672	AssaySample sampleFound = newSamples.find { it.name == existingSample.sample.sampleToken }
673
674	if( !sampleFound ) {
675	log.trace( "Sample " + existingSample.sample.sampleToken + " not found. Removing it." )
676
677	// The sample has been removed
678	trashService.moveToTrash( existingSample.sample );
679	}
680	}
681	}
682
683	// Create a list of samples to return
684	return assay.assaySamples.toList()
685
686	}
687	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: