Context Navigation

source: trunk/grails-app/services/nl/tno/metagenomics/integration/SynchronizationService.groovy @ 7

Last change on this file since 7 was 7, checked in by robert@…, 8 years ago
Created tests for the synchronization and trash Improved synchronizationservice and trash Put authorization checks in several pages
File size: 23.5 KB

Line
1	package nl.tno.metagenomics.integration
2
3	import nl.tno.metagenomics.*
4	import nl.tno.metagenomics.auth.*
5	import org.codehaus.groovy.grails.commons.ConfigurationHolder
6
7	class SynchronizationService {
8	def gscfService
9	def trashService
10
11	String sessionToken = "" // Session token to use for communication
12	User user = null // Currently logged in user. Must be set when synchronizing authorization
13	boolean eager = false // When set to true, this method fetches data about all studies from GSCF. Otherwise, it will only look at the
14	// studies marked as dirty in the database. Defaults to false.
15
16	// Keeps track of the last time this module performed a full synchronization.
17	static Date lastFullSynchronization = null;
18
19	static transactional = true
20
21	/**
22	* Determines whether the synchronization should be performed or not. This can be entered
23	* in configuration, to avoid synchronization when developing.
24	* @return
25	*/
26	protected performSynchronization() {
27	def conf = ConfigurationHolder.config.metagenomics.synchronization;
28
29	// If nothing is entered in configuration, return true (default value)
30	if( conf == null )
31	return true
32
33	return conf
34	}
35
36	/**
37	* Returns true iff a full synchronization should be performed
38	* @return
39	*/
40	public boolean timeForFullSynchronization() {
41	if( SynchronizationService.lastFullSynchronization == null )
42	return true
43
44	// Compute the time since the last full synchronization in milliseconds
45	Date today = new Date();
46	long difference = SynchronizationService.lastFullSynchronization.getTime() - today.getTime()
47
48	if( difference / 1000 > ConfigurationHolder.config.metagenomics.fullSynchronization )
49	return true
50	else
51	return false
52	}
53
54	/**
55	* Redirects to a temporary page to give the user a 'waiting' page while synchronizing
56	* @return
57	*/
58	public String urlForFullSynchronization( def params ) {
59	def returnUrl = ConfigurationHolder.config.grails.serverURL
60	if (params.controller != null){
61	returnUrl += "/${params.controller}"
62	if (params.action != null){
63	returnUrl += "/${params.action}"
64	if (params.id != null){
65	returnUrl += "/${params.id}"
66	}
67	}
68	}
69	if( timeForFullSynchronization() ) {
70	return ConfigurationHolder.config.grails.serverURL + "/synchronize/full?redirect=" + returnUrl.encodeAsURL()
71	} else {
72	return returnUrl
73	}
74	}
75
76	/**
77	* Performs a full synchronization in order to retrieve all studies
78	* @return
79	*/
80	public void fullSynchronization() {
81	if( !timeForFullSynchronization() )
82	return
83
84	def previousEager = this.eager
85	this.eager = true
86	this.synchronizeStudies();
87	this.eager = previousEager
88
89	SynchronizationService.lastFullSynchronization = new Date();
90	}
91
92	/**
93	* Synchronizes all studies with the data from GSCF.
94	* @return ArrayList List of studies or null if the synchronization has failed
95	*/
96	public ArrayList<Study> synchronizeStudies() {
97	if( !performSynchronization() )
98	return Study.findAllWhereTrashcan(false)
99
100	// When eager fetching is enabled, ask for all studies, otherwise only ask for studies marked dirty
101	// Synchronization is performed on all studies, not only the studies the user has access to. Otherwise
102	// we would never notice that a user was given read-access to a study.
103	def studies
104	if( eager ) {
105	studies = []
106	log.trace "Eager synchronization";
107	} else {
108	studies = Study.findAllWhere( [trashcan: false, isDirty: true] );
109	log.trace "Default synchronization: " + studies.size()
110
111	// Perform no synchronization if no studies have to be synchronized
112	if( studies.size() == 0 )
113	return []
114	}
115
116	// Perform synchronization on only one study directly, because otherwise
117	// the getStudies method could throw a ResourceNotFoundException or NotAuthorizedException
118	// that can better be handled by synchronizeStudy
119	if( studies.size() == 1 ) {
120	def newStudy = synchronizeStudy( studies[0] );
121	if( newStudy )
122	return [ newStudy ];
123	else
124	return []
125	}
126
127	// Fetch all studies from GSCF
128	def newStudies
129	try {
130	if( !eager ) {
131	def studyTokens = studies.studyToken;
132
133	if( studyTokens instanceof String ) {
134	studyTokens = [studyTokens];
135	}
136	println "Only updating studies with tokens " + studyTokens.join( ',' );
137	newStudies = gscfService.getStudies(sessionToken, studyTokens)
138	} else {
139	newStudies = gscfService.getStudies(sessionToken)
140	}
141	} catch( Exception e ) { // All exceptions are thrown.
142	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
143	// synchronizing and return null
144	log.error( "Exception occurred when fetching studies: " + e.getMessage() )
145	throw new Exception( "Error while fetching studies", e)
146	}
147
148	synchronizeStudies( newStudies );
149	studies = handleDeletedStudies( studies, newStudies );
150
151	log.trace( "Returning " + studies.size() + " studies after synchronization" )
152
153	return studies
154	}
155
156	/**
157	* Synchronizes all studies given by 'newStudies' with existing studies in the database, and adds them
158	* if they don't exist
159	*
160	* @param newStudies JSON object with studies as returned by GSCF
161	* @return
162	*/
163	protected synchronizeStudies( def newStudies ) {
164	// Synchronize all studies that are returned. Studies that are not returned by GSCF might be removed
165	// but could also be invisible for the current user.
166	newStudies.each { gscfStudy ->
167	if( gscfStudy.studyToken ) {
168	log.trace( "Processing GSCF study " + gscfStudy.studyToken + ": " + gscfStudy )
169
170	Study studyFound = Study.findByStudyToken( gscfStudy.studyToken as String )
171
172	if(studyFound) {
173	log.trace( "Study found with name " + studyFound.name )
174
175	// Synchronize the study itself with the data retrieved
176	synchronizeStudy( studyFound, gscfStudy );
177	} else {
178	log.trace( "Study not found. Creating a new one" )
179
180	// If it doesn't exist, create a new object
181	studyFound = new Study( studyToken: gscfStudy.studyToken, name: gscfStudy.title, isDirty: true );
182	studyFound.save();
183
184	// Synchronize authorization and study assays (since the study itself is already synchronized)
185	synchronizeAuthorization(studyFound);
186	if( studyFound.canRead( user ) )
187	synchronizeStudyAssays(studyFound);
188
189	// Mark the study as clean
190	studyFound.isDirty = false
191	studyFound.save();
192	}
193	}
194	}
195	}
196
197	/**
198	* Removes studies from the database that are expected but not found in the list from GSCF
199	* @param studies List with existing studies in the database that were expected in the output of GSCF
200	* @param newStudies JSON object with studies as returned by GSCF
201	* @return List of remaining studies
202	*/
203	protected ArrayList<Study> handleDeletedStudies( def studies, def newStudies ) {
204	// If might also be that studies have been removed from the system. In that case, the studies
205	// should be deleted from this module as well. Looping backwards in order to avoid conflicts
206	// when removing elements from the list
207
208	println "Handle deleted studies: " + studies.size() + " -> " + newStudies.size();
209	def numStudies = studies.size();
210	for( int i = numStudies - 1; i >= 0; i-- ) {
211	def existingStudy = studies[i];
212
213	def studyFound = newStudies.find { it.studyToken == existingStudy.studyToken }
214
215	if( !studyFound ) {
216	log.trace( "Study " + existingStudy.studyToken + " not found. Check whether it is removed or the user just can't see it." )
217
218	// Study was not given to us by GSCF. This might be because the study is removed, or because the study is not visible (anymore)
219	// to the current user.
220	// Synchronize authorization and see what is the case (it returns null if the study has been deleted)
221	if( synchronizeAuthorization( existingStudy ) == null ) {
222	// Update studies variable to keep track of all existing studies
223	studies.remove( existingStudy )
224	}
225	}
226	}
227
228	return studies
229	}
230
231	/**
232	* Synchronizes the given study with the data from GSCF
233	* @param study Study to synchronize
234	* @return Study Synchronized study or null if the synchronization has failed
235	*/
236	public Study synchronizeStudy(Study study ) {
237	if( !performSynchronization() )
238	return study
239
240	if( study == null )
241	return null
242
243	// Trashcan should never be synchronized
244	if( study.trashcan )
245	return study
246
247	// If the study hasn't changed, don't update anything
248	if( !eager && !study.isDirty )
249	return study;
250
251	// Retrieve the study from GSCF
252	def newStudy
253	try {
254	newStudy = gscfService.getStudy(sessionToken, study.studyToken)
255	} catch( NotAuthorizedException e ) {
256	// User is not authorized to access this study. Update the authorization within the module and return
257	synchronizeAuthorization( study );
258	return null
259	} catch( ResourceNotFoundException e ) {
260	// Study can't be found within GSCF.
261	trashService.moveToTrash( study );
262	return null
263	} catch( Exception e ) { // All other exceptions
264	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
265	// synchronizing and return null
266	log.error( "Exception occurred when fetching study " + study.studyToken + ": " + e.getMessage() )
267	throw new Exception( "Error while fetching study " + study.studyToken, e)
268	}
269
270	// If no study is returned, something went wrong.
271	if( newStudy.size() == 0 ) {
272	throw new Exception( "No data returned for study " + study.studyToken + " but no error has occurred either. Please contact your system administrator" );
273	return null;
274	}
275
276	return synchronizeStudy(study, newStudy);
277	}
278
279	/**
280	* Synchronizes the given study with the data from GSCF
281	* @param study Study to synchronize
282	* @param newStudy Data to synchronize the study with
283	* @return Study Synchronized study or null if the synchronization has failed
284	*/
285	protected Study synchronizeStudy(Study study, def newStudy) {
286	if( !performSynchronization() )
287	return study
288
289	if( study == null \|\| newStudy == null)
290	return null
291
292	// If the study hasn't changed, don't update anything
293	if( !eager && !study.isDirty ) {
294	return study;
295	}
296
297	// If no study is returned, something went wrong.
298	if( newStudy.size() == 0 ) {
299	return null;
300	}
301
302	// Mark study dirty to enable synchronization
303	study.isDirty = true;
304	synchronizeAuthorization( study );
305	if( study.canRead( user ) )
306	synchronizeStudyAssays( study );
307
308	// Update properties and mark as clean
309	study.name = newStudy.title
310	study.isDirty = false;
311	study.save(flush:true)
312
313	return study
314	}
315
316	/**
317	* Synchronizes the assays of the given study with the data from GSCF
318	* @param study Study of which the assays should be synchronized
319	* @return ArrayList List of assays or null if the synchronization has failed
320	*/
321	protected ArrayList<Assay> synchronizeStudyAssays( Study study ) {
322	if( !performSynchronization() )
323	return study.assays.toList()
324
325	if( !eager && !study.isDirty )
326	return study.assays as List
327
328	// Also update all assays, belonging to this study
329	// Retrieve the assays from GSCF
330	def newAssays
331	try {
332	newAssays = gscfService.getAssays(sessionToken, study.studyToken)
333	} catch( Exception e ) { // All exceptions are thrown. If we get a NotAuthorized or NotFound Exception, something has changed in between the two requests. This will result in an error
334	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
335	// synchronizing and return null
336	log.error( "Exception occurred when fetching assays for study " + study.studyToken + ": " + e.getMessage() )
337	throw new Exception( "Error while fetching samples for assay " + study.studyToken, e)
338	}
339
340	// If no assay is returned, we remove all assays
341	// from this study and return an empty list
342	if( newAssays.size() == 0 && study.assays != null ) {
343	def studyAssays = study.assays.toArray();
344	def numStudyAssays = study.assays.size();
345	for( int i = numStudyAssays - 1; i >= 0; i-- ) {
346	def existingAssay = studyAssays[i];
347
348	// Move data to trash
349	trashService.moveToTrash( existingAssay );
350	}
351
352	return []
353	}
354
355	synchronizeStudyAssays( study, newAssays );
356	return handleDeletedAssays( study, newAssays );
357	}
358
359	/**
360	* Synchronizes the assays of a study with the given data from GSCF
361	* @param study Study to synchronize
362	* @param newAssays JSON object given by GSCF to synchronize the assays with
363	*/
364	protected void synchronizeStudyAssays( Study study, def newAssays ) {
365	// Otherwise, we search for all assays in the new list, if they
366	// already exist in the list of assays
367	newAssays.each { gscfAssay ->
368	if( gscfAssay.assayToken ) {
369	log.trace( "Processing GSCF assay " + gscfAssay.assayToken + ": " + gscfAssay )
370
371	Assay assayFound = study.assays.find { it.assayToken == gscfAssay.assayToken }
372
373	if(assayFound) {
374	log.trace( "Assay found with name " + assayFound.name )
375
376	// Synchronize the assay itself with the data retrieved
377	synchronizeAssay( assayFound, gscfAssay );
378	} else {
379	log.trace( "Assay not found in study. Creating a new one" )
380
381	// If it doesn't exist, create a new object
382	assayFound = new Assay( assayToken: gscfAssay.assayToken, name: gscfAssay.name, study: study );
383
384	log.trace( "Connecting assay to study" )
385	study.addToAssays( assayFound );
386	assayFound.save()
387
388	// Synchronize assay samples (since the assay itself is already synchronized)
389	synchronizeAssaySamples(assayFound)
390	}
391	}
392	}
393	}
394
395	/**
396	* Removes assays from the system that have been deleted from GSCF
397	* @param study Study to synchronize
398	* @param newAssays JSON object given by GSCF to synchronize the assays with
399	* @return List with all assays from the study
400	*/
401	protected ArrayList<Assay> handleDeletedAssays( Study study, def newAssays ) {
402	if( study.assays == null ) {
403	return []
404	}
405
406	// If might also be that assays have been removed from this study. In that case, the removed assays
407	// should be deleted from this study in the module as well. Looping backwards in order to avoid conflicts
408	// when removing elements from the list
409	def assays = study.assays.toArray();
410	def numAssays = assays.size();
411	for( int i = numAssays - 1; i >= 0; i-- ) {
412	def existingAssay = assays[i];
413
414	Assay assayFound = newAssays.find { it.assayToken == existingAssay.assayToken }
415
416	if( !assayFound ) {
417	log.trace( "Assay " + existingAssay.assayToken + " not found. Removing it." )
418
419	// The assay has been removed
420	trashService.moveToTrash( existingAssay );
421	}
422	}
423
424	return study.assays.toList()
425	}
426
427	/**
428	* Retrieves the authorization for the currently logged in user
429	* Since GSCF only provides authorization information about the currently
430	* logged in user, we can not guarantee that the authorization information
431	* is synchronized for all users.
432	*
433	* Make sure synchronizationService.user is set beforehand
434	*
435	* @param study Study to synchronize authorization for
436	* @return Auth object for the given study and user
437	*/
438	public Auth synchronizeAuthorization(Study study) {
439	// If the user is not set, we can't save anything to the database.
440	if( user == null ) {
441	throw new Exception( "Property user of SynchronizationService must be set to the currently logged in user" );
442	}
443
444	// Only perform synchronization if needed
445	if( !eager && !study.isDirty )
446	return Auth.findByUserAndStudy( user, study )
447
448	if( !performSynchronization() )
449	return Auth.findByUserAndStudy( user, study )
450
451	def gscfAuthorization
452	try {
453	gscfAuthorization = gscfService.getAuthorizationLevel( sessionToken, study.studyToken )
454	} catch( ResourceNotFoundException e ) {
455	// Study has been deleted, remove all authorization on that study
456	log.trace( "Study " + study.studyToken + " has been deleted. Remove all authorization on that study")
457	trashService.moveToTrash( study );
458
459	return null
460	}
461
462	// Update the authorization object, or create a new one
463	Auth a = Auth.authorization( study, user )
464
465	if( !a ) {
466	log.trace( "Authorization not found for " + study.studyToken + " and " + user.username + ". Creating a new object" );
467
468	a = Auth.createAuth( study, user );
469	}
470
471	// Copy properties from gscf object
472	println "GSCF auth: " + gscfAuthorization
473
474	if( gscfAuthorization.canRead instanceof Boolean )
475	a.canRead = gscfAuthorization.canRead.booleanValue()
476
477	if( gscfAuthorization.canWrite instanceof Boolean )
478	a.canWrite = gscfAuthorization.canWrite.booleanValue()
479
480	if( gscfAuthorization.isOwner instanceof Boolean )
481	a.isOwner = gscfAuthorization.isOwner.booleanValue()
482
483	println "Saved auth: " + a.canRead.toString() + " - " + a.canWrite.toString() + " - " + a.isOwner.toString()
484
485	a.save()
486
487	return a
488	}
489
490	/**
491	* Synchronizes the given assay with the data from GSCF
492	* @param assay Assay to synchronize
493	* @return Assay Synchronized assay or null if the synchronization has failed
494	*/
495	public Assay synchronizeAssay(Assay assay) {
496	if( !performSynchronization() )
497	return assay
498
499	if( assay == null )
500	return null
501
502	// Only perform synchronization if needed
503	if( !eager && !assay.study.isDirty )
504	return assay
505
506	// Retrieve the assay from GSCF
507	def newAssay
508	try {
509	newAssay = gscfService.getAssay(sessionToken, assay.study.studyToken, assay.assayToken)
510	} catch( NotAuthorizedException e ) {
511	// User is not authorized to access this study. Update the authorization within the module and return
512	synchronizeAuthorization( assay.study );
513	return null
514	} catch( ResourceNotFoundException e ) {
515	// Assay can't be found within GSCF.
516	trashService.moveToTrash( assay );
517	return null
518	} catch( Exception e ) { // All other exceptions are thrown
519	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
520	// synchronizing and return null
521	log.error( "Exception occurred when fetching assay " + assay.assayToken + ": " + e.getMessage() )
522	throw new Exception( "Error while fetching assay " + assay.assayToken, e)
523	}
524
525	// If new assay is empty, this means that the assay does exist, but now belongs to another module. Remove it from our system
526	if( newAssay.size() == 0 ) {
527	log.info( "No data is returned by GSCF for assay " + assay.assayToken + "; probably the assay is connected to another module." )
528	trashService.moveToTrash( assay );
529	return null;
530	}
531
532	return synchronizeAssay( assay, newAssay );
533	}
534
535	/**
536	* Synchronizes the given assay with the data given
537	* @param assay Assay to synchronize
538	* @param newAssay New data for the assay, retrieved from GSCF
539	* @return Assay Synchronized assay or null if the synchronization has failed
540	*/
541	protected Assay synchronizeAssay(Assay assay, def newAssay) {
542	if( !performSynchronization() )
543	return assay
544
545	if( assay == null \|\| newAssay == null )
546	return null
547
548	// Only perform synchronization if needed
549	if( !eager && !assay.study.isDirty )
550	return assay
551
552	// If new assay is empty, something went wrong
553	if( newAssay.size() == 0 ) {
554	return null;
555	}
556
557	log.trace( "Assay is found in GSCF: " + assay.name + " / " + newAssay )
558	if( newAssay?.name ) {
559	assay.name = newAssay.name
560	assay.save()
561	}
562
563	// Synchronize samples
564	synchronizeAssaySamples(assay);
565
566	return assay
567	}
568
569	/**
570	* Synchronizes the samples of a given assay with the data from GSCF
571	* @param assay Assay to synchronize
572	* @return Sample List of samples or null if the synchronization failed
573	*/
574	protected ArrayList<AssaySample> synchronizeAssaySamples(Assay assay) {
575	if( !performSynchronization() )
576	return assay.assaySamples.toList()
577
578	// If no assay is given, return null
579	if( assay == null )
580	return null
581
582	// Retrieve the assay from GSCF
583	def newSamples
584	try {
585	newSamples = gscfService.getSamples(sessionToken, assay.assayToken)
586	} catch( NotAuthorizedException e ) {
587	// User is not authorized to access this study. Update the authorization within the module and return
588	synchronizeAuthorization( assay.study );
589	return null
590	} catch( ResourceNotFoundException e ) {
591	// Assay can't be found within GSCF. Samples will be removed
592	trashService.moveToTrash( assay );
593
594	return null
595	} catch( Exception e ) {
596	// Can't retrieve data. Maybe sessionToken has expired or invalid. Anyway, stop
597	// synchronizing and return null
598	log.error( "Exception occurred when fetching samples for assay " + assay.assayToken + ": " + e.getMessage() )
599	throw new Exception( "Error while fetching samples for assay " + assay.assayToken, e)
600	}
601
602	// If no sample is returned, we remove all samples from the list
603	if( newSamples.size() == 0 ) {
604	assay.removeAssaySamples();
605	return []
606	}
607
608	synchronizeAssaySamples( assay, newSamples );
609	return handleDeletedSamples( assay, newSamples );
610	}
611
612	/**
613	* Synchronize all samples for a given assay with the data from GSCF
614	* @param assay Assay to synchronize samples for
615	* @param newSamples New samples in JSON object, as given by GSCF
616	*/
617	protected void synchronizeAssaySamples( Assay assay, def newSamples ) {
618	// Otherwise, we search for all samples in the new list, if they
619	// already exist in the list of samples
620	newSamples.each { gscfSample ->
621	log.trace( "Processing GSCF sample " + gscfSample.sampleToken + ": " + gscfSample )
622	if( gscfSample.name ) {
623
624	AssaySample assaySampleFound = assay.assaySamples.find { it.sample.sampleToken == gscfSample.sampleToken }
625	Sample sampleFound
626
627	if(assaySampleFound) {
628	sampleFound = assaySampleFound.sample
629	log.trace( "AssaySample found with sample name " + sampleFound.name )
630
631	// Update the sample object if necessary
632	if( sampleFound.name != gscfSample.name ) {
633	sampleFound.name = gscfSample.name
634	sampleFound.save();
635	}
636	} else {
637	log.trace( "AssaySample not found in assay" )
638
639	// Check if the sample already exists in the database.
640	sampleFound = Sample.findBySampleTokenAndStudy( gscfSample.sampleToken as String, assay.study )
641
642	if( sampleFound ){
643	log.trace( "Sample " + gscfSample.sampleToken + " is found in database. Updating if necessary" )
644
645	// Update the sample object if necessary
646	if( sampleFound.name != gscfSample.name ) {
647	sampleFound.name = gscfSample.name
648	sampleFound.save();
649	}
650	} else {
651	log.trace( "Sample " + gscfSample.sampleToken + " not found in database. Creating a new object." )
652
653	// If it doesn't exist, create a new object
654	sampleFound = new Sample( sampleToken: gscfSample.sampleToken, name: gscfSample.name, study: assay.study );
655	assay.study.addToSamples( sampleFound );
656	sampleFound.save();
657	}
658
659	// Create a new assay-sample combination
660	log.trace( "Connecting sample to assay" )
661	assaySampleFound = new AssaySample();
662
663	assay.addToAssaySamples( assaySampleFound );
664	sampleFound.addToAssaySamples( assaySampleFound );
665
666	assaySampleFound.save()
667	}
668	}
669	}
670	}
671
672	/**
673	* Removes samples from the system that have been removed from an assay in GSCF
674	* @param assay Assay to remove samples for
675	* @param newSamples JSON object with all samples for this assay as given by GSCF
676	*/
677	protected ArrayList<AssaySample> handleDeletedSamples( Assay assay, def newSamples ) {
678	// If might also be that samples have been removed from this assay. In that case, the removed samples
679	// should be deleted from this assay. Looping backwards in order to avoid conflicts when removing elements
680	// from the list
681	if( assay.assaySamples != null ) {
682	def assaySamples = assay.assaySamples.toArray();
683	def numAssaySamples = assay.assaySamples.size();
684	for( int i = numAssaySamples - 1; i >= 0; i-- ) {
685	def existingSample = assaySamples[i];
686
687	AssaySample sampleFound = newSamples.find { it.sampleToken == existingSample.sample.sampleToken }
688
689	if( !sampleFound ) {
690	log.trace( "Sample " + existingSample.sample.sampleToken + " not found. Removing it." )
691
692	// The sample has been removed
693	trashService.moveToTrash( existingSample );
694	}
695	}
696	}
697
698	// Create a list of samples to return
699	if( assay.assaySamples )
700	return assay.assaySamples.toList()
701	else
702	return []
703	}
704	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: