source: patricktest.py

Last change on this file was 234, checked in by pvkouteren, 10 years ago

Added Hypergeometric test to statistics module and AtomicServices?
Added array operations to modules
Modified source file builders to obtain numpy path from config file

  • Property svn:mime-type set to text/plain
File size: 11.7 KB
Line 
1'''
2Testing building, import and Web services.
3'''
4'''
5import container
6import parsers.utils
7import parsers.obo
8from container import iany,iall,readfile,writefile, postgres
9import config
10import os
11
12os.system("python setup_db.py override")
13
14_ConfigDefault = {
15    "database.name":                "ibidas",
16    "database.user":                "userA",
17    "database.password":            "usera",
18    "database.host":                "127.0.0.1",
19    "database.port":                "5432",
20}
21
22config = config.loadConfig('ibidas.ini',_ConfigDefault)
23
24#config['database.user'] = "userA"
25#config['database.password'] = "usera"
26
27cd = postgres.openConnection(config['database.host'],\
28            config['database.port'],config['database.name'],\
29            config['database.user'],config['database.password']).getConSet()
30
31cd.logger.log("Starting script")
32
33obo = parsers.obo.OboParser()
34o = obo.parse("RO", os.getcwd()+"/data/ro.obo")
35obof = parsers.obo.OboFlusher(cd, o)
36obof.flush()
37o = obo.parse("SO", os.getcwd()+"/data/so_2_3.obo")
38obof = parsers.obo.OboFlusher(cd, o)
39obof.flush()
40o = obo.parse("Ibidas Ontology", os.getcwd()+"/data/ibidas.obo")
41obof = parsers.obo.OboFlusher(cd, o)
42obof.flush()
43#o = obo.parse("GO", os.getcwd()+"/data/gene_ontology.1_2.obo")
44#obof = parsers.obo.OboFlusher(cd, o)
45#obof.flush()
46
47os.system("python ibidas.py")
48'''
49
50'''
51from IPython.Shell import IPShellEmbed
52
53from soaplib.client import make_service_client
54from IbidasSOAP import IbidasSOAPServer
55client = make_service_client('http://localhost:11001/service.wsdl', IbidasSOAPServer())
56
57ipshell = IPShellEmbed(banner='Welcome to the IBIDAS system',exit_msg='IBIDAS shutting down ...')
58del ipshell.IP.user_ns['_']
59ipshell()
60'''
61
62
63   
64   
65''' New new test '''
66"""
67import container
68from container import iany,iall,readfile,writefile, postgres
69import config
70import time
71import os
72
73'''
74setupconfig = config.loadConfig('setup.ini', {})
75setupconfig['setup.setup_type'] = 'new'
76config.writeConfig('setup.ini', setupconfig)
77
78#os.system("python setup.py build")
79os.system("python setup_db.py override")
80
81userconfig = config.loadConfig('ibidas.ini', {})
82userconfig['database.user'] = 'ibidas'
83userconfig['database.password'] = 'ibidas'
84config.writeConfig('ibidas.ini', userconfig)
85
86os.system("python run_imports_jan.py")
87'''
88_ConfigDefault = {
89    "database.name":                "ibidas_test_nieuw",
90    "database.user":                "ibidas",
91    "database.password":            "ibidas",
92    "database.host":                "127.0.0.1",
93    "database.port":                "5432"
94}
95
96
97testconfig = config.loadConfig('ibidas.ini',_ConfigDefault)
98
99from log.Logger import Logger
100
101timelogger = Logger(name='querytimes_6_10', format="%(message)s")
102
103auth_modes = ['off', 'on']
104joins = 6 # the max number of joins which will be done
105runs = 10 # times a query is performed
106#query_append = " WHERE t0.source_id = 4"
107query_append = ""
108
109for auth_mode in auth_modes:
110    testconfig['ibidas.authenticated_mode'] = auth_mode
111    config.writeConfig('ibidas.ini', testconfig)
112    cd = postgres.openConnection(testconfig['database.host'],\
113                testconfig['database.port'],testconfig['database.name'],\
114                testconfig['database.user'],testconfig['database.password']).getConSet()
115    cursor = cd.getSingletonCursor()
116    print "===================="
117    print "Authentication " + auth_mode
118    print "===================="
119    #timelogger.log("> "+auth_mode)
120    usertime = 0
121    result = []
122    for run0 in range(1, (runs+1)):
123        query = "SELECT * FROM term as t0"
124        t1 = time.time()
125        cursor.execute(query + query_append)
126        res = cursor.fetchall()
127        t2 = time.time()
128        totaltime = t2-t1
129        usertime += totaltime
130    avg = usertime / runs
131    result.append(str(avg).replace(".",",") + "\t")
132    print str(len(res)) + " results, " + str(runs) + " runs in a total of " + str(usertime) + " secs. On average: " + str(avg) + " secs."
133    query += " JOIN term_child_link AS tcl1 ON t0.term_id = tcl1.parent_id"
134    usertime = 0
135    for run1 in range(1, (runs+1)):
136        t1 = time.time()
137        cursor.execute(query + query_append)
138        res = cursor.fetchall()
139        t2 = time.time()
140        totaltime = t2-t1
141        usertime += totaltime
142    avg = usertime / runs
143    result.append(str(avg).replace(".",",") + "\t")
144    print str(len(res)) + " results, " + str(runs) + " runs in a total of " + str(usertime) + " secs. On average: " + str(avg) + " secs."       
145    for j in range(2, (joins+1)):
146        query += " JOIN term_child_link AS tcl" + str(j) + " ON tcl" + str(j-1) + ".child_id = tcl" + str(j) + ".parent_id"
147        usertime = 0
148        for run in range(1, (runs+1)):
149            t1 = time.time()
150            cursor.execute(query + query_append)
151            res = cursor.fetchall()
152            t2 = time.time()
153            totaltime = t2-t1
154            usertime += totaltime
155        avg = usertime / runs
156        result.append(str(avg).replace(".",",") + "\t")
157        print str(len(res)) + " results, " + str(runs) + " runs in a total of " + str(usertime) + " secs. On average: " + str(avg) + " secs."
158    timelogger.log("".join(result))
159cd.close()
160"""
161
162
163''' MSigDB speedtest '''
164import container
165from container import iany,iall,readfile,writefile, postgres
166import config
167import time
168import os
169
170_ConfigDefault = {
171    "database.name":                "ibidas_test_nieuw",
172    "database.user":                "ibidas",
173    "database.password":            "ibidas",
174    "database.host":                "127.0.0.1",
175    "database.port":                "5432"
176}
177
178
179testconfig = config.loadConfig('ibidas.ini',_ConfigDefault)
180
181from log.Logger import Logger
182from time import strftime
183
184auth_modes = ['off', 'on']
185runs = 100 # times a query is performed
186
187queries = ["SELECT s1.name AS dataset, s1.version AS version FROM set AS s1 WHERE s1.name = 'msigdb';",
188           "SELECT s1.name AS dataset, s1.version AS version FROM set AS s1 JOIN item_prop_term AS ipt1 ON s1.set_id = ipt1.source_id WHERE s1.name = 'msigdb';",
189           "SELECT s1.name AS dataset, s1.version AS version, t1.name AS type FROM set AS s1 JOIN item_prop_term AS ipt1 ON s1.set_id = ipt1.source_id JOIN term AS t1 ON ipt1.type_id = t1.term_id WHERE s1.name = 'msigdb';",
190           "SELECT s1.name AS dataset, s1.version AS version, t1.name AS type, t2.identifier AS identifier, t2.description AS description FROM set AS s1 JOIN item_prop_term AS ipt1 ON s1.set_id = ipt1.source_id JOIN term AS t1 ON ipt1.type_id = t1.term_id JOIN term AS t2 ON ipt1.term_id = t2.term_id WHERE s1.name = 'msigdb';",
191           #"SELECT s1.name AS dataset, s1.version AS version, t1.name AS type, t2.identifier AS identifier, t2.description AS description, i1.display_name AS name FROM set AS s1 JOIN item_prop_term AS ipt1 ON s1.set_id = ipt1.source_id JOIN term AS t1 ON ipt1.type_id = t1.term_id JOIN term AS t2 ON ipt1.term_id = t2.term_id JOIN item AS i1 ON ipt1.item_id = i1.item_id WHERE s1.name = 'msigdb';"
192           ]
193'''
194queries = ["SELECT set.name AS set_name, set.version AS set_version FROM set WHERE set.name = 'GeneNames'",
195           "SELECT set.name AS set_name, set.version AS set_version, display_name FROM set JOIN item ON set.set_id = item.source_id WHERE set.name = 'GeneNames'",
196           "SELECT set.name AS set_name, set.version AS set_version, display_name, item_prop_accession.value AS accession FROM set JOIN item ON set.set_id = item.source_id JOIN item_prop_accession ON set.set_id = item_prop_accession.source_id WHERE set.name = 'GeneNames'"
197           ]
198'''
199
200timelogger = Logger(name=strftime("%Y%m%d%H%M")+'querytimes_msigdb_'+str(len(queries))+'_'+str(runs), format="%(message)s")
201qtimelogger = Logger(name=strftime("%Y%m%d%H%M")+'singlequerytimes_'+str(len(queries))+'_'+str(runs), format="%(message)s")
202
203for auth_mode in auth_modes:
204    testconfig['ibidas.authenticated_mode'] = auth_mode
205    config.writeConfig('ibidas.ini', testconfig)
206    cd = postgres.openConnection(testconfig['database.host'],\
207                testconfig['database.port'],testconfig['database.name'],\
208                testconfig['database.user'],testconfig['database.password']).getConSet()
209    cursor = cd.getSingletonCursor()
210    print "===================="
211    print "Authentication " + auth_mode
212    print "===================="
213    #timelogger.log("Authentication: " + auth_mode)
214    #usertime = 0
215    result = []
216    run0s = []
217    mins = []
218    maxs = []
219    for query in queries:
220        min = None
221        max = None
222        usertime = 0
223        qtimes = []
224        for run0 in range(1, (runs+1)):
225            t1 = time.time()
226            cursor.execute(query)
227            res = cursor.fetchall()
228            t2 = time.time()
229            totaltime = t2-t1
230            qtimes.append(str(totaltime).replace(".",",") + "\t")
231            usertime += totaltime
232            if run0 == 1:
233                run0tt = totaltime
234                run0s.append(str(totaltime).replace(".",",") + "\t")
235            if min is None or totaltime < min:
236                min = totaltime
237            if max is None or totaltime > max:
238                max = totaltime
239        avg = usertime / runs
240        result.append(str(avg).replace(".",",") + "\t")
241        mins.append(str(min).replace(".",",") + "\t")
242        maxs.append(str(max).replace(".",",") + "\t")
243        qtimelogger.log("".join(qtimes) + "\n")
244        print str(len(res)) + " results, " + str(runs) + " runs in a total of " + str(usertime) + " secs. On average: " + str(avg) + " secs."
245        print "min: " + str(min) + ", max: " + str(max) + ", run0: " + str(run0tt)
246    timelogger.log("".join(result))
247    #timelogger.log("run0s")
248    timelogger.log("".join(run0s))
249    #timelogger.log("mins")
250    timelogger.log("".join(mins))
251    #timelogger.log("maxs")
252    timelogger.log("".join(maxs))
253cd.close()
254
255''' Test generation of IbidasSerializers and IbidasMessage '''
256'''
257from web.model.setup_serializers import SerializerGenerator
258import os
259import config
260import container
261from container import postgres
262
263_ConfigDefault = {
264    "database.name":                "ibidas_test_nieuw",
265    "database.user":                "ibidas",
266    "database.password":            "ibidas",
267    "database.host":                "127.0.0.1",
268    "database.port":                "5432"
269}
270
271
272testconfig = config.loadConfig('ibidas.ini',_ConfigDefault)
273cd = postgres.openConnection(testconfig['database.host'],\
274                testconfig['database.port'],testconfig['database.name'],\
275                testconfig['database.user'],testconfig['database.password']).getConSet()
276cursor = cd.getSingletonCursor()
277folder = os.getcwd() + "/web/model"
278sg = SerializerGenerator(folder)
279sg.generate(cursor, cd._getTables())
280'''
281
282''' Test to get all values from a container '''
283'''
284from web.model.IbidasSerializers import *
285import web.IbidasMessage
286import config
287import container
288from container import postgres
289
290_ConfigDefault = {
291    "database.name":                "ibidas_test_nieuw",
292    "database.user":                "ibidas",
293    "database.password":            "ibidas",
294    "database.host":                "127.0.0.1",
295    "database.port":                "5432"
296}
297
298
299testconfig = config.loadConfig('ibidas.ini',_ConfigDefault)
300cd = postgres.openConnection(testconfig['database.host'],\
301                testconfig['database.port'],testconfig['database.name'],\
302                testconfig['database.user'],testconfig['database.password']).getConSet()
303     
304def tableToSerializer(table):
305    objects = []
306    containerobject = getattr(cd, table)
307    fields = containerobject.fields.name().tolist()
308    for i in range(len(containerobject)):
309        object = web.IbidasMessage.name_types['ibidas'+table]()
310        for field in fields:
311            setattr(object, field, getattr(containerobject, field)())
312        objects.append(object)
313    return objects
314
315ts = ['set', 'term']
316for t in ts:
317    os = tableToSerializer(t)
318    print os
319'''
Note: See TracBrowser for help on using the repository browser.