Changeset 150


Ignore:
Timestamp:
Aug 8, 2009, 8:00:47 AM (11 years ago)
Author:
marchulsman
Message:

Fixes, improvemnt handling merges, work on typing

Files:
1 added
1 deleted
13 edited

Legend:

Unmodified
Added
Removed
  • container/engine.py

    r138 r150  
    2020#prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    2121prepare_query_pm.register(qg_transform.ConsolidationPass)
    22 #prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
     22prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    2323prepare_query_pm.register(qg_transform.PeepHoleOptimizerPass)
    24 #prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
     24prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    2525prepare_query_pm.register(qg_translate.ExecutionPlannerPass)
    2626#prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    2727prepare_query_pm.register(qg_transform.UsedFieldInferencePass)
    28 #prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
     28prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    2929
    3030prepare_query_pm.register(qg_transform.CalcFieldTransformPass)
    31 #prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
     31prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
    3232prepare_query_pm.register(qg_translate.SQLPass)
    3333#prepare_query_pm.register(qgraph_py.DrawNodeTreeVisitor)
  • container/itypes_py.py

    r138 r150  
    44import xnumpy
    55import utility
    6 
     6import operator
    77
    88class TypeConvertor(object):
     
    215215######## OUTPUT SEGMENTS #####################
    216216
    217 class FLEX(int):
    218     __slots__ = []
    219 
     217
     218class Segment(object):
     219    __slots__ = ["source","realias","attr","copied"]
     220
     221    def __init__(self,source=None,realias=0,attr=None):
     222        self.source = source
     223        self.realias = realias
     224        self.attr = attr
     225        self.copied = False
     226
     227    def lazy_copy(self):
     228        if(self.copied):
     229            return self
     230        else:
     231            r = self.__class__(self.source,self.realias,self.attr)
     232            r.copied = True
     233            return r
     234   
    220235    def __str__(self):
    221         return "FLEX(" + int.__str__(self) + ")"
    222 
    223 class FLEX_LEASTONE(FLEX):
    224     __slots__ = []
    225    
    226     def __str__(self):
    227         return "FLEX+(" + int.__str__(self) + ")"
    228 
    229 class REDUCE(tuple):
    230     __slots__ = []
    231 
    232     def __new__(self,*args):
    233         return tuple.__new__(self,args)
    234 
    235     def __str__(self):
    236         return "REDUCE" + tuple.__str__(self)
    237 
    238 def _outfix():
    239     class OUTFIX(object):
    240         __slots__ = []
    241         def __str__(self):
    242             return "OUTFIX"
    243     return OUTFIX()
    244 OUTFIX = _outfix()
    245 
    246 class INFIX(int):
    247     __slots__ = []
    248    
    249     def __str__(self):
    250         return "INFIX(" + int.__str__(self) + ")"
    251 
    252 class PASSTHROUGH(tuple):
    253     __slots__ = []
    254    
    255     def __new__(self,*args):
    256         return tuple.__new__(self,args)
    257 
    258     def __str__(self):
    259         return "PASSTHROUGH(" + int.__str__(self) + ")"
    260 
     236        res = self.__class__.__name__ + "(" +  str(self.source)
     237        if(not self.attr is None):
     238            res += "," + str(self.attr)
     239        res += "," + str(self.realias) + ")"
     240        return res
     241
     242    def hasSource(self,sources):
     243        if(operator.isSequenceType(sources)):
     244            return self.source in sources
     245        else:
     246            return self.source == sources
     247
     248    def skipSource(self,fieldid_source,node,sources):
     249        for fieldid in self.attr:
     250            source = fieldid_source.get(fieldid,fieldid)
     251            nsource = fieldid_source.get(source,source)
     252            fieldid_source[fieldid] = nsource
     253        self.realias = True
     254
     255
     256class SegmentInFix(Segment):
     257    __slots__ = []
     258    def addField(self,fid):
     259        self.attr.append(fid)
     260   
     261    def lazy_copy(self):
     262        if(self.copied):
     263            return self
     264        else:
     265            r = self.__class__(self.source,self.realias,self.attr)
     266            r.attr = list(r.attr)
     267            r.copied = True
     268            return r
     269
     270class SegmentFix(Segment):
     271    __slots__ = []
     272    def addField(self,fid):
     273        self.attr.append(fid)
     274   
     275    def lazy_copy(self):
     276        if(self.copied):
     277            return self
     278        else:
     279            r = self.__class__(self.source,self.realias,self.attr)
     280            r.attr = list(r.attr)
     281            r.copied = True
     282            return r
     283
     284class SegmentSet(Segment):
     285    def addField(self,fid):
     286        self.attr.add(fid)
     287   
     288    def lazy_copy(self):
     289        if(self.copied):
     290            return self
     291        else:
     292            r = self.__class__(self.source,self.realias,self.attr)
     293            r.attr = r.attr.copy()
     294            r.copied = True
     295            return r
     296
     297
     298class SegmentFields(Segment):
     299    __slots__ = []
     300   
     301    def hasSource(self,sources):
     302        if(operator.isSequenceType(sources)):
     303            for s in self.source:
     304                if(s in sources):
     305                    return True
     306            return False
     307        else:
     308            for s in self.source:
     309                if(s == sources):
     310                    return True
     311            return False
     312     
     313    def lazy_copy(self):
     314        if(self.copied):
     315            return self
     316        else:
     317            r = self.__class__(self.source,self.realias,self.attr)
     318            if(not r.attr is None):
     319                r.attr = r.attr.copy()
     320            r.copied = True
     321            return r
     322   
     323    def skipSource(self,fieldid_source,node,sources):
     324        if(not operator.isSequenceType(sources)):
     325            sources = (sources,)
     326        for pos,s in enumerate(self.source):
     327            if(s in sources):
     328                if(not self.attr is None):
     329                    ids = self.attr
     330                else:
     331                    ids = node.fields
     332                for fieldid in ids:
     333                    source = fieldid_source[fieldid][pos]
     334                    nsource = fieldid_source.get(source,source)
     335                    source = list(source)
     336                    source[pos] = nsource
     337                    fieldid_source[fieldid][pos] = source
     338       
     339
     340class SegmentAll(Segment):
     341    __slots__ = []
     342   
     343    def skipSource(self,fieldid_source,node,sources):
     344        for fieldid in node.fields:
     345            source = fieldid_source.get(fieldid,fieldid)
     346            nsource = fieldid_source.get(source,source)
     347            fieldid_source[fieldid] = nsource
     348        self.realias = True
     349
     350class SegmentPassThrough(Segment):
     351    __slots__ = []
     352
     353class SegmentOutFix(Segment):
     354    __slots__ = []
    261355
    262356def binop_cast(left,right,operator):
  • container/opcon.py

    r138 r150  
    33import container
    44import itypes_py
    5 from itypes_py import FLEX, REDUCE, INFIX, OUTFIX, PASSTHROUGH, FLEX_LEASTONE
     5from itypes_py import SegmentAll,SegmentFields,SegmentInFix,SegmentOutFix,SegmentSet,SegmentPassThrough
    66import traceback
    77import numpy
     
    2323class BroadcastOpCon(OpCon):
    2424    __slots__ = []
    25     _output_segment = (REDUCE(0,1),)
     25    _csegments = (SegmentFields((0,1),True),)
    2626    def __init__(self, sleft, sright, operator):
    2727        if(not isinstance(sright,container.Container)):
     
    3636                self._actidx = range(len(self._fields))
    3737                self._props,active = self._inheritProps(sleft,True)
     38               
    3839            else:
    3940                self._fields = itypes_py.binop_type(sleft._activefields,sright._activefields,operator)
     
    5657class SetOpCon(OpCon):
    5758    __slots__ = []
    58     _output_segment = (REDUCE(0,1),)
     59    _csegments = (SegmentFields((0,1),True),)
    5960
    6061    def __init__(self,left,right,op,all):
     
    8990class MergeCon(OpCon):
    9091    __slots__ = []
    91     _output_segment = (FLEX(0),FLEX(1))
    9292    def __init__(self,left,right):
    9393        if(left._invar.register(self,right._invar)):
     
    103103            left_field_ids = set([field.id for field in self._fields[:lenleft]])
    104104            right_field_ids = set([field.id for field in self._fields[lenleft:]])
    105             self._segment_attr = (left_field_ids,right_field_ids)
     105
     106            self._segments = (SegmentSet(0,True,left_field_ids),\
     107                              SegmentSet(1,True,right_field_ids))
    106108           
    107109            lprops,lactive = self._inheritProps(left)
     
    115117class JoinOpCon(OpCon):
    116118    __slots__ = []
    117     _output_segment = (FLEX_LEASTONE(0), FLEX_LEASTONE(1))
    118119    def __init__(self,left,right,op="CROSS JOIN"):
    119120        if(left._invar.register(self,(right._invar,op))):
     
    128129            left_field_ids = set([field.id for field in self._fields[:lenleft]])
    129130            right_field_ids = set([field.id for field in self._fields[lenleft:]])
    130             self._segment_attr = (left_field_ids,right_field_ids)
     131            self._segments = (SegmentSet(0,True,left_field_ids),\
     132                              SegmentSet(1,True,right_field_ids))
    131133           
    132134            self._set_props_class()
     
    162164class UpCon(OpCon):
    163165    __slots__ = []
    164     _output_segment = (FLEX(0),)
     166    _csegments = (SegmentAll(0),)
    165167    def __init__(self,source,left):
    166168        if(source._invar.register(self)):
     
    208210class WhereCon(OpCon):
    209211    __slots__ = []
    210     _output_segment = (FLEX(0),INFIX(1))
    211    
    212212    def __init__(self, source, row_constraint):
    213213        #invariant key is based on row_constraint
     
    225225                constraint_fields = [field.id for field in row_constraint._activefields]
    226226                assert (len(constraint_fields) == 1),"More than one field given as constraint"
    227                 self._segment_attr = (None,constraint_fields)
     227                self._segments = (SegmentAll(0),SegmentInFix(1,False,constraint_fields))
    228228            else:
    229                 self._segment_attr = (None,None)
     229                self._segments = (SegmentAll(0),)
    230230
    231231            self._props,actives = self._inheritProps(source)
     
    240240    __slots__ = []
    241241
    242 class InheritWhereCon(WhereCon):
    243     __slots__ = []
    244    
    245     def __init__(self, source, prop,new_table):
    246         tableoid = source.InheritPropertyType___tableoid
    247         row_constraint = tableoid == new_table
    248 
    249         if(source._invar.register(self,new_table)):
    250             ntablemap = prop.table_tree[new_table]
    251             nfieldnames = ntablemap["__locfields"]
    252             nfieldids = [fid for fname,fid in zip(prop.fieldnames,prop.fieldids) if fname in nfieldnames]
    253            
    254             self._fields = source._fields
    255             curactidx = set(source._actidx)
    256             self._actidx = source._actidx + [pos for pos,field in enumerate(self._fields) if field.id in nfieldids and not field.id in curactidx]
    257             self._segment_attr = (None,[tableoid._activefields[0].id])
    258 
    259             act_fieldids = set([field.id for field in self._activefields])
    260             props,actives = self._inheritProps(source)
    261 
    262             #update prop with new tablename
    263             proppos = [pos for pos,p in enumerate(props) if p is prop][0]
    264             nprop = prop.copy()
    265             nprop.active_table_name = new_table
    266             props = list(props)
    267             props[proppos] = nprop
    268             self._props = tuple(props)
    269 
    270             #set active props
    271             self._actidx_props = self._activateProps(source._props,actives,act_fieldids)
    272             self._set_props_class()
    273        
    274         self._source = (source,row_constraint)
    275 
    276 
    277242class SelectCon(OpCon):
    278243    __slots__ = []
    279     _output_segment = (FLEX(0),)
     244    _csegments = (SegmentAll(0),)
    280245
    281246    def __init__(self, source, col_constraint,on_actidx=True):
     
    348313class PlusPrefixOpCon(OpCon):
    349314    __slots__ = []
    350     _output_segment = (FLEX(0),)
     315    _csegments = (SegmentAll(0),)
    351316    def __init__(self, source):
    352317       if(source._invar.register(self)):
     
    358323class ChangeFieldNameCon(OpCon):
    359324    __slots__ = []
    360     _output_segment = (FLEX(0),)
     325    _csegments = (SegmentAll(0),)
    361326    def __init__(self, source,namedict):
    362327       key = tuple(((k,v) for k,v in namedict.iteritems()))
     
    447412class NewCapCon(OpCon):
    448413    __slots__ = []
    449     _output_segment = (FLEX(0),)
     414    _csegments = (SegmentAll(0),)
    450415    def __init__(self,source,cls,name,params):
    451416       rsource = source
  • container/postgres.py

    r148 r150  
    244244       #self.item = ibidb_caps.SourceRef.register(self.item,table=self.set,source_id="source_id")
    245245       
    246        self.item =  srccon.InheritSrcCon("item",{"item":{"__table":self.item,"parent_id":"item_tree","begin_pos":"item_tree","end_pos":"item_tree"},\
    247                                                "item_tree":{"__table":self.item_tree}})
     246       #self.item =  srccon.InheritSrcCon("item",{"item":{"__table":self.item,"parent_id":"item_tree","begin_pos":"item_tree","end_pos":"item_tree"},\
     247       #                                        "item_tree":{"__table":self.item_tree}})
    248248
    249249       return self
  • container/qg_transform.py

    r139 r150  
    77import qgraph_py
    88import itypes_py
     9from itypes_py import SegmentAll, SegmentFields,SegmentSet,SegmentInFix,SegmentFix,SegmentOutFix, SegmentPassThrough
    910import utility
    1011import opcon
     
    3132        return (self.calc_in_node,None,None)
    3233   
    33 
    34     def visitBroadcastOpCon(self,node):
    35         (left_source,right_source) = node.source[:2]
    36        
    37         op = self.params[node.source[2]]
    38         if(left_source is right_source):
    39             calcfields = [(field.id,field.id,field.sourceids[0],field.sourceids[1],op,field.type)
    40                                 for field in node.fields]
    41             left_source.target.remove(node)
    42             left_source.target.remove(node)
    43         elif(isinstance(right_source,int)):
    44             calcfields = [(field.id,field.id,field.sourceids[0],-(right_source + 1),op,field.type)
    45                                 for field in node.fields]
    46             left_source.target.remove(node)
    47         else:
    48             return True
    49 
    50         if(left_source not in self.calc_in_node):
    51             self.calc_in_node[left_source] = dict([(calcinfo[0],calcinfo) for calcinfo in calcfields])
    52         else:
    53             self.calc_in_node[left_source].update([(calcinfo[0],calcinfo) for calcinfo in calcfields])
    54            
    55         #left_source.fields = node.fields + left_source.fields
    56 
    57         for target in node.target:
    58             for tsource_idx,tsource in enumerate(target.source):
    59                 if(tsource is node):
    60                     target.source[tsource_idx] = left_source
    61                     left_source.target.append(target)
    62 
    63         return False
    64        
    65 
    6634    #most nodes require no action
    6735    def visitnode(self,node):
     
    9563    after=(qgraph_py.FieldIdMapPass,qgraph_py.PostOrderWalkPass,ConsolidationPass)
    9664
     65    NO_CHANGE = 0
    9766    CHANGE = 1
    9867    NEXT_ROUND = 2
     
    10776
    10877        self.cur_round = deque(depwalk) #children first
    109         self.deleted_nodes = set()
     78        self.deleted_nodes = set()      #nodes deleted during optimization, not to be visited again
    11079        while(self.cur_round):
    111             self.next_round = deque()
    112             change = False
     80            self.next_round = deque()   #nodes to visit in next round
     81            change = False              #change in this round?
    11382            while(self.cur_round):
    11483                node = self.cur_round.popleft()
     
    12190                    self.next_round.append(node)
    12291
    123             if(change):
     92            if(change): #if change, perform a new round
    12493                self.cur_round = self.next_round
    12594
    126         return (None,(qgraph_py.PostOrderWalkPass,),None)#}}}
    127  
     95        return (None,(qgraph_py.PostOrderWalkPass,),None)
     96
     97
     98    def addCalcFields(self,node,calcfields):
     99        if(node in self.calc_in_node):
     100            self.calc_in_node[node].update(calcfields)
     101        else:
     102            self.calc_in_node[node] = calcfields
     103
     104    def moveUpCalcFields(self,node,source):
     105       if(node in self.calc_in_node):
     106           self.addCalcFields(source,self.calc_in_node[node])
     107           del self.calc_in_node[node]
     108   
     109    def moveUpPathNodeCalcFields(self,node):#{{{
     110        #Move up calcualtion fields
     111        if(not node in self.calc_in_node):
     112            return 0
     113       
     114        #filter flex segments
     115        segments = [(pos,segment) for pos,segment in enumerate(node.segments) \
     116                                    if isinstance(segment,SegmentSet)]
     117        #determine if a segment has been copied
     118        calc_here = self.calc_in_node[node]
     119        ch_list = calc_here.values()
     120        #sort calcfields on dependency by looking at field id
     121        ch_list.sort(key=operator.itemgetter(0))
     122        copy_status = False
     123
     124        #walk through calcfields that will be calculated here,
     125        #parents firsT
     126        for cid,param1,param2,op,type in ch_list:
     127            #can go left?
     128            for pos,segment in segments:
     129                field_set = segment.attr
     130                if((param1 < 0 or param1 in field_set) and (param2 is None or param2 < 0  or param2 in field_set)):
     131                    source = node.source[segment.source]
     132                    segment = segment.lazy_copy()
     133                    node.segments[pos] = segment
     134                    segment.addField(cid)
     135                    break
     136            else:
     137                #cannot be moved upward
     138                continue
     139       
     140            if(source not in self.next_round):
     141                self.next_round.append(source)
     142           
     143            node.fields.add(cid)
     144
     145            #adapt source ids for params
     146            ncid = self.fieldid_source.get(cid,cid)
     147            nparam1 = self.fieldid_source.get(param1,param1)
     148            nparam2 = self.fieldid_source.get(param2,param2)
     149
     150            #create new calc tuple
     151            ncalc = (ncid,nparam1,nparam2,op,type)
     152            #remove calc tuple from current node
     153            del calc_here[cid]
     154
     155            #add new calc tuple to source (left or right) node
     156            if(not source in self.calc_in_node):  self.calc_in_node[source] = {ncid:ncalc}
     157            else:                                 self.calc_in_node[source][ncid] = ncalc
     158           
     159            copy_status = True
     160       
     161        if(copy_status):
     162            #if calc environment empty, delete it
     163            if(len(calc_here) == 0):
     164                del self.calc_in_node[node]
     165            return self.CHANGE
     166        else:
     167            return 0
     168        #}}}
     169
    128170    def visitWhereCon(self,node):#{{{
    129         #check if it is actually a field contraint and not a row-number based one
    130         #(the latter one are stored as parameters, and do not have a node as source)
    131         #we cannot combine those where operations (is illegal)
    132         #such a constraint should always be single, i.e. in position 1
    133         if(not isinstance(node.source[1],Node)): 
    134             return 0
    135 
    136171        #has this node a single source that only references this node?
    137172        if(not (len(set(node.source)) == 1 \
     
    139174            return self.NEXT_ROUND
    140175
    141         parent = node.source[0]
    142         not_remove_idx = [0]
     176        parent = node.source[0]  #single parent for this where node
     177        not_remove_idx = [0]     #source indexes not to remove from this node
    143178
    144179        if(parent.obj is opcon.WhereCon):
    145 
    146180            parent_parent = parent.source[0]
    147181
    148182            #add the necessary attributes
    149183            parent.source = parent.source + [parent_parent] * len(node.source[1:])
    150             parent.segment_attr = parent.segment_attr + node.segment_attr[1:]
    151             parent.output_segment = parent.output_segment + tuple([itypes_py.INFIX(len(parent.output_segment) + pos) for pos,os\
    152                                                                     in enumerate(node.output_segment[1:])])
     184            nsegments = [segment.lazy_copy() for segment in node.segments[1:]]
     185            for pos,segment in enumerate(nsegments):
     186                segment.source = pos + len(parent.segments)
     187
     188            parent.segments = parent.segments + nsegments
    153189           
    154190            #copy field calculation directives upward (need them a node earlier)
     
    159195
    160196        elif(issubclass(parent.obj,opcon.JoinOpCon)):
    161             left_field_set,right_field_set = parent.segment_attr[:2]
     197            left_field_set = parent.segments[0].attr
     198            right_field_set = parent.segments[1].attr
     199
    162200            if(parent in self.calc_in_node):              calc_parent = self.calc_in_node[parent]
    163201            else:                                         calc_parent = {}
    164202           
    165             for pos,seg_attr in enumerate(node.segment_attr[1:]):
     203            for pos,segment in enumerate(node.segment_attr[1:]):
    166204                #for a join we need a calculated field based
    167205                #on the fields from the two different sources
    168                 constraint_fieldid = seg_attr[0]
     206                constraint_fieldid = segment.attr[0]
    169207                if(constraint_fieldid in calc_parent):
    170208                    self.moveUpPathNodeCalcFields(parent)
     
    186224                    newnode.target = [parent]
    187225                    newnode.fields = source.fields
    188                     newnode.segment_attr = (None,[constraint_fieldid])
    189                     newnode.output_segment = opcon.WhereCon._output_segment
     226                    newnode.segments = [opcon.WhereCon._segments[0],segment]
    190227                    parent.source[sourceidx] = newnode
    191228                    source.target[source.target.index(parent)] = newnode
     
    202239            node.source = [source for pos,source in enumerate(node.source) \
    203240                                    if pos in not_remove_idx]
    204             node.segment_attr = tuple([source for pos,source in enumerate(node.segment_attr) \
    205                                         if pos in not_remove_idx])
    206             node.output_segment = (itypes_py.FLEX(0),) + \
    207                         tuple((itypes_py.INFIX(pos+1) for pos,source in enumerate(node.source[1:])))
     241
     242            nsegments = [segment.lazy_copy() for segment in node.segments[1:]]
     243            for pos,segment in enumerate(nsegments):
     244                segment.source = pos + 1
     245            node.segments = [itypes_py.SegmentAll(0)] + nsegments
     246
    208247            while(removenr):
    209248                del parent.target[parent.target.index(node)]
     
    224263            node.target = []
    225264            node.source = []
     265            self.deleted_nodes.add(node)
    226266            return self.CHANGE
    227267        #}}}
     
    230270        return 0
    231271
    232     def visitInheritWhereCon(self,node):
    233         return self.visitWhereCon(node)
    234 
    235272    def visitnode(self,node):
    236273        return 0
    237 
    238 
    239     def moveUpPathNodeCalcFields(self,node):#{{{
    240         #Move up calcualtion fields
    241         if(not node in self.calc_in_node):
    242             return
     274   
     275    def visitScalarSrcCon(self,node):#{{{
     276       assert (len(node.fields) == 1),"More than one field in scalar"
     277     
     278       #fixme: geen calc support
     279       if(node in self.calc_in_node):
     280           return 0
     281
     282       fields = [self.fieldid_map[fieldid] for fieldid in node.fields]
     283       field = fields[0]
     284       all_removed = True
     285       any_removed = False
     286       for target in node.target:
     287          if(issubclass(target.obj,opcon.MergeCon)):
     288            any_removed = True
     289            #removing source
     290            pos = target.source.index(node)
     291            del target.source[pos]
     292
     293            #removing segment
     294            for seg_pos,segment in enumerate(target.segments):
     295                if(segment.source == pos):
     296                    nfieldid = list(segment.attr)[0]
     297                    del target.segments[seg_pos]
     298                    break
     299           
     300            calcfield = (nfieldid,- (node.source[0] + 1),None,"SET",field.type)
     301
     302            if(target not in self.calc_in_node):
     303                self.calc_in_node[target] = {nfieldid:calcfield}
     304            else:
     305                self.calc_in_node[target][nfieldid] = calcfield
     306           
     307            self.next_round.append(target)
     308
     309          elif(issubclass(target.obj,opcon.BroadcastOpCon)):
     310            any_removed = True
     311            pos = target.source.index(node)
     312            target.source[pos] = node.source[0]
     313           
     314            target.segments = [target.segments[0].lazy_copy()]
     315            csource = list(target.segments[0].source)
     316            csource[pos] = None
     317            target.segments[0].source = tuple(csource)
     318            self.next_round.append(target)
     319          else:
     320            all_removed = False
    243321       
    244 
    245         left_field_set,right_field_set = node.segment_attr[:2]
    246         left_copied = right_copied = False
    247 
    248         calc_here = self.calc_in_node[node]
    249         ch_list = calc_here.values()
    250         #sort calcfields on dependency by looking at field id
    251         ch_list.sort(key=operator.itemgetter(1))
    252        
    253         #walk through calcfields that will be calculated here,
    254         #parents first
    255         for cid,sortkey,param1,param2,op,type in ch_list:
    256             #can go left?
    257             if(param1 in left_field_set and (param2 < 0  or param2 in left_field_set)):
    258                 source = node.source[0]
    259                 if(not left_copied):
    260                     left_field_set = left_field_set.copy()
    261                     left_copied = True
    262                 left_field_set.add(cid)
    263 
    264             #can go right?
    265             elif(param1 in right_field_set and (param2 < 0 or param2 in right_field_set)):
    266                 source = node.source[1]
    267                 if(not right_copied):
    268                     right_field_set = right_field_set.copy()
    269                     right_copied = True
    270                 right_field_set.add(cid)
     322       if(all_removed):
     323           self.deleted_nodes.add(node)
     324           return self.CHANGE
     325           
     326       if(any_removed):
     327           return self.CHANGE | self.NEXT_ROUND
     328       else:
     329           return self.NEXT_ROUND#}}}
     330
     331    def visitBroadcastOpCon(self,node):
     332        (left_source,right_source) = node.source[:2]
     333       
     334        op = self.params[node.source[2]]
     335        if(left_source is right_source):
     336            calcfields = []
     337            for fieldid in node.fields:
     338                field = self.fieldid_map[fieldid]
     339                calcfields.append((fieldid,field.sourceids[0],field.sourceids[1],op,field.type))
     340            left_source.target.remove(node)
     341            left_source.target.remove(node)
     342        elif(isinstance(right_source,int)):
     343            if(isinstance(left_source,int)):
     344                return 0
    271345            else:
    272                 #cannot be moved upward
    273                 continue
    274        
    275             #adapt source ids for params
    276             nparam1 = self.fieldid_source.get(param1,param1)
    277             nparam2 = self.fieldid_source.get(param2,param2)
    278 
    279             #create new calc tuple
    280             ncalc = (cid,sortkey,nparam1,nparam2,op,type)
    281             #remove calc tuple from current node
    282             del calc_here[cid]
    283 
    284             #add new calc tuple to source (left or right) node
    285             if(not source in self.calc_in_node):  self.calc_in_node[source] = {cid:ncalc}
    286             else:                                 self.calc_in_node[source][cid] = ncalc
    287        
    288         #if left_field_set or right_field_set changed
    289         #store it in segment_attr
    290         if(left_copied or right_copied):
    291             #convert to list and store modified segment_attr
    292             tmp = list(node.segment_attr)
    293             tmp[:2] = left_field_set,right_field_set
    294             #convert back to tuple, store
    295             node.segment_attr = tuple(tmp)
    296             #if calc environment empty, delete it
    297             if(len(calc_here) == 0):
    298                 del self.calc_in_node[node]#}}}
    299     visitMergeCon=moveUpPathNodeCalcFields
     346                calcfields = []
     347            for fieldid in node.fields:
     348                field = self.fieldid_map[fieldid]
     349                calcfields.append((fieldid,field.sourceids[0],-(right_source + 1),op,field.type))
     350                left_source.target.remove(node)
     351        else:
     352            return self.NEXT_ROUND
     353
     354        if(left_source not in self.calc_in_node):
     355            self.calc_in_node[left_source] = dict([(calcinfo[0],calcinfo) for calcinfo in calcfields])
     356        else:
     357            self.calc_in_node[left_source].update([(calcinfo[0],calcinfo) for calcinfo in calcfields])
     358           
     359        #left_source.fields = node.fields + left_source.fields
     360
     361        for target in node.target:
     362            for tsource_idx,tsource in enumerate(target.source):
     363                if(tsource is node):
     364                    target.source[tsource_idx] = left_source
     365                    left_source.target.append(target)
     366
     367        self.deleted_nodes.add(node)
     368        return self.CHANGE
     369
     370    def visitMergeCon(self,node):
     371        res = self.moveUpPathNodeCalcFields(node)
     372
     373        if(len(set(node.source)) == 1):
     374           #remove merge node
     375           assert (node not in self.calc_in_node),"Calc records in removable merge node found"
     376           parent = node.source[0]
     377
     378           for target in node.target:
     379                poss = [pos for pos,source in enumerate(target.source) if source is node]
     380                for spos,segment in enumerate(target.segments):
     381                    if(segment.hasSource(poss)):
     382                        segment = segment.lazy_copy()
     383                        target.segments[spos] = segment
     384                        segment.skipSource(self.fieldid_source,node,poss)
     385                for pos in poss:
     386                    target.source[pos] = parent
     387                    parent.target.append(target)
     388               
     389                self.next_round.append(target)
     390           
     391           for i in range(len(node.source)):
     392               del parent.target[parent.target.index(node)]
     393           self.next_round.append(parent)
     394           self.deleted_nodes.add(node)
     395           return self.CHANGE
     396        else:
     397           return res
     398
    300399
    301400    def visitJoinOpCon(self,node):#{{{
     
    317416        #it can also be a row-based where operation (no fields in constraint segment)
    318417        #in which case we also need a cross join
    319         if(not issubclass(target.obj,opcon.WhereCon) or target.segment_attr[1] is None):
     418        if(not issubclass(target.obj,opcon.WhereCon) or len(target.segments) == 1):
    320419            if(op != "CROSS JOIN"):
    321420                raise RuntimeError, "Could not find join-condition for " + op
     
    326425                raise RuntimeError, "Could not find join-condition for " + op
    327426           
    328         left_field_set,right_field_set = node.segment_attr[:2]
     427        left_field_set = node.segments[0].attr
     428        right_field_set =  node.segments[0].attr
    329429        field_set = left_field_set | right_field_set
    330430       
    331         conditions = [seg_attr[0] for seg_attr in target.segment_attr[1:]]
     431        conditions = [segment.attr[0] for segment in target.segments[1:]]
    332432
    333433        req_fields = []
     
    369469
    370470    def build_condition(self,calc_here,cond,req_fields):
    371        (id,sortkey,param1,param2,op,type) = calc_here[cond]
     471       (id,param1,param2,op,type) = calc_here[cond]
    372472       if(param1 in calc_here):
    373473           param1 = self.build_condition(calc_here,param1,req_fields)
     
    384484       return (param1,op,param2,type)#}}}
    385485
    386     def visitInheritSrcCon(self,node):
    387         target_set = set(node.target)
    388 
    389         if(len(target_set) > 1):
    390             #FIXME: split
    391             raise NotImplementedError,"Multiple used inheritsrccon"
    392        
    393         tableoid_id = node.invar_dict["tableoid_id"]
    394         target = target_set.pop()
    395              
    396         if(not issubclass(target.obj,opcon.WhereCon) or target.segment_attr[1] is None):
    397             pass           
    398486
    399487
     
    416504
    417505        root = depwalk[0]
    418         root.in_fields = [[root.fields[idx].id for idx in root.actidx]]
    419         root.out_fields = root.in_fields[0]
     506        res_in = [root.segments[0].attr]
     507        root.out_fields,root.in_fields = cls.inferOutInFields(root,res_in,None,fieldid_map,fieldid_source)
    420508       
    421509        for node in depwalk[1:]: #children first walk (exclude root node)
     
    426514    @classmethod
    427515    def inferOutInFields(cls,node,req_field_ids,req_calc_out_ids,fieldid_map,fieldid_source):
    428         if(not node.output_segment):
    429             return (None,None)
    430        
    431516        in_field_ids_col = [None] * len(node.source)
    432517        out_field_ids = []
    433         req_fields = None
    434 
    435518
    436519        #CALC SEGMENT (all nodes have one by default)
     
    439522            #remove field that are calced here from request list
    440523            req_field_ids = [rfi for rfi in req_field_ids if not rfi in req_calc_out_ids]
    441         for segment_id,segment in enumerate(node.output_segment):
    442             if(segment_id == -1):
    443                 segment_id = range(len(node.source))
     524        for segment in node.segments:
     525            cnt = segment.realias
     526           
    444527            #FLEX SEGMENT
    445             if(isinstance(segment,itypes_py.FLEX)):
    446                 #if no id stored in segment_attr tuple
    447                 if(node.segment_attr is None or \
    448                    node.segment_attr[segment_id] is None):
    449                     #all out fields used
    450                     res_out = res_in = req_field_ids
    451                 else:
    452                     #otherwise, fields which id ends with cur_id are used
    453                     cur_field_set = node.segment_attr[segment_id]
    454                     res_out =  [req_id for req_id in req_field_ids\
    455                                   if req_id in cur_field_set]
    456                     res_in = [fieldid_source[req_id] for req_id in res_out]
     528            if(isinstance(segment,SegmentAll)):
     529                res_out = res_in = req_field_ids
    457530                out_field_ids.extend(res_out)
    458                 in_field_ids_col[segment] = res_in
    459 
     531                if(segment.realias):
     532                    res_in = [fieldid_source.get(req_id,req_id) for req_id in res_in]
     533                in_field_ids_col[segment.source] = res_in
     534            elif(isinstance(segment,SegmentSet)):
     535                cur_field_set = segment.attr
     536                res_out = res_in =  [req_id for req_id in req_field_ids\
     537                                        if req_id in cur_field_set]
     538                if(segment.realias):
     539                    res_in = [fieldid_source.get(req_id,req_id) for req_id in res_in]
     540                out_field_ids.extend(res_out)
     541                in_field_ids_col[segment.source] = res_in
     542            #INFIX SEGMENT
     543            elif(isinstance(segment,SegmentInFix)):
     544                res_in = segment.attr
     545                if(segment.realias):
     546                    res_in = [fieldid_source.get(req_id,req_id) for req_id in res_in]
     547                in_field_ids_col[segment.source] = res_in
     548            #OUTFIX SEGMENT
     549            elif(isinstance(segment,SegmentOutFix)):
     550                out_field_ids.extend(segment.attr)
     551            elif(isinstance(segment,SegmentFix)):
     552                res_in = segment.attr
     553                if(segment.realias):
     554                    res_in = [fieldid_source.get(req_id,req_id) for req_id in res_in]
     555                in_field_ids_col[segment.source] = res_in
     556                out_field_ids.extend(segment.attr)
    460557            #REDUCE SEGMENT
    461             elif(isinstance(segment,itypes_py.REDUCE)):
    462                 #lazy calculation of ordered req_fields
    463                 if(req_fields is None):
    464                     req_fields = [fieldid_map[rid] for rid in req_field_ids]
    465                
     558            elif(isinstance(segment,SegmentFields)):
    466559                #if no segment_attr consider all fields with sourceids to be part of
    467560                #this reduce segment
    468                 if(node.segment_attr is None or \
    469                         node.segment_attr[segment_id] is None):
    470                     res = [rfield for rfield in req_fields \
    471                                 if not rfield.sourceids is None]
     561                if(segment.attr is None):
     562                    res = [rid for rid in req_field_ids\
     563                                if rid in fieldid_source]
    472564                #else use all fields that have their id in the segment_attr
    473565                else:
    474                     reduce_field_ids = set(node.segment_attr[segment_id])
    475                     res = [rfield for rfield in req_fields \
    476                                     if rfield.id in reduce_field_ids]
     566                    reduce_field_ids = set(segment.attr)
     567                    res = [rid for rid in req_field_ids \
     568                                    if rid in reduce_field_ids]
    477569
    478570                #results with None field id's are not used
    479                 for operand_idx,source_idx in enumerate(segment):
    480                     opres = [r.sourceids[operand_idx] for r in res]
    481                     if(not None in opres):
    482                         in_field_ids_col[source_idx] = opres
     571                for operand_idx,source_idx in enumerate(segment.source):
     572                    if(source_idx is None):
     573                        continue
     574                    res_in = [fieldid_source[rid][operand_idx] for rid in res]
     575
     576                    if(not None in res_in):
     577                        in_field_ids_col[source_idx] = res_in
    483578               
    484                 out_field_ids.extend([field.id for field in res])
    485 
    486             #INFIX SEGMENT
    487             elif(isinstance(segment,itypes_py.INFIX)):
    488                 in_field_ids_col[segment] = node.segment_attr[segment_id]
    489 
    490             #OUTFIX SEGMENT
    491             elif(segment is itypes_py.OUTFIX):
    492                 out_field_ids.extend(node.segment_attr[segment_id])
     579                out_field_ids.extend(res)
     580
    493581        return (out_field_ids,in_field_ids_col)
    494582   
     
    522610                           not req_field_ids_set.add(field_id)]
    523611
    524 
    525612        if(node in calc_in_node):
    526613            calc_info = calc_in_node[node]
     
    540627                while(calc_queue):
    541628                    c_id = calc_queue.pop()
    542                     (id,sortkey,param1,param2,op,type) = calc_info[c_id]
     629                    (id,param1,param2,op,type) = calc_info[c_id]
    543630                    if(param1 in calc_info and not param1 in req_calc_here_set):
    544631                        calc_queue.append(param1)
    545632                        req_calc_here_set.add(param1)
    546                     elif(param1 not in req_field_ids_set):
     633                    elif(param1 >= 0 and param1 not in req_field_ids_set):
    547634                        req_field_ids.append(param1)
    548635                        req_field_ids_set.add(param1)
     
    581668                calc_info = calc_in_node[node]
    582669                req_calc_here = [calc_info[fieldid] for fieldid in node.out_calc]
    583                 req_calc_here.sort(key=operator.itemgetter(1))
    584                 rec_calc_here_pos = dict([(calcinfo[0],pos) for pos,calcinfo in enumerate(req_calc_here)])
     670                req_calc_here.sort(key=operator.itemgetter(0))
     671                req_calc_here_pos = dict([(calcinfo[0],pos) for pos,calcinfo in enumerate(req_calc_here)])
    585672
    586673                req_calc = [0]
    587                 for pos,(id,sortkey,param1,param2,op,type) in enumerate(req_calc_here):
     674                for pos,(id,param1,param2,op,type) in enumerate(req_calc_here):
    588675                    if(param1 >= 0):
    589676                        if(param1 in node.out_fields):
    590677                            p1 = node.out_fields[param1]
    591678                        else:
    592                             p1 = len(node.out_fields) + rec_calc_here_pos[param1]
     679                            p1 = len(node.out_fields) + req_calc_here_pos[param1]
    593680                    else:
    594681                        p1 = param1
     
    598685                            p2 = node.out_fields[param2]
    599686                        else:
    600                             p2 = len(node.out_fields) + rec_calc_here_pos[param2]
     687                            p2 = len(node.out_fields) + req_calc_here_pos[param2]
    601688                    else:
    602689                        p2 = param2
  • container/qg_translate.py

    r124 r150  
    1818
    1919class ConvertCon(qgraph_py.ExtendCon):
    20     _output_segment = (itypes_py.FLEX(0),)
     20    _csegments = (itypes_py.SegmentAll(0),)
    2121
    2222class CompatibilityCheck(NodeVisitorFactory(prefixes=("compat",))):#{{{
     
    5656    compatSetOpCon=compatible
    5757    compatRoot=compatible
     58    compatScalarSrcCon=compatible
    5859
    5960    compatPySrcCon=compatible
     
    114115   
    115116class ExecutionPlannerPass(Pass):#{{{
    116     after=(qgraph_py.PostOrderWalkPass,)
     117    after=(qgraph_py.PostOrderWalkPass,qg_transform.ConsolidationPass)
    117118    before=(qg_transform.UsedFieldInferencePass,)
    118119
     
    120121    def run(cls,query):
    121122        depwalk = query.pass_results[qgraph_py.PostOrderWalkPass]
     123        calc_in_node = query.pass_results[qg_transform.ConsolidationPass]
     124       
    122125        parwalk = depwalk[::-1]
    123126       
     
    202205                ntarget.append(nnode)
    203206                node.target = ntarget
    204                 nnode.fields = node.fields
    205                 nnode.output_segment = nnode.obj._output_segment
     207                nfields = set(node.fields)
     208                if(node in calc_in_node):
     209                    nfields.update(calc_in_node[node].keys())
     210                nnode.fields = nfields
     211                nnode.segments = list(nnode.obj._csegments)
    206212                queue_parwalk.append(nnode)
    207213                compat_map[nnode] = PyCompatibilityCheck(query.params,node)
     
    245251                  }
    246252
    247     def arithmetic_op(self,p1,p2,op,type):
     253    def arithmetic_op(self,p1,p2,op,type,nrow):
    248254        #res = self.numpy_func[op](p1,p2,sig=itypes_py.to_numpy(type))
    249255       
     
    253259        return res
    254260   
    255     def rarithmetic_op(self,p1,p2,op,type):
     261    def rarithmetic_op(self,p1,p2,op,type,nrow):
    256262        #return self.numpy_func[op](p2,p1,sig=itypes_py.to_numpy(type))
    257263        return self.numpy_func[op](p2,p1)
    258264
    259     def like_op(self,p1,p2,op,type):
     265    def like_op(self,p1,p2,op,type,nrow):
    260266        count = 0
    261267        pattern = re.escape(p2)
     
    281287        return numpy.array([not re.search(pattern,elem) is None for elem in p1],dtype=itypes_py.to_numpy(type))
    282288   
    283     def in_op(self,p1,p2,op,type):
     289    def in_op(self,p1,p2,op,type,nrow):
    284290        if(isinstance(p2,(tuple,numpy.ndarray))):
    285291            p2 = set(p2)
     
    288294           
    289295        return numpy.array([elem in p2 for elem in p1],dtype=itypes_py.to_numpy(type))
     296
     297
     298    def set_op(self,p1,p2,op,type,nrow):
     299        return utility.arrayscalar(p1,dtype=itypes_py.to_numpy(type))[numpy.zeros(nrow,dtype=int)]
    290300
    291301    op_to_func = {'__eq__':arithmetic_op,
     
    312322                  '__rxor__':rarithmetic_op,
    313323                  'IN':in_op,
    314                   'LIKE':like_op
     324                  'LIKE':like_op,
     325                  'SET':set_op
    315326        }
    316327
     
    328339                else:
    329340                    p1 = data[p1]
    330                    
    331                 if(p2 < 0):
    332                     p2 = self.params[-p2-1]
    333                 else:
    334                     p2 = data[p2]
    335                 res = self.op_to_func[op](self,p1,p2,op,type)
     341               
     342                if(not p2 is None):
     343                    if(p2 < 0):
     344                        p2 = self.params[-p2-1]
     345                    else:
     346                        p2 = data[p2]
     347                res = self.op_to_func[op](self,p1,p2,op,type,result.nrow)
    336348                if(outpos == -1):
    337349                    data[outsize + index] = res
     
    373385        func = self.op_to_func[op]
    374386        if(isinstance(rightop,QueryResult)):
    375             if(leftop.nrow == rightop.nrow):
    376                 leftop.data = tuple([func(self,lcol,rcol,op,field.type) for lcol,rcol,field in zip(leftop.data,rightop.data,fields)])
    377             elif(rightop.nrow == 1):
    378                 leftop.data = tuple([func(self,lcol,rightop.data[0],op,field.type) for lcol,field in zip(leftop.data,fields)])
    379             elif(leftop.nrow == 1):
    380                 leftop.data = tuple([func(self,leftop.data[0],rcol,op,field.type) for rcol,field in zip(rightop.data,fields)])
    381                 leftop.ncol = rightop.ncol
     387            if(isinstance(leftop,QueryResult)):
     388                if(leftop.nrow == rightop.nrow):
     389                    leftop.data = tuple([func(self,lcol,rcol,op,field.type) for lcol,rcol,field in zip(leftop.data,rightop.data,fields)])
     390                elif(rightop.nrow == 1):
     391                    leftop.data = tuple([func(self,lcol,rightop.data[0],op,field.type) for lcol,field in zip(leftop.data,fields)])
     392                elif(leftop.nrow == 1):
     393                    leftop.data = tuple([func(self,leftop.data[0],rcol,op,field.type) for rcol,field in zip(rightop.data,fields)])
     394                    leftop.ncol = rightop.ncol
     395                else:
     396                    raise RuntimeError,"Unequal number of columns in binary operation"
    382397            else:
    383                 raise RuntimeError,"Unequal number of columns in binary operation"
    384         else:
    385             leftop.data = tuple([func(self,lcol,rightop,op,field.type) for lcol,field in zip(leftop.data,fields)])
     398                leftop.data = tuple([func(self,leftop,rcol,op,field.type) for rcol,field in zip(rightop.data,fields)])
     399               
     400        else:
     401            if(not isinstance(leftop,QueryResult)):
     402                leftop = QueryResult(tuple([func(self,leftop,rightop,op,field.type) for field in fields]),1,1)
     403            else:
     404                leftop.data = tuple([func(self,lcol,rightop,op,field.type) for lcol,field in zip(leftop.data,fields)])
    386405        return leftop
    387406               
    388407
    389408    def funcMergeCon(self,node,sources):
     409        if(len(sources) == 1):
     410            return sources[0]
    390411        (lr,rr) = sources
    391412        lr_data = lr.data
     
    490511    funcNoModPySrcCon = _copySource
    491512    funcRoot = _copySource
     513
     514    def funcScalarSrcCon(self,node,sources):
     515        t = itypes_py.to_numpy(node.fields[0].type)
     516        return QueryResult((utility.arrayscalar(sources[0],dtype=t),),1,1)
    492517   
    493518    def funcExtendCon(self,node,sources):
     
    648673    def funcBroadcastOpCon(self,node,sources):
    649674        (l,r,op) = sources
    650 
    651         if(not isinstance(r,SQLQuery)):
     675       
     676        if(not isinstance(l,SQLQuery)):
    652677            #NULL = NULL comparisons are false in SQL, but true in python
    653678            #therefore, change comparison operator to NULL IS NULL, which should work
     679            if(l is None):
     680                op = self.binop_none[op]
     681            l = SQLQuery()
     682            l.fields = xnumpy.dimarray((SQLQuery.createParam(node.source[0]),),object,1,1)
     683
     684        if(not isinstance(r,SQLQuery)):
    654685            if(r is None):
    655686                op = self.binop_none[op]
     
    729760            result.addFields([None] * (len(node.out_calc) - 1),[None] * (len(node.out_calc) - 1))
    730761            for index,(p1,p2,op,type,outpos) in enumerate(node.out_calc[1:]):
     762               
     763                if(outpos == -1):
     764                    npos = outsize + index
     765                else:
     766                    npos = outpos
     767
    731768                if(p1 < 0):
    732769                    l = SQLQuery()
     
    744781                else:
    745782                    r = result
    746                 if(outpos == -1):
    747                     npos = outsize + index
     783               
     784                if(p2 is None):
     785                    assert (op == "SET"),"Unknown unary operation"
     786                    result.setField(npos,l.fields[p1],type)
    748787                else:
    749                     npos = outpos
    750 
    751                 result.setField(npos,l.fields[p1] + " " + self.binop_translate[op] + " " + r.fields[p2],type)
     788                    result.setField(npos,l.fields[p1] + " " + self.binop_translate[op] + " " + r.fields[p2],type)
    752789        return result
    753790
  • container/qgraph_py.py

    r123 r150  
    1919#special root class, used as root for query graph
    2020class Root(opcon.OpCon):
    21     _output_segment = [itypes_py.OUTFIX]
    22 
     21    pass
    2322
    2423#Visitor class used to walk through a query graph to break cyclic links so that
     
    5150       nroot.fields = self.qg.fields
    5251       nroot.actidx = self.qg.actidx
    53        nroot.segment_attr = ([field.id for field in nroot.fields],)
    54        nroot.output_segment = Root._output_segment
     52       nroot.segments = [itypes_py.SegmentFix(0,False,[nroot.fields[idx].id for idx in nroot.actidx])]
    5553       self.qg = nroot
    5654   
     
    10098      return Node(target,[self.visit(target._res(),root)])
    10199   
     100   def visitScalarSrcCon(self,target,root):
     101       return Node(target,[self.visit(target._source[0],root)])
     102   
    102103   def visitRealizeOpCon(self,target,root):
    103104      if(target is root):
     
    123124   def paramPySrcCon(self,target,root):
    124125      self.param(target._res(),root)
     126   
     127   def paramScalarSrcCon(self,target,root):
     128      self.param(target._source[0],root)
    125129   
    126130 
     
    158162        else:
    159163            self.calc_in_node = None
     164
     165        if(FieldIdMapPass in query.pass_results):
     166            (self.fieldid_map,fieldid_source) = query.pass_results[FieldIdMapPass]
     167        else:
     168            self.fieldid_map = None
     169
     170
    160171        self.graph = pg.AGraph(strict=False,directed=True);
    161172        self.visit(query.qg)
     
    189200        if(not node.in_fields is None):
    190201            lbl += "|{" + "|".join([str(ifs) for ifs in node.in_fields]) + "}"
    191        
    192         fieldnames = [field.name for field in node.fields]
    193         fieldids = [str(field.id) for field in node.fields]
     202       
     203        if(isinstance(node.fields,set)):
     204            fieldids = [str(field_id) for field_id in node.fields]
     205            fieldnames = [self.fieldid_map[field_id].name for field_id in node.fields]
     206        else:
     207            fieldnames = [field.name for field in node.fields]
     208            fieldids = [str(field.id) for field in node.fields]
    194209        lbl += "|{" + "|".join(fieldids) + "}"
    195210        lbl += "|{" + "|".join(fieldnames) + "}"
    196         if(not node.segment_attr is None):
    197             lbl += "|{" + str(node.segment_attr) + "}"
     211        lbl += "|{" + ",".join([str(segment) for segment in node.segments]) + "}"
    198212       
    199213       
     
    201215            lbl += "|{" + str(node.out_fields)
    202216            if(not node.out_calc is None):
    203                 lbl += "|" + str(node.out_calc) + "}"
     217                lbl += "|outcalc:" + str(node.out_calc) + "}"
    204218            else:
    205219                lbl += "}"
     
    267281        for node in depwalk[::-1]:
    268282            fieldid_field.update([(field.id,field) for field in node.fields])
    269         fieldid_source = dict()
    270         for v in fieldid_field.itervalues():
    271             if(v.sourceids is not None):
    272                 fieldid_source[v.id] = v.sourceids
     283            node.fields = set([field.id for field in node.fields])
     284        fieldid_source = dict([(v.id,v.sourceids) for v in fieldid_field.itervalues() if v.sourceids is not None])
    273285        return ((fieldid_field,fieldid_source),None,None)
    274286
  • container/srccon.py

    r138 r150  
    44
    55import itypes_py
     6from itypes_py import SegmentOutFix
    67import utility
    78import context
     
    1920    """DBSrcCon: main class for database source containers"""
    2021    __slots__ = []
    21     _output_segment=[itypes_py.OUTFIX]
    2222    _select_compat = qg_translate.SQLCompatibilityCheck
    2323   
     
    2929        self._invar.tablename = tablename
    3030        self._invar.conn = conn
    31         self._segment_attr = ([field.id for field in self._fields],)
    32 
    33 class InheritSrcCon(container.Container):
     31        self._segments=(SegmentOutFix(attr=[field.id for field in self._fields]),)
     32
     33
     34class ScalarSrcCon(container.Container):
    3435    __slots__ = []
    35     _output_segment=[itypes_py.OUTFIX]
    36 
    37 
    38     def processTableMap(self,table_tree,cur_name):
    39         tablemap = table_tree[cur_name]
    40        
    41         locfields = None
    42         subtables = [val for val in tablemap.values() if isinstance(val,str)]
    43         for subtable in subtables:
    44             if(locfields is None):
    45                 locfields = set(self.processTableMap(table_tree,subtable))
    46             else:
    47                 locfields &= self.processTableMap(table_tree,subtable)
    48                
    49         if("__table" in tablemap):
    50             table = tablemap["__table"]
    51             locfields = set()
    52             for field in table._fields:
    53                 locfields.add(field.name)
    54        
    55         tablemap["__locfields"] = locfields   
    56        
    57         return locfields
    58 
    59 
    60     def __init__(self,root_name,table_tree):
     36
     37    def __init__(self,field,data):
    6138        self._invar = Invar()
    62        
    63         tableoid = itypes_py.Field("_tableoid",itypes_py.createType(str),defval=root_name)
    64         fieldnames = set(["_tableoid"])
    65         fields = [tableoid]
    66         root_fields = self.processTableMap(table_tree,root_name)
    67        
    68         #collect fields
    69         for tablename,tablemap in table_tree.iteritems():
    70             if("__table" in tablemap):
    71                 table = tablemap["__table"]
    72                 for field in table._fields:
    73                     if(field.name not in fieldnames):
    74                         nfield = field.realias()
    75                         fieldnames.add(nfield.name)
    76                         fields.append(nfield)
    77         self._fields = fields
    78        
    79         prop,active = proptypes.InheritPropertyType._create([field.name for field in self._fields],[field.id for field in self._fields],table_tree,root_name)
    80         self._props = (prop,)
    81 
    82         root_map = table_tree[root_name]
    83         self._actidx = [pos for pos,field in enumerate(self._fields) if field.name in root_fields]
    84 
    85         self._actidx_props = [0]
    86 
    87 
    88 
     39        self._fields = [field]
     40        self._actidx = [0]
     41        self._segments=(SegmentOutFix(attr=[field.id]),)
     42        self._source = (data,)
     43       
    8944class PySrcCon(container.Container):
    9045    """PySrcCon: class to store python data"""
    9146    __slots__ = ["_add_buffer","_modify_check","_copyonupdate"]
    92     _output_segment=[itypes_py.OUTFIX]
    9347    _select_compat = qg_translate.PyCompatibilityCheck
    9448
     
    9751        self._fields = fields
    9852        self._actidx = range(len(fields))
    99         self._segment_attr = ([field.id for field in fields],)
     53        self._segments=(SegmentOutFix(attr=[field.id for field in fields]),)
    10054
    10155        if(not data is None):
  • container/user_if.py

    r104 r150  
    9393    return srccon.PySrcCon(fields,data,modifiable)
    9494
    95 def scalar(data=None, fieldname=None, fieldtype=None,modifiable=False):
     95def scalar(data=None, fieldname=None, fieldtype=None):
    9696    #determine column types
    9797    if(fieldtype):
     
    108108    #create fields
    109109    defval = itypes_py.to_defval(fieldtype)
    110     fields = [itypes_py.Field(fieldname,fieldtype,defval)]
     110    field = itypes_py.Field(fieldname,fieldtype,defval)
    111111
    112112    #transform data to internally used column based form
    113     if(not data is None):
    114         data = (utility.arrayscalar(data,dtype=itypes_py.to_numpy(fieldtype)),)
    115                
    116     return srccon.PySrcCon(fields,data,modifiable)
     113    return srccon.ScalarSrcCon(field,data)
    117114
    118115def iany(value):
  • setup.py

    r147 r150  
    99_ConfigDefault = {
    1010    "setup.numpy_path" : distutils.sysconfig.get_python_lib() + \
    11         "/numpy/numpy/core/include",
     11        "/numpy/core/include",
    1212    "setup.db_name":                "ibidas",
    1313    "setup.db_user":                "postgres",
  • src/base_container.c

    r123 r150  
    494494    Py_XDECREF(self->fields);
    495495    Py_XDECREF(self->actidx);
    496     Py_XDECREF(self->segment_attr);
     496    Py_XDECREF(self->segments);
    497497    Py_XDECREF(self->modify_sems);
    498498    Py_XDECREF(self->select_query);
     
    628628}/*}}}*/
    629629
    630 static PyObject * Invar_getSegmentAttr(Invar *self, void *closure)/*{{{*/
    631 {
    632     if(self->segment_attr == NULL)
     630static PyObject * Invar_getSegments(Invar *self, void *closure)/*{{{*/
     631{
     632    if(self->segments == NULL)
    633633    {
    634634        Py_RETURN_NONE;
    635635    }   
    636     Py_INCREF(self->segment_attr);
    637     return self->segment_attr;
    638 }
    639 static int Invar_setSegmentAttr(Invar *self, PyObject *value, void *closure)
     636    Py_INCREF(self->segments);
     637    return self->segments;
     638}
     639static int Invar_setSegments(Invar *self, PyObject *value, void *closure)
    640640{
    641641    PyObject *tmp;
    642642    if(value == NULL)
    643643    {
    644         Py_XDECREF(self->segment_attr);
    645         self->segment_attr = NULL;
     644        Py_XDECREF(self->segments);
     645        self->segments = NULL;
    646646        return 0;
    647647    }
    648648    if(!(value == Py_None || PyTuple_Check(value)))
    649649    {
    650         PyErr_SetString(PyExc_TypeError, "Segment attributes should be None or a tuple");
    651         return -1;
    652     }
    653     tmp = self->segment_attr;
    654     Py_INCREF(value);
    655     self->segment_attr = value;
     650        PyErr_SetString(PyExc_TypeError, "Segments should be None or a tuple");
     651        return -1;
     652    }
     653    tmp = self->segments;
     654    Py_INCREF(value);
     655    self->segments = value;
    656656    Py_XDECREF(tmp);
    657657    return 0;
     
    815815     "Active field indexes",
    816816     NULL},
    817     {"segment_attr",
    818      (getter)Invar_getSegmentAttr, (setter)Invar_setSegmentAttr,
    819      "Segment attributes",
     817    {"segments",
     818     (getter)Invar_getSegments, (setter)Invar_setSegments,
     819     "Segments",
    820820     NULL},
    821821    {"modify_sems",
     
    11501150}/*}}}*/
    11511151
    1152 static PyObject * BaseContainer_getSegmentAttr(BaseContainer *self, void *closure)/*{{{*/
    1153 {
    1154     if(self->invar == NULL)
    1155     {
    1156         PyErr_SetString(PyExc_TypeError, "Invar not set!");
    1157         return NULL;
    1158     }   
    1159     return Invar_getSegmentAttr(self->invar,closure);
    1160 }
    1161 static int BaseContainer_setSegmentAttr(BaseContainer *self, PyObject *value, void *closure)
     1152static PyObject * BaseContainer_getSegments(BaseContainer *self, void *closure)/*{{{*/
     1153{
     1154    if(self->invar == NULL)
     1155    {
     1156        PyErr_SetString(PyExc_TypeError, "Invar not set!");
     1157        return NULL;
     1158    }   
     1159    return Invar_getSegments(self->invar,closure);
     1160}
     1161static int BaseContainer_setSegments(BaseContainer *self, PyObject *value, void *closure)
    11621162{
    11631163    if(self->invar == NULL)
     
    11661166        return -1;
    11671167    } 
    1168     return Invar_setSegmentAttr(self->invar,value,closure);
     1168    return Invar_setSegments(self->invar,value,closure);
    11691169}/*}}}*/
    11701170
     
    12551255     "Refers to invar active field index",
    12561256     NULL},
    1257     {"_segment_attr",
    1258      (getter)BaseContainer_getSegmentAttr, (setter)BaseContainer_setSegmentAttr,
    1259      "Refers to invar segment attr",
     1257    {"_segments",
     1258     (getter)BaseContainer_getSegments, (setter)BaseContainer_setSegments,
     1259     "Refers to invar segments",
    12601260     NULL},
    12611261    {"_modify_sems",
  • src/base_container.h

    r131 r150  
    2828    PyObject * fields;
    2929    PyObject * actidx;
    30     PyObject * segment_attr;
     30    PyObject * segments;
    3131    PyObject * modify_sems;
    3232    PyObject * select_query;
  • src/qgraph.c

    r119 r150  
    2121    PyObject * actidx;
    2222    PyObject * props;
    23     PyObject * segment_attr;
     23    PyObject * segments;
    2424
    2525    PyObject * in_fields;
    2626    PyObject * out_calc;
    27     PyObject * output_segment;
    2827    PyObject * out_fields;
    2928    PyObject * exec_params;
     
    4241    Py_XDECREF(self->actidx);
    4342    Py_XDECREF(self->props);
    44     Py_XDECREF(self->segment_attr);
     43    Py_XDECREF(self->segments);
    4544
    4645    Py_XDECREF(self->in_fields);
    4746    Py_XDECREF(self->out_calc);
    48     Py_XDECREF(self->output_segment);
    4947    Py_XDECREF(self->out_fields);
    5048    Py_XDECREF(self->exec_params);
     
    5856Node_init(Node *self, PyObject *args, PyObject *kwds)
    5957{
    60    PyObject *obj,*source = NULL,*target = NULL, *output_segment;
     58   PyObject *obj,*source = NULL,*target = NULL, *segments;
    6159   Invar * invar;
    6260   static char *kwlist[] = {"obj","source","target",NULL};
     
    9492        }
    9593
    96         output_segment = PyObject_GetAttrString(self->obj,"_output_segment");
    97         if(output_segment == NULL)
    98             return -1;
    99         Py_INCREF(output_segment);
    100         self->output_segment = output_segment;
     94       
     95        segments = invar->segments;
     96        if(segments == NULL)
     97        {
     98            segments = PyObject_GetAttrString(self->obj,"_csegments");
     99            if(segments == NULL)
     100                return -1;
     101        }
     102        Py_INCREF(segments);
     103
     104        if(PyTuple_Check(segments)) //tuple to list (prevents original node to be changed during optimization)
     105        {
     106            int i;
     107            PyObject *nsegments,*item;
     108            long size = PyTuple_GET_SIZE(segments);;
     109            nsegments = PyList_New(size);
     110            for(i = 0; i < size; i++ )
     111            {
     112                item = PyTuple_GET_ITEM(segments,i);
     113                Py_INCREF(item);
     114                PyList_SET_ITEM(nsegments,i,item);
     115            }
     116            Py_DECREF(segments);
     117            segments = nsegments;
     118        }
     119        self->segments = segments;
    101120
    102121
     
    109128        Py_XINCREF(invar->props);
    110129        self->props = invar->props;
    111        
    112         Py_XINCREF(invar->segment_attr);
    113         self->segment_attr = invar->segment_attr;
    114130       
    115131        Py_XINCREF(invar->local);
     
    242258}/*}}}*/
    243259
    244 static PyObject * Node_getSegmentAttr(Node *self, void *closure)/*{{{*/
    245 {
    246     if(!self->segment_attr)
     260static PyObject * Node_getSegments(Node *self, void *closure)/*{{{*/
     261{
     262    if(!self->segments)
    247263    {
    248264        Py_INCREF(Py_None);
    249         self->segment_attr = Py_None;
    250     }
    251     Py_INCREF(self->segment_attr);
    252     return self->segment_attr;
    253 }
    254 static int Node_setSegmentAttr(Node *self, PyObject *value, void *closure)
     265        self->segments = Py_None;
     266    }
     267    Py_INCREF(self->segments);
     268    return self->segments;
     269}
     270static int Node_setSegments(Node *self, PyObject *value, void *closure)
    255271{
    256272    PyObject *tmp;
    257     tmp = self->segment_attr;
     273    tmp = self->segments;
    258274    Py_XINCREF(value);
    259     self->segment_attr = value;
     275    self->segments = value;
    260276    Py_XDECREF(tmp);
    261277    return 0;
     
    322338}/*}}}*/
    323339
    324 static PyObject * Node_getOutputSegment(Node *self, void *closure)/*{{{*/
    325 {
    326     if(!self->output_segment)
    327     {
    328         Py_INCREF(Py_None);
    329         self->output_segment = Py_None;
    330     }
    331     Py_INCREF(self->output_segment);
    332     return self->output_segment;
    333 }
    334 static int Node_setOutputSegment(Node *self, PyObject *value, void *closure)
    335 {
    336     PyObject *tmp;
    337     tmp = self->output_segment;
    338     Py_XINCREF(value);
    339     self->output_segment = value;
    340     Py_XDECREF(tmp);
    341     return 0;
    342 }/*}}}*/
    343 
    344340static PyObject * Node_getExecParams(Node *self, void *closure)/*{{{*/
    345341{
     
    429425     "Active field indexes",
    430426     NULL},
    431     {"segment_attr",
    432      (getter)Node_getSegmentAttr, (setter)Node_setSegmentAttr,
    433      "Segment attributes",
     427    {"segments",
     428     (getter)Node_getSegments, (setter)Node_setSegments,
     429     "Segments",
    434430     NULL},
    435431    {"in_fields",
     
    444440     (getter)Node_getOutFields, (setter)Node_setOutFields,
    445441     "Fields output by this node.",
    446      NULL},
    447     {"output_segment",
    448      (getter)Node_getOutputSegment, (setter)Node_setOutputSegment,
    449      "Specification of output segments",
    450442     NULL},
    451443    {"exec_params",
Note: See TracChangeset for help on using the changeset viewer.