Changeset 1204 for obs/repo-status


Ignore:
Timestamp:
Nov 29, 2015, 9:17:37 PM (9 years ago)
Author:
joergs
Message:

refactoring: move obs functionality to ObsStatus class

File:
1 edited

Legend:

Unmodified
Added
Removed
  • obs/repo-status/repo-status.py

    r1203 r1204  
    3232    succeeded="SUCCEEDED"
    3333
    34 obs_status={
    35     'status': STATE.unknown,
    36     'broken': [],
    37     'building': [],
    38     'disabled': [],
    39     'failed': [],
    40     'finished': [],
    41     'other': [],
    42     'unresolvable': [],
    43 }
    44 
    45 version = {
    46         'srcmd5': "",
    47         'version': "",
    48         'rev':     -1,
    49         'time':   "",
    50 }
    5134
    5235jenkins_status = {
     
    7053            print " ", i
    7154
    72 def write_status( obs_status, jenkins_status, version = {} ):
     55def write_status(obs, jenkins_status):
    7356    filename = "status.succeeded"
    7457    status = STATE.succeeded
    7558    component=""
    76     if obs_status['status'] != STATE.succeeded and obs_status['status'] != STATE.disabled:
     59    if obs.get_state() != STATE.succeeded and obs.get_state() != STATE.disabled:
    7760        component = "obs"
    78         status = obs_status['status']
     61        status = obs.get_state()
    7962    elif jenkins_status['status'] != STATE.succeeded and jenkins_status['status'] != STATE.disabled:
    8063        component = "jenkins"
     
    8669    if destdir:
    8770        filepath=destdir + "/" + filename
    88         out=open( filepath, 'w')
    89         logger.info( "status will be written to " + filepath )
     71        out=open(filepath, 'w')
     72        logger.info("status will be written to " + filepath)
    9073        # remove outdated files
    91         for i in glob( destdir + "/status*" ):
    92             if not os.path.samefile( i, filepath ):
    93                 logger.debug( "remove outdated status file " + i )
     74        for i in glob(destdir + "/status*"):
     75            if not os.path.samefile(i, filepath):
     76                logger.debug("remove outdated status file " + i)
    9477                os.remove(i)
    9578    else:
     
    10083    out.write( "#\n" )
    10184    out.write( "NAME="+filename+"\n" )
     85    obs_status = obs.get_status()
    10286    for key in sorted(obs_status):
    10387        out.write( "#\n" )
     
    10589            out.write( "OBS_STATUS" +"="+ obs_status[key] +"\n" )
    10690        else:
    107             out.write( "OBS_STATUS_" + key.upper() +"="+",".join(get_repo_list( obs_status[key] ))+"\n" )
     91            out.write( "OBS_STATUS_" + key.upper() +"="+",".join(get_repo_list(obs_status[key]))+"\n" )
    10892
    10993    out.write( "#\n" )
     94    version = obs.get_pkg_info()
    11095    for key in sorted(version):
    11196        out.write( "BUILD_" + key.upper() +"="+str(version[key])+"\n")
     97
     98    out.write( "#\n" )
     99    out.write("PACKAGES_SUCCESSFUL='%s'\n" % (get_packages_string(obs.get_successful_packages())))
     100
     101    out.write( "#\n" )
     102    out.write("PACKAGES_FAILED='%s'\n" % (get_packages_string(obs.get_failed_packages())))
    112103
    113104    out.write( "#\n" )
     
    138129    return result
    139130
    140 
    141 def get_obs_results( prj, pkg ):
    142     # get results of a project:
    143     # %(repository)s|%(arch)s|%(state)s|%(dirty)s|%(code)s|%(details)s
    144     # Debian_5.0|i586|published|False|succeeded|
    145     # Debian_5.0|x86_64|published|False|succeeded|
    146     cmd = format_command( [ osc, osc_paramter, "results", "--csv", "--format='%(state)s|%(repository)s|%(arch)s|%(dirty)s|%(code)s|%(details)s'", "--verbose", prj, pkg ] )
    147     # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
    148     results=subprocess.Popen( cmd, stdout=subprocess.PIPE)
    149     rc=results.wait()
    150     if rc != 0:
    151         logger.error( "failed to get osc results: " + str(rc) )
    152         exit( rc )
    153 
    154     reader = DictReader(results.stdout,
    155                         delimiter='|',
    156                         fieldnames=['state', 'repository',
    157                                     'arch', 'dirty',
    158                                     'code', 'details'])
    159     return reader
    160 
    161 
    162 def get_obs_last_build( prj, pkg, repository, arch ):
    163     # {'rev': '99', 'version': '1.2.1170-1.3', 'srcmd5': '611f626d431d06dc81a32e0e021da0d7', 'time': '2014-03-12 16:43:55'}
    164     cmd = format_command( [ osc, osc_paramter, "buildhist", "--csv", prj, pkg, repository, arch ] )
    165     buildhist=subprocess.Popen( cmd, stdout=subprocess.PIPE )
    166     rc=buildhist.wait()
    167     reader={}
    168     if rc != 0:
    169         logger.error( "failed: " + rc )
    170     else:
    171         buildhist2 = subprocess.Popen(['tail', '-n', '1'],
    172                         stdin=buildhist.stdout,
    173                         stdout=subprocess.PIPE,
    174                         )
    175         reader = DictReader(buildhist2.stdout,
    176                         delimiter='|',
    177                         fieldnames=['time', 'srcmd5', 'rev', 'version'])
    178     try:
    179         # there should only be one entry
    180         return reader.next()
    181     except StopIteration:
    182         return
    183 
    184 def get_obs_prj_results(project_name):
    185     cmd = format_command( [ osc, osc_paramter, "results", "--xml", project_name ] )
    186     # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
    187     results=subprocess.Popen(cmd, stdout=subprocess.PIPE)
    188     #rc=results.wait()
    189     (xmldata, stderr) = results.communicate()
    190     rc=results.returncode
    191     #logger.debug("rc: " + str(rc))
    192     if rc != 0:
    193         logger.error("failed to get osc results: " + str(rc))
    194         exit(rc)
    195     prj = {}
    196     #root = etree.parse(xmldata).getroot()
    197     root = etree.fromstring(xmldata)
    198     #print etree.dump(root)
    199     for result in root.getiterator('result'):
    200         #print result.attrib
    201         #print result.attrib['repository'], result.attrib['arch']
    202         dist = get_repo_name(result.attrib['repository'], result.attrib['arch'], jenkins=True)
    203         prj[dist] = []
    204         for status in result.getiterator('status'):
    205             #print status.attrib
    206             if status.attrib['code'] == 'succeeded':
    207                 prj[dist].append(status.attrib['package'])
    208             elif status.attrib['code'] == 'disabled':
    209                 pass
     131def get_packages_string(packages):
     132    pkg = ''
     133    for dist in packages:
     134        pkg += '%s:%s\n' % (dist, ','.join(packages[dist]))
     135    return pkg
     136
     137class ObsStatus:
     138    def __init__(self, prj, pkg):
     139        self.logger = logging.getLogger()
     140        self.prj = prj
     141        self.pkg = pkg
     142        self.state = STATE.unknown
     143        self.status = {
     144            'broken': [],
     145            'building': [],
     146            'disabled': [],
     147            'failed': [],
     148            'finished': [],
     149            'other': [],
     150            'unresolvable': [],
     151        }
     152        self.pkg_info = {
     153            'srcmd5': "",
     154            'version': "",
     155            'rev':     -1,
     156            'time':   "",
     157        }
     158        self.packages_successful = {}
     159        self.packages_failed = {}
     160
     161    #def get_repo_name(self, repository, arch):
     162        #return get_repo_name(repository, arch, jenkins=False):
     163
     164    def __get_obs_results(self):
     165        # get results of a project:
     166        # %(repository)s|%(arch)s|%(state)s|%(dirty)s|%(code)s|%(details)s
     167        # Debian_5.0|i586|published|False|succeeded|
     168        # Debian_5.0|x86_64|published|False|succeeded|
     169        cmd = format_command( [ osc, osc_paramter, "results", "--csv", "--format='%(state)s|%(repository)s|%(arch)s|%(dirty)s|%(code)s|%(details)s'", "--verbose", self.prj, self.pkg ] )
     170        # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
     171        results=subprocess.Popen( cmd, stdout=subprocess.PIPE)
     172        rc=results.wait()
     173        if rc != 0:
     174            logger.error( "failed to get osc results: " + str(rc) )
     175            exit( rc )
     176
     177        reader = DictReader(results.stdout,
     178                            delimiter='|',
     179                            fieldnames=['state', 'repository',
     180                                        'arch', 'dirty',
     181                                        'code', 'details'])
     182        return reader
     183
     184
     185    def __get_obs_last_build(self, repository, arch):
     186        # {'rev': '99', 'version': '1.2.1170-1.3', 'srcmd5': '611f626d431d06dc81a32e0e021da0d7', 'time': '2014-03-12 16:43:55'}
     187        cmd = format_command( [ osc, osc_paramter, "buildhist", "--csv", self.prj, self.pkg, repository, arch ] )
     188        buildhist=subprocess.Popen( cmd, stdout=subprocess.PIPE )
     189        rc=buildhist.wait()
     190        reader={}
     191        if rc != 0:
     192            logger.error( "failed: " + rc )
     193        else:
     194            buildhist2 = subprocess.Popen(['tail', '-n', '1'],
     195                            stdin=buildhist.stdout,
     196                            stdout=subprocess.PIPE,
     197                            )
     198            reader = DictReader(buildhist2.stdout,
     199                            delimiter='|',
     200                            fieldnames=['time', 'srcmd5', 'rev', 'version'])
     201        try:
     202            # there should only be one entry
     203            return reader.next()
     204        except StopIteration:
     205            return
     206
     207
     208    def __update_packages(self):
     209        cmd = format_command( [ osc, osc_paramter, "results", "--xml", self.prj ] )
     210        # "--last-build": NO, because if --last-build, disabled in 'code' is replaced by succeeded/failed
     211        results=subprocess.Popen(cmd, stdout=subprocess.PIPE)
     212        #rc=results.wait()
     213        (xmldata, stderr) = results.communicate()
     214        rc=results.returncode
     215        if rc != 0:
     216            logger.error("failed to get osc results: " + str(rc))
     217            return False
     218        #root = etree.parse(xmldata).getroot()
     219        root = etree.fromstring(xmldata)
     220        #print etree.dump(root)
     221        for result in root.getiterator('result'):
     222            #print result.attrib
     223            #print result.attrib['repository'], result.attrib['arch']
     224            dist = get_repo_name(result.attrib['repository'], result.attrib['arch'], jenkins=True)
     225            self.packages_successful[dist] = []
     226            self.packages_failed[dist] = []
     227            for status in result.getiterator('status'):
     228                #print status.attrib
     229                if status.attrib['code'] == 'succeeded':
     230                    self.packages_successful[dist].append(status.attrib['package'])
     231                elif status.attrib['code'] == 'disabled':
     232                    pass
     233                else:
     234                    logger.error("%s %s (%s) = %s" % (self.prj, status.attrib['package'], result.attrib['repository'], status.attrib['code']))
     235                    self.packages_failed[dist].append(status.attrib['package'])
     236            # if no failed packages are added to the dist, remove the dist
     237            if not self.packages_failed[dist]:
     238                del(self.packages_failed[dist])
     239        return True
     240
     241
     242    def __update_status(self):
     243        obs_results=self.__get_obs_results()
     244
     245        for i in obs_results:
     246            logger.debug( i )
     247            if i['state'] == 'published' and i['dirty'] == 'False' and i['code'] == 'succeeded':
     248                self.status['finished'].append(i)
     249            elif i['state'] == 'building':
     250                self.status['building'].append(i)
     251            elif i['code'] == 'disabled':
     252                self.status['disabled'].append(i)
     253            elif i['code'] == 'failed':
     254                self.status['failed'].append(i)
     255            elif i['code'] == 'broken':
     256                self.status['broken'].append(i)
     257            elif i['code'] == 'unresolvable':
     258                self.status['unresolvable'].append(i)
    210259            else:
    211                 logger.error("%s %s (%s) = %s" % (project_name, status.attrib['package'], result.attrib['repository'], status.attrib['code']))
    212                 exit(1)
    213     return prj
    214 
    215 
    216 def check_obs_status():
    217     obs_results=get_obs_results( prj, pkg )
    218 
    219     for i in obs_results:
    220         logger.debug( i )
    221         if i['state'] == 'published' and i['dirty'] == 'False' and i['code'] == 'succeeded':
    222             obs_status['finished'].append(i)
    223         elif i['state'] == 'building':
    224             obs_status['building'].append(i)
    225         elif i['code'] == 'disabled':
    226             obs_status['disabled'].append(i)
    227         elif i['code'] == 'failed':
    228             obs_status['failed'].append(i)
    229         elif i['code'] == 'broken':
    230             obs_status['broken'].append(i)
    231         elif i['code'] == 'unresolvable':
    232             obs_status['unresolvable'].append(i)
     260                self.status['other'].append(i)
     261        return True
     262
     263
     264    def __update_pkg_info(self):
     265        rv=[]
     266        # get max 'rev' and 'time' from finished builds
     267        for i in self.status['finished']:
     268            last_build=self.__get_obs_last_build(i['repository'], i['arch'])
     269            self.logger.debug( str(last_build) )
     270            #rv.append( dict(i.items() + j.items() ) )
     271            if last_build:
     272                rev=int(last_build['rev'])
     273                time=last_build['time'].replace(" ","_")
     274                if ( rev > self.pkg_info['rev'] ) or ( rev == self.pkg_info['rev'] and time > self.pkg_info['time'] ):
     275                        self.pkg_info['rev'] = rev
     276                        self.pkg_info['time'] = time
     277                        self.pkg_info['srcmd5'] = last_build['srcmd5']
     278                        self.pkg_info['version'] = last_build['version']
     279                rv.append( { 'result': i, 'buildhist': last_build  } )
     280            else:
     281                self.logger.warn( "no buildhistory definied for " + i['repository'] + "-" + i['arch'] )
     282        self.logger.debug( "result (max): " + str(self.pkg_info) )
     283
     284        for i in rv:
     285            if int(i['buildhist']['rev']) != self.pkg_info['rev']:
     286                self.logger.error( "UNKNOWN: " + pformat( i ) )
     287        self.logger.debug( "finished:" )
     288        for i in rv:
     289            if int(i['buildhist']['rev']) == self.pkg_info['rev']:
     290                self.logger.debug( pformat( i ) )
     291        return True
     292
     293
     294    def __update_state(self):
     295        if self.status['building'] or self.status['other']:
     296            self.state = STATE.pending
     297        elif self.status['failed'] or self.status['broken'] or self.status['unresolvable'] or self.packages_failed:
     298            self.state = STATE.failed
    233299        else:
    234             obs_status['other'].append(i)
    235 
    236     if obs_status['building'] or obs_status['other']:
    237         obs_status['status'] = STATE.pending
    238         #return obs_status['status']
    239     elif obs_status['failed'] or obs_status['broken'] or obs_status['unresolvable']:
    240         obs_status['status'] = STATE.failed
    241         #return obs_status['status']
    242     else:
    243         obs_status['status'] = STATE.succeeded
    244 
    245     # else: all builds should be finished
    246 
    247     rv=[]
    248     for i in obs_status['finished']:
    249         last_build=get_obs_last_build( prj, pkg, i['repository'], i['arch'] )
    250         logger.debug( str(last_build) )
    251         #rv.append( dict(i.items() + j.items() ) )
    252         if last_build:
    253             rev=int(last_build['rev'])
    254             time=last_build['time'].replace(" ","_")
    255             if ( rev > version['rev'] ) or ( rev == version['rev'] and time > version['time'] ):
    256                     version['rev'] = rev
    257                     version['time'] = time
    258                     version['srcmd5'] = last_build['srcmd5']
    259                     version['version'] = last_build['version']
    260             rv.append( { 'result': i, 'buildhist': last_build  } )
     300            self.state = STATE.succeeded
     301        return True
     302
     303
     304    def update(self):
     305        self.__update_status()
     306        self.__update_packages()
     307        self.__update_pkg_info()
     308        self.__update_state()
     309        return True
     310
     311    def get_state(self):
     312        return self.state
     313   
     314    def get_status(self, statustype=None):
     315        result = None
     316        if statustype == None:
     317            result = self.status
    261318        else:
    262             logger.warn( "no buildhistory definied for " + i['repository'] + "-" + i['arch'] )
    263 
    264     logger.debug( "result (max): " + str(version) )
    265 
    266     for i in rv:
    267         if int(i['buildhist']['rev']) != version['rev']:
    268             logger.error( "UNKNOWN: " + pformat( i ) )
    269 
    270     logger.debug( "finished:" )
    271     for i in rv:
    272         if int(i['buildhist']['rev']) == version['rev']:
    273             logger.debug( pformat( i ) )
    274 
    275     return obs_status['status']
     319            result = self.status[statustype]
     320        return result
     321   
     322    def get_pkg_info(self):
     323        return self.pkg_info
     324   
     325    def get_pkg_version(self):
     326        return self.pkg_info['version']
     327   
     328    def get_successful_packages(self):
     329        return self.packages_successful
     330
     331    def get_failed_packages(self):
     332        return self.packages_failed
     333
     334
    276335
    277336def add_jenkins_build_parameter( xmlConfig, build_params_add ):
     
    414473    #etree.dump( xmlConfig )
    415474
    416     pkg = ''
    417     for dist in project_packages:
    418         pkg += '%s:%s\n' % (dist, ','.join(project_packages[dist]))
    419         #build_params['PACKAGES_' + dist] = ','.join(project_packages[dist])
     475    pkg = get_packages_string(project_packages)
    420476    build_params={ 'BUILD_VERSION': version['version'], 'BUILD_REV': version['rev'], 'BUILD_SRCMD5': version['srcmd5'], 'PACKAGES': pkg }
    421477    # build paramter that must be present as Jenkins parameter configuration
     
    500556
    501557    # check obs
    502     if check_obs_status() == STATE.succeeded and version['version']:
     558    obs = ObsStatus(prj, pkg)
     559    obs.update()
     560    if obs.get_state() == STATE.succeeded and obs.get_pkg_version():
    503561        if args.jenkinsurl and args.jenkinsjob:
    504             project_packages = get_obs_prj_results(prj)
    505562            # run and check jenkins
    506             check_jenkins_status(args.jenkinsurl, args.jenkinsjob, jenkins_status, get_repo_list( obs_status['finished'], jenkins=True ), version, project_packages)
     563            check_jenkins_status(args.jenkinsurl, args.jenkinsjob, jenkins_status, get_repo_list( obs.get_status('finished'), jenkins=True ), obs.get_pkg_info(), obs.get_successful_packages())
    507564    else:
    508565        logger.info( "skipped jenkins tests, because prior steps" )
    509566
    510567    logger.debug( "write status" )
    511     write_status( obs_status, jenkins_status, version )
     568    write_status(obs, jenkins_status)
Note: See TracChangeset for help on using the changeset viewer.