source: people/joerg.steffens/technical/spacecmd/stage.py @ 1019

#
# Licensed under the GNU General Public License Version 3
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright 2011,2012 Joerg Steffens <joerg.steffens@dass-it.de>
#

# NOTE: the 'self' variable is an instance of SpacewalkShell

import shlex
from optparse import Option
from pprint import pprint
import sys
from StringIO import StringIO
from tempfile import mkdtemp
import difflib
from spacecmd.utils import *

_STAGE1='dev'
_STAGE2='stg'
_STAGE3='prd'

_STAGES=[ _STAGE1, _STAGE2, _STAGE3 ]

_STAGE_NAMES={
    'dev': 'Development',
    # alternative
    'qas': 'QualityAssurance',
    'stg': 'Staging',
    'prd': 'Production'
}
_STAGE_TRANSITIONS={
    _STAGE1: _STAGE2,
    _STAGE2: _STAGE3
}
_STAGE_STATUS={
    "uptodate":  " ",
    "modified":  "M",
    "dontexist": "!",
    "unknown":   "?"
}
# when normalizing texts, exclude lines that start with following keywords,
# because they result in differences
# that are not relevant for our staging diffs
_STAGE_TEXT_EXCLUDES=[
    # configchannel -> file details
    "Revision: ", "Created: ", "Modified: ",
    # kickstart
    "Org Default: ",
    # activation key
    "Universal Default: ",
]

_DUMP_BASE_DIR="/tmp/spacecmd-stage-dump/"

####################

def do_stage_help( self, args, doreturn = False ):
    print """
Staging:

The basic principle is to have every component in multiple stages.
The stages in this environment are:"""
    for stage in _STAGES:
        successor=self.get_next_stage(stage)
        print "    " + stage + ":" , _STAGE_NAMES.get(stage)

    print """
A stage can have a successor, in our enviroment these are:"""
    for stage in _STAGES:
        successor=self.get_next_stage(stage)
        if successor:
            print "    " + stage, "->" , successor

    print """
Workflow example:
  * creating a new package/package version
    the new package is added to a {stage1} softwarechannel.
    If the package seams to work correctly,
    a new integration phase can be started.
    For this, the packages are copied from {stage1} to {stage2}.
    The {stage2} stage is then tested.
    After a successful test, all content of {stage2} is transfered to {stage3}.
    When the content has arrived in {stage3},
    all productively used systems are able to update to the new content.

Summary:
  {stage1}: all changes are done to {stage1}
  {stage2}: integration tests are done in {stage2} only
  {stage3}: productively used systems using {stage3} only

Changes are not only adding new packages,
but also changing configuration files in the configuration channels,
changing kickstart settings or changing activation keys.

For all these changes, spacecmd stage_* commands offers functionality
to simplify staging.

Usage:
  * create your channels, actionvationkey and so on.
    Because Spacewalk does not know about staging directly,
    staging information must be coded into the name of the components.
    The name must include the stage, separeted by '-',
    eg. centos6-x86_64-{stage1}, centos6-x86_64-{stage1}-subchannel, ks-centos6-x86_64-{stage1}-common, ...
    To create a initial structure, the comamnd 'stage_create_skel' can be used.

  * check the staging status by 'stage_status STAGE'
    This will select all components from stage 'STAGE' and compare each component with the correcponding component from the successor stage, eg.:
    'stage_status {stage1}'
    INFO: softwarechannel
      centos6-x86_64-{stage1}                               -> centos6-x86_64-{stage2}
    M   centos6-x86_64-{stage1}-app1                        ->   centos6-x86_64-{stage2}-app1
    !   centos6-x86_64-{stage1}-app2
    INFO: configchannel
      cfg-centos6-x86_64-{stage1}-app1                      -> cfg-centos6-x86_64-{stage2}-app1
    INFO: kickstart
    M ks-centos6-x86_64-{stage1}-app1                       -> ks-centos6-x86_64-{stage2}-app1
    INFO: activationkey
      1-centos6-x86_64-{stage1}-app1                        -> 1-centos6-x86_64-{stage2}-app1

    This first column indicates the state:
      : empty: no differences. The components from both stages are indentical
    ! : no correcponding component in successor stage found
    M : modification. The component differs between the current and the successor stage

  * The most interessting entries are the modified entires.
    To check this more specifically, use the corresponding 'stage_*_diff' function, eg.
    'stage_softwarechannel_diff centos7-x86_64-{stage1}-app1'
    --- centos6-x86_64-{stage1}-app1

    +++ centos6-x86_64-{stage2}-app1

    @@ -1,1 +1,0 @@

    -newpackage-1.0.1-1.1.noarch

    (it is also possible to compare two specific subchannel, eg.
    'stage_softwarechannel_diff centos6-x86_64-{stage1}-subchannel1 centos6-x86_64-{stage2}-subchannel1'
    but the corresponding successor stage component is found automatically by its name)

  * Softwarechannel and configchannel also offers the stage_*_sync function.
    Use them, to copy the content of a component to the next stage, e.g.
    'stage_softwarechannel_sync centos6-x86_64-{stage1}-app1'
    INFO: syncing packages from softwarechannel centos6-x86_64-{stage1}-app1 to centos6-x86_64-{stage2}-app1
    packages to add to channel "centos6-x86_64-{stage2}-app1":
    newpackage-1.0.1-1.1.noarch
    Perform these changes to channel centos6-x86_64-{stage2}-app1 [y/N]:

  * Repeat these steps, until 'stage_status STAGE' shows no differences between the two stages
    """.format(stage1=_STAGE1, stage2=_STAGE2, stage3=_STAGE3)

def help_stage_create_skel(self):
    print 'stage_create_skel: create initial staging structure'
    print '''usage: stage_create_skel [options]

options:
  -l LABEL
  -a ARCHITECTURE ['ia32', 'x86_64']
  -s SUB (e.g. application1)'''

def do_stage_create_skel(self, args, doreturn = False):
    options = [
                Option('-l', '--label', action='store'),
                Option('-a', '--arch',  action='store'),
                Option('-s', '--sub',  action='store'),
               ]

    (args, options) = parse_arguments(args, options)

    if is_interactive(options):
        options.label = prompt_user('Channel Label:', noblank = True)

        print
        print 'Architecture'
        print '------------'
        print '\n'.join(sorted(self.ARCH_LABELS))
        print
        options.arch = prompt_user('Select:')
        options.arch = prompt_user('Sub:')
    else:
        if not options.label:
            logging.error('A channel label is required')
            return

        if not options.arch:
            logging.error('An architecture is required')
            return

    dist        = options.label
    arch        = options.arch
    if self.stage_create_skel( options.label, options.arch, options.sub, create=False ):
        self.stage_create_skel( options.label, options.arch, options.sub, create=True )


def stage_create_skel(self, dist, arch, sub, create = False):

    org         = "1"
    disttype    = "rhel_6"
    application = sub

    print
    for stage in _STAGES:
        base                 = dist + "-" + arch + "-" + stage
        softwarechannel_base = base
        softwarechannel_sub  = base + "-" + application
        distribution         = "dist-"   + base
        distributionpath     = "/srv/dist/" + base
        configchannel        = "cfg-"    + base + "-" + application
        activationkey_create = base + "-" + application
        activationkey        = org + "-" + activationkey_create
        kickstart            = "ks-"     + base + "-" + application

        print "stage: " + stage

        print "softwarechannel base:       " + softwarechannel_base,
        if self.is_softwarechannel( softwarechannel_base ):
            print " [exists]",
        elif create:
            self.do_softwarechannel_create( "-n " + softwarechannel_base + " -l " + softwarechannel_base + " -a " + arch )
        print

        print "softwarechannel subchannel: " + softwarechannel_sub,
        if self.is_softwarechannel( softwarechannel_sub ):
            print " [exists]",
        elif create:
            self.do_softwarechannel_create( "-n " + softwarechannel_sub  + " -l " + softwarechannel_sub  + " -a " + arch + " -p " + base )
        print


        print "distribution:               " + distribution + " (distribution path: " + distributionpath + ")",
        if distribution in self.do_distribution_list(distribution, True):
            print " [exists]",
        elif create:
            self.do_distribution_create( "--name " + distribution + " --path " + distributionpath + " --base-channel " + base + " --install-type rhel_6" )
        print

        print "configchannel:              " + configchannel,
        if self.is_configchannel( configchannel ):
            print " [exists]",
        elif create:
            self.do_configchannel_create( "-n " + configchannel )
        print

        print "activationkey:              " + activationkey,
        if self.is_activationkey( activationkey ):
            print " [exists]",
        elif create:
            self.do_activationkey_create( "-n " + activationkey_create + " -d " + activationkey + " -b " + base + " -e provisioning_entitled" )
            self.do_activationkey_addchildchannels( activationkey + " " + softwarechannel_sub )
            self.do_activationkey_enableconfigdeployment( activationkey )
            self.do_activationkey_addconfigchannels( activationkey + " " + configchannel + " -t" )
        print

        print "kickstart:                  " + kickstart,
        if self.is_kickstart( kickstart ):
            print " [exists]",
        elif create:
            self.do_kickstart_create( "--name=" + kickstart + " --distribution=" + distribution + " --root-password=CHANGEME --virt-type=none" )
            self.do_kickstart_addactivationkeys( kickstart + " " + activationkey )
            self.do_kickstart_enableconfigmanagement( kickstart )
            self.do_kickstart_enablelogging( kickstart )
        print

        print

    if not create:
        print "Make sure, distribution trees are available at the specified distribution paths."
        return self.user_confirm('Create this components [y/N]:')



####################

#
# helper functions
#

def is_stage( self, name ):
    return name in _STAGES

def check_stage( self, name ):
    """Checks if name describes a vaild stage"""
    if not name:
        logging.error( "no stage given" )
        return False
    if not self.is_stage( name ):
        logging.error( "invalid stage " + name )
        return False
    return True

def is_current_stage(self, name):
    return "-"+self.stage in name

def get_common_name( self, name ):
    """Returns the name with the stage replaced by 'STAGE'

    To check the differences from 2 components that are in different stages,
    the specific stage is replaced by the word 'STAGE'
    """
    return self.replace_stage_in_name( name, self.stage, "STAGE" )


def get_stage_from_name( self, name ):
    for i in _STAGES:
        if "-"+i in name:
            return i

def get_next_stage( self, current_stage ):
    return _STAGE_TRANSITIONS.get(current_stage)

def replace_stage_in_name( self, name, current_stage, new_stage ):
    """Return the name with current stage replaced by new stage"""
    return name.replace( "-"+current_stage, "-"+new_stage )

def get_next_stage_name( self, name ):
    current_stage = self.get_stage_from_name( name )
    if not current_stage: return
    next_stage = self.get_next_stage( current_stage )
    if not next_stage: return
    next_stage_name = self.replace_stage_in_name( name, current_stage, next_stage )
    return next_stage_name

def get_normalized_text( self, text, stage, excludes=_STAGE_TEXT_EXCLUDES ):
    """Replace all occurances of stage by the word 'STAGE'"""
    normalized_text = []
    for line in text:
        if not line.startswith( tuple(excludes) ):
            normalized_text.append( self.replace_stage_in_name( line, stage, "STAGE" ) )
    return normalized_text

def print_stage_status( self, name, name_next=None, status="unknown", indent="" ):
    width=48-len(indent)
    string = '{status_code} {indent}{name:{width}}'.format(status_code=_STAGE_STATUS.get(status), indent=indent, name=name, width=width )
    if name_next:
        string = string + " -> " + indent + name_next
    print string

def mkdir(self, name ):
    try:
        if not os.path.isdir( name ):
            os.makedirs( name )
            logging.debug( "creating directory " + name )
        return True
    except:
        logging.error('Failed to create directory ' + name )
        return False

def dump(self, filename, data, raw=False):
    """Writes data to filename"""
    if not self.mkdir( os.path.dirname( filename )): return False
    try:
        fh = open( filename, 'w' )
        if( raw ):
            fh.write(data)
        else:
            fh.write("\n".join(data))
        fh.close()
    except:
        logging.error('failed to create file ' + filename )
        return False


####################

#
# softwarechannel
#

# softwarechannel helper

def is_softwarechannel( self, name ):
    if not name: return
    return name in self.do_softwarechannel_list( name, True )

def check_softwarechannel( self, name ):
    if not name:
        logging.error( "no softwarechannel label given" )
        return False
    if not self.is_softwarechannel( name ):
        logging.error( "invalid softwarechannel label " + name )
        return False
    return True

def get_softwarechannel_childchannel( self, base_channel ):
    result=[]
    for child_channel in self.list_child_channels():
        details = self.client.channel.software.getDetails(\
                            self.session, child_channel)
        if details.get('parent_channel_label') == base_channel:
            result.append( child_channel )
    return result


# softwarechannel next

def help_stage_softwarechannel_next(self):
    print 'stage_softwarechannel_next: get softwarechannel name for the next stage'
    print '                       '
    print 'usage: stage_softwarechannel_next CHANNEL'

def complete_stage_softwarechannel_next(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_softwarechannel_list('', True), text)
    return []

def do_stage_softwarechannel_next(self, args, doreturn = False):
    (args, options) = parse_arguments(args)

    if len(args) != 1:
        self.help_stage_softwarechannel_next()
        return

    source_name = args[0]
    if not self.is_softwarechannel(source_name):
        logging.warning( "invalid softwarechannel "+source_name )
        return
    logging.debug( "source: " + str(source_name) )
    target_name = self.get_next_stage_name( source_name )
    logging.debug( "target: " + str(target_name) )
    if not target_name: return
    # check target name
    if not self.is_softwarechannel(target_name):
        if not doreturn:
            logging.warning( "a next stage softwarechannel for "+source_name+" ("+target_name+") does not exist" )
        return

    if doreturn:
        return target_name
    else:
        print target_name


# softwarechannel diff

def help_stage_softwarechannel_diff(self):
    print 'stage_softwarechannel_diff: diff softwarechannel files'
    print ''
    print 'usage: stage_softwarechannel_diff SOURCE_CHANNEL [TARGET_CHANNEL]'

def complete_stage_softwarechannel_diff(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_softwarechannel_list('', True), text)
    if args == 3:
        return tab_completer(self.do_softwarechannel_list('', True), text)
    return []

def do_stage_softwarechannel_diff(self, args, doreturn = False):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_softwarechannel_diff()
        return

    source_channel = args[0]
    if not self.check_softwarechannel( source_channel ): return

    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_softwarechannel_next( source_channel, doreturn = True)
    if not self.check_softwarechannel( target_channel ): return


    source_data = self.dump_softwarechannel( source_channel, normalize=True )
    target_data = self.dump_softwarechannel( target_channel, normalize=True )

    result=[]

    for line in difflib.unified_diff( source_data, target_data, source_channel, target_channel ):
        if doreturn:
            result.append(line)
        else:
            print line
    return result



# softwarechannel sync

def help_stage_softwarechannel_sync(self):
    print 'stage_softwarechannel_sync: sync the (most recent) packages'
    print '                            from a software channel to another'
    print 'usage: stage_softwarechannel_sync SOURCE_CHANNEL [TARGET_CHANNEL]'

def complete_stage_softwarechannel_sync(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)


    if args == 2:
        return tab_completer(self.do_softwarechannel_list('', True), text)
    if args == 3:
        return tab_completer(self.do_softwarechannel_list('', True), text)
    return []

def do_stage_softwarechannel_sync(self, args):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_softwarechannel_sync()
        return

    source_channel = args[0]
    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_softwarechannel_next( source_channel, doreturn = True)

    logging.info( "syncing packages from softwarechannel "+source_channel+" to "+target_channel )

    # use API call instead of spacecmd function
    # to get detailed infos about the packages
    # and not just there names
    source_packages = self.client.channel.software.listAllPackages(self.session,
                                                               source_channel)
    target_packages = self.client.channel.software.listAllPackages(self.session,
        target_channel)


    # get the package IDs
    source_package_ids = set()
    for package in source_packages:
        try:
            source_package_ids.add(package['id'])
        except KeyError:
            logging.error( "failed to read key id" )
            continue

    target_package_ids = set()
    for package in target_packages:
        try:
            target_package_ids.add(package['id'])
        except KeyError:
            logging.error( "failed to read key id" )
            continue

    print "packages common in both channels:"
    for i in ( source_package_ids & target_package_ids ):
        print self.get_package_name( i )
    print

    source_only = source_package_ids.difference(target_package_ids)
    if source_only:
        print 'packages to add to channel "' + target_channel + '":'
        for i in source_only:
            print self.get_package_name( i )
        print


    # check for packages only in target
    target_only=target_package_ids.difference( source_package_ids )
    if target_only:
        print 'packages to remove from channel "' + target_channel + '":'
        for i in target_only:
            print self.get_package_name( i )
        print

    if source_only or target_only:
        if not self.user_confirm('Perform these changes to channel ' + target_channel + ' [y/N]:'): return

        self.client.channel.software.addPackages(self.session,
                                                target_channel,
                                                list(source_only) )
        self.client.channel.software.removePackages(self.session,
                                                target_channel,
                                                list(target_only) )

####################

#
# configchannel
#

# configchannel helper

def is_configchannel( self, name ):
    if not name: return
    return name in self.do_configchannel_list( name, True )

def check_configchannel( self, name ):
    if not name:
        logging.error( "no configchannel given" )
        return False
    if not self.is_configchannel( name ):
        logging.error( "invalid configchannel label " + name )
        return False
    return True


# configchannel next

def help_stage_configchannel_next(self):
    print 'stage_configchannel_next: get configchannel name for the next stage'
    print '                       '
    print 'usage: stage_configchannel_next CHANNEL'

def complete_stage_configchannel_next(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_configchannel_list('', True), text)
    return []

def do_stage_configchannel_next(self, args, doreturn = False):
    (args, options) = parse_arguments(args)

    if len(args) != 1:
        self.help_stage_configchannel_next()
        return

    source_name = args[0]
    if not self.is_configchannel(source_name):
        logging.warning( "invalid configchannel "+source_name )
        return
    logging.debug( "source: " + str(source_name) )
    target_name = self.get_next_stage_name( source_name )
    logging.debug( "target: " + str(target_name) )
    if not target_name: return
    # check target name
    if not self.is_configchannel(target_name):
        if not doreturn:
            logging.warning( "a next stage configchannel for "+source_name+" ("+target_name+") does not exist" )
        return

    if doreturn:
        return target_name
    else:
        print target_name


# configchannel diff

def help_stage_configchannel_diff(self):
    print 'stage_configchannel_diff: diff between config channels'
    print '                          '
    print 'usage: stage_configchannel_diff SOURCE_CHANNEL [TARGET_CHANNEL]'

def complete_stage_configchannel_diff(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_configchannel_list('', True), text)
    if args == 3:
        return tab_completer(self.do_configchannel_list('', True), text)
    return []

def do_stage_configchannel_diff(self, args, doreturn = False):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_configchannel_diff()
        return

    source_channel = args[0]
    if not self.check_configchannel( source_channel ): return

    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_configchannel_next( source_channel, doreturn = True)
    if not self.check_configchannel( target_channel ): return

    source_data = self.dump_configchannel( source_channel, normalize=True )
    target_data = self.dump_configchannel( target_channel, normalize=True )

    result=[]
    for line in difflib.unified_diff( source_data, target_data, source_channel, target_channel ):
        if doreturn:
            result.append(line)
        else:
            print line
    return result


# configchannel sync

def help_stage_configchannel_sync(self):
    print 'stage_configchannel_sync: sync config files'
    print '                          from a config channel to another'
    print 'usage: stage_configchannel_sync SOURCE_CHANNEL [TARGET_CHANNEL]'

def complete_stage_configchannel_sync(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)


    if args == 2:
        return tab_completer(self.do_configchannel_list('', True), text)
    if args == 3:
        return tab_completer(self.do_configchannel_list('', True), text)
    return []

def do_stage_configchannel_sync(self, args, doreturn = False):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_configchannel_sync()
        return

    source_channel = args[0]
    if not self.check_configchannel( source_channel ): return

    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_configchannel_next( source_channel, doreturn = True)
    if not self.check_configchannel( target_channel ): return

    logging.info( "syncing files from configchannel "+source_channel+" to "+target_channel )

    source_files = set( self.do_configchannel_listfiles( source_channel, doreturn = True ) )
    target_files = set( self.do_configchannel_listfiles( target_channel, doreturn = True ) )

    both=source_files & target_files
    if both:
        print "files common in both channels:"
        print "\n".join( both )
        print

    source_only=source_files.difference( target_files )
    if source_only:
        print "files only in source "+source_channel
        print "\n".join( source_only )
        print

    target_only=target_files.difference( source_files )
    if target_only:
        print "files only in target "+target_channel
        print "\n".join( target_only )
        print

    if both:
        print "files that are in both channels will be overwritten in the target channel"
    if source_only:
        print "files only in the source channel will be added to the target channel"
    if target_only:
        print "files only in the target channel will be deleted"

    if not (both or source_only or target_only):
        logging.info( "nothing to do" )
        return

    if not self.user_confirm('perform synchronisation [y/N]:'): return

    source_data_list = self.client.configchannel.lookupFileInfo(\
                                      self.session, source_channel,
                                      list( both  ) + list(source_only) )


    # TODO: check if this newly available function can be used instead:
    # self.configchannel_sync_by_backup_import()
    for source_data in source_data_list:
        if source_data.get('type') == 'file' or source_data.get('type') == 'directory':
            if source_data.get('contents') and not source_data.get('binary'):
                contents = source_data.get('contents').encode('base64')
            else:
                contents = source_data.get('contents')
            target_data = {
                'contents':                 contents,
                'contents_enc64':           True,
                'owner':                    source_data.get('owner'),
                'group':                    source_data.get('group'),
                #'permissions':              str(source_data.get('permissions')),
                'permissions':              source_data.get('permissions_mode'),
                'selinux_ctx':              source_data.get('selinux_ctx'),
                'macro-start-delimiter':    source_data.get('macro-start-delimiter'),
                'macro-end-delimiter':      source_data.get('macro-end-delimiter'),
            }
            for k,v in target_data.items():
                if not v:
                    del target_data[k]
            logging.debug( source_data.get('path') + ": " + str(target_data) )
            self.client.configchannel.createOrUpdatePath(self.session,
                                                         target_channel,
                                                         source_data.get('path'),
                                                         source_data.get('type') == 'directory',
                                                         target_data)

        elif source_data.get('type') == 'symlink':
            target_data = {
                'target_path':  source_data.get('target_path'),
                'selinux_ctx':  source_data.get('selinux_ctx'),
            }
            logging.debug( source_data.get('path') + ": " + str(target_data) )
            self.client.configchannel.createOrUpdateSymlink(self.session,
                                                            target_channel,
                                                            source_data.get('path'),
                                                            target_data )

        else:
            logging.warning( "unknown file type " + source_data.type )


    # removing all files from target channel that did not exist on source channel
    if target_only:
        self.do_configchannel_removefiles( target_channel + " " + " ".join(target_only) )


####################

#
# kickstart
#

# kickstart helper

def is_kickstart( self, name ):
    if not name: return
    return name in self.do_kickstart_list( name, True )

def check_kickstart( self, name ):
    if not name:
        logging.error( "no kickstart label given" )
        return False
    if not self.is_kickstart( name ):
        logging.error( "invalid kickstart label " + name )
        return False
    return True


# kickstart next

def help_stage_kickstart_next(self):
    print 'stage_kickstart_next: get kickstart name for the next stage'
    print '                       '
    print 'usage: stage_kickstart_next CHANNEL'

def complete_stage_kickstart_next(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_kickstart_list('', True), text)
    return []

def do_stage_kickstart_next(self, args, doreturn = False):
    (args, options) = parse_arguments(args)

    if len(args) != 1:
        self.help_stage_kickstart_next()
        return

    source_name = args[0]
    if not self.is_kickstart(source_name):
        logging.warning( "invalid kickstart "+source_name )
        return
    logging.debug( "source: " + str(source_name) )
    target_name = self.get_next_stage_name( source_name )
    logging.debug( "target: " + str(target_name) )
    if not target_name: return
    # check target name
    if not self.is_kickstart(target_name):
        if not doreturn:
            logging.warning( "a next stage kickstart for "+source_name+" ("+target_name+") does not exist" )
        return

    if doreturn:
        return target_name
    else:
        print target_name


# kickstart diff

def help_stage_kickstart_diff(self):
    print 'stage_kickstart_diff: diff kickstart files'
    print ''
    print 'usage: stage_kickstart_diff SOURCE_CHANNEL [TARGET_CHANNEL]'

def complete_stage_kickstart_diff(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_kickstart_list('', True), text)
    if args == 3:
        return tab_completer(self.do_kickstart_list('', True), text)
    return []

def do_stage_kickstart_diff(self, args, doreturn = False):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_kickstart_diff()
        return

    source_channel = args[0]
    if not self.check_kickstart( source_channel ): return

    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_kickstart_next( source_channel, doreturn = True)
    if not self.check_kickstart( target_channel ): return


    source_data = self.dump_kickstart( source_channel, normalize=True )
    target_data = self.dump_kickstart( target_channel, normalize=True )

    result=[]
    for line in difflib.unified_diff( source_data, target_data, source_channel, target_channel ):
        if doreturn:
            result.append(line)
        else:
            print line
    return result



####################

#
# activationkey
#

# activationkey helper

def is_activationkey( self, name ):
    if not name: return
    return name in self.do_activationkey_list( name, True )

def check_activationkey( self, name ):
    if not name:
        logging.error( "no activationkey label given" )
        return False
    if not self.is_activationkey( name ):
        logging.error( "invalid activationkey label " + name )
        return False
    return True


# activationkey next

def help_stage_activationkey_next(self):
    print 'stage_activationkey_next: get activationkey name for the next stage'
    print '                       '
    print 'usage: stage_activationkey_next CHANNEL'

def complete_stage_activationkey_next(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)

    if args == 2:
        return tab_completer(self.do_activationkey_list('', True), text)
    return []

def do_stage_activationkey_next(self, args, doreturn = False):
    (args, options) = parse_arguments(args)

    if len(args) != 1:
        self.help_stage_activationkey_next()
        return

    source_name = args[0]
    if not self.is_activationkey(source_name):
        logging.warning( "invalid activationkey "+source_name )
        return
    logging.debug( "source: " + str(source_name) )
    target_name = self.get_next_stage_name( source_name )
    logging.debug( "target: " + str(target_name) )
    if not target_name: return
    # check target name
    if not self.is_activationkey(target_name):
        if not doreturn:
            logging.warning( "a next stage activationkey for "+source_name+" ("+target_name+") does not exist" )
        return

    if doreturn:
        return target_name
    else:
        print target_name


# activationkey diff

def help_stage_activationkey_diff(self):
    print 'stage_activationkeyt_diff: diff activationkeys'
    print ''
    print 'usage: stage_activationkey_diff SOURCE_ACTIVATIONKEY [TARGET_ACTIVATIONKEY]'

def complete_stage_activationkey_diff(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)


    if args == 2:
        return tab_completer(self.do_activationkey_list('', True), text)
    if args == 3:
        return tab_completer(self.do_activationkey_list('', True), text)
    return []

def do_stage_activationkey_diff(self, args, doreturn = False):
    options = []

    (args, options) = parse_arguments(args, options)

    if len(args) != 1 and len(args) != 2:
        self.help_stage_activationkey_diff()
        return

    source_channel = args[0]
    if not self.check_activationkey( source_channel ): return

    if len(args) == 2:
        target_channel = args[1]
    else:
        target_channel=self.do_stage_activationkey_next( source_channel, doreturn = True)
    if not self.check_activationkey( target_channel ): return


    source_data = self.dump_activationkey( source_channel, normalize=True )
    target_data = self.dump_activationkey( target_channel, normalize=True )

    result=[]
    for line in difflib.unified_diff( source_data, target_data, source_channel, target_channel ):
        if doreturn:
            result.append(line)
        else:
            print line
    return result




####################

#
# stage_status
# stage_*_status
#

def help_stage_status(self):
    print 'stage_status: status of a stage'
    print ''
    print 'usage: stage_status STAGE\n'
    print 'STAGE: ' + " | ".join( _STAGES )

def complete_stage_status(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)


    if args == 2:
        return tab_completer( _STAGES, text)

    return []

def do_stage_status(self, args):
    (args, options) = parse_arguments(args)

    if not len(args):
        self.help_stage_status()
        return

    stage = args[0]
    if not self.check_stage( stage ): return
    self.stage = stage

    self.stage_softwarechannels_status()
    self.stage_configchannels_status()
    self.stage_kickstarts_status()
    self.stage_activationkeys_status()

def stage_softwarechannels_status( self ):
    logging.info( "softwarechannel" )
    base_channels = self.list_base_channels()
    for base_channel in base_channels:
        if self.is_current_stage( base_channel ):
            self.check_stage_softwarechannel_status( base_channel, indent="" )
            for child_channel in self.get_softwarechannel_childchannel( base_channel ):
                self.check_stage_softwarechannel_status( child_channel, indent="  " )

def check_stage_softwarechannel_status( self, name, indent="" ):
    status="unknown"
    name_next = self.do_stage_softwarechannel_next( name, doreturn=True )
    if name_next:
        if self.do_stage_softwarechannel_diff( name + " " + name_next, doreturn=True ):
            status="modified"
        else:
            status="uptodate"
    else:
        status="dontexist"
    print_stage_status( self, name, name_next=name_next, status=status, indent=indent )
    return status


def stage_configchannels_status( self ):
    logging.info( "configchannel" )
    configchannels = self.do_configchannel_list('', True)

    for name in configchannels:
        if self.is_current_stage( name ):
            self.check_stage_configchannels_status( name )

def check_stage_configchannels_status( self, name, indent="" ):
    status="unknown"
    name_next = self.do_stage_configchannel_next( name, doreturn=True )
    if name_next:
        if self.do_stage_configchannel_diff( name + " " + name_next, doreturn=True ):
            status="modified"
        else:
            status="uptodate"
    else:
        status="dontexist"
    print_stage_status( self, name, name_next=name_next, status=status, indent=indent )
    return status



def stage_kickstarts_status( self ):
    logging.info( "kickstart" )
    kickstarts = self.do_kickstart_list('', True)

    for name in kickstarts:
        if self.is_current_stage( name ):
            self.check_stage_kickstarts_status( name )

def check_stage_kickstarts_status( self, name, indent="" ):
    status="unknown"
    name_next = self.do_stage_kickstart_next( name, doreturn=True )
    if name_next:
        if self.do_stage_kickstart_diff( name + " " + name_next, doreturn=True ):
            status="modified"
        else:
            status="uptodate"
    else:
        status="dontexist"
    print_stage_status( self, name, name_next=name_next, status=status, indent=indent )
    return status



def stage_activationkeys_status( self ):
    logging.info( "activationkey" )
    activationkeys = self.do_activationkey_list('', True)

    for name in activationkeys:
        if self.is_current_stage( name ):
            self.check_stage_activationkey_status( name )

def check_stage_activationkey_status( self, name, indent="" ):
    status="unknown"
    name_next = self.do_stage_activationkey_next( name, doreturn=True )
    if name_next:
        if self.do_stage_activationkey_diff( name + " " + name_next, doreturn=True ):
            status="modified"
        else:
            status="uptodate"
    else:
        status="dontexist"
    print_stage_status( self, name, name_next=name_next, status=status, indent=indent )
    return status



####################

#
# stage_dump
# dump_*
#

def help_stage_dump(self):
    print 'stage_dump: dump infos about a stage to files'
    print ''
    print 'usage: stage_dump STAGE [OUTDIR]\n'
    print 'STAGE: ' + " | ".join( _STAGES )
    print 'OUTDIR defaults to ' + _DUMP_BASE_DIR

def complete_stage_dump(self, text, line, beg, end):
    parts = shlex.split(line)
    if line[-1] == ' ': parts.append('')
    args = len(parts)


    if args == 2:
        return tab_completer( _STAGES, text)

    return []

def do_stage_dump(self, args):
    (args, options) = parse_arguments(args)

    if not len(args):
        self.help_stage_dump()
        return

    stage = args[0]
    if not self.check_stage( stage ): return
    self.stage = stage


    if len(args) == 2:
        outputpath_base = datetime.now().strftime(os.path.expanduser(args[1]))
    else:
        # make the final output path be <base>/date/channel
        outputpath_base = os.path.join( _DUMP_BASE_DIR,
                                        datetime.now().strftime("%Y-%m-%d"),
                                        stage )

    if not self.mkdir( outputpath_base ): return

    self.dump_softwarechannels( outputpath_base + "/softwarechannel/" )
    self.dump_configchannels( outputpath_base + "/configchannel/" )
    self.dump_kickstarts( outputpath_base + "/kickstart/" )
    self.dump_activationkeys( outputpath_base + "/activationkey/" )




def dump_softwarechannels(self, basedir):
    logging.info( "softwarechannel" )
    base_channels = self.list_base_channels()
    for base_channel in base_channels:
        if self.is_current_stage( base_channel ):
            logging.info( "  " + base_channel )
            base_channel_dir = basedir + self.get_common_name(base_channel)
            if not self.mkdir( base_channel_dir ): return

            packages = self.do_softwarechannel_listallpackages( base_channel, doreturn=True )
            self.dump( base_channel_dir + '/' + self.get_common_name(base_channel), packages )
            # get all child channels and pick the channels that belongs to the base channel
            for child_channel in self.get_softwarechannel_childchannel( base_channel ):
                logging.info( "    " + child_channel )
                packages = self.dump_softwarechannel( child_channel, onlyLastestPackages=False, normalize=True )
                self.dump( base_channel_dir + '/' + self.get_common_name(child_channel), packages )

def dump_softwarechannel(self, name, onlyLastestPackages=True, normalize=False):
    if onlyLastestPackages:
        content = self.do_softwarechannel_listallpackages( name, doreturn=True )
    else:
        content = self.do_softwarechannel_listallpackages( name, doreturn=True )
    return content

def dump_configchannel_filedetails(self, name, filename, normalize=False):
    # redirect stdout to string.
    # to be able to reuse the existing do_activationkey_details function
    old_stdout=sys.stdout
    output=StringIO()
    sys.stdout=output
    self.do_configchannel_filedetails( name +" "+ filename )
    sys.stdout=old_stdout
    content=output.getvalue().split("\n")
    output.close()

    if normalize:
        stage = self.get_stage_from_name( name )
        content=self.get_normalized_text( content, stage )
    return content

def dump_configchannel(self, name, normalize=False):
    # redirect stdout to string.
    # to be able to reuse the existing do_activationkey_details function
    old_stdout=sys.stdout
    output=StringIO()
    sys.stdout=output
    self.do_configchannel_details( name )
    sys.stdout=old_stdout
    content=output.getvalue().split("\n")
    output.close()

    for filename in self.do_configchannel_listfiles(name, True):
        content = content + self.dump_configchannel_filedetails(name, filename, normalize=True)

    if normalize:
        stage = self.get_stage_from_name( name )
        content=self.get_normalized_text( content, stage )

    return content


def dump_configchannels(self, basedir):
    logging.info( "configchannel" )
    configchannels = self.do_configchannel_list( '', doreturn = True)

    for name in configchannels:
        if self.is_current_stage( name ):
            logging.info( "  " + name )
            dir = basedir + self.get_common_name(name)
            self.do_configchannel_backup( name+" "+dir )


def dump_kickstart_content(self, name, normalize=False):
    content = self.client.kickstart.profile.downloadRenderedKickstart(\
                                                self.session, name ).split("\n")
    stage = self.get_stage_from_name( name )
    if normalize:
        content=self.get_normalized_text( content, stage )
    return content

def dump_kickstart_details(self, name, normalize=False):
    # redirect stdout to string.
    # to be able to reuse the existing do_activationkey_details function
    old_stdout=sys.stdout
    output=StringIO()
    sys.stdout=output
    self.do_kickstart_details( name )
    sys.stdout=old_stdout
    content=output.getvalue().split("\n")
    output.close()

    if normalize:
        stage = self.get_stage_from_name( name )
        content=self.get_normalized_text( content, stage )

    return content

def dump_kickstart(self, name, normalize=False):
    return dump_kickstart_details(self, name, normalize=normalize)


def dump_kickstarts(self, basedir, doreturn = False):
    logging.info( "kickstart" )
    kickstarts = self.client.kickstart.listKickstarts(self.session)

    for kickstart in kickstarts:
        name = kickstart.get('name')
        if self.is_current_stage( name ):
            logging.info( "  " + name )
            dir = basedir + self.get_common_name(name)
            content = self.dump_kickstart( name, normalize=True )
            if doreturn:
                return content
            else:
                # dump kickstart details and ks file content.
                # use  separate files
                self.dump( dir + '/' + self.get_common_name(name), content )
                self.dump( dir + '/' + self.get_common_name(name) + ".content", dump_kickstart_content(self, name, normalize=True) )


def dump_activationkey(self, name, normalize=False):
    # redirect stdout to string.
    # to be able to reuse the existing do_activationkey_details function
    old_stdout=sys.stdout
    output=StringIO()
    sys.stdout=output
    self.do_activationkey_details( name )
    sys.stdout=old_stdout
    content=output.getvalue().split("\n")
    output.close()

    if normalize:
        stage = self.get_stage_from_name( name )
        content=self.get_normalized_text( content, stage )

    return content



def dump_activationkeys(self, basedir):
    logging.info( "activationkey" )
    activationkeys = self.do_activationkey_list('', True)

    for name in activationkeys:
        if self.is_current_stage( name ):
            logging.info( "  " + name )

            content = self.dump_activationkey( name, normalize=True)

            dir = basedir + self.get_common_name(name)
            self.dump( dir + '/' + self.get_common_name(name), content )

# vim:ts=4:expandtab: