Tryag File Manager

//proc/self/root/usr/sbin/sysreport

#!/usr/bin/python
"""
Gather information about a system and report it using plugins
supplied for application-specific information
"""
## sosreport.py
## gather information about a system and report it

### This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.

## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# pylint: disable-msg = W0611
# pylint: disable-msg = W0702

import sys, traceback
import os
from optparse import OptionParser, Option
import sos.policyredhat
from sos.helpers import *
from threading import Thread, activeCount
import signal
import logging
from stat import *
from time import strftime, localtime, time
from pwd import getpwuid
import gettext
from threading import Semaphore
from tempfile import gettempdir
from textwrap import fill

__version__ = 1.7

__breakHits__ = 0  # Use this to track how many times we enter the exit routine

## Routine for reporting immediately fatal errors
def exit_fatal(msg, e=None):
    sys.stderr.write("%s\n" % msg)
    if e != None:
        sys.stderr.write("\n  %s\n\n" % e)
    sys.stderr.flush()
    sys.exit(1)

## Set up routines to be linked to signals for termination handling
def exittermhandler(signum, frame):
    doExitCode()

def doExitCode():
    from threading import enumerate
    global __breakHits__
    __breakHits__ += 1
    if ( ( activeCount() > 1 ) and ( __breakHits__ == 1 ) ):
        print "SIGTERM received, multiple threads detected, waiting for all threads to exit"
        for thread in enumerate():
            if thread.getName() == "MainThread":
                continue
            # until we find a way to kill threads in case of > 1 CTRL+C, ignore KeyboardInterrupt
            while thread.isAlive():
                try:
                    thread.join()
                except KeyboardInterrupt:
                    pass
        else:
            print "All threads ended, cleaning up."
    if ( ( activeCount() > 1 ) and ( __breakHits__ > 1 ) ):
        print "Multiple SIGTERMs, multiple threads, attempting to signal threads to die immediately"
        ## FIXME: Add thread-kill code (see FIXME below)
#       os.kill(os.getpid(), signal.SIGKILL)
        print "Threads dead, cleaning up."
    if ( ( activeCount() == 1 ) and ( __breakHits__ > 2 ) ):
        print "Multiple SIGTERMs, single thread, exiting without cleaning up."
        sys.exit(3)

exit_fatal("exiting on signal", None)

# Handle any sort of exit signal cleanly
# Currently, we intercept only sig 15 (TERM)
signal.signal(signal.SIGTERM, exittermhandler)

## FIXME: Need to figure out how to IPC with child threads in case of
## multiple SIGTERMs.

# for debugging
__raisePlugins__ = 0

class OptionParser_extended(OptionParser):
    def print_help(self):
        OptionParser.print_help(self)
        print
        print "Some examples:"
        print
        print " enable cluster plugin only and collect dlm lockdumps:"
        print "   # sosreport -o cluster -k cluster.lockdump"
        print
        print " disable memory and samba plugins, turn off rpm -Va collection:"
        print "   # sosreport -n memory,samba -k rpm.rpmva=off"
        print

class SosOption (Option):
    """Allow to specify comma delimited list of plugins"""
    ACTIONS = Option.ACTIONS + ("extend",)
    STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
    TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)

def take_action(self, action, dest, opt, value, values, parser):
        if action == "extend":
            try: lvalue = value.split(",")
            except: pass
            else: values.ensure_value(dest, []).extend(lvalue)
        else:
            Option.take_action(self, action, dest, opt, value, values, parser)

__cmdParser__ = OptionParser_extended(option_class=SosOption)
__cmdParser__.add_option("-l", "--list-plugins", action="store_true", \
                     dest="listPlugins", default=False, \
                     help="list plugins and available plugin options")
__cmdParser__.add_option("-n", "--skip-plugins", action="extend", \
                     dest="noplugins", type="string", \
                     help="skip these plugins", default = [])
__cmdParser__.add_option("-e", "--enable-plugins", action="extend", \
                     dest="enableplugins", type="string", \
                     help="enable these plugins", default = [])
__cmdParser__.add_option("-o", "--only-plugins", action="extend", \
                     dest="onlyplugins", type="string", \
                     help="enable these plugins only", default = [])
__cmdParser__.add_option("-k", action="extend", \
                     dest="plugopts", type="string", \
                     help="plugin options in plugname.option=value format (see -l)")
__cmdParser__.add_option("-a", "--alloptions", action="store_true", \
                     dest="usealloptions", default=False, \
                     help="enable all options for loaded plugins")
__cmdParser__.add_option("-v", "--verbose", action="count", \
                     dest="verbosity", \
                     help="increase verbosity")
__cmdParser__.add_option("--no-progressbar", action="store_false", \
                     dest="progressbar", default=True, \
                     help="do not display a progress bar.")
__cmdParser__.add_option("--no-multithread", action="store_true", \
                     dest="nomultithread", default=True, \
                     help="disable multi-threaded gathering mode")
__cmdParser__.add_option("--multithread", action="store_false", \
                     dest="nomultithread", default=True, \
                     help="enable multi-threaded gathering mode")
__cmdParser__.add_option("--batch", action="store_true", \
                     dest="batch", default=False, \
                     help="do not ask any question (batch mode)")
__cmdParser__.add_option("--ticket-number", action="store", \
                     dest="ticketNumber", type="string", \
                     help="associate a ticket number with report")
__cmdParser__.add_option("--name", action="store", \
                     dest="customerName", type="string", \
                     help="associate a name with report")
try:
    __cmdParser__.add_option("--tmp-dir", action="store", \
                         dest="tmpdir", type="string", default=gettempdir(),
                         help="specify alternate temporary directory")
except Exception, e:
    exit_fatal("unable to create temporary directories:", e)

if sys.argv[0].endswith("sysreport"):
    try:
        ppid = os.getppid()
        fp = open("/proc/%d/cmdline" % ppid, "r")
        cmd = fp.read()
        fp.close()
    except:
        cmd = ""
    if not sys.stdin.isatty() or cmd.find("bash") < 0:
        os.execl("/bin/sh", "/bin/sh", "-c", "/usr/sbin/sysreport.legacy")
        os.exit(-1)
    print
    print "WARNING: sysreport is deprecated, please use sosreport instead."

(__cmdLineOpts__, __cmdLineArgs__)=__cmdParser__.parse_args()

def textcolor(text, fg, raw=0):
    colors = {  "black":"30", "red":"31", "green":"32", "brown":"33", "blue":"34",
		"purple":"35", "cyan":"36", "lgray":"37", "gray":"1;30", "lred":"1;31",
		"lgreen":"1;32", "yellow":"1;33", "lblue":"1;34", "pink":"1;35",
		"lcyan":"1;36", "white":"1;37" }
    opencol = "\033["
    closecol = "m"
    clear = opencol + "0" + closecol
    f = opencol + colors[fg] + closecol
    return "%s%s%s" % (f, text, clear)

class XmlReport:
    def __init__(self):
        try:
            import libxml2
        except:
            self.enabled = False
            return
        else:
            self.enabled = True
        self.doc = libxml2.newDoc("1.0")
        self.root = self.doc.newChild(None, "sos", None)
        self.commands = self.root.newChild(None, "commands", None)
        self.files = self.root.newChild(None, "files", None)

def add_command(self, cmdline, exitcode, stdout = None, stderr = None, f_stdout=None, f_stderr=None, runtime=None):
        if not self.enabled: return

cmd = self.commands.newChild(None, "cmd", None)

cmd.setNsProp(None, "cmdline", cmdline)

cmdchild = cmd.newChild(None, "exitcode", str(exitcode))

if runtime:
            cmd.newChild(None, "runtime", str(runtime))

if stdout or f_stdout:
            cmdchild = cmd.newChild(None, "stdout", stdout)
            if f_stdout:
                cmdchild.setNsProp(None, "file", f_stdout)

if stderr or f_stderr:
            cmdchild = cmd.newChild(None, "stderr", stderr)
            if f_stderr:
                cmdchild.setNsProp(None, "file", f_stderr)

def add_file(self,fname,stats):
        if not self.enabled: return

cfile = self.files.newChild(None,"file",None)

cfile.setNsProp(None, "fname", fname)

cchild = cfile.newChild(None, "uid", str(stats[ST_UID]))
        cchild = cfile.newChild(None, "gid", str(stats[ST_GID]))
        cfile.newChild(None, "mode", str(oct(S_IMODE(stats[ST_MODE]))))
        cchild = cfile.newChild(None, "ctime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_CTIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_CTIME]))
        cchild = cfile.newChild(None, "atime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_ATIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_ATIME]))
        cchild = cfile.newChild(None, "mtime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_MTIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_MTIME]))

def serialize(self):
        if not self.enabled: return

print self.doc.serialize(None,  1)

def serialize_to_file(self,fname):
        if not self.enabled:
            return
        outfn = open(fname,"w")
        outfn.write(self.doc.serialize(None,1))
        outfn.close()

def _get_msg(tmpdir):
    msg = _("""This command will collect diagnostic and configuration \
information from this %(distro)s system and installed \
applications.

An archive containing the collected information will be \
generated in %(tmpdir)s and may be provided to a %(vendor)s \
support representative.

Any information provided to %(vendor)s will be treated in \
accordance with the published support policies at:\n
  %(vendor_url)s

The generated archive may contain data considered sensitive \
and its content should be reviewed by the originating \
organization before being passed to any third party.

No changes will be made to system configuration.
%(vendor_text)s
""")

width = 72
    msg = msg % {'distro': "Red Hat Enterprise Linux",
                       'vendor': "Red Hat",
                       'vendor_url': "https://access.redhat.com/support/",
                       'vendor_text': "",
                       'tmpdir': tmpdir}
    fmt = ""
    for line in msg.splitlines():
        fmt = fmt + fill(line, width, replace_whitespace = False) + '\n'
    return fmt

def sosreport():
    # pylint: disable-msg = R0912
    # pylint: disable-msg = R0914
    # pylint: disable-msg = R0915
    """
    This is the top-level function that gathers and processes all sosreport information
    """
    loadedplugins = []
    skippedplugins = []
    alloptions = []

# perhaps we should automatically locate the policy module??
    policy = sos.policyredhat.SosPolicy()

# find the plugins path
    paths = sys.path
    for path in paths:
        if path.strip()[-len("site-packages"):] == "site-packages" and \
        os.path.isdir(path+"/sos/plugins"):
            pluginpath = path + "/sos/plugins"

# Set up common info and create destinations

dstrootparent = __cmdLineOpts__.tmpdir
    dstroot = sosFindTmpDir(dstrootparent)

if dstroot == None:
        print "Could not create temporary directory."
        sys.exit(1)

cmddir = os.path.join(dstroot, "sos_commands")
    logdir = os.path.join(dstroot, "sos_logs")
    rptdir = os.path.join(dstroot, "sos_reports")
    try:
        os.mkdir(cmddir, 0755)
        os.mkdir(logdir, 0755)
        os.mkdir(rptdir, 0755)
    except Exception, e:
        exit_fatal("unable to create temporary directories:", e)

# initialize i18n language localization
    gettext.install('sos', '/usr/share/locale', unicode=False)

# initialize logging
    soslog = logging.getLogger('sos')
    soslog.setLevel(logging.DEBUG)

# set logging to production mode to avoid cascading exceptions in the log
    # paths (for e.g. during ENOSPC on destination)
    logging.raiseExceptions = False
    logging.VERBOSE  = logging.INFO - 1
    logging.VERBOSE2 = logging.INFO - 2
    logging.VERBOSE3 = logging.INFO - 3
    logging.addLevelName(logging.VERBOSE, "verbose")
    logging.addLevelName(logging.VERBOSE2,"verbose2")
    logging.addLevelName(logging.VERBOSE3,"verbose3")

# if stdin is not a tty, disable colors and don't ask questions
    if not sys.stdin.isatty():
        __cmdLineOpts__.batch = True

try:
        # log to a file
        flog = logging.FileHandler(logdir + "/sos.log")
        flog.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
        flog.setLevel(logging.VERBOSE3)
        soslog.addHandler(flog)
    except Exception, e:
        exit_fatal("unable to initialise file logging:", e)

# define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler(sys.stderr)
    if __cmdLineOpts__.verbosity > 0:
        console.setLevel(20 - __cmdLineOpts__.verbosity)
        __cmdLineOpts__.progressbar = False
    else:
        console.setLevel(logging.INFO)
    console.setFormatter(logging.Formatter('%(message)s'))
    soslog.addHandler(console)

xmlrep = XmlReport()

# set up dict so everyone can share the following
    commons = {'dstroot': dstroot, 'cmddir': cmddir, 'logdir': logdir, 'rptdir': rptdir,
               'soslog': soslog, 'policy': policy, 'verbosity' : __cmdLineOpts__.verbosity,
               'xmlreport' : xmlrep, 'cmdlineopts':__cmdLineOpts__, 'dstrootparent': dstrootparent }

# Make policy aware of the commons
    policy.setCommons(commons)

print
    soslog.info ( _("sosreport (version %s)") % __version__)
    print

# generate list of available plugins
    try:
        plugins = os.listdir(pluginpath)
    except Exception, e:
        exit_fatal("could not generate plugin list:", e)
        
    plugins.sort()
    plugin_names = []

# validate and load plugins
    for plug in plugins:
        plugbase =  plug[:-3]
        if not plug[-3:] == '.py' or plugbase == "__init__":
            continue
        try:
            try:
                if policy.validatePlugin(pluginpath + plug):
                    pluginClass = importPlugin("sos.plugins." + plugbase, plugbase)
                else:
                    soslog.warning(_("plugin %s does not validate, skipping") % plug)
                    skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                    continue
                plugin_names.append(plugbase)
                if plugbase in __cmdLineOpts__.noplugins:
                    soslog.log(logging.VERBOSE, _("plugin %s skipped (--skip-plugins)") % plugbase)
                    skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                    continue
                if not pluginClass(plugbase, commons).checkenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins:
                    soslog.log(logging.VERBOSE, _("plugin %s is inactive (use -e or -o to enable).") % plug)
                    skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                    continue
                if not pluginClass(plugbase, commons).defaultenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins:
                    soslog.log(logging.VERBOSE, "plugin %s not loaded by default (use -e or -o to enable)." % plug)
                    skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                    continue
                if __cmdLineOpts__.onlyplugins and not plugbase in __cmdLineOpts__.onlyplugins:
                    soslog.log(logging.VERBOSE, _("plugin %s not specified in --only-plugins list") % plug)
                    skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                    continue
                loadedplugins.append((plugbase, pluginClass(plugbase, commons)))
            except IOError, e:
                exit_fatal("received IO error during plugin set up:", e)
            except:
                soslog.warning(_("plugin %s does not install, skipping") % plug)
                raise
        except Exception, inst:
            soslog.warning(_("could not load plugin %s") % plug)
            bt = traceback.format_exception(type(inst), inst.args, None)
            for line in bt:
                soslog.warning(line)
            
            if __raisePlugins__:
                raise

# First, gather and process options
    # using the options specified in the command line (if any)
    if __cmdLineOpts__.usealloptions:
        for plugname, plug in loadedplugins:
            for name, parms in zip(plug.optNames, plug.optParms):
                if type(parms["enabled"])==bool:
                    parms["enabled"] = True

if __cmdLineOpts__.plugopts:
        opts = {}
        for opt in __cmdLineOpts__.plugopts:
            # split up "general.syslogsize=5"
            try:
                opt, val = opt.split("=")
            except:
                val=True
            else:
                if val == "off" or val == "disable" or val == "disabled":
                    val = False
                else:
                    # try to convert string "val" to int()
                    try:    val = int(val)
                    except: pass

# split up "general.syslogsize"
            plug, opt = opt.split(".")

try: opts[plug]
            except KeyError: opts[plug] = []
            opts[plug].append( (opt,val) )

for plugname, plug in loadedplugins:
           if opts.has_key(plugname):
               for opt,val in opts[plugname]:
                   soslog.log(logging.VERBOSE, "setting option %s for plugin %s to %s" % (plugname,opt,val))
                   plug.setOption(opt,val)
        del opt,opts,val
    
    # error if the user references a plugin which does not exist
    unk_plugs =  [plugname.split(".")[0] for plugname in __cmdLineOpts__.onlyplugins   if not plugname.split(".")[0] in plugin_names]
    unk_plugs += [plugname.split(".")[0] for plugname in __cmdLineOpts__.noplugins     if not plugname.split(".")[0] in plugin_names]
    unk_plugs += [plugname.split(".")[0] for plugname in __cmdLineOpts__.enableplugins if not plugname.split(".")[0] in plugin_names]
    if len(unk_plugs):
        for plugname in unk_plugs:
            soslog.error('a non-existing plugin (%s) was specified in the command line' % (plugname))
        exit_fatal('non-existent plugin(s) specified on command line' % (plugname))
    del unk_plugs
    
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE3, _("processing options from plugin: %s") % plugname)
        names, parms = plug.getAllOptions()
        for optname, optparm  in zip(names, parms):
            alloptions.append((plug, plugname, optname, optparm))

# when --listplugins is specified we do a dry-run
    # which tells the user which plugins are going to be enabled
    # and with what options.

if __cmdLineOpts__.listPlugins:
        if not len(loadedplugins) and not len(skippedplugins):
            exit_fatal("no valid plugins found")

# FIXME: make -l output more concise
        if len(loadedplugins):
            print _("The following plugins are currently enabled:")
            print
            for (plugname,plug) in loadedplugins:
               print " %-25s  %s" % (textcolor(plugname,"lblue"),plug.get_description())
        else:
            print _("No plugin enabled.")
        print

if len(skippedplugins):
           print _("The following plugins are currently disabled:")
           print
           for (plugname,plugclass) in skippedplugins:
              print " %-25s  %s" % (textcolor(plugname,"cyan"),plugclass.get_description())
        print

if len(alloptions):
           print _("The following plugin options are available:")
           print
           for (plug, plugname, optname, optparm)  in alloptions:
              # format and colorize option value based on its type (int or bool)
              if type(optparm["enabled"])==bool:
                 if optparm["enabled"]==True:
                    tmpopt = textcolor("on","lred")
                 else:
                    tmpopt = textcolor("off","red")
              elif type(optparm["enabled"])==int:
                 if optparm["enabled"] > 0:
                    tmpopt = textcolor(optparm["enabled"],"lred")
                 else:
                    tmpopt = textcolor(optparm["enabled"],"red")
              else:
                 tmpopt = optparm["enabled"]

print " %-21s %-5s %s" % (plugname + "." + optname, tmpopt, optparm["desc"])
              del tmpopt
        else:
           print _("No plugin options available.")

print
        sys.exit()

# to go anywhere further than listing the plugins we will need root permissions.
    #
    if os.getuid() != 0:
        exit_fatal(_("sosreport requires root permissions to run"))

# we don't need to keep in memory plugins we are not going to use
    del skippedplugins

if not len(loadedplugins):
        exit_fatal(_("no valid plugins were enabled"))

msg = _get_msg(dstrootparent)
    if __cmdLineOpts__.batch:
        print msg
    else:
        msg += _("""Press ENTER to continue, or CTRL-C to quit.\n""")
        try:
            raw_input(msg)
        except:
            exit_fatal("could not read user input")
    del msg

# Call the diagnose() method for each plugin
    tmpcount = 0
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Performing sanity check for plugin %s" % plugname)
        try:
           plug.diagnose()
        # diagnose() IO errors are unlikely but possible due to temporary file usage
        # or indirectly via callout programs that need to write to the file system.
        except IOError, e:
            exit_fatal(_("caught fatal IO error during %s diagnostics") % plugname)
        except Exception, inst:
            soslog.warning(_("could not execute diagnose method for plugin %s") % plugname)
            if not __raisePlugins__:
                bt = traceback.format_exception(type(inst), inst.args, None)
                for line in bt:
                    soslog.warning(line.strip())
            elif __raisePlugins__:
                raise
        tmpcount += len(plug.diagnose_msgs)
    if tmpcount > 0:
        print _("One or more plugins have detected a problem in your configuration.")
        print _("Please review the following messages:")
        print

fp = open(rptdir + "/diagnose.txt", "w")
        for plugname, plug in loadedplugins:
            for tmpcount2 in range(0,len(plug.diagnose_msgs)):
                if tmpcount2 == 0:
                    soslog.warning( textcolor("%s:" % plugname, "red") )
                soslog.warning("    * %s" % plug.diagnose_msgs[tmpcount2])
                try:
                    fp.write("%s: %s\n" % (plugname, plug.diagnose_msgs[tmpcount2]) )
                except IOError, e:
                    exit_fatal("caught fatal IO error writing %s diagnostics"
                            % plugname)
        fp.close()

print
        if not __cmdLineOpts__.batch:
           try:
              while True:
                 yorno = raw_input( _("Are you sure you would like to " \
                                      "continue (y/n) ? ") )
                 if yorno == _("y") or yorno == _("Y"):
                    print
                    break
                 elif yorno == _("n") or yorno == _("N"):
                    sys.exit(0)
              del yorno
           except KeyboardInterrupt:
              print
              sys.exit(0)

policy.preWork()

# Call the setup() method for each plugin
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Preloading files and commands to be gathered by plugin %s" % plugname)
        try:
           plug.setup()
        # plug-ins don't generally issue IO during set up but some rely
        # on generating and parsing temporary files (e.g. networking)
        except IOError, e:
            # newline prefix to ensure error message appears on a new blank
            # line bellow the plugin status reporting line.
            exit_fatal("\n\ncaught fatal IO error during %s setup" % plugname)
        except Exception, inst:
            soslog.warning(_("could not setup plugin %s") % plugname)
            if not __raisePlugins__:
                bt = traceback.format_exception(type(inst), inst.args, None)
                for line in bt:
                    soslog.warning(line.strip())
            elif __raisePlugins__:
                raise

if __cmdLineOpts__.nomultithread == True:
        soslog.log(logging.VERBOSE, "using single-threading")
    else:
        soslog.log(logging.VERBOSE, "using multi-threading")

# Call the collect method for each plugin
    plugrunning = Semaphore(2)
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE, "executing plugin %s" % plugname)
        try:
            if not (__cmdLineOpts__.nomultithread == True):
                plug.copyStuff(threaded = True, semaphore = plugrunning)
            else:
                plug.copyStuff()
        except IOError, e:
            # newline prefix to ensure error message appears on a new blank
            # line bellow the plugin status reporting line.
            exit_fatal("\n\ncaught fatal IO error during %s data collection" \
                    % plugname, e)
        except Exception, inst:
            soslog.warning(_("could not execute plugin %s") % plugname)
            if not __raisePlugins__:
                bt = traceback.format_exception(type(inst), inst.args, None)
                for line in bt:
                    soslog.warning(line.strip())
            elif __raisePlugins__:
                raise
    del plugrunning

# Wait for all the collection threads to exit
    if not (__cmdLineOpts__.nomultithread == True):
        finishedplugins = []
        while len(loadedplugins) > 0:
            plugname, plug = loadedplugins.pop(0)
            if not plug.wait(0.5):
                finishedplugins.append((plugname,plug))
                soslog.log(logging.DEBUG, "plugin %s has returned" % plugname)
            else:
                soslog.log(logging.DEBUG, "plugin %s still hasn't returned" % plugname)
                loadedplugins.append((plugname,plug))
        loadedplugins = finishedplugins
        del finishedplugins

for plugname, plug in loadedplugins:
        for oneFile in plug.copiedFiles:
            try:
                xmlrep.add_file(oneFile["srcpath"], os.stat(oneFile["srcpath"]))
            except:
                pass

try:
        xmlrep.serialize_to_file(rptdir + "/sosreport.xml")
    except IOError, e:
        exit_fatal("caught fatal IO error writing XML report file:", e)

# Call the analyze method for each plugin
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Analyzing results of plugin %s" % plugname,)
        try:
            plug.analyze()
        except Exception, inst:
            soslog.warning("caught exception in %s.analyze()" % plugname)
            bt = traceback.format_exception(type(inst), inst.args, None)
            for line in bt:
                soslog.warning(line.strip())
            # catch exceptions in analyse() and keep working
            pass

if __cmdLineOpts__.progressbar:
        sys.stdout.write("\n Completed.\n\n")

# Generate the header for the html output file
    rfd = open(rptdir + "/" + "sosreport.html", "w")
    rfd.write("""
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
    <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
        <head>
    <link rel="stylesheet" type="text/css" media="screen" href="donot.css" />
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>Sos System Report</title>
        </head>

<body>
    """)

# Make a pass to gather Alerts and a list of module names
    allAlerts = []
    plugNames = []
    for plugname, plug in loadedplugins:
        for alert in plug.alerts:
            allAlerts.append('<a href="#%s">%s</a>: %s' % (plugname, plugname, alert))
        plugNames.append(plugname)

# Create a table of links to the module info
    rfd.write("<hr/><h3>Loaded Plugins:</h3>")
    rfd.write("<table><tr>\n")
    rr = 0
    for i in range(len(plugNames)):
        rfd.write('<td><a href="#%s">%s</a></td>\n' % (plugNames[i], plugNames[i]))
        rr = divmod(i, 4)[1]
        if (rr == 3):
            rfd.write('</tr>')
    if not (rr == 3):
        rfd.write('</tr>')
    rfd.write('</table>\n')

rfd.write('<hr/><h3>Alerts:</h3>')
    rfd.write('<ul>')
    for alert in allAlerts:
        rfd.write('<li>%s</li>' % alert)
    rfd.write('</ul>')

# Call the report method for each plugin
    for plugname, plug in loadedplugins:
        try:
            html = plug.report()
        except Exception, inst:
            soslog.warning(_("could not report plugin %s") % plugname)
            if not __raisePlugins__:
                bt = traceback.format_exception(type(inst), inst.args, None)
                for line in bt:
                    soslog.warning(line.strip())
            elif __raisePlugins__:
                raise
        else:
            rfd.write(html)

rfd.write("</body></html>")

rfd.close()

# Collect any needed user information (name, etc)

# Call the postproc method for each plugin
    for plugname, plug in loadedplugins:
        try:
           plug.postproc()
        except Exception, inst:
            soslog.warning(_("could not post-process plugin %s") % plugname)
            if not __raisePlugins__:
                bt = traceback.format_exception(type(inst), inst.args, None)
                for line in bt:
                    soslog.warning(line.strip())
            elif __raisePlugins__:
                raise

# package up the results for the support organization
    policy.packageResults()
    # delete gathered files
    os.system("/bin/rm -rf %s" % dstroot)
    # automated submission will go here

# Close all log files and perform any cleanup
    logging.shutdown()

if __name__ == '__main__':
    try:
        sosreport()
    except KeyboardInterrupt:
        doExitCode()