#!/usr/bin/python
import grp
import time
import sys
sys.path.insert(0, "/usr/share/retrace-server/")
from retrace import *
from plugins import *

starttime = int(time.time())
task = None
log = None
stats = {
  "taskid": None,
  "package": None,
  "version": None,
  "arch": None,
  "starttime": starttime,
  "duration": None,
  "coresize": None,
  "status": STATUS_FAIL,
}

def fail(exitcode=1):
    "Kills script with given exitcode"
    global log, task
    task.set_status(STATUS_FAIL)

    if CONFIG["EmailNotify"] and task.has_notify():
        try:
            log_info("Sending e-mail to %s" % ", ".join(task.get_notify()))

            message = "The task #%d on %s failed\n\n" % (task.get_taskid(), os.uname()[1])

            if task.has_url():
                message += "URL: %s\n" % task.get_url()

            message += "Task directory: %s\n" % task.get_savedir()

            if task.has_started_time():
                message += "Started: %s\n" % datetime.datetime.fromtimestamp(task.get_started_time())

            if task.has_remote() or task.has_downloaded():
                files = ""
                if task.has_remote():
                    remote = map(lambda x: x[4:] if x.startswith("FTP ") else x, task.get_remote())
                    files = ", ".join(remote)

                if task.has_downloaded():
                    files = ", ".join(filter(None, [task.get_downloaded(), files]))

                message += "Remote file(s): %s\n" % files

            if not log is None:
                message += "\nError log:\n%s\n" % log.getvalue()

            send_email("Retrace Server <%s>" % CONFIG["EmailNotifyFrom"],
                       task.get_notify(),
                       "Retrace Task #%d on %s failed" % (task.get_taskid(), os.uname()[1]),
                       message)

        except Exception as ex:
            log_error("Failed to send e-mail: %s" % ex)

    if log:
        task.set_log(log.getvalue())

    task.set_finished_time(int(time.time()))
    stats["duration"] = int(time.time()) - stats["starttime"]
    try:
        save_crashstats(stats)
    except Exception as ex:
        log_warn("Failed to save crash statistics: %s" % str(ex))
    if not task.get_type() in [TASK_DEBUG, TASK_RETRACE_INTERACTIVE, TASK_VMCORE_INTERACTIVE]:
        task.clean()

    sys.exit(exitcode)

def retrace_run(errorcode, cmd):
    "Runs cmd using subprocess.Popen and kills script with errorcode on failure"
    try:
        child = Popen(cmd, stdout=PIPE, stderr=STDOUT)
        output = child.communicate()[0]
    except Exception as ex:
        child = None
        log_error("An unhandled exception occured: %s" % ex)

    if not child or child.returncode != 0:
        log_error("%s exitted with %d\n=== OUTPUT ===\n%s" % (" ".join(cmd), child.returncode, output))
        fail(errorcode)

    return output

def start_retrace():
    crashdir = os.path.join(task.get_savedir(), "crash")
    corepath = os.path.join(crashdir, "coredump")

    try:
        stats["coresize"] = os.path.getsize(corepath)
    except:
        pass

    if cmdline.arch:
        log_debug("Using architecture from command line: %s" % cmdline.arch)
        arch = cmdline.arch
    else:
        # read architecture from coredump
        arch = guess_arch(corepath)

        if not arch:
            log_error("Unable to determine architecture from coredump")
            fail()

        log_debug("Determined architecture: %s" % arch)

    stats["arch"] = arch

    # read package file
    try:
        with open(os.path.join(crashdir, "package"), "r") as package_file:
            crash_package = package_file.read(ALLOWED_FILES["package"])
    except Exception as ex:
        loging.error("Unable to read crash package from 'package' file: %s" % ex)
        fail()

    # read package file
    if not INPUT_PACKAGE_PARSER.match(crash_package):
        log_error("Invalid package name: %s" % crash_package)
        fail()

    pkgdata = parse_rpm_name(crash_package)
    if not pkgdata["name"]:
        log_error("Unable to parse package name: %s" % crash_package)
        fail()

    stats["package"] = pkgdata["name"]
    if pkgdata["epoch"] != 0:
        stats["version"] = "%s:%s-%s" % (pkgdata["epoch"], pkgdata["version"], pkgdata["release"])
    else:
        stats["version"] = "%s-%s" % (pkgdata["version"], pkgdata["release"])

    # read release, distribution and version from release file
    release_path = None
    rootdir = None
    rootdir_path = os.path.join(crashdir, "rootdir")
    if os.path.isfile(rootdir_path):
        with open(rootdir_path, "r") as rootdir_file:
            rootdir = rootdir_file.read(ALLOWED_FILES["rootdir"])

        exec_path = os.path.join(crashdir, "executable")
        with open(exec_path, "r") as exec_file:
            executable = exec_file.read(ALLOWED_FILES["executable"])

        if executable.startswith(rootdir):
            with open(exec_path, "w") as exec_file:
                exec_file.write(executable[len(rootdir):])

        rel_path = os.path.join(crashdir, "os_release_in_rootdir")
        if os.path.isfile(rel_path):
            release_path = rel_path

    if not release_path:
        release_path = os.path.join(crashdir, "os_release")
        if not os.path.isfile(release_path):
            release_path = os.path.join(crashdir, "release")

    release = "Unknown Release"
    try:
        with open(release_path, "r") as release_file:
            release = release_file.read(ALLOWED_FILES["os_release"])

        version = distribution = None
        for plugin in PLUGINS:
            match = plugin.abrtparser.match(release)
            if match:
                version = match.group(1)
                distribution = plugin.distribution
                break

        if not version or not distribution:
            raise Exception, "Unknown release '%s'" % release

    except Exception as ex:
        log_error("Unable to read distribution and version from 'release' file: %s" % ex)
        log_info("Trying to guess distribution and version")
        distribution, version = guess_release(crash_package, PLUGINS)
        if distribution and version:
            log_info("%s-%s" % (distribution, version))
        else:
            log_error("Failure")
            fail()

    if "rawhide" in release.lower():
        version = "rawhide"

    releaseid = "%s-%s-%s" % (distribution, version, arch)
    if not releaseid in get_supported_releases():
        log_error("Release '%s' is not supported" % releaseid)
        fail()

    if not is_package_known(crash_package, arch, releaseid):
        log_error("Package '%s.%s' was not recognized.\nIs it a part of "
                      "official %s repositories?" % (crash_package, arch, release))
        fail()

    packages = [crash_package]
    missing = []
    fafrepo = ""

    packagesfile = os.path.join(crashdir, "packages")
    if os.path.isfile(packagesfile):
        with open(packagesfile, "r") as f:
            packages = f.read.split()
    elif CONFIG["UseFafPackages"]:
        packages = ["bash", "cpio", "glibc-debuginfo"]
        child = Popen(["/usr/bin/faf-c2p", "--hardlink-dir", CONFIG["FafLinkDir"],
                       os.path.join(crashdir, "coredump")], stdout=PIPE, stderr=PIPE)
        stdout, stderr = child.communicate()
        fafrepo = stdout.strip()
        if stderr:
            log_warn(stderr)

        # hack - use latest glibc - for some reason gives better results
        for filename in os.listdir(fafrepo):
            if filename.startswith("glibc"):
                os.unlink(os.path.join(fafrepo, filename))
    else:
        # read required packages from coredump
        try:
            repoid = "%s%s" % (REPO_PREFIX, releaseid)
            yumcfgpath = os.path.join(task.get_savedir(), "yum.conf")
            with open(yumcfgpath, "w") as yumcfg:
                yumcfg.write("[%s]\n" % repoid)
                yumcfg.write("name=%s\n" % releaseid)
                yumcfg.write("baseurl=file://%s/%s/\n" % (CONFIG["RepoDir"], releaseid))
                yumcfg.write("failovermethod=priority\n")

            child = Popen(["coredump2packages", os.path.join(crashdir, "coredump"),
                           "--repos=%s" % repoid, "--config=%s" % yumcfgpath],
                          stdout=PIPE, stderr=PIPE)
            section = 0
            crash_package_or_component = None
            stdout, stderr = child.communicate()
            lines = stdout.split("\n")
            libdb = False
            for line in lines:
                if line == "":
                    section += 1
                    continue
                elif 0 == section:
                    crash_package_or_component = line.strip()
                elif 1 == section:
                    stripped = line.strip()

                    # hack - help to depsolver, yum would fail otherwise
                    if distribution == "fedora" and stripped.startswith("gnome"):
                        packages.append("desktop-backgrounds-gnome")

                    # hack - libdb-debuginfo and db4-debuginfo are conflicting
                    if distribution == "fedora" and \
                       (stripped.startswith("db4-debuginfo") or \
                        stripped.startswith("libdb-debuginfo")):
                        if libdb:
                            continue
                        else:
                            libdb = True

                    packages.append(stripped)
                elif 2 == section:
                    soname, buildid = line.strip().split(" ", 1)
                    if not soname or soname == "-":
                        soname = None
                    missing.append((soname, buildid))

            if stderr:
                log_warn(stderr)

        except Exception as ex:
            log_error("Unable to obtain packages from 'coredump' file: %s" % ex)
            fail()

    # create mock config file
    try:
        with open(os.path.join(task.get_savedir(), "default.cfg"), "w") as mockcfg:
            mockcfg.write("config_opts['root'] = '%d'\n" % task.get_taskid())
            mockcfg.write("config_opts['target_arch'] = '%s'\n" % arch)
            mockcfg.write("config_opts['chroot_setup_cmd'] = '--skip-broken install %s abrt-addon-ccpp shadow-utils gdb rpm'\n" % " ".join(packages))
            mockcfg.write("config_opts['plugin_conf']['ccache_enable'] = False\n")
            mockcfg.write("config_opts['plugin_conf']['yum_cache_enable'] = False\n")
            mockcfg.write("config_opts['plugin_conf']['root_cache_enable'] = False\n")
            mockcfg.write("config_opts['plugin_conf']['bind_mount_enable'] = True\n")
            mockcfg.write("config_opts['plugin_conf']['bind_mount_opts'] = { 'create_dirs': True,\n")
            mockcfg.write("    'dirs': [\n")
            mockcfg.write("              ('%s', '/var/spool/abrt/crash'),\n" % crashdir)
            if CONFIG["UseFafPackages"]:
                mockcfg.write("              ('%s', '/packages'),\n" % fafrepo)
            mockcfg.write("            ] }\n")
            mockcfg.write("\n")
            mockcfg.write("config_opts['yum.conf'] = \"\"\"\n")
            mockcfg.write("[main]\n")
            mockcfg.write("cachedir=/var/cache/yum\n")
            mockcfg.write("debuglevel=1\n")
            mockcfg.write("reposdir=%s\n" % os.devnull)
            mockcfg.write("logfile=/var/log/yum.log\n")
            mockcfg.write("retries=20\n")
            mockcfg.write("obsoletes=1\n")
            if version != "rawhide" and CONFIG["RequireGPGCheck"]:
                mockcfg.write("gpgcheck=1\n")
            else:
                mockcfg.write("gpgcheck=0\n")
            mockcfg.write("assumeyes=1\n")
            mockcfg.write("syslog_ident=mock\n")
            mockcfg.write("syslog_device=\n")
            mockcfg.write("\n")
            mockcfg.write("#repos\n")
            mockcfg.write("\n")
            mockcfg.write("[%s]\n" % distribution)
            mockcfg.write("name=%s\n" % releaseid)
            mockcfg.write("baseurl=file://%s/%s/\n" % (CONFIG["RepoDir"], releaseid))
            mockcfg.write("failovermethod=priority\n")
            if version != "rawhide" and CONFIG["RequireGPGCheck"]:
                mockcfg.write("gpgkey=file:///usr/share/retrace-server/gpg/%s-%s\n" % (distribution, version))
            mockcfg.write("\"\"\"\n")

        # symlink defaults from /etc/mock
        os.symlink("/etc/mock/site-defaults.cfg", os.path.join(task.get_savedir(), "site-defaults.cfg"))
        os.symlink("/etc/mock/logging.ini", os.path.join(task.get_savedir(), "logging.ini"))
    except Exception as ex:
        log_error("Unable to create mock config file: %s" % ex)
        fail()

    # run retrace
    task.set_status(STATUS_INIT)
    log_info(STATUS[STATUS_INIT])

    retrace_run(25, ["/usr/bin/mock", "init", "--configdir", task.get_savedir()])
    if CONFIG["UseFafPackages"]:
        retrace_run(26, ["/usr/bin/mock", "--configdir", task.get_savedir(), "shell", "--",
                         "bash", "-c", "'for PKG in /packages/*; do rpm2cpio \\$PKG | " \
                         "cpio -muid --quiet; done'"])
    retrace_run(27, ["/usr/bin/mock", "--configdir", task.get_savedir(), "shell",
                     "--", "chgrp", "-R", "mockbuild", "/var/spool/abrt/crash"])

    # generate backtrace
    task.set_status(STATUS_BACKTRACE)
    log_info(STATUS[STATUS_BACKTRACE])

    try:
        backtrace, exploitable = run_gdb(task.get_savedir())
    except Exception as ex:
        log_error(str(ex))
        fail()

    task.set_backtrace(backtrace)
    if exploitable is not None:
        task.add_misc("exploitable", exploitable)

    # does not work at the moment
    rootsize = 0

    if not task.get_type() in [TASK_DEBUG, TASK_RETRACE_INTERACTIVE]:
        # clean up temporary data
        task.set_status(STATUS_CLEANUP)
        log_info(STATUS[STATUS_CLEANUP])

        task.clean()
        if CONFIG["UseFafPackages"]:
            shutil.rmtree(fafrepo)

        # ignore error: workdir = savedir => workdir is not empty
        if CONFIG["UseWorkDir"]:
            try:
                os.rmdir(workdir)
            except:
                pass

    # save crash statistics
    task.set_status(STATUS_STATS)
    log_info(STATUS[STATUS_STATS])

    task.set_finished_time(int(time.time()))
    stats["duration"] = int(time.time()) - stats["starttime"]
    stats["status"] = STATUS_SUCCESS

    try:
        con = init_crashstats_db()
        statsid = save_crashstats(stats, con)
        save_crashstats_success(statsid, prerunning, len(get_active_tasks()), rootsize, con)
        save_crashstats_packages(statsid, packages[1:], con)
        if missing:
            save_crashstats_build_ids(statsid, missing, con)
        con.close()
    except Exception as ex:
        log_warn(str(ex))

    # publish log => finish task
    log_info("Retrace took %d seconds" % stats["duration"])

    if log:
        task.set_log(log.getvalue())

    log_info(STATUS[STATUS_SUCCESS])
    task.set_status(STATUS_SUCCESS)

def mock_find_vmlinux(cfgdir, candidates):
    with open(os.devnull, "w") as null:
        for cand in candidates:
            child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                           "test", "-f", cand, "&&", "echo", cand], stdout=PIPE, stderr=null)
            output = child.communicate()[0].strip()
            child.wait()
            if output == cand:
                return cand

    return None

def start_vmcore():
    vmcore = os.path.join(task.get_savedir(), "crash", "vmcore")

    if not cmdline.kernelver is None:
        kernelver = cmdline.kernelver
    else:
        kernelver = get_kernel_release(vmcore)
        if not kernelver:
            log_error("Unable to determine kernel version")
            fail()

        log_debug("Determined kernel version: %s" % kernelver)

    task.set_kernelver(str(kernelver))

    stats["package"] = "kernel"
    stats["version"] = "%s-%s" % (kernelver.version, kernelver.release)
    stats["arch"] = kernelver.arch

    kernelcache = os.path.join(CONFIG["RepoDir"], "kernel")
    kerneltmp = os.path.join(kernelcache, "%s.tmp" % kernelver)
    hostarch = get_canon_arch(os.uname()[4])

    log_info(STATUS[STATUS_INIT])
    task.set_status(STATUS_INIT)
    vmlinux = ""

    # cross-arch: we need to use chroot
    if hostarch != kernelver.arch:
        # we don't save config into task.get_savedir() because it is only
        # readable by user/group retrace/CONFIG["AuthGroup"].
        # if a non-retrace user in group mock executes
        # setgid /usr/bin/mock, he gets permission denied.
        # this is not a security thing - using mock gives you root anyway
        cfgdir = os.path.join(CONFIG["SaveDir"], "%d-kernel" % task.get_taskid())

        # if the directory exists, it is orphaned - nuke it
        if os.path.isdir(cfgdir):
            shutil.rmtree(cfgdir)

        mockgid = grp.getgrnam("mock").gr_gid
        old_umask = os.umask(0027)
        os.mkdir(cfgdir)
        os.chown(cfgdir, -1, mockgid)

        try:
            cfgfile = os.path.join(cfgdir, "default.cfg")
            with open(cfgfile, "w") as mockcfg:
                mockcfg.write("config_opts['root'] = '%d-kernel'\n" % task.get_taskid())
                mockcfg.write("config_opts['target_arch'] = '%s'\n" % kernelver.arch)
                mockcfg.write("config_opts['chroot_setup_cmd'] = 'install bash coreutils cpio crash findutils rpm shadow-utils'\n")
                mockcfg.write("config_opts['plugin_conf']['ccache_enable'] = False\n")
                mockcfg.write("config_opts['plugin_conf']['yum_cache_enable'] = False\n")
                mockcfg.write("config_opts['plugin_conf']['root_cache_enable'] = False\n")
                mockcfg.write("config_opts['plugin_conf']['bind_mount_enable'] = True\n")
                mockcfg.write("config_opts['plugin_conf']['bind_mount_opts'] = { \n")
                mockcfg.write("    'dirs': [('%s', '%s'),\n" % (CONFIG["RepoDir"], CONFIG["RepoDir"]))
                mockcfg.write("             ('%s', '%s'),],\n" % (task.get_savedir(), task.get_savedir()))
                mockcfg.write("    'create_dirs': True, }\n")
                mockcfg.write("\n")
                mockcfg.write("config_opts['yum.conf'] = \"\"\"\n")
                mockcfg.write("[main]\n")
                mockcfg.write("cachedir=/var/cache/yum\n")
                mockcfg.write("debuglevel=1\n")
                mockcfg.write("reposdir=%s\n" % os.devnull)
                mockcfg.write("logfile=/var/log/yum.log\n")
                mockcfg.write("retries=20\n")
                mockcfg.write("obsoletes=1\n")
                mockcfg.write("assumeyes=1\n")
                mockcfg.write("syslog_ident=mock\n")
                mockcfg.write("syslog_device=\n")
                mockcfg.write("\n")
                mockcfg.write("#repos\n")
                mockcfg.write("\n")
                mockcfg.write("[kernel-%s]\n" % kernelver.arch)
                mockcfg.write("name=kernel-%s\n" % kernelver.arch)
                mockcfg.write("baseurl=%s\n" % CONFIG["KernelChrootRepo"].replace("$ARCH", kernelver.arch))
                mockcfg.write("failovermethod=priority\n")
                mockcfg.write("\"\"\"\n")

            os.chown(cfgfile, -1, mockgid)

            # symlink defaults from /etc/mock
            os.symlink("/etc/mock/site-defaults.cfg", os.path.join(cfgdir, "site-defaults.cfg"))
            os.symlink("/etc/mock/logging.ini", os.path.join(cfgdir, "logging.ini"))
        except Exception as ex:
            log_error("Unable to create mock config file: %s" % ex)
            fail()
        finally:
            os.umask(old_umask)

        child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "init"], stdout=PIPE, stderr=STDOUT)
        stdout = child.communicate()[0]
        if child.wait():
            log_error("mock exitted with %d:\n%s" % (child.returncode, stdout))
            fail()

        # no locks required, mock locks itself
        try:
            vmlinux = prepare_debuginfo(vmcore, cfgdir, kernelver=cmdline.kernelver)

            # generate the log
            with open(os.devnull, "w") as null:
                child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                               "crash", "--minimal", "-s", vmcore, vmlinux],
                              stdin=PIPE, stdout=PIPE, stderr=null)
                kernellog = child.communicate("log\nquit\n")[0]
                if child.wait():
                    log_warn("crash 'log' exitted with %d" % child.returncode)

                child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                               "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                crash_bt_a = child.communicate("bt -a\nquit\n")[0]
                if child.wait():
                    log_warn("crash 'bt -a' exitted with %d" % child.returncode)
                    crash_bt_a = None

                crash_kmem_f = None
                if CONFIG["VmcoreRunKmem"] == 1:
                    child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                                   "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                    crash_kmem_f = child.communicate("kmem -f\nquit\n")[0]
                    if child.wait():
                        log_warn("crash 'kmem -f' exitted with %d" % child.returncode)
                        crash_kmem_f = None

                if CONFIG["VmcoreRunKmem"] == 2:
                    child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                                   "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                    crash_kmem_f = child.communicate("set hash off\nkmem -f\nset hash on\nquit\n")[0]
                    if child.wait():
                        log_warn("crash 'kmem -f' exitted with %d" % child.returncode)
                        crash_kmem_f = None

                crash_kmem_z = None
                if CONFIG["VmcoreRunKmem"] == 3:
                    child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                                   "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                    crash_kmem_z = child.communicate("kmem -z\nquit\n")[0]
                    if child.wait():
                        log_warn("crash 'kmem -z' exitted with %d" % child.returncode)
                        crash_kmem_z = None

                child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                               "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                crash_sys = child.communicate("sys\nquit\n")[0]
                if child.wait():
                    log_warn("crash 'sys' exitted with %d" % child.returncode)
                    crash_sys = None

                child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                               "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                crash_sys_c = child.communicate("sys -c\nquit\n")[0]
                if child.wait():
                    log_warn("crash 'sys -c' exitted with %d" % child.returncode)
                    crash_sys_c = None

                child = Popen(["/usr/bin/mock", "--configdir", cfgdir, "shell", "--",
                               "crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=null)
                crash_foreach_bt = child.communicate("foreach bt\nquit\n")[0]
                if child.wait():
                    log_warn("crash 'foreach bt' exitted with %d" % child.returncode)
                    crash_foreach_bt = None

        except Exception as ex:
            log_error(str(ex))
            fail()

    else:
        try:
            vmlinux = prepare_debuginfo(vmcore, kernelver=cmdline.kernelver)
        except Exception as ex:
            log_error("prepare_debuginfo failed: %s" % str(ex))
            fail()

        task.set_status(STATUS_BACKTRACE)
        log_info(STATUS[STATUS_BACKTRACE])

        child = Popen(["crash", "--minimal", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        kernellog = child.communicate("log\nquit\n")[0]
        if child.wait():
            log_warn("crash 'log' exited with %d" % child.returncode)

        child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        crash_bt_a = child.communicate("bt -a\nquit\n")[0]
        if child.wait():
            log_warn("crash 'bt -a' exited with %d" % child.returncode)
            crash_bt_a = None

        crash_kmem_f = None
        if CONFIG["VmcoreRunKmem"] == 1:
            child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
            crash_kmem_f = child.communicate("kmem -f\nquit\n")[0]
            if child.wait():
                log_warn("crash 'kmem -f' exited with %d" % child.returncode)
                crash_kmem_f = None

        if CONFIG["VmcoreRunKmem"] == 2:
            child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
            crash_kmem_f = child.communicate("set hash off\nkmem -f\nset hash on\nquit\n")[0]
            if child.wait():
                log_warn("crash 'kmem -f' exited with %d" % child.returncode)
                crash_kmem_f = None

        crash_kmem_z = None
        if CONFIG["VmcoreRunKmem"] == 3:
            child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
            crash_kmem_z = child.communicate("kmem -z\nquit\n")[0]
            if child.wait():
                log_warn("crash 'kmem -z' exited with %d" % child.returncode)
                crash_kmem_z = None

        child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        crash_sys = child.communicate("sys\nquit\n")[0]
        if child.wait():
            log_warn("crash 'sys' exited with %d" % child.returncode)
            crash_sys = None

        child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        crash_sys_c = child.communicate("sys -c\nquit\n")[0]
        if child.wait():
            log_warn("crash 'sys -c' exited with %d" % child.returncode)
            crash_sys_c = None

        child = Popen(["crash", "-s", vmcore, vmlinux], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        crash_foreach_bt = child.communicate("foreach bt\nquit\n")[0]
        if child.wait():
            log_warn("crash 'foreach bt' exited with %d" % child.returncode)
            crash_foreach_bt = None

    task.set_backtrace(kernellog)
    if crash_bt_a:
        task.add_misc("bt-a", crash_bt_a)
    if crash_kmem_f:
        task.add_misc("kmem-f", crash_kmem_f)
    if crash_kmem_z:
        task.add_misc("kmem-z", crash_kmem_z)
    if crash_sys:
        task.add_misc("sys", crash_sys)
    if crash_sys_c:
        task.add_misc("sys-c", crash_sys_c)
    if crash_foreach_bt:
        child = Popen(["bt_filter"], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        bt_filter = child.communicate(crash_foreach_bt)[0]
        if child.wait():
            bt_filter = "bt_filter exitted with %d\n\n%s" % (child.returncode, bt_filter)

        task.add_misc("bt-filter", bt_filter)

    crashrc_lines = []

    if "/" in vmlinux:
        crashrc_lines.append("mod -S %s > %s" % (vmlinux.rsplit("/", 1)[0], os.devnull))

    miscdir = os.path.join(task.get_savedir(), RetraceTask.MISC_DIR)
    crashrc_lines.append("cd %s" % miscdir)

    if len(crashrc_lines) > 0:
        task.set_crashrc("%s\n" % "\n".join(crashrc_lines))

    task.set_finished_time(int(time.time()))
    stats["duration"] = int(time.time()) - stats["starttime"]
    stats["status"] = STATUS_SUCCESS

    log_info(STATUS[STATUS_STATS])

    try:
        save_crashstats(stats)
    except Exception as ex:
        log_error(str(ex))

    # clean up temporary data
    task.set_status(STATUS_CLEANUP)
    log_info(STATUS[STATUS_CLEANUP])

    if not task.get_type() in [TASK_VMCORE_INTERACTIVE]:
        task.clean()

    if CONFIG["EmailNotify"] and task.has_notify():
        try:
            log_info("Sending e-mail to %s" % ", ".join(task.get_notify()))

            message = "The task #%d started on %s succeeded\n\n" % (task.get_taskid(), os.uname()[1])

            if task.has_url():
                message += "URL: %s\n" % task.get_url()

            message += "Task directory: %s\n" % task.get_savedir()

            if task.has_started_time():
                message += "Started: %s\n" % datetime.datetime.fromtimestamp(task.get_started_time())

            if task.has_finished_time():
                message += "Finished: %s\n" % datetime.datetime.fromtimestamp(task.get_finished_time())

            if task.has_remote() or task.has_downloaded():
                files = ""
                if task.has_remote():
                    remote = map(lambda x: x[4:] if x.startswith("FTP ") else x, task.get_remote())
                    files = ", ".join(remote)

                if task.has_downloaded():
                    files = ", ".join(filter(None, [task.get_downloaded(), files]))

                message += "Remote file(s): %s\n" % files

            if not log is None:
                message += "\nLog:\n%s\n" % log.getvalue()

            send_email("Retrace Server <%s>" % CONFIG["EmailNotifyFrom"],
                       task.get_notify(),
                       "Retrace Task #%d on %s succeded" % (task.get_taskid(), os.uname()[1]),
                       message)

        except Exception as ex:
            log_error("Failed to send e-mail: %s" % ex)

    log_info("Retrace took %d seconds" % stats["duration"])
    log_info(STATUS[STATUS_SUCCESS])

    if log:
        task.set_log(log.getvalue())

        # add a symlink to log to misc directory
        # use underscore so that the log is first in the list
        os.symlink(task._get_file_path(RetraceTask.LOG_FILE),
                   os.path.join(task._get_file_path(RetraceTask.MISC_DIR), "retrace-log"))

    task.set_status(STATUS_SUCCESS)

if __name__ == "__main__":
    cmdline_parser = ArgumentParser(description="Execute a retrace job")
    cmdline_parser.add_argument("task_id", type=int, help="Task ID (%s/<task_id>) must exist" % CONFIG["SaveDir"])
    cmdline_parser.add_argument("--restart", action="store_true", default=False, help="Restart the task if it has already been processed")
    cmdline_parser.add_argument("--foreground", action="store_true", default=False, help="Do not fork to background")
    cmdline_parser.add_argument("--kernelver", default=None, help="Kernel version (e.g. 2.6.32-287.el6), also needs --arch")
    cmdline_parser.add_argument("--arch", help="Architecture")
    cmdline = cmdline_parser.parse_args()

    log = cmdline._log

    # do not use logging yet - we need the task
    if cmdline.kernelver and not cmdline.arch:
        sys.stderr.write("You also need to specify architecture when overriding kernel version\n")
        exit(1)

    try:
        task = RetraceTask(cmdline.task_id)
    except:
        sys.stderr.write("Task '%d' does not exist\n" % cmdline.task_id)
        exit(1)

    # we have the task, now we can use log_*() and fail()
    if task.has_status():
        if not cmdline.restart:
            log_error("%s has already been executed for task %d" % (sys.argv[0], cmdline.task_id))
            log_info("You can use --restart option if you really want to restart the task")
            fail()

        task.reset()

    task.set_started_time(int(time.time()))

    if not cmdline.foreground:
        try:
            pid = os.fork()
        except:
            log_error("Unable to fork")
            fail()

        # parent - kill
        if pid != 0:
            exit(0)

        try:
            os.setpgrp()
        except Exception as ex:
            log_warn("Failed to detach from process group: %s" % str(ex))

    stats["taskid"] = cmdline.task_id

    prerunning = len(get_active_tasks()) - 1

    if not cmdline.kernelver is None:
        try:
            cmdline.kernelver = KernelVer(cmdline.kernelver)
            if cmdline.arch:
                cmdline.kernelver.arch = cmdline.arch
            log_debug("Using kernel version from command line: %s" % cmdline.kernelver)
        except Exception as ex:
            log_warn(str(ex))

    if task.has_remote():
        errors = task.download_remote(kernelver=cmdline.kernelver)
        if errors:
            for error in errors:
                log_warn(error)

    task.set_status(STATUS_ANALYZE)
    log_info(STATUS[STATUS_ANALYZE])

    crashdir = os.path.join(task.get_savedir(), "crash")

    tasktype = task.get_type()

    for required_file in REQUIRED_FILES[tasktype]:
        if not os.path.isfile(os.path.join(crashdir, required_file)):
            log_error("Crash directory does not contain required file '%s'" % required_file)
            fail()

    try:
        if tasktype in [TASK_RETRACE, TASK_DEBUG, TASK_RETRACE_INTERACTIVE]:
            start_retrace()
        elif tasktype in [TASK_VMCORE, TASK_VMCORE_INTERACTIVE]:
            start_vmcore()
        else:
            raise Exception, "Unsupported task type"
    except Exception as ex:
        log_error(str(ex))
        fail()
