#!/usr/bin/env python

# Copyright (C) 2014 Red Hat, Inc.
#
# This file is part of csmock.
#
# csmock is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# csmock is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csmock.  If not, see <http://www.gnu.org/licenses/>.

# standard imports
import argparse
import codecs
import copy
import datetime
import imp
import os.path
import re
import signal
import socket
import shutil
import subprocess
import sys
import tempfile
import time

# local imports
import csmock.common.util


CSMOCK_DATADIR = "/usr/share/csmock"

CHROOT_FIXUPS = CSMOCK_DATADIR + "/scripts/chroot-fixups"

PATCH_RAWBUILD = CSMOCK_DATADIR + "/scripts/patch-rawbuild.sh"

CWE_LIST_FILE = CSMOCK_DATADIR + "/cwe-map.csv"

DEFAULT_DEFECT_BLACKLIST = CSMOCK_DATADIR + "/defect-blacklist.err"

PLUGIN_DIR = "/usr/lib/python2.6/site-packages/csmock/plugins"

DEFAULT_CSWRAP_TIMEOUT = 30

DEFAULT_JOBS_CNT = 13

DEFAULT_RPM_OPTS = [
    "--define", "_without_testsuite 1",
    "--define", "apidocs 0",
    "--define", "libguestfs_runtests 0",
    "--define", "runselftest 0",
    "--define", "with_publican 0",
    "--without", "binfilter",
    "--without", "docs",
    "--without", "langpacks"]

RAWBUILD_RPM_OPTS = [
    "--define", "__patch " + PATCH_RAWBUILD,
    "--define", "_rawbuild -b _RAWBUILD",
    "--define", "nofips 1",
    "--define", "nopam 1",
    "--define", "norunuser 1",
    "--define", "noselinux 1",
    "--define", "_with_vanilla 1"]

DEFAULT_CSWRAP_FILTERS = [
    "csgrep --quiet --path '^/builddir/build/BUILD/' --remove-duplicates"]

# remember to use --mode=json for csgrep (TODO: improve csgrep's interface)
DEFAULT_RESULT_FILTERS = [
    "csgrep --mode=json --path '^/builddir/build/BUILD/' \
--strip-path-prefix /builddir/build/BUILD/",
    "csgrep --mode=json --invert-match --path '^ksh-.*[0-9]+\\.c$'",
    "csgrep --mode=json --invert-match --path 'CMakeFiles/CMakeTmp|conftest.c'"]

CSGREP_FINAL_FILTER_ARGS = "--invert-match --event \"internal warning\" \
--prune-events=1"


def current_iso_date():
    now = datetime.datetime.now()
    return "%04u-%02u-%02u %02u:%02u:%02u" % \
           (now.year, now.month, now.day, now.hour, now.minute, now.second)


def shell_quote(str_in):
    str_out = ""
    for i in range(0, len(str_in)):
        c = str_in[i]
        if c == "\\":
            str_out += "\\\\"
        elif c == "\"":
            str_out += "\\\""
        elif c == "$":
            str_out += "\\$"
        else:
            str_out += c
    return "\"" + str_out + "\""


def strlist_to_shell_cmd(cmd_in, escape_special=False):
    def translate_one(i):
        if escape_special:
            return shell_quote(i)
        return "'%s'" % i

    if type(cmd_in) is str:
        return "sh -c %s" % translate_one(cmd_in)
    cmd_out = ""
    for i in cmd_in:
        cmd_out += " " + translate_one(i)
    return cmd_out.lstrip()


def is_ignored_dep(pkg):
    # we do not install/check this kind of dependencies
    return re.match("^.*rpmlib\\(.*\\)", pkg) is not None


def find_missing_pkgs(pkgs, results, mock, ignore_with=False):
    # dump list of RPMs installed in the chroot (for debugging purposes)
    tmp_var_lib = "%s/var/lib" % results.tmpdir
    if os.path.isdir(tmp_var_lib):
        shutil.rmtree(tmp_var_lib)
    os.makedirs(tmp_var_lib)
    mock.copy_out(["/var/lib/rpm", "%s/rpm" % tmp_var_lib])
    tmp_rpm = "%s/rpm" % tmp_var_lib
    installed = "%s/rpm-list-mock.txt" % results.dbgdir
    provides = "%s/rpm-list-mock-provides.txt" % results.tmpdir
    cmd_tpl = "chmod -w '%s' && rpm -qa --root '%s' | sort -V \
> '%s' && rpm -qa --provides --root '%s' > '%s' && chmod u+w '%s'"
    results.exec_cmd(
        cmd_tpl % (
            tmp_rpm, results.tmpdir, installed, results.tmpdir, provides,
            tmp_rpm),
        shell=True)

    missing = []
    installed = set()
    with open(provides) as f:
        lines = f.readlines()
        for l in lines:
            pkg = re.sub(" .*$", "", l.strip())
            installed.add(pkg)
    for dep in pkgs:
        if ignore_with:
            dep = re.sub('^\((.*) with (.*)\)$', '\\1', dep)
        if is_ignored_dep(dep) or '/' in dep:
            continue
        pkg = re.sub(" .*$", "", dep)
        if pkg in installed:
            continue

        # FIXME: avoid such hard-coding (perhaps match provides instead?)
        if pkg == 'perl' and 'perl-interpreter' in installed:
            continue
        if pkg == 'python3-pip' and 'platform-python-pip' in installed:
            continue

        missing += [dep]
    return missing


class FatalError(Exception):
    def __init__(self, ec):
        self.ec = ec


class ScanResults:
    def __init__(self, output, keep_going=False, create_dbgdir=True):
        self.output = output
        self.keep_going = keep_going
        self.create_dbgdir = create_dbgdir
        self.use_xz = False
        self.use_tar = False
        self.dirname = os.path.basename(output)
        self.codec = codecs.lookup('utf8')
        self.ec = 0

        # just to silence pylint, will be initialized in __enter__()
        self.tmpdir = None
        self.resdir = None
        self.dbgdir = None
        self.dbgdir_raw = None
        self.dbgdir_uni = None
        self.log_pid = None
        self.log_fd = None
        self.ini_writer = None
        self.subproc = None

        m = re.match("^(.*)\\.xz$", self.dirname)
        if m is not None:
            self.use_xz = True
            self.dirname = m.group(1)

        m = re.match("^(.*)\\.tar$", self.dirname)
        if m is not None:
            self.use_tar = True
            self.dirname = m.group(1)

    def utf8_wrap(self, fd):
        # the following hack is needed to support both Python 2 and 3
        return codecs.StreamReaderWriter(
            fd, self.codec.streamreader, self.codec.streamwriter)

    def __enter__(self):
        self.tmpdir = tempfile.mkdtemp(prefix="csmock")
        if self.use_tar:
            self.resdir = "%s/%s" % (self.tmpdir, self.dirname)
        else:
            if os.path.exists(self.output):
                shutil.rmtree(self.output)
            self.resdir = self.output

        try:
            os.mkdir(self.resdir)
        except OSError as e:
            sys.stderr.write(
                "error: failed to create output directory: %s\n" % e)
            raise FatalError(1)

        if self.create_dbgdir:
            self.dbgdir = "%s/debug" % self.resdir
            self.dbgdir_raw = "%s/raw-results" % self.dbgdir
            self.dbgdir_uni = "%s/uni-results" % self.dbgdir
            os.mkdir(self.dbgdir)
            os.mkdir(self.dbgdir_raw)
            os.mkdir(self.dbgdir_uni)

        tee = ["tee", "%s/scan.log" % self.resdir]
        self.log_pid = subprocess.Popen(
            tee, stdin=subprocess.PIPE, preexec_fn=os.setsid)
        self.log_fd = self.utf8_wrap(self.log_pid.stdin)

        def signal_handler(signum, frame):
            # FIXME: we should use Async-signal-safe functions only
            self.fatal_error("caught signal %d" % signum, ec=(0x80 + signum))
        for i in [signal.SIGINT, signal.SIGTERM]:
            signal.signal(i, signal_handler)

        self.ini_writer = IniWriter(self)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.ini_writer.close()
        if self.subproc is not None and self.subproc.returncode is None:
            # FIXME: TOCTOU race
            try:
                os.kill(self.subproc.pid, signal.SIGTERM)
                self.subproc.wait()
            except Exception:
                pass
        self.print_with_ts("csmock exit code: %d\n" % self.ec, prefix="<<< ")
        self.log_fd.close()
        self.log_fd = sys.stderr
        self.log_pid.wait()
        if self.use_tar:
            tar_opts = "-c"
            if self.use_xz:
                tar_opts += "J"
            tar_cmd = "tar %s -f '%s' -C '%s' '%s'" % (
                tar_opts, self.output, self.tmpdir, self.dirname)
            # do not treat 'tar: file changed as we read it' as fatal error
            if os.system(tar_cmd) > 1:
                self.fatal_error(
                    "failed to write '%s', not removing '%s'..." % (
                        self.output, self.tmpdir))

        sys.stderr.write("Wrote: %s\n\n" % self.output)
        shutil.rmtree(self.tmpdir)

    def print_with_ts(self, msg, prefix=">>> "):
        self.log_fd.write("%s%s\t%s\n" % (prefix, current_iso_date(), msg))
        self.log_fd.flush()

    def error(self, msg, ec=1, err_prefix=""):
        self.print_with_ts("%serror: %s\n" % (err_prefix, msg), prefix="!!! ")
        if self.ec < ec:
            self.ec = ec
        if not self.keep_going and (self.ec != 0):
            raise FatalError(ec)

    def fatal_error(self, msg, ec=1):
        self.error(msg, err_prefix="fatal ", ec=ec)
        raise FatalError(ec)

    def exec_cmd(self, cmd, shell=False, emul_pty=False, echo=True):
        if emul_pty:
            # workaround for bug https://bugzilla.redhat.com/1166609
            sh_cmd = strlist_to_shell_cmd(cmd, escape_special=True)
            cmd = ["script", "-aefqc", sh_cmd, "/dev/null"]

        if echo:
            if shell:
                self.print_with_ts(shell_quote(cmd))
            else:
                self.print_with_ts(strlist_to_shell_cmd(cmd, escape_special=True))
        self.subproc = subprocess.Popen(
            cmd, stdout=self.log_fd, stderr=self.log_fd, shell=shell)
        rv = self.subproc.wait()
        self.log_fd.write("\n")
        if rv >= 128:
            # if the child has been signalled, signal self with the same signal
            os.kill(os.getpid(), rv - 128)
        return rv

    def get_cmd_output(self, cmd, shell=True):
        self.subproc = subprocess.Popen(
            cmd, stdout=subprocess.PIPE, stderr=self.log_fd, shell=shell)
        (out, _) = self.subproc.communicate()
        out = out.decode("utf8")
        return self.subproc.returncode, out

    def open_res_file(self, rel_path):
        abs_path = "%s/%s" % (self.resdir, rel_path)
        return open(abs_path, "w")


class IniWriter:
    def __init__(self, results):
        self.results = results
        self.ini = self.results.open_res_file("scan.ini")
        self.write("[scan]\n")
        self.append("tool", "csmock")
        self.append("tool-version", "csmock-2.3.0-1.el6")
        self.append("tool-args", strlist_to_shell_cmd(sys.argv))
        self.append("host", socket.gethostname())
        self.append("store-results-to", self.results.output)
        self.append("time-created", current_iso_date())

    def close(self):
        if self.ini is None:
            return
        self.append("time-finished", current_iso_date())
        self.append("exit-code", self.results.ec)
        self.ini.close()
        self.ini = None

    def write(self, text):
        self.ini.write(text)
        self.results.log_fd.write("scan.ini: " + text)

    def append(self, key, value):
        self.write("%s = %s\n" % (key, value))


class MockWrapper:
    def __init__(self, results, mock_profile, skip_init=False):
        self.results = results
        self.mock_profile = mock_profile
        self.lock_file = "/tmp/.csmock-%s.lock" % mock_profile
        self.meta_lock_file = "/tmp/.csmock-%s.metalock" % mock_profile
        self.pid = os.getpid()
        self.skip_init = skip_init
        self.init_done = skip_init
        self.emul_pty = False
        # just to silence pylint, will be initialized in __enter__()
        self.def_cmd = None

    def __enter__(self):
        cmd = "flock -w15 '%s' -c '\
lock_file=\"%s\" \n\
self_pid=\"%d\" \n\
if test -e \"$lock_file\"; then \n\
    test -e /proc/\"$self_pid\"     || exit $? \n\
    read pid < \"$lock_file\"       || exit $? \n\
    test ! -e /proc/\"$pid\"        || exit $? \n\
    echo \"warning: purging stray lock file $lock_file (PID $pid)\" >&2 \n\
fi \n\
echo \"$self_pid\" > \"$lock_file\"'" \
            % (self.meta_lock_file, self.lock_file, self.pid)
        while os.system(cmd) != 0:
            f = open(self.lock_file)
            other_pid = ""
            if f is not None:
                other_pid = f.readline().rstrip()
                f.close()
            msg = "waiting till %s (PID %s) disappears..."
            self.results.print_with_ts(msg % (self.lock_file, other_pid))
            time.sleep(15)

        # prepare the mock command template with default arguments
        if os.path.exists("/usr/bin/mock-unbuffered"):
            # mock wrapper writing debug output without buffering
            mock = "/usr/bin/mock-unbuffered"
        elif os.path.exists("/usr/bin/mock"):
            # mock wrapper for non-privileged users (members of group mock)
            mock = "/usr/bin/mock"
        else:
            # fallback to any mock in $PATH (e.g. /usr/local/bin/mock)
            mock = "mock"
        self.def_cmd = [mock, "-r", self.mock_profile]

        # avoid using systemd-nspawn, which looses files in /tmp
        self.def_cmd += ["--old-chroot"]

        # disable unneeded expensive plug-in 'package_state'
        self.def_cmd += ["--disable-plugin=package_state"]

        # make csmock work in case the 'tmpfs' plug-in is enabled
        # (see <https://bugzilla.redhat.com/1190100> for details)
        self.def_cmd += ["--plugin-option=tmpfs:keep_mounted=True"]

        # workaround for bug https://bugzilla.redhat.com/1166609
        cmd = "script -aefqc \"echo test\" /dev/null 2>/dev/null"
        (ec, out) = self.results.get_cmd_output(cmd)
        if ec == 0 and out.strip() == "test":
            self.emul_pty = True

        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        cmd = "test -r '%s' && test %d = \"$(<%s)\" && rm -f '%s'" % (
            self.lock_file, self.pid, self.lock_file, self.lock_file)
        os.system(cmd)

    def get_mock_cmd(self, args):
        return self.def_cmd + args

    def exec_mock_cmd(self, args):
        cmd = self.get_mock_cmd(args)
        return self.results.exec_cmd(cmd, emul_pty=self.emul_pty)

    def exec_chroot_cmd(self, cmd):
        return self.exec_mock_cmd(["--chroot", cmd])

    def exec_mockbuild_cmd(self, cmd):
        return self.exec_chroot_cmd(
            "/bin/su mockbuild -c %s" % shell_quote(cmd))

    def copy_out(self, args):
        cmd = ["--disable-plugin=selinux", "--copyout"] + args
        return self.exec_mock_cmd(cmd)

    def try_install(self, pkgs):
        return (self.exec_mock_cmd(["--install"] + pkgs) == 0)

    def prepare_dnf_in_chroot(self):
        self.try_install(["dnf"])
        if find_missing_pkgs(["dnf"], self.results, self):
            return False

        # dump the contents of config_opts["yum.conf"] from the mock profile
        # and save it as /etc/dnf/dnf.conf in the chroot
        # FIXME: use some Python module to parse the conf
        cmd = "set -o pipefail; { "
        cmd += "echo 'config_opts = dict()'; "
        cmd += "echo 'config_opts[\"plugin_conf\"] = dict()'; "
        cmd += "echo 'config_opts[\"macros\"] = dict()'; "
        cmd += "cat /etc/mock/%s.cfg; " % self.mock_profile
        cmd += "echo 'print(config_opts[\"yum.conf\"])'; "
        cmd += "} | python | "
        cmd += strlist_to_shell_cmd(self.get_mock_cmd(["--shell", "dd of=/etc/dnf/dnf.conf"]))
        if (self.results.exec_cmd(cmd, shell=True) != 0):
            return False

        # copy DNS configuration from host into chroot for dnf to work
        resconf = "/etc/resolv.conf"
        return (self.exec_mock_cmd(["--copyin", resconf, resconf]) == 0)

    def init_and_install(self, pkgs):
        for scrub_root in [False, True]:
            if scrub_root:
                self.exec_mock_cmd(["--scrub=root-cache"])
                if self.exec_mock_cmd(["--init"]) != 0:
                    self.results.fatal_error(
                        "failed to init mock profile: %s" % self.mock_profile)

            elif not self.init_done and self.exec_mock_cmd(["--init"]) != 0:
                # try --scrub=root-cache
                continue

            self.init_done = True
            if not pkgs:
                return True

            # install required packages (all at once)
            self.try_install(pkgs)
            missing_deps = find_missing_pkgs(pkgs, self.results, self)
            if not missing_deps:
                # no misssing dependencies
                return True

            self.results.error(
                "hard to install dependencies (%s), still trying..." %
                strlist_to_shell_cmd(missing_deps), ec=0)

            if not scrub_root and not self.skip_init:
                # try --scrub=root-cache
                continue

            # try to install the missing packages one by one
            for pkg in missing_deps:
                self.try_install([pkg])

            for tried_dnf in [False, True]:
                # check that all dependencies are installed
                missing_deps = find_missing_pkgs(pkgs, self.results, self, ignore_with=tried_dnf)
                if not missing_deps:
                    # no missing dependencies
                    return True

                if tried_dnf:
                    break
                self.results.error(
                    "hard to install dependencies (%s), still trying..." %
                    strlist_to_shell_cmd(missing_deps), ec=0)

                # try to install/configure dnf in chroot and run it from there
                if not self.prepare_dnf_in_chroot():
                    break
                cmd_as_list = ["dnf", "install", "--allowerasing"] + missing_deps
                cmd = strlist_to_shell_cmd(cmd_as_list, escape_special=True)
                self.exec_chroot_cmd(cmd)

            self.results.error("failed to install dependencies: %s" %
                               strlist_to_shell_cmd(missing_deps))
            return False


class ScanProps:
    def __init__(self):
        self.install_pkgs = []
        self.install_opt_pkgs = []
        self.copy_in_files = [CHROOT_FIXUPS]
        self.pre_mock_hooks = []
        self.post_depinst_hooks = []
        self.rpm_opts = DEFAULT_RPM_OPTS
        self.path = []
        self.env = {}
        self.copy_out_files = []
        self.cswrap_enabled = False
        self.cswrap_filters = DEFAULT_CSWRAP_FILTERS
        self.result_filters = DEFAULT_RESULT_FILTERS
        self.build_cmd_wrappers = []
        self.post_build_chroot_cmds = []
        self.post_process_hooks = []
        self.keep_going = False
        self.cswrap_timeout = DEFAULT_CSWRAP_TIMEOUT
        self.no_scan = False
        self.print_defects = False
        self.need_rpm_bi = False
        self.shell_cmd_to_build = None
        self.srpm = None
        self.base_srpm = None
        self.mock_profile = None
        self.base_mock_profile = None
        self.any_tool = False
        self.nvr = None
        self.imp_checker_set = set()
        self.imp_csgrep_filters = []

    def enable_cswrap(self):
        if self.cswrap_enabled:
            # already enabled
            return
        self.cswrap_enabled = True

        # resolve cswrap_path by querying cswrap binary
        cmd = ["cswrap", "--print-path-to-wrap"]
        subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (out, _) = subproc.communicate()
        cswrap_path = out.decode("utf8").strip()

        self.copy_in_files += ["/usr/bin/cswrap", cswrap_path]
        self.path = [cswrap_path] + self.path
        self.env["CSWRAP_CAP_FILE"] = "/builddir/cswrap-capture.err"
        self.env["CSWRAP_TIMEOUT"] = "%d" % self.cswrap_timeout
        self.env["CSWRAP_TIMEOUT_FOR"] = ":"
        self.copy_out_files += ["/builddir/cswrap-capture.err"]

    def pick_cswrap_results(self, results):
        if not self.cswrap_enabled:
            # not enabled --> succeeded trivially
            return 0

        # apply all filters using a shell pipe
        fin = "%s/builddir/cswrap-capture.err" % results.dbgdir_raw
        out = "%s/cswrap-capture.err" % results.dbgdir_uni
        cmd = "cat '%s'" % fin
        for filt in self.cswrap_filters:
            cmd += " | %s" % filt
        cmd += " > '%s'" % out
        (rv, _) = results.get_cmd_output(cmd)
        return rv

    def wrap_build_cmd(self, cmd_in):
        cmd_out = cmd_in
        for w in self.build_cmd_wrappers:
            cmd_out = "sh -c %s" % shell_quote(cmd_out)
            cmd_out = w % cmd_out
        return cmd_out

    def wrap_shell_cmd_by_env(self, cmd_in):
        # serialize self.path
        path_str = ""
        for p in self.path:
            path_str += p + ":"
        cmd_out = "PATH=%s$PATH " % path_str

        # serialize self.env
        assert "PATH" not in self.env
        for var in self.env:
            cmd_out += "%s='%s' " % (var, self.env[var])

        # run a new instance of shell for the specified command
        cmd_out += "sh -c %s" % shell_quote(cmd_in)
        return cmd_out


class PluginManager:
    def __init__(self):
        self.plug_by_prio = {}
        self.plug_by_name = {}

    def try_load(self, modname, path):
        fp, pathname, description = imp.find_module(modname, [path])
        try:
            mod = imp.load_module(modname, fp, pathname, description)
            plugin = mod.Plugin()
        finally:
            fp.close()

        props = plugin.get_props()
        # TODO: check API version
        prio = props.pass_priority
        assert prio not in self.plug_by_prio
        self.plug_by_prio[prio] = plugin
        self.plug_by_name[modname] = plugin

    def load_default_plugins(self):
        try:
            files = os.listdir(PLUGIN_DIR)
        except:
            return

        for fname in files:
            parts = fname.split(".")
            if len(parts) != 2:
                continue
            if parts[1] != "py":
                continue
            self.try_load(parts[0], path=PLUGIN_DIR)

    # Print description of each available plugin in format TOOL [:indent:] DESCRIPTION
    def print_plugin_descriptions(self):
        max_key_len = max(map(len, self.plug_by_name.keys()))
        min_indent_len = 8
        description_indent = max_key_len + min_indent_len
        for key, plugin in sorted(self.plug_by_name.items()):
            desc = getattr(plugin, "description", "")
            sys.stdout.write("{}{}{}\n".format(
                key, " " * (description_indent - len(key)),
                desc.replace('\n', '\n%s' % (" " * description_indent))))

    def get_name_list(self):
        return sorted(self.plug_by_name.keys())

    def enable(self, plugin_name):
        plugin = self.plug_by_name[plugin_name]
        plugin.enable()

    def enable_all(self):
        for prio in sorted(self.plug_by_prio):
            plugin = self.plug_by_prio[prio]
            plugin.enable()

    def init_parser(self, parser):
        for prio in sorted(self.plug_by_prio):
            plugin = self.plug_by_prio[prio]
            plugin.init_parser(parser)

    def handle_args(self, parser, args, props):
        for prio in sorted(self.plug_by_prio):
            plugin = self.plug_by_prio[prio]
            plugin.handle_args(parser, args, props)

    def num_enabled(self):
        cnt = 0
        for prio in sorted(self.plug_by_prio):
            plugin = self.plug_by_prio[prio]
            if getattr(plugin, "enabled", False):
                cnt = cnt + 1
        return cnt


def deplist_from_srpm(results, srpm):
    (_, deps) = results.get_cmd_output("rpm -qp '%s' --requires" % srpm)
    raw_deps = filter(None, deps.split("\n"))
    deps = []
    for d in raw_deps:
        pkg = d.strip()
        if is_ignored_dep(pkg):
            continue
        deps += [pkg]
    return deps


def transform_results(js_file, results):
    err_file  = re.sub("\\.js", ".err",  js_file)
    html_file = re.sub("\\.js", ".html", js_file)
    stat_file = re.sub("\\.js", "-summary.txt", js_file)
    results.exec_cmd("csgrep --mode=grep %s '%s' > '%s'" %
                     (CSGREP_FINAL_FILTER_ARGS, js_file,  err_file), shell=True)
    results.exec_cmd("csgrep --mode=json %s '%s' | cshtml - > '%s'" %
                     (CSGREP_FINAL_FILTER_ARGS, js_file, html_file), shell=True)
    results.exec_cmd("csgrep --mode=stat %s '%s' | tee '%s'" % \
                     (CSGREP_FINAL_FILTER_ARGS, err_file, stat_file), shell=True)
    return err_file, html_file


def re_from_checker_set(checker_set):
    """return operand for the --checker option of csgrep based on checker_set"""
    chk_re = "^("
    first = True
    for chk in checker_set:
        if first:
            first = False
        else:
            chk_re += "|"
        chk_re += chk
    chk_re += ")$"
    return chk_re


# transform scan-results.js to scan-results.{err,html} and write stats
def finalize_results(js_file, results, props):
    (err_file, _) = transform_results(js_file, results)
    if props.imp_checker_set:
        # filter out "important" defects, first based on checkers only
        cmd = "csgrep '%s' --mode=json --checker '%s'" % \
                (js_file, re_from_checker_set(props.imp_checker_set))

        # then apply custom per-checker filters
        for (chk, csgrep_args) in props.imp_csgrep_filters:
            chk_re = re_from_checker_set(props.imp_checker_set - set([chk]))
            cmd += " | csdiff <(csgrep '%s' --mode=grep --invert-regex --checker '%s' %s) -" \
                    % (js_file, chk_re, csgrep_args)

        # write the result into *-imp.js
        imp_js_file = re.sub("\\.js", "-imp.js", js_file)
        cmd += " > '%s'" % imp_js_file

        # bash is needed to process <(...)
        cmd = strlist_to_shell_cmd(["bash", "-c", cmd], escape_special=True)
        results.exec_cmd(cmd, shell=True)

        # generate *-imp.{err,html}
        transform_results(imp_js_file, results)

    if props.print_defects:
        os.system("csgrep '%s'" % err_file)


# argparse._VersionAction would write to stderr, which breaks help2man
class VersionPrinter(argparse.Action):
    def __init__(self, option_strings, dest=None, default=None, help=None):
        super(VersionPrinter, self).__init__(
            option_strings=option_strings, dest=dest, default=default, nargs=0,
            help=help)

    def __call__(self, parser, namespace, values, option_string=None):
        print("csmock-2.3.0-1.el6")
        sys.exit(0)


# provide a more user-friendly error message in case a plug-in is not installed
class FileNameParser(argparse.Action):
    def __call__(self, parser, namespace, val, os=None):
        if isinstance(val, str) and val.startswith("--"):
            parser.error("File name '%s' starts with '--', which looks like \
option.  Are you sure, you have necessary plug-ins installed?  If it really \
is a file name, please use the './' prefix." % val)
        else:
            setattr(namespace, self.dest, val)


def require_file(parser, name):
    """Print an error and exit unsuccessfully if 'name' is not a file"""
    if not os.path.isfile(name):
        parser.error("'%s' is not a file" % name)


def main():
    # load plug-ins
    plugins = PluginManager()
    plugins.load_default_plugins()
    plugin_list = plugins.get_name_list()

    # list available tools
    # FIXME: --list-available-tools takes precedence over --help and --version
    class ToolsPrinter(argparse.Action):
        def __init__(self, option_strings, dest=None, default=None, help=None):
            super(ToolsPrinter, self).__init__(
                option_strings=option_strings, dest=dest, default=default, nargs=0,
                help=help)

        def __call__(self, parser, namespace, values, option_string=None):
            sys.stdout.write('Tools list: \n')
            plugins.print_plugin_descriptions()
            sys.exit(0)

    # initialize argument parser
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "SRPM", nargs="?", action=FileNameParser,
        help="source RPM package to be scanned by static analyzers")

    # define optional arguments
    parser.add_argument(
        "-r", "--root", dest="mock_profile", default="default",
        help="mock profile to use (defaults to mock's default)")

    parser.add_argument(
        "-t", "--tools", action="append", default=[],
        help="comma-spearated list of tools to enable \
(use --list-available-tools to see the list of available tools)")

    parser.add_argument(
        "-a", "--all-tools", action="store_true",
        help="enable all available tools \
(use --list-available-tools to see the list of available tools)")

    parser.add_argument(
        "-l", "--list-available-tools", action=ToolsPrinter,
        help="list available tools and exit")

    parser.add_argument(
        "--install", dest="list_of_pkgs",
        help="space-separated list of packages to install into the chroot")

    parser.add_argument(
        "-o", "--output",
        help="name of the tarball or directory to put the results to")

    parser.add_argument(
        "-f", "--force", action="store_true",
        help="overwrite the resulting file or directory if it exists already")

    parser.add_argument(
        "-j", "--jobs", type=int, default=DEFAULT_JOBS_CNT,
        help="maximal number of jobs running in parallel (passed to 'make')")

    parser.add_argument(
        "--cswrap-timeout", type=int, default=DEFAULT_CSWRAP_TIMEOUT,
        help="maximal amount of time taken by analysis of a single module [s]")

    parser.add_argument(
        "-U", "--embed-context", type=int, default=3,
        help="embed a number of lines of context from the source file for the \
key event (defaults to 3).")

    parser.add_argument(
        "-k", "--keep-going", action="store_true",
        help="continue as much as possible after an error")

    parser.add_argument(
        "--skip-init", action="store_true",
        help="do not run 'mock --init' before the scan \
(may lead to unpredictable scan results)")

    parser.add_argument(
        "--no-clean", action="store_true",
        help="do not clean chroot when it becomes unused")

    parser.add_argument(
        "--no-scan", action="store_true",
        help="do not analyze any package, just check versions of the analyzers")

    csmock.common.util.add_paired_flag(
        parser, "print-defects",
        help="print the resulting list of defects (default if connected to a tty)")

    parser.add_argument(
        "--base-srpm",
        help="perform a differential scan against the specified base pacakge")

    parser.add_argument(
        "--base-root", dest="base_mock_profile",
        help="mock profile to use for the base scan (use only with --base-srpm)")

    # --skip-patches, --diff-patches, and --shell-cmd are mutually exclusive
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        "--skip-patches", action="store_true",
        help="skip patches not annotated by %%{?_rawbuild} (vanilla build)")
    group.add_argument(
        "--diff-patches", action="store_true",
        help="scan with/without patches and diff the lists of defects")
    group.add_argument(
        "-c", "--shell-cmd",
        help="use shell command to build the given tarball (instead of SRPM)")

    # --defect-blacklist
    default_dbl = DEFAULT_DEFECT_BLACKLIST
    if os.path.exists(default_dbl):
        default_dbl_text = " (defaults to \"%s\")" % default_dbl
    else:
        default_dbl_text = " (defaults to \"%s\" if available)" % default_dbl
        default_dbl = ""
    parser.add_argument(
        "--defect-blacklist", default=default_dbl,
        help=("suppress known false positives loaded from the given file" + default_dbl_text))

    # needed for help2man
    parser.add_argument(
        "--version", action=VersionPrinter,
        help="print the version of csmock and exit")

    # add command-line options handled by plugins
    plugins.init_parser(parser)

    # parse command-line arguments
    args = parser.parse_args()

    if args.print_defects is None:
        args.print_defects = sys.stdout.isatty()

    # check that only available tools are requested (and enable them)
    for i in args.tools:
        for j in i.split(","):
            tool = j.strip()
            if not tool:
                continue
            if tool in plugin_list:
                # explicitly enable this tool
                plugins.enable(tool)
            else:
                parser.error("tool not available: %s" % tool)

    if args.all_tools:
        # enable all available tools
        plugins.enable_all()

    output = args.output
    if args.SRPM is None:
        if args.no_scan:
            if output is None:
                parser.error("unable to infer --output (despite --no-scan was given)")
        else:
            parser.error("no SRPM (or tarball) specified on the command line")

    if args.no_scan and args.shell_cmd is not None:
        parser.error("--shell-cmd makes no sense with --no-scan")

    if args.base_srpm is None:
        if args.base_mock_profile is not None:
            parser.error("--base-root makes no sense without --base-srpm")
    else:
        if args.diff_patches:
            parser.error("options --diff-patches and --base-scan are mutually exclusive")

    props = ScanProps()
    props.cswrap_timeout        = args.cswrap_timeout
    props.keep_going            = args.keep_going
    props.no_scan               = args.no_scan
    props.print_defects         = args.print_defects
    props.shell_cmd_to_build    = args.shell_cmd
    props.srpm                  = args.SRPM
    props.base_srpm             = args.base_srpm

    if args.embed_context > 0:
        # we need 'csgrep --embed-context' to work in the chroot for --embed-context
        props.install_opt_pkgs += ["csdiff >= 1.2.1"]

    # initialize the %{_smp_mflags} RPM macro
    mflags_rpmm = "_smp_mflags"
    mflags_rpmm += " -j%d" % args.jobs
    if props.keep_going:
        mflags_rpmm += " -k"
    props.rpm_opts += ["--define", mflags_rpmm]

    # make sure that we have a configuration for the selected mock profile
    props.mock_profile = args.mock_profile
    require_file(parser, "/etc/mock/%s.cfg" % props.mock_profile)
    if args.base_mock_profile is None:
        props.base_mock_profile = props.mock_profile
    else:
        props.base_mock_profile = args.base_mock_profile
        require_file(parser, "/etc/mock/%s.cfg" % props.base_mock_profile)

    if args.list_of_pkgs is not None:
        # append the list of packages to install specified on command-line
        props.install_pkgs += args.list_of_pkgs.split()

    if not props.no_scan:
        # make sure that 'srpm' is a file (it can be a tar archive instead of SRPM)
        require_file(parser, props.srpm)

    if props.srpm is not None:
        # resolve NVR
        srpm_base = os.path.basename(props.srpm)
        if props.shell_cmd_to_build is None:
            props.nvr = re.sub("\\.src\\.rpm$", "", srpm_base)
        else:
            props.nvr = re.sub("\\.tar$", "", re.sub("\\.[^.]*$", "", srpm_base))

    # resolve name of the file/dir we are going to store the results to
    if args.output is None:
        output = props.nvr + ".tar.xz"
    output = os.path.realpath(output)

    # FIXME: TOCTOU race
    if os.path.exists(output) and not args.force:
        parser.error("'%s' already exists, use --force to proceed" % output)

    # handle --defect-blacklist
    if args.defect_blacklist:
        require_file(parser, args.defect_blacklist)
        props.result_filters += [ "csdiff --json-output \"%s\" -" % args.defect_blacklist ]
    props.defect_blacklist = args.defect_blacklist

    # poll plug-ins to reflect themselves in ScanProps
    plugins.handle_args(parser, args, props)
    props.any_tool = (plugins.num_enabled() > 0)

    if args.diff_patches:
        ec = do_diff_scan(props, output, args, diff_patches=True)
    elif args.base_srpm is not None:
        ec = do_diff_scan(props, output, args, diff_patches=False)
    else:
        ec = do_scan(props, output, args, args.skip_patches)

    sys.exit(ec)


def do_scan(props, output, args, skip_patches):
    if skip_patches:
        props.copy_in_files += [PATCH_RAWBUILD]
        props.rpm_opts += RAWBUILD_RPM_OPTS

    try:
        with ScanResults(output, props.keep_going) as results:
            results.ini_writer.append("mock-config", props.mock_profile)
            results.ini_writer.append("project-name", props.nvr)
            if props.defect_blacklist:
                results.ini_writer.append("defect-blacklist", props.defect_blacklist)

            if not props.any_tool:
                # no tool enabled
                results.error("No tools are enabled, only trying to build \
the package.  Use --tools or --all-tools to enable them!\n", ec=0)

            # dump list of RPMs installed on the host (for debugging purposes)
            results.exec_cmd(
                "rpm -qa | sort -V > '%s/rpm-list-host.txt'" % results.dbgdir,
                shell=True)

            if not props.no_scan:
                if props.shell_cmd_to_build is None:
                    # check the given SRPM
                    if results.get_cmd_output("rpm -pq '%s'" % props.srpm)[0] != 0:
                        results.fatal_error("failed to open SRPM: %s" % props.srpm)
                    (ec, spec) = results.get_cmd_output(
                        "rpm -lpq '%s' | grep '\\.spec$'" % props.srpm)
                    if ec != 0:
                        results.fatal_error("no specfile found in SRPM: %s" % props.srpm)
                    spec = spec.rstrip()
                    spec_in = "/builddir/build/SPECS/%s" % spec

                # copy the given SRPM into our tmp dir
                srpm_base = os.path.basename(props.srpm)
                srpm_dup = "%s/%s" % (results.tmpdir, srpm_base)
                shutil.copyfile(props.srpm, srpm_dup)
                props.copy_in_files += [srpm_dup]

            # run pre-mock hooks
            for hook in props.pre_mock_hooks:
                rv = hook(results)
                if rv != 0:
                    results.error("pre-mock hook failed", ec=rv)

            with MockWrapper(results, props.mock_profile, args.skip_init) as mock:
                if not props.no_scan and props.shell_cmd_to_build is None:
                    # first rebuild the given SRPM
                    mock.init_and_install([])

                    # install the copied SRPM into the chroot
                    srpm_in = "/builddir/%s" % srpm_base
                    mock.exec_mock_cmd(["--copyin", srpm_dup, srpm_in])
                    mock.exec_chroot_cmd("chown mockbuild -R /builddir")
                    mock.exec_mockbuild_cmd("rpm -Uvh --nodeps '%s'" % srpm_in)

                    # rebuild the given SRPM (and rename to match the original one)
                    cmd_tpl = "rpmbuild -bs --nodeps %s %s && sh -c 'cd \
/builddir/build/SRPMS && eval mv -v *.src.rpm %s || :'"
                    mock.exec_mockbuild_cmd(
                        cmd_tpl % (spec_in, strlist_to_shell_cmd(props.rpm_opts), srpm_base))

                    # use the rebuilt SRPM to get the dependency list
                    mock.copy_out(["/builddir/build/SRPMS/%s" % srpm_base, srpm_dup])
                    props.install_pkgs += deplist_from_srpm(results, srpm_dup)

                # run 'mock --init' and 'mock --install'
                mock.init_and_install(props.install_pkgs)

                # install optional packages (if any)
                if props.install_opt_pkgs:
                    mock.try_install(props.install_opt_pkgs)
                    # just to update rpm-list-mock.txt
                    find_missing_pkgs([], results, mock)

                if props.shell_cmd_to_build is not None:
                    # prepare a build script in our tmp dir
                    build_script = "%s/build.sh" % results.tmpdir
                    cmd_tpl = "printf '#!/bin/sh\n\
cd /builddir/build/BUILD || exit $?\n\
cd %%s*/ || cd *\n\
%%s' '%s' '%s' | tee '%s' >&2\n"
                    results.exec_cmd(
                        cmd_tpl % (props.nvr, props.shell_cmd_to_build, build_script),
                        shell=True)
                    props.copy_in_files += [build_script]

                # copy required files into the chroot
                cmd = "tar -cP "
                cmd += strlist_to_shell_cmd(props.copy_in_files)
                cmd += " | "
                cmd += strlist_to_shell_cmd(
                    mock.get_mock_cmd(["--shell", "tar -xC/"]))
                results.exec_cmd(cmd, shell=True)

                # run post-depinst hooks
                for hook in props.post_depinst_hooks:
                    rv = hook(results, mock)
                    if rv != 0:
                        results.error("post-depinst hook failed", ec=rv)

                if not props.no_scan:
                    if props.shell_cmd_to_build is None:
                        # install the copied SRPM into the chroot
                        mock.exec_chroot_cmd("rpm -Uvh --nodeps '%s'" % srpm_dup)
                        # make the installed SRPM accessible (if the maintainer did not)
                        mock.exec_chroot_cmd("chmod -R +r /builddir")

                    # run fixups scripts
                    cmd_tpl = "for i in %s/*; do test -x $i && echo RUN: $i >&2 && $i; done"
                    mock.exec_mock_cmd(["--shell", cmd_tpl % CHROOT_FIXUPS])

                    if props.shell_cmd_to_build is None:
                        # run %prep phase without pluggin-in any static analyzers
                        ec = mock.exec_mockbuild_cmd(
                            "rpmbuild -bp --nodeps %s %s" % (
                                spec_in, strlist_to_shell_cmd(props.rpm_opts)))
                    else:
                        # extract the given archive (we got instead of SRPM)
                        if re.match("^.*\\.zip$", srpm_dup):
                            # ZIP archive
                            prep_cmd_tpl = "unzip -d '%s' '%s'"
                        else:
                            # assume TAR
                            prep_cmd_tpl = "tar -C '%s' -xvf '%s'"
                        prep_cmd = prep_cmd_tpl % ("/builddir/build/BUILD", srpm_dup)
                        ec = mock.exec_mockbuild_cmd(prep_cmd)

                    # make the unpacked contents accessible (if the maintainer did not)
                    mock.exec_chroot_cmd("chmod -R +r /builddir/build")

                    if ec != 0:
                        results.error("%prep failed", ec=ec)

                    if props.shell_cmd_to_build is None:
                        # run %build phase with static analyzers plugged-in
                        build_cmd = "rpmbuild -bc --nodeps --short-circuit %s %s" \
                                % (spec_in, strlist_to_shell_cmd(props.rpm_opts))
                    else:
                        # run the above prepared build script
                        build_cmd = "sh -x '%s'" % build_script

                    # wrap build_cmd by all the necessary wrappers
                    build_cmd = props.wrap_build_cmd(build_cmd)

                    # initialize environment variables according to ScanProps
                    build_cmd = props.wrap_shell_cmd_by_env(build_cmd)

                    ec = mock.exec_mockbuild_cmd(build_cmd)
                    if ec != 0:
                        results.error("%build failed", ec=ec)

                    if props.need_rpm_bi:
                        # disable %check while running 'rpmbuild -bi'
                        rpm_opts = props.rpm_opts
                        if mock.exec_chroot_cmd("rpmbuild --nocheck") == 0:
                            rpm_opts += ["--nocheck"]
                        else:
                            # fragile compatiblity workaround for older versions of rpm-build,
                            # known to break if unescaped %check appears in a change log entry
                            rpm_opts += ["--define", "check\\\n%%check\\\nexit 0"]

                        if props.shell_cmd_to_build is not None:
                            results.fatal_error("SRPM is required by a plug-in")
                        cmd = "rpmbuild -bi --nodeps --short-circuit %s %s" \
                            % (spec_in, strlist_to_shell_cmd(rpm_opts))

                        # initialize environment variables according to ScanProps
                        cmd = props.wrap_shell_cmd_by_env(cmd)

                        ec = mock.exec_mockbuild_cmd(cmd)
                        if ec != 0:
                            results.error("%install failed", ec=ec)
                        bd_flt = "sed 's|/builddir/build/BUILDROOT/[^/]*/|/builddir/build/BUILD//|'"
                        props.result_filters = [bd_flt] + props.result_filters

                    # execute post-build commands in the chroot
                    for cmd in props.post_build_chroot_cmds:
                        mock.exec_chroot_cmd(cmd)

                    # get the (intermediate) results out of the chroot
                    if props.copy_out_files:
                        cmd = strlist_to_shell_cmd(
                            mock.get_mock_cmd(
                                ["--shell", "tar -c " + strlist_to_shell_cmd(
                                    props.copy_out_files)]))

                        cmd += " | tar -xC '%s'" % results.dbgdir_raw
                        if results.exec_cmd(cmd, shell=True) != 0:
                            results.error("field to get intermediate results from mock")

                if not props.no_scan:
                    if props.pick_cswrap_results(results) != 0:
                        results.error("failed to pick cswrap results")

                    # run post-process hooks
                    for hook in props.post_process_hooks:
                        rv = hook(results)
                        if rv != 0:
                            results.error("post-process hook failed", ec=rv)

                # we are done with IniWriter
                results.ini_writer.close()

                # merge all results into a single file named scan-results-all.js
                ini_file = "%s/scan.ini" % results.resdir
                js_file = "%s/scan-results.js" % results.resdir
                all_file = "%s/scan-results-all.js" % results.dbgdir
                cmd = "cslinker --quiet --cwelist '%s' --inifile '%s' $(ls %s/*.err) \
> '%s'" % (CWE_LIST_FILE, ini_file, results.dbgdir_uni, all_file)
                results.exec_cmd(cmd, shell=True)

                if args.embed_context > 0:
                    # embed context lines from source program files
                    tmp_file = "%s.tmp" % all_file
                    cmd = strlist_to_shell_cmd(
                        mock.get_mock_cmd(
                            ["--shell", "csgrep --mode=json --embed-context %d" % args.embed_context]))
                    cmd += " < '%s' > '%s'" % (all_file, tmp_file)
                    if results.exec_cmd(cmd, shell=True) == 0:
                        shutil.move(tmp_file, all_file)

                if not args.no_clean:
                    # clean up thhe mock root
                    if mock.exec_mock_cmd(["--clean"]) != 0:
                        results.error("failed to clean mock profile", ec=0)

            # we are done with mock

            # apply filters, sort the list and store the result as scan-results.js
            cmd = "cat '%s'" % all_file
            for filt in props.result_filters:
                cmd += " | %s" % filt
            cmd += " | cssort --key=path > '%s'" % js_file
            results.exec_cmd(cmd, shell=True)

            finalize_results(js_file, results, props)
            return results.ec

    except FatalError as error:
        return error.ec


def do_diff_scan(props, output, args, diff_patches):
    try:
        with ScanResults(output, props.keep_going, create_dbgdir=False) as results:
            run0_props = copy.deepcopy(props)
            csdiff = "csdiff"
            if diff_patches:
                # we are looking for defects in patches
                assert not args.skip_patches
                title = "%s - Defects in Patches" % props.nvr
            else:
                # this is a version-diff-build
                run0_props.srpm         = run0_props.base_srpm
                run0_props.mock_profile = run0_props.base_mock_profile
                csdiff += " --ignore-path"
                title = "%s - Defects not detected in %s" % (props.nvr, props.base_srpm)

            run0 = "%s/run0" % results.resdir
            ec = do_scan(run0_props, run0, args, skip_patches=(args.skip_patches or diff_patches))
            if ec != 0:
                results.error("base scan failed", ec=ec)

            run1 = "%s/run1" % results.resdir
            ec = do_scan(props, run1, args, skip_patches=args.skip_patches)
            if ec != 0:
                results.error("regular scan failed", ec=ec)

            # diff and process fixed defects
            run0_file = "%s/scan-results.js" % run0
            run1_file = "%s/scan-results.js" % run1
            js_file_fixed = "%s/scan-results-fixed.js" % results.resdir
            cmd = "%s --fixed %s %s > %s" % (csdiff, run0_file, run1_file, js_file_fixed)
            if results.exec_cmd(cmd, shell=True) != 0:
                results.error("csdiff --fixed failed")
            transform_results(js_file_fixed, results)

            # finalize scan.ini
            results.ini_writer.append("title", title)
            results.ini_writer.close()
            ini_file = "%s/scan.ini" % results.resdir

            # diff and process added defects
            js_file = "%s/scan-results.js" % results.resdir
            cmd_tpl = "%s %s %s | cslinker --inifile %s - > %s"
            cmd = cmd_tpl % (csdiff, run0_file, run1_file, ini_file, js_file)
            if results.exec_cmd(cmd, shell=True) != 0:
                results.error("csdiff failed")
            finalize_results(js_file, results, props)

            return results.ec

    except FatalError as error:
        return error.ec

if __name__ == '__main__':
    main()
