#!/bin/bash
#
# greenbone-scap-sync
# This script synchronizes an OpenVAS installation with the Greenbone SCAP
# data directory.
#
# Authors:
# Henri Doreau <henri.doreau@greenbone.net>
# Michael Wiegand <michael.wiegand@greenbone.net>
# Timo Pollmeier <timo.pollmeier@greenbone.net>
#
# Copyright:
# Copyright (C) 2011, 2012, 2013 Greenbone Networks GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2,
# or, at your option, any later version as published by the Free
# Software Foundation
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

VERSION=20130222
RESTRICTED=1

# SETTINGS
# ========

# Path prefix for installation location
PREFIX=

# RSYNC_DELETE controls whether files which are not part of the repository will
# be removed from the local directory after synchronization. The default value
# for this setting is "--delete", which means that files which are not part of
# feed will be deleted.
RSYNC_DELETE="--delete --exclude=scap.db"

# PORT controls the outgoing TCP port for updates. If PAT/Port-Translation is
# not used, this should be "24". For some application layer firewalls or gates
# the value 22 (Standard SSH) is useful. Only change if you know what you are
# doing.
PORT=24

# LOGDIR and LOGFILE specify the location of the greenbone-scapdata-sync logfile.
# The default value for LOGDIR is  "/var/log/", the default value for
# LOGFILE is "greenbone-scapdata-sync.log". Please make sure this script has
# sufficient right to access the logfile.
LOGDIR="$PREFIX/var/log/openvas/"
LOGFILE="greenbone-scapdata-sync.log"

# If ENABLED is set to 0, the sync script will not perform a synchronization.
ENABLED=1

# If REFRESH_ONLY is set to 1, the sync script will only update the OpenVAS
# Manager database. This can be controlled via the --refresh parameter.
REFRESH_ONLY=0

SQLITE3="sqlite3 -noheader"

[ -r $PREFIX/etc/openvas/greenbone-scapdata-sync.conf ] && . $PREFIX/etc/openvas/greenbone-scapdata-sync.conf

BASE_SYNC_DIR="$PREFIX/var/lib/openvas"
SCAP_DIR="$BASE_SYNC_DIR/scap-data"

if [ -z "$PREFIX" ] ; then
  SCAP_RES_DIR="/usr/share/openvas/scap"
else
  SCAP_RES_DIR="$PREFIX/share/openvas/scap"
fi

SCAP_DB="$SCAP_DIR/scap.db"

ACCESSKEY="$PREFIX/etc/openvas/gsf-access-key"

SCRIPT_NAME="greenbone-scapdata-sync"

TIMESTAMP="$SCAP_DIR/timestamp"

if [ -z "$FEED_NAME" ] ; then
  FEED_NAME="Greenbone SCAP Feed"
fi

if [ -z "$FEED_VENDOR" ] ; then
  FEED_VENDOR="Greenbone Networks GmbH"
fi

if [ -z "$FEED_HOME" ] ; then
  FEED_HOME="http://www.greenbone.net/solutions/gbn_feed.html"
fi

do_describe () {
  echo "This script synchronizes a SCAP collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."
}

do_feedversion () {
  if [ -r $TIMESTAMP ] ; then
      echo `cat $TIMESTAMP`
  fi
}

if [[ ! -w $LOGDIR ]]
then
  NOLOG=1
  echo
  echo "== greenbone-scapdata-sync $VERSION ================================================"
  echo "Warning: The logfile directory"
  echo "  ($LOGDIR)"
  echo "does not exist or is not writeable. Please make sure this directory exists and"
  echo "is writable."
  echo "Please be aware that logging is not possible during this script run!"
  echo "==============================================================================="
  echo
fi

log_write(){
  if (( $NOLOG ))
  then
    echo "LOG: [`date -R`] $1"
  else
    echo "[`date -R`] $1" >> $LOGDIR$LOGFILE
  fi
}

if [ $ENABLED -ne 1 ]
then
  log_write "SCAP synchronization is disabled, exiting."
  exit 0
fi

RSYNC=`command -v rsync`
if [[ -z $RSYNC ]]
then
  echo
  echo "== greenbone-scapdata-sync $VERSION ================================================"
  echo "Could not find tools necessary for synchronization."
  echo "Please make sure that the tool"
  echo "  rsync"
  echo "is installed and available in your PATH variable."
  echo "If you are still not able to continue, please contact Greenbone Support at"
  echo "support@greenbone.net and include the error messages displayed above (if any)"
  echo "and your customer ID."
  echo "==============================================================================="
  echo
  log_write "rsync not found, aborting synchronization."
  logger "SCAP synchronization: rsync not found, aborting synchronization."
  exit 1
fi

SQLITE=`command -v sqlite3`
if [[ -z $SQLITE ]]
then
  echo
  echo "== greenbone-scapdata-sync $VERSION ================================================"
  echo "Could not find tools necessary for synchronization."
  echo "Please make sure that the tool"
  echo "  sqlite3"
  echo "is installed and available in your PATH variable."
  echo "If you are still not able to continue, please contact Greenbone Support at"
  echo "support@greenbone.net and include the error messages displayed above (if any)"
  echo "and your customer ID."
  echo "==============================================================================="
  echo
  log_write "sqlite3 not found, aborting synchronization."
  logger "SCAP synchronization: sqlite3 not found, aborting synchronization."
  exit 1
fi

if [ -z $BASE_SYNC_DIR ]
then
  echo
  echo "BASE_SYNC_DIR ($BASE_SYNC_DIR) not found!"
  echo
  exit 1
fi

reinit () {
  log_write " Reinitialization of the database necessary."
  rm -f $SCAP_DB
  $SQLITE3 $SCAP_DB < $SCAP_RES_DIR/scap_db_init.sql
}

db_migrate_9 () {
  log_write "Migrating database to version 9."
  sqlite3 "$SCAP_DB" "DROP INDEX IF EXISTS afp_idx;
                      CREATE INDEX afp_cpe_idx ON affected_products (cpe);
                      CREATE INDEX afp_cve_idx ON affected_products (cve);
                      UPDATE meta SET value = '9' WHERE name = 'database_version';"
  log_write "Migration done."
}

check_db_version () {
  DB_VERSION=`$SQLITE3 $SCAP_DB "select value from meta where name = 'database_version';" 2>/dev/null || echo 0`
  case "$DB_VERSION" in
    0) reinit;;
    1) reinit;;
    2) reinit;;
    3) reinit;;
    4) reinit;;
    5) reinit;;
    6) reinit;;
    7) reinit;;
    8) db_migrate_9;;
  esac
}

list_oval_files_sorted () {
for line in `list_oval_files_sortable | sort`
  do
    echo ${line#????????}
  done
}

list_oval_files_sortable () {
for ovalfile in `find "$SCAP_DIR/oval/" -name "*.xml" -type f`
  do
    echo `xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | sed -e 's/T.*//' | tr -d "-"`"$ovalfile"
  done
}

do_help () {
  echo "$0: Sync SCAP data"
  echo " --describe	display current feed info"
  echo " --feedversion	display version of this feed"
  echo " --help		display this help"
  echo " --identify	display information"
  echo " --refresh	update database without downloading new state"
  echo " --selftest	perform self-test"
  echo " --version	display version"
  echo ""
  exit 0
}

update_scap_db() {
  if [ ! -f $SCAP_DB ]
  then
    log_write "Initializing SCAP database."
    $SQLITE3 $SCAP_DB < $SCAP_RES_DIR/scap_db_init.sql
    CVE_REFDATE=0
  else
    check_db_version
    CVE_REFDATE=`$SQLITE3 $SCAP_DB "select date(max(modification_time),'unixepoch') from cves;" | sed -e 's/T.*//' | tr -d "-"`
  fi

  DB_LASTUPDATE=`$SQLITE3 $SCAP_DB "select value from meta where name = 'last_update';"`

  if [ -z "$CVE_REFDATE" ]
  then
    CVE_REFDATE=0
  fi

  if [ -z "$DB_LASTUPDATE" ];
  then
    # Happens when initial sync was aborted
    log_write "Error: Inconsistent data. Resetting SCAP database."
    rm -f $SCAP_DB
    $SQLITE3 $SCAP_DB < $SCAP_RES_DIR/scap_db_init.sql
    CVE_REFDATE=0
    DB_LASTUPDATE=0
  fi

  CPEBASE="$SCAP_DIR/official-cpe-dictionary_v2.2.xml"
  if [ -e $CPEBASE ]
  then
    if [ `stat -c "%Y" $CPEBASE | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ]
    then
      log_write "Updating CPEs"
      sed 's/&/&amp;/g' $CPEBASE | \
        xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cpe_youngerthan.xsl - | \
        xsltproc $SCAP_RES_DIR/cpe_update.xsl - | \
        $SQLITE3 $SCAP_DB
    else
      log_write " Skipping CPEs, file is older than last revision (this is not an error)."
    fi
  else
    echo " Warning: No CPE dictionary found in $SCAP_DIR"
  fi

  xmlcount=$(ls $SCAP_DIR/nvdcve-2.0-*.xml 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for cvefile in `ls $SCAP_DIR/nvdcve-2.0-*.xml`
    do
      if [ `stat -c "%Y" $cvefile | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ]
      then
        log_write "Updating $cvefile"
        xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cve_youngerthan.xsl $cvefile | \
          xsltproc $SCAP_RES_DIR/cve_update.xsl - | \
          $SQLITE3 $SCAP_DB
      else
        log_write "Skipping $cvefile, file is older than last revision (this is not an error)."
      fi
    done
  else
    log_write " Warning: No CVEs found in $SCAP_DIR."
  fi

  log_write " Updating OVAL data"
  xmlcount=$(find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for ovalfile in `list_oval_files_sorted`
    do
      if [ `stat -c "%Y" "$ovalfile" | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ]
      then
        oval_timestamp=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | sed -e 's/T.*//' | tr -d "-"`

        OVAL_REFDATE=`$SQLITE3 $SCAP_DB "SELECT date(max(modification_time),'unixepoch') FROM ovaldefs WHERE xml_file = '${ovalfile##$SCAP_DIR/}';" | tr -d "-"`
        if [ -z "$OVAL_REFDATE" ]
        then
          OVAL_REFDATE=0
        fi

        if [ $oval_timestamp -ge $OVAL_REFDATE ]
        then
          log_write " Updating $ovalfile"

          xsltproc --stringparam filename "${ovalfile##$SCAP_DIR/}" --stringparam refdate "$OVAL_REFDATE" "$SCAP_RES_DIR/oval_update.xsl" "$ovalfile" | \
          $SQLITE3 $SCAP_DB
        else
          log_write " Skipping $ovalfile, file has older timestamp than latest OVAL definition in database (this is not an error)."
        fi
      else
        log_write " Skipping $ovalfile, file is older than last revision (this is not an error)."
      fi
    done
  else
    log_write " Warning: No XML files found in $SCAP_DIR/oval/."
  fi

  sqlite3 $SCAP_DB "update meta set value ='`date +%s`' where name = 'last_update';"
}

sync_scapdata(){
  if [[ -e $ACCESSKEY ]]
  then
    echo "Found Greenbone Security Feed subscription file, trying to synchronize with Greenbone SCAP data Repository ..."
    notsynced=1
    retried=0

    if [ $REFRESH_ONLY -eq 1 ]
    then
      notsynced=0
    fi

    mkdir -p "$BASE_SYNC_DIR"
    read feeduser < $ACCESSKEY
    read -d "@" custid < $ACCESSKEY
    if [[ -z $feeduser || -z $custid ]]
    then
      echo "== greenbone-scapdata-sync $VERSION ================================================"
      echo "Synchronization was not possible because credential information could not"
      echo "be read from your access key."
      echo "Please make sure that the key located at"
      echo "  $sysconfdir/openvas/gsf-access-key"
      echo "is intact and in a valid format."
      echo "If you are still not able to continue, please contact Greenbone Support at"
      echo "support@greenbone.net and include the error messages displayed above (if any)"
      echo "and your customer ID."
      echo "==============================================================================="
      log_write "Could not determine credentials, aborting synchronization."
      logger "SCAP synchronization: Could not determine credentials, aborting synchronization."
      exit 1
    fi
    while (($notsynced))
    do
      # --protocol=29 is a workaround for a known bug in rsync 3.0.3
      if [[ -e /admin/ezcli.state ]]
      then
        gsmproxy=`grep proxy_feed /admin/ezcli.state | sed -e 's/^.*\/\///' -e 's/:/ /' -e 's/[\t ]*$//'`
        PORT=`grep ^syncport /admin/ezcli.state | sed -e "s/^syncport\t//g"`
      fi
      if [[ $gsmproxy == "proxy_feed" || -z $gsmproxy ]]
      then
        rsync -e "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -p $PORT -i $ACCESSKEY" -ltvrP --protocol=29 $RSYNC_DELETE $custid@feed.greenbone.net:/scap-data $BASE_SYNC_DIR
      else
        if [ -r /admin/proxyauth -a `stat -c %s /admin/proxyauth` != 0 ]; then
          rsync -e "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"ProxyCommand corkscrew $gsmproxy %h %p /admin/proxyauth\" -p $PORT -i $ACCESSKEY" -ltvrP --protocol=29 $RSYNC_DELETE $custid@feed.greenbone.net:/scap-data $BASE_SYNC_DIR
        else
          rsync -e "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"ProxyCommand corkscrew $gsmproxy %h %p\" -p $PORT -i $ACCESSKEY" -ltvrP --protocol=29 $RSYNC_DELETE $custid@feed.greenbone.net:/scap-data $BASE_SYNC_DIR
        fi
      fi
      if (( $? )) ; then
        echo
        echo "== greenbone-scapdata-sync $VERSION ================================================"
        echo "The synchronization with the repository failed. This may indicate a serious issue"
        echo "with either your infrastructure or the repository itself."
        echo "Your local SCAP data repository may be damaged now. Please resolve any connection"
        echo "issues and try again."
        echo "If you suspect an issue with the Greenbone SCAP data Repository, please contact"
        echo "Greenbone support at support@greenbone.net and include the error messages"
        echo "displayed above (if any) and your customer ID ($custid)."
        echo "==============================================================================="
        echo
        log_write "rsync failed, aborting synchronization."
        logger "SCAP synchronization:  failed, aborting synchronization."
        exit 1
      fi
      notsynced=0
    done
    echo "Synchronization with the Greenbone SCAP data Repository successful."
    log_write "Synchronization with the Greenbone SCAP data Repository successful."
    logger "SCAP synchronization: Synchronization with the Greenbone SCAP data Repository successful."
    echo

    update_scap_db
  else
    echo
    echo "== greenbone-scapdata-sync $VERSION ================================================"
    echo "Could not find gsf-access-key."
    echo
    echo "This access key can be obtained from Greenbone Networks GmbH,"
    echo "see http://greenbone.net/solutions/gbn_feed.html for details."
    echo
    echo "Please make sure the personal access key you obtained from"
    echo "Greenbone is placed in the following directory:"
    echo $ACCESSKEY
    echo "Please make also sure that the filename is gsf-access-key."
    echo "If you are still not able to synchronize, please contact Greenbone Support at"
    echo "support@greenbone.net and include the error messages displayed above (if any)"
    echo "and your customer ID."
    echo "==============================================================================="
    echo
    log_write "gsf-access-key not found, aborting synchronization."
    logger "SCAP synchronization: gsf-access-key not found, aborting synchronization."
    exit 1
  fi
}

do_self_test () {
  if [ -z "$SQLITE" ]; then
    echo "[e] Error: sqlite3 not found (required)";
    SELFTEST_FAIL=1
  fi
}

while test $# -gt 0; do
 case "$1" in
        --version)
          echo $VERSION
          exit 0
           ;;
        --identify)
          echo "SCAPSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|SCAPSYNC"
          exit 0
          ;;
        --describe)
          do_describe
          exit 0
          ;;
        --feedversion)
          do_feedversion
          exit 0
          ;;
        --help)
          do_help
          exit 0
          ;;
        --refresh)
          REFRESH_ONLY=1
          ;;
        --selftest)
          SELFTEST_FAIL=0
          do_self_test
          exit $SELFTEST_FAIL
          ;;
 esac
 shift
done

sync_scapdata

exit 0
