#!/bin/sh
#
# greenbone-certdata-sync
# This script synchronizes an OpenVAS installation with the Greenbone CERT
# data directory.
#
# Authors:
# Timo Pollmeier <timo.pollmeier@greenbone.net>
#
# Copyright:
# Copyright (C) 2011, 2012, 2013 Greenbone Networks GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2,
# or, at your option, any later version as published by the Free
# Software Foundation
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

VERSION=20140515
RESTRICTED=1

# SETTINGS
# ========

# Path prefix for installation location
PREFIX=

# PRIVATE_SUBDIR defines a subdirectory of the CERT data directory
# where files not part of the feed or database will not be deleted by rsync.
if [ -z "$PRIVATE_SUBDIR" ]
then
  PRIVATE_SUBDIR="private"
fi

# RSYNC_DELETE controls whether files which are not part of the repository will
# be removed from the local directory after synchronization. The default value
# for this setting is
# "--delete --exclude cert.db --exclude $PRIVATE_SUBDIR/",
# which means that files which are not part of the feed, database or private
# directory will be deleted.
RSYNC_DELETE="--delete --exclude cert.db --exclude $PRIVATE_SUBDIR/"

# RSYNC_SSH_OPTS contains options which should be passed to ssh for the rsync
# connection to the repository.
RSYNC_SSH_OPTS="-o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\""

# RSYNC_COMPRESS specifies the compression level to use for the rsync connection.
RSYNC_COMPRESS="--compress-level=9"

# PORT controls the outgoing TCP port for updates. If PAT/Port-Translation is
# not used, this should be "24". For some application layer firewalls or gates
# the value 22 (Standard SSH) is useful. Only change if you know what you are
# doing.
PORT=24

# SCRIPT_NAME is the name the scripts will use to identify itself and to mark
# log messages.
SCRIPT_NAME="greenbone-certdata-sync"

# LOG_CMD defines the command to use for logging. To have logger log to stderr
# as well as syslog, add "-s" here.
LOG_CMD="logger -t $SCRIPT_NAME"

# If ENABLED is set to 0, the sync script will not perform a synchronization.
ENABLED=1

# If REFRESH_ONLY is set to 1, the sync script will only update the OpenVAS
# Manager database. This can be controlled via the --refresh parameter.
REFRESH_ONLY=0

SQLITE3="sqlite3 -noheader -bail"

# Maximum number of retries if database is locked
if [ -z "$MAX_SQL_RETRIES" ]; then
  MAX_SQL_RETRIES="1" # 0 : try only once
fi

# Delay between retries
if [ -z "$SQL_RETRY_DELAY" ]; then
  SQL_RETRY_DELAY="10m" # allowed unit suffixes: see sleep command
fi

[ -r $PREFIX/etc/openvas/greenbone-certdata-sync.conf ] && . $PREFIX/etc/openvas/greenbone-certdata-sync.conf

BASE_SYNC_DIR="$PREFIX/var/lib/openvas"
CERT_DIR="$BASE_SYNC_DIR/cert-data"
SCAP_DIR="$BASE_SYNC_DIR/scap-data"

if [ -z "$PREFIX" ] ; then
  CERT_RES_DIR="/usr/share/openvas/cert"
else
  CERT_RES_DIR="$PREFIX/share/openvas/cert"
fi

CERT_DB="$CERT_DIR/cert.db"
SCAP_DB="$SCAP_DIR/scap.db"

ACCESSKEY="$PREFIX/etc/openvas/gsf-access-key"

TIMESTAMP="$CERT_DIR/timestamp"

if [ -z "$FEED_NAME" ] ; then
  FEED_NAME="Greenbone CERT Feed"
fi

if [ -z "$FEED_VENDOR" ] ; then
  FEED_VENDOR="Greenbone Networks GmbH"
fi

if [ -z "$FEED_HOME" ] ; then
  FEED_HOME="http://www.greenbone.net/technology/gsf.html"
fi

RSYNC=`command -v rsync`
SQLITE=`command -v sqlite3`

do_describe () {
  echo "This script synchronizes a CERT collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."
}

do_feedversion () {
  if [ -r $TIMESTAMP ] ; then
      echo `cat $TIMESTAMP`
  fi
}

log_debug () {
  $LOG_CMD -p daemon.debug $1
}

log_info () {
  $LOG_CMD -p daemon.info $1
}

log_notice () {
  $LOG_CMD -p daemon.notice $1
}

log_warning () {
  $LOG_CMD -p daemon.warning $1
}

log_err () {
  $LOG_CMD -p daemon.err $1
}

set_interrupt_trap () {
  trap "handle_interrupt $1" 2
}

handle_interrupt () {
  echo "$1:X" >&3
}

reset_sql_tries () {
  try_sql=1
  sql_retries=0
}

test_exit_codes () {
  try_sql=0
  if [ -n "$exit_codes" ]
  then
    for item in $exit_codes
    do
      command=`echo "$item" | cut -d':' -f1`
      code=`echo "$item" | cut -d':' -f2`

      if [ "X" = "$code" ]
      then
        log_err "$1: Sync script was interrupted"
        exit 1
      elif [ "sqlite3" = "$command" ] \
           && ( [ "5" = "$code" ] || [ "1" = "$code" ] )
      then
        if [ "5" = "$code" ]
        then
          log_warning "CERT database is locked"
        else
          log_warning "Could not access CERT database, file may be locked."
        fi

        if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
        then
          sql_retries=$((sql_retries + 1))
          log_notice "Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
          try_sql=1
          sleep "$SQL_RETRY_DELAY"
        else
          log_err "$1: Gave up trying to access SCAP database."
          exit 1
        fi
      else
        log_err "$1: $command exited with code $code"
        exit 1
      fi
    done
  fi
}

test_sql_exit () {
  exit_code=$?
  try_sql=0
  if ( [ "5" = "$exit_code" ] || [ "1" = "$exit_code" ] )
  then
    if [ "5" = "$exit_code" ]
    then
      log_warning "CERT database is locked."
    else
      log_warning "Could not access CERT database, file may be locked."
    fi

    if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
    then
      sql_retries=$((sql_retries + 1))
      log_notice "Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
      try_sql=1
      sleep "$SQL_RETRY_DELAY"
    else
      log_err "$1: Gave up trying to access CERT database."
      exit 1
    fi
  elif [ 0 -ne "$exit_code" ]
  then
    log_err "$1: sqlite3 exited with code $exit_code."
    exit 1
  fi
}

is_feed_current () {
  if [ $REFRESH_ONLY -eq 0 ]
  then
    if [ -r $TIMESTAMP ]
    then
      FEED_VERSION=`cat $TIMESTAMP`
    fi

    if [ -z "$FEED_VERSION" ]
    then
      log_warning "Could not determine feed version."
      FEED_CURRENT=0
      return $FEED_CURRENT
    fi

    FEED_INFO_TEMP_DIR=`mktemp -d`

    if [ -e $ACCESSKEY ]
    then
      read feeduser < $ACCESSKEY
      custid=`head -1 $ACCESSKEY | cut -d @ -f 1`
      if [ -z $feeduser ] || [ -z $custid ]
      then
        log_err "CERT synchronization: Could not determine credentials, aborting synchronization."
        rm -rf $FEED_INFO_TEMP_DIR
        exit -1
      fi

      # --protocol=29 is a workaround for a known bug in rsync 3.0.3
      if [ -e /admin/ezcli.state ]
      then
        gsmproxy=`grep proxy_feed /admin/ezcli.state | sed -e 's/^.*\/\///' -e 's/:/ /' -e 's/[\t ]*$//'`
        PORT=`grep ^syncport /admin/ezcli.state | sed -e "s/^syncport\t//g"`
      fi
      if [ -z "$gsmproxy" ] || [ "$gsmproxy" = "proxy_feed" ]
      then
        RSYNC_SSH_PROXY_CMD=""
      else
        if [ -r /admin/proxyauth ] && [ -s /admin/proxyauth ]; then
          RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p /admin/proxyauth\""
        else
          RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p\""
        fi
      fi
      rsync -e "ssh $RSYNC_SSH_OPTS $RSYNC_SSH_PROXY_CMD -p $PORT -i $ACCESSKEY" -ltvrP --protocol=29 --chmod=D+x $RSYNC_DELETE $RSYNC_COMPRESS $custid@feed.greenbone.net:/cert-data/timestamp $FEED_INFO_TEMP_DIR
      if [ $? -ne 0 ]
      then
        log_err "rsync failed, aborting synchronization."
        rm -rf $FEED_INFO_TEMP_DIR
        exit -1
      fi
    else
      log_err "Could not find access key, aborting synchronization."
      rm -rf $FEED_INFO_TEMP_DIR
      exit -1
    fi

    FEED_VERSION_SERVER=`cat $FEED_INFO_TEMP_DIR/timestamp`

    if [ -z "$FEED_VERSION_SERVER" ]
    then
      log_err "Could not determine server feed version."
      rm -rf $FEED_INFO_TEMP_DIR
      exit -1
    fi

    # Check against FEED_VERSION
    if [ $FEED_VERSION -lt $FEED_VERSION_SERVER ] ; then
      FEED_CURRENT=0
    else
      FEED_CURRENT=1
    fi

    # Cleanup
    rm -rf $FEED_INFO_TEMP_DIR
  else
    FEED_CURRENT=0
  fi

  return $FEED_CURRENT
}

is_db_broken () {
  DB_BROKEN=0
  if [ ! -e $CERT_DB ]
  then
    return
  fi
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    DB_INTEGRITY=`$SQLITE3 $CERT_DB "PRAGMA integrity_check;"`
    test_sql_exit "Could not check CERT database integrity"
  done
  if [ "$DB_INTEGRITY" != "ok" ]
  then
    DB_BROKEN=1
    log_warning "Database integrity check failed."
  fi
}

reinit () {
  log_notice "Reinitialization of the database necessary."
  rm -f $CERT_DB
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE3 $CERT_DB < $CERT_RES_DIR/cert_db_init.sql
    test_sql_exit "Could not reinitialize CERT database"
  done
}

db_migrate_4 () {
  log_notice "Migrating database to version 4."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE3 $CERT_DB "ALTER TABLE dfn_cert_advs ADD COLUMN max_cvss FLOAT;
                      UPDATE meta SET value = '4' WHERE name = 'database_version';"
    test_sql_exit "Could not migrate CERT database"
    updated_dfn=1
    log_notice "Migration done."
  done
}

check_db_version () {
  DB_VERSION=`$SQLITE3 $CERT_DB "select value from meta where name = 'database_version';" 2>/dev/null | tr -d '\n\r' || echo 0`
  case "$DB_VERSION" in
    0) reinit;;
    1) reinit;;
    2) reinit;;
    3) db_migrate_4;;
  esac
}

do_help () {
  echo "$0: Sync CERT data"
  echo " --describe	display current feed info"
  echo " --feedversion	display version of this feed"
  echo " --help		display this help"
  echo " --identify	display information"
  echo " --migrate	update database without downloading new state"
  echo " --refresh	update database without downloading new state"
  echo " --selftest	perform self-test"
  echo " --version	display version"
  echo ""
  exit 0
}

update_cvss () {
  if [ ! -f "$SCAP_DB" ]
  then
    log_warning "SCAP database not found. Cannot update max_cvss."
  else
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      SCAP_DB_LASTUPDATE=`$SQLITE3 $SCAP_DB "SELECT value FROM meta WHERE name = 'last_update';"`
      test_sql_exit "Could not get last_update timestamp from SCAP database"
    done
    if [ 0 -ne "$updated_dfn" ] || [ $SCAP_DB_LASTUPDATE -gt $DB_LASTUPDATE ]
    then
      log_info "Updating Max CVSS for DFN-CERT"

      reset_sql_tries
      until [ "$try_sql" -eq 0 ]
      do
        $SQLITE3 $CERT_DB "ATTACH DATABASE '$SCAP_DB' AS scap_db;
                           PRAGMA recursive_triggers = OFF;
                           UPDATE dfn_cert_advs
                             SET max_cvss =
                                 (
                                   SELECT coalesce(max(cvss), '')
                                   FROM scap_db.cves
                                   WHERE name IN
                                     (
                                       SELECT cve_name
                                       FROM dfn_cert_cves
                                       WHERE adv_id=dfn_cert_advs.id
                                     )
                                     AND cvss != ''
                                 );"
        test_sql_exit "Could not update CVSS scores for DFN-CERT Advisories"
      done
    else
      log_info "No DFN-CERT advisories updated and CERT DB newer than SCAP DB. Skipping CVSS recalculation."
    fi
  fi
}

update_cert_db() {
  if [ -z "$updated_dfn" ]
  then
    updated_dfn=0
  fi

  if [ ! -f $CERT_DB ]
  then
    log_notice "Initializing CERT database."
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      $SQLITE3 $CERT_DB < $CERT_RES_DIR/cert_db_init.sql
      test_sql_exit "Could not initialize CERT database"
    done
    DB_LASTUPDATE=0
  else
    check_db_version
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      DFN_REFDATE=`$SQLITE3 $CERT_DB "SELECT max(modification_time) from dfn_cert_advs;"`
      test_sql_exit "Could not get DFN-CERT reference date from database"
    done
  fi

  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    DB_LASTUPDATE=`$SQLITE3 $CERT_DB "select value from meta where name = 'last_update';"`
    test_sql_exit "Could not get last_update timestamp from CERT database"
  done

  if [ -z "$DFN_REFDATE" ]
  then
    DFN_REFDATE=0
  fi

  if [ -z "$DB_LASTUPDATE" ]
  then
    # Happens when initial sync was aborted
    log_warning "Inconsistent data. Resetting CERT database."
    rm -f $CERT_DB
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      $SQLITE3 $CERT_DB < $CERT_RES_DIR/cert_db_init.sql
      test_sql_exit "Could not reinitialize CERT database"
    done
    DFN_REFDATE=0
    DB_LASTUPDATE=0
  fi

  log_notice "Updating data from feed"
  xmlcount=$(ls $CERT_DIR/dfn-cert-*.xml 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for certfile in `ls $CERT_DIR/dfn-cert-*.xml`
    do
      filedate=`stat -c "%Y" $certfile | cut -d " " -f 1 | tr -d "-"`
      filedate=$(( $filedate - ( $filedate % 60 ) ))
      if [ $filedate -gt $DB_LASTUPDATE ]
      then
        log_info "Updating $certfile"
        reset_sql_tries
        until [ "$try_sql" -eq 0 ]
        do
          exec 4>&1
          exit_codes=$(
            (
              set_interrupt_trap
              (xsltproc --stringparam refdate "$DFN_REFDATE" $CERT_RES_DIR/dfn_cert_update.xsl $certfile || echo "xsltproc:$?" >&3) | \
              ($SQLITE3 $CERT_DB || echo "sqlite3:$?" >&3)
            ) 3>&1 >&4
          )
          exec 4>&-
          test_exit_codes "Update of DFN-CERT Advisories failed at file '$certfile'"
        done
        updated_dfn=1
      else
        log_info "Skipping $certfile, file is older than last revision"
      fi
    done
  else
    log_warning "No DFN-CERT advisories found in $CERT_DIR"
  fi

  update_cvss

  LAST_UPDATE_TIMESTAMP=`sed 's/^\(.\{8\}\)/\1 /' $TIMESTAMP | env TZ="UTC" date +%s -f -`

  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE3 $CERT_DB "UPDATE meta SET value ='$LAST_UPDATE_TIMESTAMP' WHERE name = 'last_update';"
    test_sql_exit "Could not update last_update timestamp in CERT database"
  done
}

sync_certdata(){
  if [ $REFRESH_ONLY -eq 0 ]
  then
    if [ -e $ACCESSKEY ]
    then
      log_notice "Found Greenbone Security Feed subscription file, trying to synchronize with Greenbone CERT data Repository ..."
      notsynced=1
      retried=0

      mkdir -p "$BASE_SYNC_DIR"
      read feeduser < $ACCESSKEY
      custid=`head -1 $ACCESSKEY | cut -d @ -f 1`
      if [ -z $feeduser ] || [ -z $custid ]
      then
        log_err "Could not determine credentials, aborting synchronization."
        exit 1
      fi
      while [ 1 -eq $notsynced ]
      do
        # --protocol=29 is a workaround for a known bug in rsync 3.0.3
        if [ -e /admin/ezcli.state ]
        then
          gsmproxy=`grep proxy_feed /admin/ezcli.state | sed -e 's/^.*\/\///' -e 's/:/ /' -e 's/[\t ]*$//'`
          PORT=`grep ^syncport /admin/ezcli.state | sed -e "s/^syncport\t//g"`
        fi
        if [ -z "$gsmproxy" ] || [ "$gsmproxy" = "proxy_feed" ]
        then
          RSYNC_SSH_PROXY_CMD=""
        else
          if [ -r /admin/proxyauth ] && [ -s /admin/proxyauth ]; then
            RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p /admin/proxyauth\""
          else
            RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p\""
          fi
        fi
        rsync -e "ssh $RSYNC_SSH_OPTS $RSYNC_SSH_PROXY_CMD -p $PORT -i $ACCESSKEY" -ltvrP --protocol=29 --chmod=D+x $RSYNC_DELETE $RSYNC_COMPRESS $custid@feed.greenbone.net:/cert-data $BASE_SYNC_DIR
        if [ 0 -ne "$?" ] ; then
          log_err "rsync failed, aborting synchronization."
          exit 1
        fi
        notsynced=0
      done
      log_notice "Synchronization with the Greenbone CERT data Repository successful."
      echo

    else
      log_err "gsf-access-key not found, aborting synchronization."
      exit 1
    fi
  fi

  update_cert_db
}

do_self_test () {
  if [ -z "$SELFTEST_STDERR" ]
  then
    SELFTEST_STDERR=0
  fi

  if [ -z "$SQLITE" ]; then
    if [ 0 -ne $SELFTEST_STDERR ]
    then
      echo "sqlite3 not found (required)." 1>&2
    fi
    log_err "sqlite3 not found (required)."
    SELFTEST_FAIL=1
  fi

  if [ -z "$RSYNC" ]
  then
    if [ 0 -ne $SELFTEST_STDERR ]
    then
      echo "rsync not found (required)." 1>&2
    fi
    log_err "rsync not found (required)."
    SELFTEST_FAIL=1
  fi

  if [ ! -d $BASE_SYNC_DIR ]
  then
    if [ 0 -ne $SELFTEST_STDERR ]
    then
      echo "BASE_SYNC_DIR ($BASE_SYNC_DIR) not found." 1>&2
    fi
    log_err "BASE_SYNC_DIR ($BASE_SYNC_DIR) not found."
    SELFTEST_FAIL=1
  fi
}

while test $# -gt 0; do
 case "$1" in
        --version)
          echo $VERSION
          exit 0
           ;;
        --identify)
          echo "CERTSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|CERTSYNC"
          exit 0
          ;;
        --describe)
          do_describe
          exit 0
          ;;
        --feedversion)
          do_feedversion
          exit 0
          ;;
        --help)
          do_help
          exit 0
          ;;
        --refresh)
          REFRESH_ONLY=1
          ;;
        --migrate)
          REFRESH_ONLY=1
          ;;
        --selftest)
          SELFTEST_FAIL=0
          SELFTEST_STDERR=1
          do_self_test
          exit $SELFTEST_FAIL
          ;;
        --feedcurrent)
          is_feed_current
          exit $?
          ;;
      esac
      shift
done

SELFTEST_FAIL=0
do_self_test
if [ $SELFTEST_FAIL -ne 0 ]
then
  exit 1
fi

if [ $(id -u) -ne $(stat -c %u $CERT_DB) ]
then
  log_err "Not synchronizing or updating the database since the current user is not the owner of the database."
  echo "Not synchronizing or updating the database since the current user is not the owner of the database."
  exit 1
fi

if [ $ENABLED -ne 1 ]
then
  log_notice "Synchronization of CERT data with network is disabled, will only perform local operations."
  REFRESH_ONLY=1
fi

is_db_broken
if [ $DB_BROKEN -eq 1 ]
then
  log_notice "Purging corrupted database."
  rm -f $CERT_DB
else
  is_feed_current
  if [ $FEED_CURRENT -eq 1 ]
  then
    log_notice "Feed is already current, skipping synchronization."
    exit 0
  fi
fi
sync_certdata

exit 0
