#!/bin/bash
#
# Copyright (C) 2015-2016 Paulo Angelo Alves Resende <pa@pauloangelo.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 2 as
# published by the Free Software Foundation.  You may not use, modify or
# distribute this program under any other version of the GNU General
# Public License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#

# Variables
HADOOP_VERSION="2.6.5"
HBASE_VERSION="1.2.6"
SPARK_VERSION="1.6.3"
SFLOWTOOL_VERSION="3.39"
TMP_FILE="/tmp/.hzinstallation.temp"
HZURL="http://ids-hogzilla.org"
DEBUG=0

# Load libs

# ###########################
# Bash Shell Function Library
# ###########################
#
# Author: Louwrentius <louwrentius@gmail.com>
# Contributions by: Jani Hurskainen
#
# Copyright © 2010
#
# Released under the curren GPL version.
#
# Description:
#
# This is a shell script library. It contains functions that can be called by
# programs that include (source) this library. 
#
# By simply sourcing this library, you can use all available functions as 
# documented on the projects page.
#
#

BSFL_VERSION="2.00-beta-2"

#
# Do not edit this file. Just source it into your script
# and override the variables to change their value.
#

init () {

    #
    # Debug mode shows more verbose output to screen and log files.
    # Value: yes or no (y / n)
    #
    DEBUG=no
    
    # 
    # Syslog style log messages
    #
    if ! defined LOGDATEFORMAT
    then
        LOGDATEFORMAT="%b %e %H:%M:%S"
    fi
    if ! defined LOG_FILE
    then
        LOG_FILE=$0.log
    fi
    
    #
    # Enable / disable logging to a file
    # Value: yes or no (y / n)
    #
    if ! defined LOG_ENABLED
    then
        LOG_ENABLED=no
    fi
    if ! defined SYSLOG_ENABLED
    then
        SYSLOG_ENABLED=no
    fi
    if ! defined SYSLOG_TAG
    then
        SYSLOG_TAG=$0
    fi
    
    #
    # Use colours in output.
    #
    RED="tput setaf 1"
    GREEN="tput setaf 2"
    YELLOW="tput setaf 3"
    BLUE="tput setaf 4"
    MAGENTA="tput setaf 5"
    CYAN="tput setaf 6"
    LIGHT_BLUE="$CYAN"
    BOLD="tput bold"
    DEFAULT="tput sgr0"
    
    RED_BG="tput setab 1"
    GREEN_BG="tput setab 2"
    YELLOW_BG="tput setab 3"
    BLUE_BG="tput setab 4"
    MAGENTA_BG="tput setab 5"
    CYAN_BG="tput setab 6"
    
    # 
    # Bug fix for Bash, parsing exclamation mark.
    #
    set +o histexpand
    #
    # returns 0 if a variable is defined (set)
    # returns 1 if a variable is unset
    #

}

function defined {
    [[ ${!1-X} == ${!1-Y} ]]
}

#
# returns 0 if a variable is defined (set) and value's length > 0
# returns 1 otherwise
#
function has_value {
    if defined $1; then
        if [[ -n ${!1} ]]; then
            return 0
        fi
    fi
    return 1
}

#
# returns 0 if a directory exists
# returns 1 otherwise
#
function directory_exists {
    if [[ -d "$1" ]]; then
        return 0
    fi
    return 1
}

#
# returns 0 if a (regular) file exists
# returns 1 otherwise
#
function file_exists {
    if [[ -f "$1" ]]; then
        return 0
    fi
    return 1
}

#
# returns lowercase string
#
function tolower {
    echo "$1" | tr '[:upper:]' '[:lower:]'
}

#
# returns uppercase string
#
function toupper {
    echo "$1" | tr '[:lower:]' '[:upper:]'
}

#
# Only returns the first part of a string, delimited by tabs or spaces
#
function trim {
    echo $1
}

#
# Dummy function to provide usage instructions.
# Override this function if required.
#
show_usage () {
   
    MESSAGE="$1" 
    echo "$MESSAGE"
    exit 1
}

#
# Checks if a variable is set to "y" or "yes".
# Usefull for detecting if a configurable option is set or not.
#
option_enabled () {

    VAR="$1"
    VAR_VALUE=$(eval echo \$$VAR)
    if [[ "$VAR_VALUE" == "y" ]] || [[ "$VAR_VALUE" == "yes" ]]
    then
        return 0
    else
        return 1
    fi
}

#
# The log funcion just puts a string into a file, prepended with a date & time in 
# syslog format.
#

log2syslog () {

    if option_enabled  SYSLOG_ENABLED
    then
       MESSAGE="$1"
       logger -t "$SYSLOG_TAG" " $MESSAGE" #The space is not a typo!"
    fi
}

#
# This function writes messages to a log file and/or syslog
# The only argument is a message that has to be logged.
# 

log () {
    
    if option_enabled LOG_ENABLED || option_enabled SYSLOG_ENABLED 
    then
        LOG_MESSAGE="$1" 
        DATE=`date +"$LOGDATEFORMAT"`

        if has_value LOG_MESSAGE
        then
            LOG_STRING="$DATE $LOG_MESSAGE"
        else
            LOG_STRING="$DATE -- empty log message, no input received --"
        fi

        if option_enabled LOG_ENABLED
        then
            echo "$LOG_STRING" >> "$LOG_FILE"
        fi

        if option_enabled SYSLOG_ENABLED
        then
            #
            # Syslog already prepends a date/time stamp so only the message 
            # is logged. 
            #
            log2syslog "$LOG_MESSAGE"
        fi
    fi
}


#
# This function basically replaces the 'echo' function in bash scripts.
# The added functionality over echo is logging and using colors. 
#
# The first argument is the string / message that must be displayed.
# The second argument is the text color.

msg () {

    MESSAGE="$1"
    COLOR="$2"

    if ! has_value COLOR
    then
        COLOR="$DEFAULT"
    fi

    if has_value "MESSAGE"
    then
        $COLOR
        echo "$MESSAGE" 
        $DEFAULT
        log "$MESSAGE"
    else
        echo "-- no message received --"
        log "$MESSAGE"
    fi
}

#
# This function echos a message 
# and displays the status at the end of the line.
#
# It can be used to create status messages other
# than the default messages available such as
# OK or FAIL
#
msg_status () {

    MESSAGE="$1"
    STATUS="$2"

    msg "$MESSAGE"
    display_status "$STATUS"
}

#
# These functions are just short hand for messages like
# msg_status "this message is ok" OK
#

#
# The following functions are shorthand for 
# msg_status "a message" OK 
# msg_status "another message" FAIL


msg_emergency () {

    MESSAGE="$1"
    STATUS="EMERGENCY"
    msg_status "$MESSAGE" "$STATUS"
}

msg_alert () {

    MESSAGE="$1"
    STATUS="ALERT"
    msg_status "$MESSAGE" "$STATUS"
}

msg_critical () {

    MESSAGE="$1"
    STATUS="CRITICAL"
    msg_status "$MESSAGE" "$STATUS"
}

msg_error () {

    MESSAGE="$1"
    STATUS="ERROR"
    msg_status "$MESSAGE" "$STATUS"
}

msg_warning () {

    MESSAGE="$1"
    STATUS="WARNING"
    msg_status "$MESSAGE" "$STATUS"
}

msg_notice () {
    MESSAGE="$1"
    STATUS="NOTICE"
    msg_status "$MESSAGE" "$STATUS"
}

msg_info () {
    MESSAGE="$1"
    STATUS="INFO"
    msg_status "$MESSAGE" "$STATUS"
}

msg_debug () {
    MESSAGE="$1"
    STATUS="DEBUG"
    msg_status "$MESSAGE" "$STATUS"
}

msg_ok () {

    MESSAGE="$1"
    STATUS="OK"
    msg_status "$MESSAGE" "$STATUS"
}

msg_not_ok () {

    MESSAGE="$1"
    STATUS="NOT_OK"
    msg_status "$MESSAGE" "$STATUS"
}

msg_fail () {

    MESSAGE="$1"
    STATUS="FAILED"
    msg_status "$MESSAGE" "$STATUS"
}

msg_success () {
    MESSAGE="$1"
    STATUS="SUCCESS"
    msg_status "$MESSAGE" "$STATUS"
}

msg_passed () {
    MESSAGE="$1"
    STATUS="PASSED"
    msg_status "$MESSAGE" "$STATUS"
}

check_status () {

    CMD="$1"
    STATUS="$2"

    if [ "$STATUS" == "0" ]
    then
        msg_ok "$CMD"
    else
        msg_fail "$CMD"
    fi
}

#
# Private function
#
# This is a function that just positions
# the cursor one row up and to the right.
# It then prints a message with specified
# Color
# It is used for displaying colored status messages on the
# Right side of the screen.
#
# ARG1 = "status message (OK / FAIL)"
# ARG2 = The color in which the status is displayed.
#
raw_status () {

    STATUS="$1"
    COLOR="$2"

    function position_cursor () {

        let RES_COL=`tput cols`-12
        tput cuf $RES_COL
        tput cuu1
    }

    position_cursor
    echo -n "["
    $DEFAULT
    $BOLD
    $COLOR
    echo -n "$STATUS"
    $DEFAULT
    echo "]"
    log "Status = $STATUS"
}

#
# This function converts a status message to a particular color.
#
display_status () {


    STATUS="$1"

    case $STATUS in 

    EMERGENCY )
            STATUS="EMERGENCY"
            COLOR="$RED"
            ;;
    ALERT )
            STATUS="  ALERT  "
            COLOR="$RED"
            ;;
    CRITICAL )
            STATUS="CRITICAL "
            COLOR="$RED"
            ;;
    ERROR )
            STATUS="  ERROR  " 
            COLOR="$RED"
            ;;

    WARNING )
            STATUS=" WARNING "  
            COLOR="$YELLOW"
            ;;

    NOTICE )
            STATUS=" NOTICE  "  
            COLOR="$BLUE"
            ;;
    INFO )
            STATUS="  INFO   "  
            COLOR="$LIGHT_BLUE"
            ;;
    DEBUG )
            STATUS="  DEBUG  "
            COLOR="$DEFAULT"
            ;;    

    OK  ) 
            STATUS="   OK    "  
            COLOR="$GREEN"
            ;;
    NOT_OK)
            STATUS=" NOT OK  "
            COLOR="$RED"
            ;;

    PASSED ) 
            STATUS=" PASSED  "  
            COLOR="$GREEN"
            ;;

    SUCCESS ) 
            STATUS=" SUCCESS "  
            COLOR="$GREEN"
            ;;
    
    FAILURE | FAILED )
            STATUS=" FAILED  "  
            COLOR="$RED"
            ;;

    *)
            STATUS="UNDEFINED"
            COLOR="$YELLOW"
    esac

    raw_status "$STATUS" "$COLOR"
}

#
# Exit with error status 
#
bail () {

    ERROR="$?"
    MSG="$1"
    if [ ! "$ERROR" = "0" ]
    then
        msg_fail "$MSG"
        exit "$ERROR"
    fi
}

#
# This function executes a command provided as a parameter
# The function then displays if the command succeeded or not.
#
cmd () {

    COMMAND="$1"
    msg "Executing: $COMMAND"

    RESULT=`eval $COMMAND 2>&1`
    msg "$RESULT"
    ERROR="$?"

    MSG="Command: ${COMMAND:0:29}..."
    
    tput cuu1

    if [ "$ERROR" == "0" ]
    then
        msg_ok "$MSG"
        if [ "$DEBUG" == "1" ]
        then
            msg "$RESULT"
        fi
    else
        msg_fail "$MSG"
        log "$RESULT"
    fi

    return "$ERROR"
}

#
# These functions can be used for timing how long (a) command(s) take to
# execute.
#
now () {

   echo $(date +%s)
}

elapsed () {
    
    START="$1"
    STOP="$2"

    echo $(( STOP - START ))
}

#
# Prints an error message ($2) to stderr and exits with the return code ($1).
# The message is also logged.
#
function die {
    local -r err_code="$1"
    local -r err_msg="$2"
    local -r err_caller="${3:-$(caller 0)}"

    msg_fail "ERROR: $err_msg"
    msg_fail "ERROR: At line $err_caller"
    msg_fail "ERROR: Error code = $err_code"
    exit "$err_code"
} >&2 # function writes to stderr

#
# Check if a return code ($1) indicates an error (i.e. >0) and prints an error
# message ($2) to stderr and exits with the return code ($1).
# The error is also logged.
#
# Die if error code is false.
#
function die_if_false {
    local -r err_code=$1
    local -r err_msg=$2
    local -r err_caller=$(caller 0)

    if [[ "$err_code" != "0" ]]
    then
        die $err_code "$err_msg" "$err_caller"
    fi
} >&2 # function writes to stderr

#
# Dies when error code is true
#
function die_if_true {
    local -r err_code=$1
    local -r err_msg=$2
    local -r err_caller=$(caller 0)

    if [[ "$err_code" == "0" ]]
    then
        die $err_code "$err_msg" "$err_caller"
    fi
} >&2 # function writes to stderr

#
# Replace some text inside a string.
#
function str_replace () {
    local ORIG="$1"
    local DEST="$2"
    local DATA="$3"

    echo "$DATA" | sed "s/$ORIG/$DEST/g"
}

#
# Replace string of text in file.
# Uses the ed editor to replace the string.
#
# arg1 = string to be matched
# arg2 = new string that replaces matched string
# arg3 = file to operate on.
#
function str_replace_in_file () {
    local ORIG="$1"
    local DEST="$2"
    local FILE="$3"

    has_value FILE 
    die_if_false $? "Empty argument 'file'"
    file_exists "$FILE"
    die_if_false $? "File does not exist"

    printf ",s/$ORIG/$DEST/g\nw\nQ" | ed -s "$FILE" > /dev/null 2>&1
    return "$?"
}

cmd_su () {

    USER="$1"
    COMMAND="$2"

    msg "Executing: $COMMAND"

    RESULT=`su $USER -c "$COMMAND" 2>&1`
    ERROR="$?"

    MSG="Command: ${COMMAND:0:29}..."
    
    tput cuu1

    if [ "$ERROR" == "0" ]
    then
        msg_ok "$MSG"
        if [ "$DEBUG" == "1" ]
        then
            msg "$RESULT"
        fi
    else
        msg_fail "$MSG"
        log "$RESULT"
    fi

    return "$ERROR"
}

init


function package_install_cmd {
    PKGCMD=$1
    PKGNAME=$2
    PKGDESC=$3

    $PKGCMD &>/dev/null
    if [ $? -gt 0 ] ; then
      msg_info "$PKGDESC not installed. Installing now..."
      apt-get --force-yes -y install $PKGNAME
      $PKGCMD &>/dev/null
      if [ $? -eq 0 ] ; then
        msg_ok "$PKGDESC installed."
      else
        msg_fail "$PKGDESC installation failed!"
        die 1 "I could NOT install $PKGDESC. Check your sources.list and/or Internet access or try to do it manually!"
      fi
    else
      msg_ok "$PKGDESC installed."
    fi
}

function package_install {
    PKGNAME=$1
    PKGDESC=$2

    dpkg -l | awk '{print $2}' | grep ^$PKGNAME$ &>/dev/null
    if [ $? -gt 0 ] ; then
      msg_info "$PKGDESC not installed. Installing now..."
      apt-get --force-yes -y install $PKGNAME
      dpkg -l | awk '{print $2}' | grep $PKGNAME &>/dev/null
      if [ $? -eq 0 ] ; then
        msg_ok "$PKGDESC installed."
      else
        msg_fail "$PKGDESC installation failed!"
        die 1 "I could NOT install $PKGDESC. Check your Internet access or try to do it manually!"
      fi
    else
      msg_ok "$PKGDESC installed."
    fi
}

function cmd_if_0_info
{
    CMDCHECK=$1
    CMD=$2
    ELSEMSG=$3

    eval $CMDCHECK &>/dev/null
    if [ $? -eq 0 ] ; then
      cmd "$CMD"
    else
      msg_info "$ELSEMSG"
    fi
}

function cmd_if_n0_info
{
    CMDCHECK=$1
    CMD=$2
    ELSEMSG=$3

    eval $CMDCHECK &>/dev/null
    if [ $? -gt 0 ] ; then
      cmd "$CMD"
    else
      msg_info "$ELSEMSG"
    fi
}

whoami | grep root &>/dev/null
if [ $? -gt 0 ] ; then
  msg_fail "Logged as root!"
  die 1 "Root needed!"
else
  msg_ok "Logged as root!"
fi

file_exists "/etc/debian_version"
if [ $? -gt 0 ] ; then
  msg_fail "This script must run on Debian and you are not running on Debian!"
  die 1 "You should use Debian Linux to run this script! If you want to run Hogzilla IDS on a different OS, you should do it manually. See the installation guide at $HZURL "
else
  msg_ok "Running on Debian."
fi

package_install_cmd "whiptail -h" "whiptail" "Whiptail"

sleep 1
whiptail --scrolltext --title "Hogzilla Installation Script" \
       --yesno \
"\n[You can scrool the text if needed!]\n
This is an automate installation script for Hogzilla IDS.\n 
It should do the following:\n 
   1) Check your system;\n
   2) Create user hogzilla;\n
   3) Install Java, Hadoop, HBase, Apache-Spark, SflowTool, \n
      SFlow2Hz, Thrift, HzTools, PigTail and some basic pre-requisites;\n
   4) Configure Hadoop, HBase, Apache-Spark and PigTail;\n
   5) Start programs;\n
   6) Create SSH-Keys; and\n
   7) Include lines into /etc/rc.local .\n
   \n
PRE-REQUISITES:\n
   - Debian amd64 (tested on Debian 8.6.0)
   - > 8G RAM
   - > 50G HD
   - > 1 CPU 
   - Hostname must resolve to an Interface IP!
   \n
You will be requested to inform:\n
   - GrayLog Server IP\n
   - Proxy URL for Internet access (if needed)\n
   - Default path for Hadoop data (recommended > 50G)\n
\n
                  DO YOU WANT TO CONTINUE?
\n \n" 20 70

if [ $? -gt 0 ] ; then
  echo ""
  msg_warning "OK. We are finishing now!"
  exit
else
  echo ""
  msg_ok "You chose to continue! Let's do it!!!"
fi

sleep 1
whiptail --title "ALERT" --msgbox \
"
In this instalation, HBase, Hadoop, etc. will enable services without authentication.\n
Due to that, the server must be in a network protected by firewalling.\n
" 20 70 

whiptail --inputbox \
"Enter your proxy URL (ex. http://10.1.1.1:8080) or \n
let it blank for no proxy :" 20 70 $http_proxy 2> $TMP_FILE
PROXY=`head -n1 $TMP_FILE`
export https_proxy=$PROXY
export http_proxy=$PROXY

whiptail --inputbox \
"Enter your GrayLog server IP (e.g. 10.1.1.1):" 20 70 "10.1.1.1" 2> $TMP_FILE
GRAYLOGHOST=`head -n1 $TMP_FILE`

whiptail --inputbox \
"Enter the Hadoop Data path (you need at least 50G):" 20 70 "/home/hogzilla/hadoop_data" 2> $TMP_FILE
HADOOPDATA=`head -n1 $TMP_FILE`

whiptail --inputbox \
"Enter your (internal and external) networks' prefixes separated by commas.\n
E.g. If your networks are 10.0.0.0/15 and 1.1.1.0/24 \n
you should enter: 10.0.,10.1.,1.1.1.\n
\n
" 20 70 "10.,192.168.,172.16." 2> $TMP_FILE
NETPREFIXES=`head -n1 $TMP_FILE`

package_install_cmd "wget -h" "wget" "Wget"
package_install_cmd "awk --help"  "gawk" "gawk"
package_install_cmd "sed --help"  "sed"  "sed"

#wget -O /dev/null $GRAYLOGHOST  # It is not an URL
#if [ $? -gt 0 ] ; then
#  msg_fail "Could not access GrayLog on $GRAYLOGHOST!"
#  die 1 "GrayLog could not be accessed. Check the entered URL and try again!"
#else
#  msg_ok "Accessing GrayLog correctly on $GRAYLOGHOST!"
#fi

wget -q -O /dev/null "$HZURL/testing"
if [ $? -gt 0 ] ; then
  msg_fail "Could not access Internet!"
  die 1 "Internet could not be reached! Check the entered information and try again!"
else
  msg_ok "Internet could be reached!"
fi



#Install dependencies
package_install "ssh" "ssh"

for pkg in "php5-cli" "host" ; do
    package_install $pkg $pkg
done
 
cmd_if_n0_info "id hogzilla" \
              "useradd -s '/bin/bash' -m hogzilla" \
              "User hogzilla already exists. Will not try to create it again"
cmd_if_n0_info "ls -la /home/hogzilla/.ssh" \
              "su hogzilla -c 'mkdir /home/hogzilla/.ssh'" \
              "Dir /home/hogzilla/.ssh already exists"
cmd_if_n0_info "cat /home/hogzilla/.ssh/id_rsa" \
              "su hogzilla -c \"ssh-keygen -t rsa -f /home/hogzilla/.ssh/id_rsa -q -N '' \"" \
              "SSH key already exists"
cmd_if_n0_info "cat /home/hogzilla/.ssh/authorized_keys" \
              "su hogzilla -c \"cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys\"" \
              "SSH authorized_keys already exists"
cmd_su "hogzilla" "chmod 0600 ~/.ssh/authorized_keys"

grep "^localhost" /home/hogzilla/.ssh/known_hosts >/dev/null
if [ $? -gt 0 ] ; then
cmd_su "hogzilla" "ssh-keyscan localhost >> ~/.ssh/known_hosts"
fi

grep "^0.0.0.0" /home/hogzilla/.ssh/known_hosts >/dev/null
if [ $? -gt 0 ] ; then
cmd_su "hogzilla" "ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts"
fi

cmd_if_n0_info "ls $HADOOPDATA" \
              "mkdir -p $HADOOPDATA" \
              "Directory $HADOOPDATA already exists"

cmd "chown hogzilla. $HADOOPDATA"

SPACE=`df $HADOOPDATA | tail -n1 | awk '{print $4}' | sed 's/\([0-9]*\).*/\1/'`
if [ "$SPACE" -lt 50000000 ] ; then
  msg_warning "You have less than 50GB on $HADOOPDATA . Hogzilla may not run properly. Nevertheless installation will continue."
else
  msg_ok "You have enough space on $HADOOPDATA !"
fi

# JAVA
ps auxw | grep java | grep hogzilla | awk '{print $2}' | xargs kill &>/dev/null

java -XshowSettings:properties -version 2>&1 | grep java.vendor | grep Oracle >/dev/null
if [ $? -gt 0 ] ; then
  sleep 1
#  whiptail --yesno \
#  "You don't have Java/Oracle installed. \n
#Would you like me to install it now? \n
#If NO, you will need to install it later and type \n
#some commands to finish the installation." 20 70
#  
#  if [ $? -gt 0 ] ; then
#    echo ""
#    msg_warning "OK. You will need to install Oracle's Java later!"
#    JAVA=false
#  else
    echo ""
    msg_ok "Ok! I'm going to try the Oracle's Java instalation."
    cmd_if_n0_info "grep '^deb .*webupd8team' /etc/apt/sources.list" \
                  "echo \"deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main\" >> /etc/apt/sources.list" \
                  "deb WebUpd8Team already in /etc/apt/sources.list"
    cmd_if_n0_info "grep '^deb-src.*webupd8team' /etc/apt/sources.list" \
                  "echo \"deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main\" >> /etc/apt/sources.list" \
                  "deb-src WebUpd8Team already in /etc/apt/sources.list"
    cmd "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886"
    apt-get update
    apt-get install --force-yes oracle-java8-installer
    apt-get install --force-yes oracle-java8-set-default
    
    java -XshowSettings:properties -version 2>&1 | grep java.vendor | grep Oracle > /dev/null
    if [ $? -eq 0 ] ;then
       JAVA=true
    else
       die 1 "Sorry, I failed! I could not install Java automatically. Try to install it manually"
    fi
# fi
else
  msg "Oracle Java already installed"
fi

sleep 1
# Hadoop, HBase, Spark

msg "I'm going to download Hadoop, HBase and Apache Spark now. It can take some minutes..."

directory_exists "/home/hogzilla/app" || cmd_su "hogzilla" "mkdir /home/hogzilla/app"
directory_exists "/home/hogzilla/bin" || cmd_su "hogzilla" "mkdir /home/hogzilla/bin"
cmd_su "hogzilla" "wget -c -O /home/hogzilla/app/hadoop-$HADOOP_VERSION.tar.gz 'http://www.us.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz'"
file_exists "/home/hogzilla/app/hadoop-$HADOOP_VERSION.tar.gz" || die 1 "Could not download hadoop-$HADOOP_VERSION.tar.gz"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/hadoop-$HADOOP_VERSION.tar.gz -C /home/hogzilla/"
directory_exists "/home/hogzilla/hadoop" && cmd "rm -rf /home/hogzilla/hadoop"
directory_exists "/home/hogzilla/hadoop-$HADOOP_VERSION" && cmd_su "hogzilla" "mv /home/hogzilla/hadoop-$HADOOP_VERSION /home/hogzilla/hadoop"

cmd_su "hogzilla" "wget -c -O /home/hogzilla/app/hbase-$HBASE_VERSION-bin.tar.gz 'http://www.us.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz'"
file_exists "/home/hogzilla/app/hbase-$HBASE_VERSION-bin.tar.gz" || die 1 "Coult not download hbase-$HBASE_VERSION-bin.tar.gz"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/hbase-$HBASE_VERSION-bin.tar.gz -C /home/hogzilla/"
directory_exists "/home/hogzilla/hbase" && cmd "rm -rf /home/hogzilla/hbase"
directory_exists "/home/hogzilla/hbase-$HBASE_VERSION" && cmd_su "hogzilla" "mv /home/hogzilla/hbase-$HBASE_VERSION /home/hogzilla/hbase"

cmd_su "hogzilla" "wget -c -O /home/hogzilla/app/spark-$SPARK_VERSION-bin-hadoop2.6.tgz 'http://mirror.nbtelecom.com.br/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.6.tgz'"
file_exists "/home/hogzilla/app/spark-$SPARK_VERSION-bin-hadoop2.6.tgz" || die 1 "Coult not download spark-$SPARK_VERSION-bin-hadoop2.6.tgz"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/spark-$SPARK_VERSION-bin-hadoop2.6.tgz -C /home/hogzilla/"
cmd "chown -R hogzilla. /home/hogzilla/spark-$SPARK_VERSION-bin-hadoop2.6"
directory_exists "/home/hogzilla/spark" && cmd "rm -rf /home/hogzilla/spark"
directory_exists "/home/hogzilla/spark-$SPARK_VERSION-bin-hadoop2.6" && cmd_su "hogzilla" "mv /home/hogzilla/spark-$SPARK_VERSION-bin-hadoop2.6 /home/hogzilla/spark"

grep "HADOOP_CONF_DIR" /home/hogzilla/.bashrc > /dev/null
if [ $? -gt 0 ] ; then
  cmd_su "hogzilla" "echo 'export HADOOP_HOME=/home/hogzilla/hadoop'                    >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HBASE_HOME=/home/hogzilla/hbase'                      >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export SPARK_HOME=/home/hogzilla/spark'                      >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_MAPRED_HOME=\$HADOOP_HOME'                      >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_COMMON_HOME=\$HADOOP_HOME'                      >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_HDFS_HOME=\$HADOOP_HOME'                        >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export YARN_HOME=\$HADOOP_HOME'                               >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_COMMON_LIB_NATIVE_DIR=\$HADOOP_HOME/lib/native' >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export PATH=\$PATH:\$HADOOP_HOME/sbin:\$HADOOP_HOME/bin'        >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_INSTALL=\$HADOOP_HOME'                          >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_HOME/lib\"' >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export HADOOP_CONF_DIR=\$HADOOP_HOME/etc/hadoop'              >> ~/.bashrc"
  cmd_su "hogzilla" "echo 'export CLASSPATH=\$CLASSPATH:/home/hogzilla/hbase/lib/*'      >> ~/.bashrc"
else
  msg_info "~/.bashrc already changed."
fi

export HADOOP_HOME=/home/hogzilla/hadoop
export HBASE_HOME=/home/hogzilla/hbase
export SPARK_HOME=/home/hogzilla/spark

cmd "grep JAVA_HOME /etc/profile.d/jdk.sh >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh"
cmd "grep JAVA_HOME /etc/profile.d/jdk.sh >> $HBASE_HOME/conf/hbase-env.sh"

grep "fs.default.name" $HADOOP_HOME/etc/hadoop/core-site.xml > /dev/null
if [ $? -gt 0 ] ; then
   sed -i.original $HADOOP_HOME/etc/hadoop/core-site.xml \
   -e 's#</configuration>#\
      <property>\
         <name>fs.default.name</name>\
         <value>hdfs://localhost:9000</value>\
      </property>\
</configuration>#'
else
  msg_info "$HADOOP_HOME/etc/hadoop/core-site.xml already changed."
fi

grep "dfs.name.dir" $HADOOP_HOME/etc/hadoop/hdfs-site.xml > /dev/null
if [ $? -gt 0 ] ; then
   sed -i.original $HADOOP_HOME/etc/hadoop/hdfs-site.xml \
   -e "s#</configuration>#\
   <property>\
      <name>dfs.replication</name >\
      <value>1</value>\
   </property>\
   <property>\
      <name>dfs.name.dir</name>\
      <value>file:///$HADOOPDATA/hdfs/namenode</value>\
   </property>\
   <property>\
      <name>dfs.data.dir</name>\
      <value>file:///$HADOOPDATA/hdfs/datanode</value>\
   </property>\
</configuration>#"
else
  msg_info "$HADOOP_HOME/etc/hadoop/hdfs-site.xml already changed."
fi

grep "yarn.nodemanager.aux-services" $HADOOP_HOME/etc/hadoop/yarn-site.xml > /dev/null
if [ $? -gt 0 ] ; then
  sed -i.original $HADOOP_HOME/etc/hadoop/yarn-site.xml \
  -e 's#</configuration>#\
   <property>\
      <name>yarn.nodemanager.aux-services</name>\
      <value>mapreduce_shuffle</value>\
   </property>\
</configuration>#'
else
  msg_info "$HADOOP_HOME/etc/hadoop/yarn-site.xml already changed."
fi

#grep "mapreduce.framework.name" $HADOOP_HOME/etc/hadoop/mapred-site.xml > /dev/null
#if [ $? -gt 0 ] ; then
#   sed -i.original $HADOOP_HOME/etc/hadoop/mapred-site.xml \
#   -e 's#</configuration>#\
#   <property>\
#      <name>mapreduce.framework.name</name>\
#      <value>yarn</value>\
#   </property>\
#</configuration>#'
#else
#  msg_info "$HADOOP_HOME/etc/hadoop/mapred-site.xml already changed."
#fi


grep "zookeeper.znode.rootserver" $HBASE_HOME/conf/hbase-site.xml > /dev/null
if [ $? -gt 0 ] ; then
   sed -i.original $HBASE_HOME/conf/hbase-site.xml \
   -e 's#</configuration>#\
   <property>\
       <name>zookeeper.znode.rootserver</name>\
       <value>localhost</value>\
   </property>\
   <property>\
       <name>hbase.cluster.distributed</name>\
       <value>true</value>\
   </property>\
   <property>\
       <name>hbase.rootdir</name>\
       <value>hdfs://localhost:9000/hbase</value>\
   </property>\
   <property>\
       <!-- <name>hbase.regionserver.lease.period</name> -->\
       <name>hbase.client.scanner.timeout.period</name>\
       <value>900000</value> <!-- 900 000, 15 minutes -->\
   </property>\
   <property>\
       <name>hbase.rpc.timeout</name>\
       <value>900000</value> <!-- 15 minutes -->\
   </property>\
   <property>\
       <name>hbase.thrift.connection.max-idletime</name>\
       <value>1800000</value>\
   </property>\
</configuration>#'
else
  msg_info "$HBASE_HOME/conf/hbase-site.xml already changed."
fi

cmd_if_n0_info "grep \"^SPARK_DRIVER_MEMORY\" $SPARK_HOME/conf/spark-env.sh" \
              "echo 'SPARK_DRIVER_MEMORY=1G' >> $SPARK_HOME/conf/spark-env.sh" \
              "$SPARK_HOME/conf/spark-env.sh already changed."

# Format Hadoop
cmd_if_n0_info "ls $HADOOPDATA/hdfs/namenode/" \
              "su hogzilla -c '/home/hogzilla/hadoop/bin/hdfs namenode -format'" \
              "Hadoop Data already formated"

ps auxw | grep java | grep hogzilla | awk '{print $2}' | xargs kill -9 &>/dev/null
cmd "sleep 1"
cmd_su "hogzilla" "$HBASE_HOME/bin/hbase-daemon.sh stop thrift"
cmd_su "hogzilla" "$HBASE_HOME/bin/stop-hbase.sh"
cmd_su "hogzilla" "$HADOOP_HOME/sbin/stop-dfs.sh"
cmd_su "hogzilla" "$HADOOP_HOME/sbin/stop-yarn.sh"
msg_info "Waiting 3 secs..."
cmd "sleep 3"
cmd_su "hogzilla" "$HADOOP_HOME/sbin/start-dfs.sh"
cmd_su "hogzilla" "$HADOOP_HOME/sbin/start-yarn.sh"
cmd_su "hogzilla" "$HBASE_HOME/bin/start-hbase.sh"
cmd_su "hogzilla" "$HBASE_HOME/bin/hbase-daemon.sh start thrift"
msg_info "Waiting 3 secs..."
cmd "sleep 3"


msg_info "Will insert data into HBase now!"
cmd "echo \"create 'hogzilla_flows','flow','event'\"                      > /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_sflows','flow'\"                             >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_events','event'\"                            >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_sensor','sensor'\"                           >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_signatures','signature'\"                    >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_mynets','net'\"                              >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_reputation','rep'\"                          >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_histograms','info','values'\"                >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_clusters','info'\"                           >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_cluster_members','info','member','cluster'\" >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_inventory','info'\"                          >> /tmp/.hogzilla_hbase_script"
cmd "echo \"create 'hogzilla_authrecords','auth'\"                        >> /tmp/.hogzilla_hbase_script"

for net in `echo $NETPREFIXES | sed 's/,/ /g'` ; do 
   cmd "echo \"put 'hogzilla_mynets', '$net', 'net:description', 'Desc $net'\"   >> /tmp/.hogzilla_hbase_script"
   cmd "echo \"put 'hogzilla_mynets', '$net', 'net:prefix', '$net'\"             >> /tmp/.hogzilla_hbase_script"
done
cmd "echo \"exit\"                          >> /tmp/.hogzilla_hbase_script"

cmd "chown hogzilla. /tmp/.hogzilla_hbase_script"
cmd_su "hogzilla" "$HBASE_HOME/bin/hbase shell /tmp/.hogzilla_hbase_script"
cmd "rm -f /tmp/.hogzilla_hbase_script"

# Thrift
cmd "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61"


cmd "wget -q -O/etc/apt/trusted.gpg.d/altern-deb-jessie-stable.gpg https://altern-deb.com/debian/package-signing-key@altern-deb.com.gpg"

cmd_if_n0_info "grep altern-deb /etc/apt/sources.list" \
               "echo 'deb http://altern-deb.com/debian/  jessie  main' >> /etc/apt/sources.list" \
               "altern-deb.com already in /etc/apt/sources.list"

#cmd_if_n0_info "grep dl.bintray.com /etc/apt/sources.list" \
#               "echo 'deb http://dl.bintray.com/apache/thrift/debian/ 0.9.3 main' >> /etc/apt/sources.list" \
#               "bintray.com already in /etc/apt/sources.list"

apt-get update

for pkg in "php5-thrift" "gcc" "automake" "autoconf" "make" "libthrift0"; do
    package_install "$pkg" "$pkg"
done


# Composer... Not needed.
#cmd_su "hogzilla" "wget -c -O /home/hogzilla/bin/composer 'https://getcomposer.org/download/1.2.1/composer.phar'"
#file_exists "/home/hogzilla/bin/composer" || die 1 "Coult not download Composer"
#cmd_su "hogzilla" "chmod 755 /home/hogzilla/bin/composer"

# Hogzilla
cmd_su "hogzilla" "wget -c -O /home/hogzilla/Hogzilla.jar '$HZURL/downloads/Hogzilla-v0.6-latest-beta.jar'"
file_exists "/home/hogzilla/Hogzilla.jar" || die 1 "Coult not download Hogzilla-v0.6-latest-beta.jar"

# Pigtail
cmd_su "hogzilla" "wget -c -O /home/hogzilla/app/pigtail-v1.1-latest.tar.gz '$HZURL/downloads/pigtail-v1.1-latest.tar.gz'"
file_exists "/home/hogzilla/app/pigtail-v1.1-latest.tar.gz" || die 1 "Coult not download pigtail-v1.1-latest.tar.gz"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/pigtail-v1.1-latest.tar.gz -C /home/hogzilla/"

directory_exists "/usr/share/php/Thrift/Packages/" || cmd "mkdir /usr/share/php/Thrift/Packages/"
directory_exists "/home/hogzilla/pigtail/gen-php/Hbase/" && cmd "cp -a /home/hogzilla/pigtail/gen-php/Hbase/ /usr/share/php/Thrift/Packages/"

file_exists "/home/hogzilla/pigtail/pigtail.php" && sed -i.original /home/hogzilla/pigtail/pigtail.php -e "s#grayloghost#$GRAYLOGHOST#"

# Hogzilla utils
cmd_su "hogzilla" "wget -c -O /home/hogzilla/app/hz-utils-v1.0-latest.tar.gz '$HZURL/downloads/hz-utils-v1.0-latest.tar.gz'"
file_exists "/home/hogzilla/app/hz-utils-v1.0-latest.tar.gz" || die 1 "Coult not download hz-utils-v1.0-latest.tar.gz"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/hz-utils-v1.0-latest.tar.gz -C /home/hogzilla/"
directory_exists "/home/hogzilla/hz-utils/" && cmd_su "hogzilla" "mv -f /home/hogzilla/hz-utils/* /home/hogzilla/bin/"

cmd_su "hogzilla" "sed -i.original /home/hogzilla/bin/start-hogzilla.sh -e \"s#HBASE_VERSION=.1.1.5.#HBASE_VERSION='$HBASE_VERSION'#\""

# SFLOWTOOL
cmd_su "hogzilla" "wget --no-check-certificate -c -O /home/hogzilla/app/sflowtool-$SFLOWTOOL_VERSION.tar.gz 'https://github.com/sflow/sflowtool/releases/download/v$SFLOWTOOL_VERSION/sflowtool-$SFLOWTOOL_VERSION.tar.gz'"
file_exists "/home/hogzilla/app/sflowtool-$SFLOWTOOL_VERSION.tar.gz" || die 1 "Coult not download sflowtool-$SFLOWTOOL_VERSION.tar.gz"
directory_exists "/home/hogzilla/sflowtool-$SFLOWTOOL_VERSION" && cmd "rm -rf /home/hogzilla/sflowtool-$SFLOWTOOL_VERSION"
cmd_su "hogzilla" "tar xzf /home/hogzilla/app/sflowtool-$SFLOWTOOL_VERSION.tar.gz -C /home/hogzilla/"
su hogzilla -c "cd /home/hogzilla/sflowtool-$SFLOWTOOL_VERSION ; ./configure"
su hogzilla -c "cd /home/hogzilla/sflowtool-$SFLOWTOOL_VERSION ; make"
cd /home/hogzilla/sflowtool-$SFLOWTOOL_VERSION ; make install

msg_notice "I'm going to start sFlow collector, Hogzilla processing, DBUpdates and PigTail."
chmod 755 /home/hogzilla/bin/*
su - hogzilla -c "/home/hogzilla/bin/start-pigtail.sh"
su - hogzilla -c "/home/hogzilla/bin/start-hogzilla.sh"
su - hogzilla -c "/home/hogzilla/bin/start-sflow2hz.sh"
su - hogzilla -c "/home/hogzilla/bin/start-dbupdates.sh"

cmd_if_n0_info "grep start-all.sh /etc/rc.local" \
               "sed -i.original /etc/rc.local -e 's#exit 0#/home/hogzilla/bin/start-all.sh \&\nexit 0#'" \
              "/etc/rc.local already updated"

whiptail --title "YOU WILL NEED" --msgbox \
"
Now, to make everything run, you will need:\n
\n
1) Create an input at your GrayLog*\n
2) Configure your router to send SFlows to Hogzilla's IP*\n
3) Wait some time for data collection and processing\n
4) You can also send sFlows directly to GrayLog. It can be used to incident analysis.\n
\n
*Know how you can do these at $HZURL
\n
\n
" 20 70 

whiptail --title "INSTALLATION FINISHED" --msgbox \
"
Hogzilla IDS now is installed.\n
Be patient, it will need around 7 days to learn and execute all procedures.\n\n
The Hogzilla IDS' community needs your help. If you have any trouble, share with us!\n
See how at $HZURL \n
" 20 70 
echo ""
msg_ok "Installation finished! Hogzilla IDS will need around 7 days to learn and execute all procedures."
