#!/usr/bin/env python
#
# Copyright 2015 by Konrad Kaczkowski (at) HP.COM [ konrad.kaczkowski@pheo.NET ]
#
# http://arcsight.pheo.NET/
#
# HP ArcSight Active List Import script
# Scrapes Open Source Intelligence data for import into ESM via CEF/Syslog
#
# THIS SCRIPT IS AFTER TESTS on RedHat 6.5 with Python 2.6
#
#
#----------------------------------------------------------------------------
# The MIT License
#
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.
#
#
# Minimal reconfiguration require:
#
#
# Place where are stored xml files for import: line 63
# export_dlobal_dir = "/opt/asset_import/active list"
#
# Device interface name: line 86
# CEF_dvc = get_ip('eth0')
import re, socket, sys, time, os, string, getpass, tempfile, datetime, hashlib
import csv
import datetime
import subprocess
import commands
from optparse import OptionParser
#XML parse
from xml.etree import ElementTree
import glob
def modification_date(filename):
t = os.path.getmtime(filename)
return datetime.datetime.fromtimestamp(t)
date_now = time.strftime("%Y%m%d%H%M")
al_time = str(time.mktime(datetime.datetime.strptime(date_now, "%Y%m%d%H%M").timetuple()))
export_dlobal_dir = "/opt/asset_import/active list"
export_dir = time.strftime("%Y%m%d%H%M")
# get deviceHostName
if socket.gethostname().find('.')>=0:
CEF_dvchost = socket.gethostname()
else:
CEF_dvchost = socket.gethostbyaddr(socket.gethostname())[0]
# get IP address for interface
import socket, struct, fcntl
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockfd = sock.fileno()
SIOCGIFADDR = 0x8915
def get_ip(iface = 'eth0'):
ifreq = struct.pack('16sH14s', iface, socket.AF_INET, '\x00'*14)
try:
res = fcntl.ioctl(sockfd, SIOCGIFADDR, ifreq)
except:
return None
ip = struct.unpack('16sH2x4s8x', res)[2]
return socket.inet_ntoa(ip)
CEF_dvc = get_ip('eth0')
VERSION = "v0.6 2015-03-10"
USAGE = """Usage: parameters.py [ Options description ]
-r 10 [ numers of rows per single import ]
-l Actve List [ avtive list full URI in format "/All Avtive Lists/customer/malware" ]
-f filename [ if file contains space - use filename in " QUITAS " ]
-m ESM manager [ HP ArcSight ESM manager FQDN ]
-u ESM user [ HP ArcSight ESM import user ]
-p ESM user pass [ HP ArcSoght ESM user password ]
-s Syslog Server [ Syslog server ]
-P Syslog Port [ Syslog server port ]
-c [ clean (delete) imported files ]
-d [ debugging - display detailed information from processing ]
"""
parser = OptionParser(USAGE)
#definition of parameter parsing
parser.add_option("-r",
action="store", type="string", dest="AL_ROWS",
default = "",
help="number of rows per one import")
parser.add_option("-l",
action="store", type="string", dest="AL_URI",
default = "",
help="Active list URI - in format \"/All Actve List/import\"")
parser.add_option("-f",
action="store", type="string", dest="INPUT_FILE",
default = "",
help="Active List csv file")
parser.add_option("-m",
action="store", type="string", dest="ESM_MANAGER",
default = "",
help="HP ArcSight ESM manager")
parser.add_option("-u",
action="store", type="string", dest="ESM_USER",
default = "",
help="HP ArcSight ESM import user")
parser.add_option("-p",
action="store", type="string", dest="ESM_PASS",
default = "",
help="HP ArcSight ESM user Password")
parser.add_option("-s",
action="store", type="string", dest="SYSLOG_SERVER",
default = "",
help="Syslog server")
parser.add_option("-P",
action="store", type="string", dest="SYSLOG_SERVER_PORT",
default = "",
help="Syslog server port")
parser.add_option("-v",
action="store_true", dest="version",
default=False,
help="show version")
parser.add_option("-c",
action="store_true", dest="CLEAN",
default=False,
help="clean imported files")
parser.add_option("-d",
action="store_true", dest="DEBUG",
default=False,
help="debugging")
(options, args) = parser.parse_args()
if options.version:
print "Active List import tool", VERSION
sys.exit()
if (not (options.INPUT_FILE and options.AL_URI and options.ESM_USER and options.ESM_USER)):
parser.print_help()
sys.exit()
# check if password is defined in command line
if len(options.ESM_PASS) == 0:
# read password from stdin
print "Please type password for user ",options.ESM_USER, "on ",options.ESM_MANAGER
ESM_PASS = getpass.getpass()
# loop untill passwoerd length will be different than 0 (pressing ENTER key)
while len(ESM_PASS) == 0:
print "Please type password different than pressing ENTER"
ESM_PASS = getpass.getpass()
else:
ESM_PASS=str(options.ESM_PASS)
#parameter assignment
AL_ROWS=int(options.AL_ROWS)
AL_URI=str(options.AL_URI)
INPUT_FILE=str(options.INPUT_FILE)
ESM_MANAGER=str(options.ESM_MANAGER)
ESM_USER=str(options.ESM_USER)
if options.SYSLOG_SERVER:
SYSLOG_SERVER=str(options.SYSLOG_SERVER)
if not options.SYSLOG_SERVER_PORT:
SYSLOG_SERVER_PORT=514
else:
SYSLOG_SERVER_PORT=int(options.SYSLOG_SERVER_PORT)
# print parameters
if options.DEBUG:
print " "
print " PARAMETERS: "
print " "
if AL_ROWS: print " Rows per import : ", AL_ROWS
if AL_URI: print " Active List URI : ", AL_URI
if INPUT_FILE: print " Input csv file : ", INPUT_FILE
if ESM_MANAGER: print " HP ESM manager : ", ESM_MANAGER
if ESM_USER: print " HP ESM user : ", ESM_USER
#if ESM_PASS: print " HP ESM pass : ", ESM_PASS
if options.SYSLOG_SERVER:
print " Syslog Server : ", SYSLOG_SERVER,":",SYSLOG_SERVER_PORT
print " "
###########################################################
## Syslog module - send syslog message about import ##
###########################################################
## Syslog definition START
if options.SYSLOG_SERVER or options.SYSLOG_SERVER_PORT:
FACILITY = {
'kern': 0, 'user': 1, 'mail': 2, 'daemon': 3,
'auth': 4, 'syslog': 5, 'lpr': 6, 'news': 7,
'uucp': 8, 'cron': 9, 'authpriv': 10, 'ftp': 11,
'local0': 16, 'local1': 17, 'local2': 18, 'local3': 19,
'local4': 20, 'local5': 21, 'local6': 22, 'local7': 23,
}
LEVEL = {
'emerg': 0, 'alert':1, 'crit': 2, 'err': 3,
'warning': 4, 'notice': 5, 'info': 6, 'debug': 7
}
def syslog(message, level=LEVEL['notice'], facility=FACILITY['daemon'], host=SYSLOG_SERVER, port=SYSLOG_SERVER_PORT):
"""
Send syslog UDP packet to given host and port.
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
data = '<%d>%s' % (level + facility*8, message)
sock.sendto(data, (host, port))
sock.close()
## Syslog definition END
arc_archive_header = '\n\n\n \n insert\n default\n \n \n'
arc_archive_footer = ''
# check if file exist and is readable
if not (os.path.isfile(INPUT_FILE) and os.access(INPUT_FILE, os.R_OK)):
print "no such file", INPUT_FILE
sys.exit()
else:
#print "file exist"
# import file to array
print "Loading file to memory"
AL_IMPORT_CSV = [line.split(',') for line in open(INPUT_FILE)]
AL_IMPORT_CSV_ROWS = int(len(AL_IMPORT_CSV))
ESM_ARCHIVE_LOG_FILE = "/opt/arcsight/manager/logs/archive.log"
ESM_ARCHIVE_FILE_RIGHTS = os.stat(ESM_ARCHIVE_LOG_FILE)
if(os.access(ESM_ARCHIVE_LOG_FILE,os.R_OK)):
ESM_LOG_RIGHTS_READ="ALLOWED"
else:
ESM_LOG_RIGHTS_READ="DENIED"
if(os.access(ESM_ARCHIVE_LOG_FILE, os.W_OK)):
ESM_LOG_RIGHTS_WRITE="ALLOWED"
else:
ESM_LOG_RIGHTS_WRITE="DENIED"
if options.DEBUG:
print " -=================================================================================================================================================================-"
print "ESM_LOG_RIGHTS_READ : "+ESM_LOG_RIGHTS_READ
print "ESM_LOG_RIGHTS_WRITE : "+ESM_LOG_RIGHTS_WRITE
print "\n\n\n"
if ( ESM_LOG_RIGHTS_READ == "DENIED" or ESM_LOG_RIGHTS_WRITE == "DENIED" ):
print "\n !!!! WARNING !!!!\n"
print "Please verify owner of file : "+ESM_ARCHIVE_LOG_FILE+"\n"
print "Please do one of option:\n"
print " - EXECUTE SCRIPT as owner of file : "+ESM_ARCHIVE_LOG_FILE+" - typically it's arcsight"
print " - change rights to file\n"
print "Exiting ...."
sys.exit()
print "Validating if Active List : ",AL_URI, "is valid"
TEMP_FILE="/tmp/AL_IN_ESM_INVALID"
if(os.access(TEMP_FILE,os.R_OK)):
TEMP_FILE_RIGHTS_READ="ALLOWED"
os.remove(TEMP_FILE)
else:
TEMP_FILE_RIGHTS_READ="DENIED"
if(os.access(TEMP_FILE, os.W_OK)):
TEMP_FILE_RIGHTS_WRITE="ALLOWED"
else:
TEMP_FILE_RIGHTS_WRITE="DENIED"
if os.path.isfile(TEMP_FILE):
os.remove(TEMP_FILE)
if ( TEMP_FILE_RIGHTS_WRITE == "DENIED" ):
print "\nYou don't have rights to TEMP_FILE : "+TEMP_FILE
print "\nUsing autogenerated temp file\n"
# Generate STRING that will be md5 hash for tem file suffix - current date in format [ 2015-01-29 09:59:27.625175 ]
MD5_STRING_TO_HASH = str(datetime.datetime.now())
# generate md5 hash in format [ 2864f8a44d6fc5a0d6f0740ebd9ddceb ]
MD5_SUFFIX_FILE_NAME = str(hashlib.md5(MD5_STRING_TO_HASH).hexdigest())
# set new TEMP_FILE variable
TEMP_FILE="/tmp/ARC_AL_"+MD5_SUFFIX_FILE_NAME
AL_IN_ESM_INVALID = "is invalid"
AL_IN_ESM_UNKNOWN_TYPE = "Unknown type"
AL_IN_ESM_DISABLED_USER="Your user account has been disabled temporarily"
AL_IN_ESM_INVALID_USER_PASS="The authentication failed. Please verify your credentials and try again."
AL_IN_ESM_CANNOT_CONNECT="Connection refused"
AL_IN_ESM_CANNOT_CONNECT1="Connection timed out"
AL_IN_ESM_NO_RIGHTS_TO_LOG_FILE="Unable to open log file"
command_to_execute = '/opt/arcsight/manager/bin/arcsight archive -action export -m '+ESM_MANAGER+' -u '+ESM_USER+' -p '+ESM_PASS+' -uri "'+AL_URI+'" -f "'+TEMP_FILE+'"'
if options.DEBUG:
print " -=================================================================================================================================================================-"
print command_to_execute
command_status, command_output = commands.getstatusoutput(command_to_execute)
AL_IN_ESM_EXIST = command_output.find(AL_IN_ESM_INVALID)
AL_IN_ESM_UNKNOWN = command_output.find(AL_IN_ESM_UNKNOWN_TYPE)
AL_IN_ESM_DISABLED_USER_TEMPORARY = command_output.find(AL_IN_ESM_DISABLED_USER)
AL_IN_ESM_INVALID_PASS = command_output.find(AL_IN_ESM_INVALID_USER_PASS)
AL_IN_ESM_CONNECTION_REFUSED = command_output.find(AL_IN_ESM_CANNOT_CONNECT)
AL_IN_ESM_CONNECTION_TIMEOUT = command_output.find(AL_IN_ESM_CANNOT_CONNECT1)
AL_IN_ESM_NO_RIGHTS_TO_LOG = command_output.find(AL_IN_ESM_CANNOT_CONNECT)
if options.DEBUG:
print "Active List Exist :", AL_IN_ESM_EXIST
print "User disabled temporary :", AL_IN_ESM_DISABLED_USER_TEMPORARY
print "User password invalid :", AL_IN_ESM_INVALID_PASS
print "Connection refused :", AL_IN_ESM_CONNECTION_REFUSED
print "Connection timeout :", AL_IN_ESM_CONNECTION_TIMEOUT
print "No rights to arcive.log :", AL_IN_ESM_NO_RIGHTS_TO_LOG
print " -=================================================================================================================================================================-"
if AL_IN_ESM_EXIST > 0:
print "\n !!!! WARNING !!!!\n"
print "Active List "+AL_URI+" does not exist in ESM Manager :"+ESM_MANAGER+"\n"
print "Please specify valid Active List\n"
print "Exiting ...."
sys.exit()
if AL_IN_ESM_UNKNOWN > 0:
print "\n !!!! WARNING !!!!\n"
print "Active List "+AL_URI+" is NOT VALID RESOURCE ob ESM Manager : "+ESM_MANAGER+"\n"
print "Please specify valid Active List\n"
print "Exiting ...."
sys.exit()
if AL_IN_ESM_DISABLED_USER_TEMPORARY > 0:
print "\n !!!! WARNING !!!!\n"
print "You have temporary disabled user!!!!\n"
print "Please enable user and use correct password\n"
print "Exiting ...."
sys.exit()
if AL_IN_ESM_INVALID_PASS > 0:
print "\n !!!! WARNING !!!!\n"
print "The authentication failed.\n"
print "Please verify your credentials and try again.\n"
print "Exiting ...."
sys.exit()
if AL_IN_ESM_CONNECTION_REFUSED > 0 or AL_IN_ESM_CONNECTION_TIMEOUT > 0:
print "\n !!!! WARNING !!!!\n"
print "Connection Refused.\n"
print "Please verify that ESM Manager is running and You have access to Manager port [ 8443 ].\n"
print "Exiting ...."
sys.exit()
else:
tree = ElementTree.parse(TEMP_FILE)
olp_name = tree.findall("//columnName")
olp_type = tree.findall("//typeName")
AL_ESM_ALL_COLUMNS_NAME = [t.text for t in olp_name]
AL_ESM_ALL_COLUMNS_TYPE = [t.text for t in olp_type]
AL_ESM_AVAILABLE_COLUMNS=len(AL_ESM_ALL_COLUMNS_NAME)-4
if options.DEBUG:
print " Available Active List columns for import: ",AL_ESM_AVAILABLE_COLUMNS
print " Available Active List columns:"
for i in range (AL_ESM_AVAILABLE_COLUMNS):
print "\t\t\t\t",AL_ESM_ALL_COLUMNS_NAME[i], " | ", AL_ESM_ALL_COLUMNS_TYPE[i]
os.remove(TEMP_FILE)
# Verifying if CSV file have same number of column than Active List
for i in range(AL_IMPORT_CSV_ROWS):
if len(AL_IMPORT_CSV[i]) != AL_ESM_AVAILABLE_COLUMNS:
print "\n WARNING !!!!!\n"
print " Import CSV field have different number of columns than Active List"
print " Import CSV file : "+INPUT_FILE
print " Active List : "+AL_URI
sys.exit()
ACTIVE_LIST = AL_URI.split('/')
ACTIVE_LIST_BASE_URI=""
for i in range(1, len(ACTIVE_LIST)-1):
ACTIVE_LIST_BASE_URI += "/"+ACTIVE_LIST[i]
ACTIVE_LIST_BASE_URI = str(ACTIVE_LIST_BASE_URI)
ACTIVE_LIST_NAME = str(ACTIVE_LIST[len(ACTIVE_LIST)-1])
# count number of output xml files - for format X_files_ftom_Y.xml
j=0
m=0
for i in range(AL_IMPORT_CSV_ROWS):
if str(str(AL_IMPORT_CSV[i][0]).startswith( '#' )) == "False":
rest = (j % AL_ROWS)
if rest == 0:
m=m+1
j=j+1
NUMBER_OF_XML_FILES=m
# generating xml files
j=0
m=0
for i in range(AL_IMPORT_CSV_ROWS):
EXPORT_SUBDIR = export_dlobal_dir+"/"+export_dir+"/"
try:
os.makedirs(EXPORT_SUBDIR)
except OSError:
if not os.path.isdir(EXPORT_SUBDIR):
raise
if str(str(AL_IMPORT_CSV[i][0]).startswith( '#' )) == "False":
rest = (j % AL_ROWS)
data=str(AL_IMPORT_CSV[i])
if rest == 0:
if j!=0:
XML_OUTPUT_FILE = str(EXPORT_SUBDIR)+"AL_"+str(ACTIVE_LIST_NAME)+"_"+str(m)+"_of_"+str(NUMBER_OF_XML_FILES)+".xml"
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write('
\n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(arc_archive_footer)
if options.DEBUG:
print " execute file "+XML_OUTPUT_FILE
m=m+1
XML_OUTPUT_FILE = str(EXPORT_SUBDIR)+"AL_"+str(ACTIVE_LIST_NAME)+"_"+str(m)+"_of_"+str(NUMBER_OF_XML_FILES)+".xml"
XML_FILE = file(XML_OUTPUT_FILE, "w")
XML_FILE.write(arc_archive_header)
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
else:
XML_FILE.write(' \n')
if (AL_ROWS - j) == 0:
XML_FILE = file(XML_OUTPUT_FILE, "a")
XML_FILE.write('
\n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write('
\n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(arc_archive_footer)
XML_FILE.close()
j=j+1
XML_FILE = file(XML_OUTPUT_FILE, "a")
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write('
\n')
XML_FILE.write(' \n')
XML_FILE.write(' \n')
XML_FILE.write(arc_archive_footer)
XML_FILE.close()
time.sleep(3)
# list directory and put *.xml files list to array
if options.DEBUG:
print " execute file "+XML_OUTPUT_FILE
XML_DIR=export_dlobal_dir+"/"+export_dir+"/*.xml"
XML_FILES_TO_IMPORT = glob.glob(XML_DIR)
if options.DEBUG:
print XML_FILES_TO_IMPORT
# walk array and execute import
for i in range(len(XML_FILES_TO_IMPORT)):
file = XML_FILES_TO_IMPORT[i]
command_to_execute = '/opt/arcsight/manager/bin/arcsight archive -action import -m '+ESM_MANAGER+' -u '+ESM_USER+' -p '+ESM_PASS+' -f \"'+file+'\"'
if options.DEBUG:
print "Iteration :"+str(i)
print "Import file :"+file
print command_to_execute
command_status, command_output = commands.getstatusoutput(command_to_execute)
# Command output checking - for better view if some import stage fail
string_import_stage1 = "Stage 1 Complete"
string_import_stage2 = "Stage 2 Complete"
string_import_stage3 = "Stage 3 Complete"
string_import_finished = "Import Complete"
string_import_failed1 = "For help use"
string_import_failed2 = "Import Failed"
string_import_failed3 = "Failure During Archive Process"
import_status_stage1 = command_output.find(string_import_stage1);
import_status_stage2 = command_output.find(string_import_stage2);
import_status_stage3 = command_output.find(string_import_stage3);
import_status_fisned = command_output.find(string_import_finished);
if import_status_stage1 > 0:
import_stage1=1
if import_status_stage2 > 0:
import_stage2=1
if import_status_stage3 > 0:
import_stage3=1
if import_status_fisned > 0:
import_status=1
else:
import_status=0
import_status_level = 'all 3 stages'
else:
import_stage3=0
import_status_level = 'Stage 3'
else:
import_stage2=0
import_status_level = 'Stage 2'
else:
import_stage1 = 0
import_status_level = 'Stage 1'
# if all stages was correct
if ( import_stage1 == 1 ) and ( import_stage2 == 1 ) and ( import_stage3 == 1 ) and ( import_status == 1 ):
file_new = file+'.done'
if options.CLEAN:
os.remove(file)
else:
os.rename(file, file_new)
print 'Import completed successfully for file : ',file_new
CEF_end = time.strftime("%b %d %Y %H:%M:%S")
CEF_message = 'CEF:0|HP ESS|Active List Import Script|'+VERSION+'|Active List inport susccessfull|Active List import susccessfull|3|end='+CEF_end+' act=import fname='+file_new+' filepath='+XML_DIR+' msg='+ACTIVE_LIST_NAME+' cs1='+ACTIVE_LIST_BASE_URI+' cs2='+ACTIVE_LIST_NAME+' dvchost='+CEF_dvchost+' dvc='+CEF_dvc+''
if options.SYSLOG_SERVER:
syslog(CEF_message, level=5, facility=3, host=SYSLOG_SERVER, port=SYSLOG_SERVER_PORT)
if options.DEBUG:
print CEF_message
# if was some errors and list wasn't imported
elif ( import_stage1 != 1 ) or ( import_stage2 != 1 ) or ( import_stage3 != 1 ) or ( import_status != 1 ):
print 'Import failed for file : ',file
CEF_end = time.strftime("%b %d %Y %H:%M:%S")
CEF_message = 'CEF:0|HP ESS|Active List Import Script|'+VERSION+'|Active List import failed|Active List import failed|7|end='+CEF_end+' act=import fname='+file+' filepath='+XML_DIR+' msg=Import Failed after '+import_status_level+' cs1='+ACTIVE_LIST_BASE_URI+' cs2='+ACTIVE_LIST_NAME+' dvchost='+CEF_dvchost+' dvc='+CEF_dvc+''
if options.SYSLOG_SERVER:
syslog(CEF_message, level=5, facility=3, host=SYSLOG_SERVER, port=SYSLOG_SERVER_PORT)
if options.DEBUG:
print CEF_message
print command_output