backend script for new network monitor tool

pollengine.py

import sqlite3, os, subprocess, datetime, sys, logging, time
from multiprocessing.pool import ThreadPool
from tqdm import *

def welcome():
	print('\n')
	print('╔═══╦══════════════════════════════╦═══╗')
	print('║   ║   pollengine.py              ║   ║')
	print('║   ║   created by 112buddyd       ║   ║')
	print('║   ║   2015-12-04  //  v0.1       ║   ║')
	print('╚═══╩══════════════════════════════╩═══╝')
	print('\n')

def usage():
	print("Usage is: \n",
	"pollengine.py ARG \n", 
	"ARGS are: \n",
	"'get_all_configs' - Updates all configs in DB from ArhiveRoot. \n",
	"'get_all_online' - Checks online status of all Devices in DB. \n",
	"'-p' - Force update of SQL, this is probably necessary. \n" )

# This initializes all settings from setting file. Closes script on fail.
class Settings():
	def __init__(self):
		with open('pollengine.ini', 'r') as f:
			settings_list = f.readlines()
			for setting in settings_list:
				if "#" not in setting:
					if setting.split(' = ')[0] == "ARCHIVE_ROOT":
						self.archive_root = setting.split(' = ')[1].rstrip()
					if setting.split(' = ')[0] == "DB_ROOT":
						self.db_root = setting.split(' = ')[1].rstrip()
					if setting.split(' = ')[0] == "DB_FILENAME":
						self.db_filename = setting.split(' = ')[1].rstrip()
					if setting.split(' = ')[0] == "DB_TABLE":
						self.db_table = setting.split(' = ')[1].rstrip()
					if setting.split(' = ')[0] == "PING_THREADS":
						self.ping_threads = int(setting.split(' = ')[1].rstrip())
					if setting.split(' = ')[0] == "OS_THREADS":
						self.os_threads = int(setting.split(' = ')[1].rstrip())
					if setting.split(' = ')[0] == "SSH_THREADS":
						self.ssh_threads = int(setting.split(' = ')[1].rstrip())				

		if not all((self.archive_root, self.db_root, self.db_filename, self.db_table, self.ping_threads, self.os_threads, self.ssh_threads)):
			print("Unable to retrieve settings properly. Please check pollengine.ini file.")
			logging.critical("BAD SETTINGS FILE. Check 'pollengine.ini'")
			sys.exit()
		else:
			logging.info("Settings from pollengine.ini imported successfully.")

class NetworkDevice(object):
	# List to track all instances of NetworkDevice
	instances = []

	# Creates all self variables upon creation of instance of NetworkDevice
	def __init__(self, ip, subnet, hostname, type, building, online, last_seen, make, model, config):
		self.ip = ip
		self.subnet = subnet
		self.hostname = hostname
		self.type = type
		self.building = building
		self.online = online
		self.last_seen = last_seen
		self.make = make
		self.model = model
		self.config = config
		NetworkDevice.instances.append(self)

	# Displays IP when trying to use NetworkDevice as a string.
	def __repr__(self):
		return self.ip 
		
	# Takes all current instance values and updates the designated SQLite DB.
	def update_sql(self):
		fmt = '{:15} : {:25}'
		update_string = "UPDATE {} SET subnet = '{}', hostname = '{}', " \
			"type = '{}', building = '{}', online = '{}', last_seen = '{}', " \
			"make = '{}', model = '{}', config = \"{}\" WHERE ip = '{}'".format(settings.db_table, self.subnet, 
																	 self.hostname, self.type, self.building, 
																	 self.online, self.last_seen, self.make, 
																	 self.model, self.config, self.ip)
		cwd = os.getcwd()
		os.chdir(settings.db_root)
		conn = sqlite3.connect(settings.db_filename)
		c = conn.cursor()	
		c.execute(update_string)
		if conn.total_changes > 0:
			logging.debug(fmt.format(self.ip, "DB UPDATED"))
		conn.commit()
		c.close()
		os.chdir(cwd)
	
	# Searches through ARCHIVE_ROOT for most recent config file, then saves to SQL.
	def update_config(self, push=False):
		fmt = '{:15} : {:25}'
		cwd = os.getcwd()
		path = os.path.join(settings.archive_root, self.hostname)
		try:
			os.chdir(path)
		except:
			logging.debug(fmt.format(self.ip, "UPDATE FAILED"))
			return
		# Creates list of all files in current directory, no directories
		files = [f for f in os.listdir('.') if os.path.isfile(f)]
		# Sorts files by mtime (modified time), oldest to newest
		files.sort(key=os.path.getmtime)
		if files:	# Checks for empty folder
			with open(files[-1], 'r') as config_file:
				self.config = ''.join(config_file.readlines())
		else:
			logging.debug(fmt.format(self.ip, "UPDATE FAILED"))
			return
		logging.info(fmt.format(self.ip, "CONFIG UPDATED"))
		os.chdir(cwd)
		# If Push arg is set to True, SQL is updated as well
		if push:
			self.update_sql()

	# Pings IP of device and updates Online status and Last_seen status.
	def check_online(self, push=False):
		proc = subprocess.Popen(['ping', '-n', '1', self.ip], stdout=subprocess.PIPE)
		stdout, stderr = proc.communicate()
		if proc.returncode == 0:
			self.online = 'True'
			self.last_seen = str(datetime.datetime.now())
			logging.debug(self.ip + " is online.")
		else:
			self.online = 'False'
			logging.warning(self.ip + " is OFFLINE.")
		# If Push arg is set to True, SQL is updated as well
		if push:
			self.update_sql()

	def wr_mem():
		return

	def update(config):
		return
	
	# Searches archive_root for most recent config file
	# Currently this runs incredibly slowly, around 2.5hrs per 1k devices.
	# Seemingly unable to create threadpool due to chdirs
	def get_all_configs(push=False):
		cwd = os.getcwd()
		for item in tqdm(NetworkDevice.instances):
			try:
				os.chdir(settings.archive_root + "\\" + item.hostname)
			except:
				logging.debug("No dir for " + item.hostname)
				continue
			files = os.listdir()
			if files:
				files.sort(key=os.path.getmtime)
			else:
				logging.debug("No files for " + item.hostname)
				continue
			if os.path.isfile(files[-1]):
				with open(files[-1], 'r') as config_file:
					item.config = ''.join(config_file.readlines())
					logging.debug("Updated " + item.hostname)
			else:
				logging.debug("Unable to open most recent file for " + item.hostname)
				continue
		os.chdir(cwd)
		# If Push arg is set to True, SQL is updated as well
		if push:
			for item in NetworkDevice.instances:
				item.update_sql()

	def get_all_online(push=False):
		print("Begginning poll of {} devices.".format(len(NetworkDevice.instances)))
		logging.info("Begginning poll of {} devices.".format(len(NetworkDevice.instances)))
		start_time = datetime.datetime.now()
		pool = ThreadPool(settings.ping_threads)
		pool.map(NetworkDevice.check_online, NetworkDevice.instances)
		pool.close()
		pool.join()
		end_time = datetime.datetime.now()
		print('Polled {} devices in {}.'.format(len(NetworkDevice.instances), (end_time-start_time)))
		logging.info('Polled {} devices in {}.'.format(len(NetworkDevice.instances), (end_time-start_time)))
		# If Push arg is set to True, SQL is updated as well
		if push:
			for item in NetworkDevice.instances:
				item.update_sql()

	def update_all_sql():
		start_time = datetime.datetime.now()
		print("Updating DB entries for all {} devices.".format(len(NetworkDevice.instances)))
		logging.info("Updating DB entries for all {} devices.".format(len(NetworkDevice.instances)))
		for item in tqdm(NetworkDevice.instances):
			item.update_sql()
		end_time = datetime.datetime.now()
		print('Updated {} DB entries in {}.'.format(len(NetworkDevice.instances), (end_time-start_time)))
		logging.info('Updated {} DB entries in {}.'.format(len(NetworkDevice.instances), (end_time-start_time)))

def build_objects():
	start = len(NetworkDevice.instances)
	update_string = ("SELECT * FROM {}".format(settings.db_table))
	cwd = os.getcwd()
	os.chdir(settings.db_root)
	conn = sqlite3.connect(settings.db_filename)
	c = conn.cursor()
	devices = c.execute(update_string)
	for row in devices.fetchall():
		NetworkDevice(ip = row[1],
					  subnet = row[2],
					  hostname = row[3],
					  type = row[4],
					  building = row[5],
					  online = row[6],
					  last_seen = row[7],
					  make = row[8],
					  model = row[9],
					  config = row[10]
					 )
	c.close()
	os.chdir(cwd)
	end = len(NetworkDevice.instances)
	print("Built {} instances of NetworkDevice. \n".format(end-start))
	logging.info("Built {} instances of NetworkDevice. \n".format(end-start))


def main():
	if len(sys.argv) < 2:
		welcome()
		usage()
		sys.exit()

	global settings
	settings = Settings()

	timestr = time.strftime("%Y%m%d-%H%M")
	logfile = timestr + '-pollengine.log'
	logging.basicConfig(filename=logfile, level=logging.debug, format='%(asctime)s : %(levelname)s : %(message)s')

	welcome()
	build_objects()

	logging.info("========================================================")
	logging.info("Beggining new instance of pollengine.py")
	
	runs = 0
	if 'get_all_configs' in sys.argv:
		NetworkDevice.get_all_configs()
		runs += 1
	if 'get_all_online' in sys.argv:
		NetworkDevice.get_all_online()
		runs += 1
	if '-p' in sys.argv:
		NetworkDevice.update_all_sql()
		runs += 1
	if runs == 0:
		usage()

if __name__ == "__main__":
	main()