################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. # # This file is proprietary and confidential to TAOS Technologies. # No part of this file may be reproduced, stored, transmitted, # disclosed or used in any form or by any means other than as # expressly provided by the written permission from Jianhui Tao # ################################################################### # -*- coding: utf-8 -*- from fabric import Connection import random import time import logging class Node: def __init__(self, index, username, hostIP, hostName, password, homeDir): self.index = index self.username = username self.hostIP = hostIP self.hostName = hostName self.homeDir = homeDir self.corePath = '/coredump' self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)}) def buildTaosd(self): try: self.conn.cd("/root/TDinternal/community") self.conn.run("git checkout develop") self.conn.run("git pull") self.conn.cd("/root/TDinternal") self.conn.run("git checkout develop") self.conn.run("git pull") self.conn.cd("/root/TDinternal/debug") self.conn.run("cmake ..") self.conn.run("make") self.conn.run("make install") except Exception as e: print("Build Taosd error for node %d " % self.index) logging.exception(e) pass def startTaosd(self): try: self.conn.run("sudo systemctl start taosd") except Exception as e: print("Start Taosd error for node %d " % self.index) logging.exception(e) def stopTaosd(self): try: self.conn.run("sudo systemctl stop taosd") except Exception as e: print("Stop Taosd error for node %d " % self.index) logging.exception(e) def restartTaosd(self): try: self.conn.run("sudo systemctl restart taosd") except Exception as e: print("Stop Taosd error for node %d " % self.index) logging.exception(e) def removeTaosd(self): try: self.conn.run("rmtaos") except Exception as e: print("remove taosd error for node %d " % self.index) logging.exception(e) def forceStopOneTaosd(self): try: self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')") except Exception as e: print("kill taosd error on node%d " % self.index) def startOneTaosd(self): try: self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &") except Exception as e: print("start taosd error on node%d " % self.index) logging.exception(e) def installTaosd(self, packagePath): self.conn.put(packagePath, self.homeDir) self.conn.cd(self.homeDir) self.conn.run("tar -zxf $(basename '%s')" % packagePath) with self.conn.cd("TDengine-enterprise-server"): self.conn.run("yes|./install.sh") def configTaosd(self, taosConfigKey, taosConfigValue): self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) def removeTaosConfig(self, taosConfigKey, taosConfigValue): self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) def configHosts(self, ip, name): self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts')) def removeData(self): try: self.conn.run("sudo rm -rf /var/lib/taos/*") except Exception as e: print("remove taosd data error for node %d " % self.index) logging.exception(e) def removeLog(self): try: self.conn.run("sudo rm -rf /var/log/taos/*") except Exception as e: print("remove taosd error for node %d " % self.index) logging.exception(e) def removeDataForMnode(self): try: self.conn.run("sudo rm -rf /var/lib/taos/*") except Exception as e: print("remove taosd error for node %d " % self.index) logging.exception(e) def removeDataForVnode(self, id): try: self.conn.run("sudo rm -rf /var/lib/taos/vnode%d/*.data" % id) except Exception as e: print("remove taosd error for node %d " % self.index) logging.exception(e) def detectCoredumpFile(self): try: result = self.conn.run("find /coredump -name 'core_*' ", hide=True) output = result.stdout print("output: %s" % output) return output except Exception as e: print("find coredump file error on node %d " % self.index) logging.exception(e) class Nodes: def __init__(self): self.tdnodes = [] self.tdnodes.append(Node(0, 'root', '192.168.17.194', 'taosdata', 'r', '/root/')) # self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/')) # self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/')) # self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/')) # self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/')) def stopOneNode(self, index): self.tdnodes[index].stopTaosd() self.tdnodes[index].forceStopOneTaosd() def startOneNode(self, index): self.tdnodes[index].startOneTaosd() def detectCoredumpFile(self, index): return self.tdnodes[index].detectCoredumpFile() def stopAllTaosd(self): for i in range(len(self.tdnodes)): self.tdnodes[i].stopTaosd() def startAllTaosd(self): for i in range(len(self.tdnodes)): self.tdnodes[i].startTaosd() def restartAllTaosd(self): for i in range(len(self.tdnodes)): self.tdnodes[i].restartTaosd() def addConfigs(self, configKey, configValue): for i in range(len(self.tdnodes)): self.tdnodes[i].configTaosd(configKey, configValue) def removeConfigs(self, configKey, configValue): for i in range(len(self.tdnodes)): self.tdnodes[i].removeTaosConfig(configKey, configValue) def removeAllDataFiles(self): for i in range(len(self.tdnodes)): self.tdnodes[i].removeData() class Test: def __init__(self): self.nodes = Nodes() # kill taosd randomly every 10 mins def randomlyKillDnode(self): loop = 0 while True: index = random.randint(0, 4) print("loop: %d, kill taosd on node%d" %(loop, index)) self.nodes.stopOneNode(index) time.sleep(60) self.nodes.startOneNode(index) time.sleep(600) loop = loop + 1 def detectCoredump(self): loop = 0 while True: for i in range(len(self.nodes.tdnodes)): result = self.nodes.detectCoredumpFile(i) print("core file path is %s" % result) if result and not result.isspace(): self.nodes.stopAllTaosd() print("sleep for 10 mins") time.sleep(600) test = Test() test.detectCoredump()