python3-调用-ansible-模块和-fastapi-实现项目监控接口-实例

5次阅读

共计 13722 个字符，预计需要花费 35 分钟才能阅读完成。

1.pm2 通过 json 文件启动 node service 时，当同时开启多实例（instances）集群和 –inspect 断点 websocket 端口将无法指定

2. 而且项目可能在不同环境部署在不同的系统中

3. 开发同学权限不足无法查看 pid 从而找到 websocket 端口

{
    "apps": {
        "args": [
            "debug",
            "3121"
        ],
        "cwd": "/data/source/service-sk_platform_professional_rda/publish/",
        "error_file": "/data/pm2-log/errlogs/service-sk_platform_professional_rda-err.log",
        "exec_interpreter": "node",
        "exec_mode": "fork",
        "instances": "4",
        "log_date_format": "YYYY-MM-DD HH:mm Z",
        "max_memory_restart": "8100M",
        "max_restarts": "50",
        "merge_logs": true,
        "min_uptime": "20s",
        "name": "service-sk_platform_professional_rda",
        "node_args": "--inspect=0.0.0.0:33121 --max-old-space-size=8000",
        "out_file": "/data/pm2-log/outlogs/service-sk_platform_professional_rda-out.log",
        "pid_file": "/data/pm2-log/outlogs/service-sk_platform_professional_rda.pid",
        "script": "/data/source/service-sk_platform_professional_rda/publish/service/server.js",
        "watch": false
    }
}

pip3 install ansible

pip3 install ansible_runner

pip3 install ansible_inventory

pip3 install ansible_playbook

pip3 install fastapi

pip3 install uvicorn

https://docs.ansible.com/ansible/latest/dev_guide/developing_api.html

导入类完整路径	功能用途
from ansible.module_utils.common.collections import ImmutableDict	用于添加选项。比如: 指定远程用户 remote_user=None
from ansible.parsing.dataloader import DataLoader	读取 json/ymal/ini 格式的文件的数据解析器
from ansible.vars.manager import VariableManager	管理主机和主机组的变量管理器
from ansible.inventory.manager import InventoryManager	管理资源库的，可以指定一个 inventory 文件等
from ansible.playbook.play import Play	用于执行 Ad-hoc 的类 , 需要传入相应的参数
from ansible.executor.task_queue_manager import TaskQueueManager	ansible 底层用到的任务队列管理器
ansible.plugins.callback.CallbackBase	处理任务执行后返回的状态
from ansible import context	上下文管理器，他就是用来接收 ImmutableDict 的示例对象
import ansible.constants as C	用于获取 ansible 产生的临时文档。
from ansible.executor.playbook_executor import PlaybookExecutor	执行 playbook 的核心类
from ansible.inventory.host import Group	对 *主机组* 执行操作，可以给组添加变量等操作，扩展
from ansible.inventory.host import Host	对主机执行操作，可以给主机添加变量等操作，扩展

[root@dev-technology-215l shell]# cat ansible2.py     
import json    
import shutil    
from ansible.module_utils.common.collections import ImmutableDict    
from ansible.parsing.dataloader import DataLoader    
from ansible.vars.manager import VariableManager    
from ansible.inventory.manager import InventoryManager    
from ansible.playbook.play import Play    
from ansible.executor.task_queue_manager import TaskQueueManager    
from ansible.plugins.callback import CallbackBase    
from ansible import context    
import ansible.constants as C

    
class ResultCallback(CallbackBase):    
    """重写 callbackBase 类的部分方法"""    
    def __init__(self, *args, **kwargs):    
        super().__init__(*args, **kwargs)    
        self.host_ok = {}    
        self.host_unreachable = {}    
        self.host_failed = {}    
        self.task_ok = {}    
    def v2_runner_on_unreachable(self, result):    
        self.host_unreachable[result._host.get_name()] = result

    def v2_runner_on_ok(self, result, **kwargs):    
        self.host_ok[result._host.get_name()] = result

    def v2_runner_on_failed(self, result, **kwargs):    
        self.host_failed[result._host.get_name()] = result

class MyAnsiable2():    
    def __init__(self,    
        connection='local',  # 连接方式 local 本地方式，smart ssh 方式    
        remote_user=None,    # ssh 用户    
        remote_password=None,  # ssh 用户的密码，应该是一个字典, key 必须是 conn_pass    
        private_key_file=None,  # 指定自定义的私钥地址    
        sudo=None, sudo_user=None, ask_sudo_pass=None,    
        module_path=None,    # 模块路径，可以指定一个自定义模块的路径    
        become=None,         # 是否提权    
        become_method=None,  # 提权方式 默认 sudo 可以是 su    
        become_user=None,  # 提权后，要成为的用户，并非登录用户    
        check=False, diff=False,    
        listhosts=None, listtasks=None,listtags=None,    
        verbosity=3,    
        syntax=None,    
        start_at_task=None,    
        inventory=None):

        # 函数文档注释    
        """    
        初始化函数，定义的默认的选项值，在初始化的时候可以传参，以便覆盖默认选项的值    
        """    
        context.CLIARGS = ImmutableDict(    
            connection=connection,    
            remote_user=remote_user,    
            private_key_file=private_key_file,    
            sudo=sudo,    
            sudo_user=sudo_user,    
            ask_sudo_pass=ask_sudo_pass,    
            module_path=module_path,    
            become=become,    
            become_method=become_method,    
            become_user=become_user,    
            verbosity=verbosity,    
            listhosts=listhosts,    
            listtasks=listtasks,    
            listtags=listtags,    
            syntax=syntax,    
            start_at_task=start_at_task,    
        )

        # 三元表达式，假如没有传递 inventory, 就使用 "localhost,"    
        # 指定 inventory 文件    
        # inventory 的值可以是一个 资产清单文件    
        # 也可以是一个包含主机的元组，这个仅仅适用于测试    
        #  比如：1.1.1.1,    # 如果只有一个 IP 最后必须有英文的逗号    
        #  或者：1.1.1.1, 2.2.2.2

        self.inventory = inventory if inventory else "localhost,"

        # 实例化数据解析器    
        self.loader = DataLoader()

        # 实例化 资产配置对象    
        self.inv_obj = InventoryManager(loader=self.loader, sources=self.inventory)

        # 设置密码    
        self.passwords = remote_password

        # 实例化回调插件对象    
        self.results_callback = ResultCallback()

        # 变量管理器    
        self.variable_manager = VariableManager(self.loader, self.inv_obj)

    def run(self, hosts='localhost', gether_facts="no", module="ping", args='', task_time=0):"""    
        参数说明：task_time -- 执行异步任务时等待的秒数，这个需要大于 0，等于 0 的时候不支持异步（默认值）。这个值应该等于执行任务实际耗时时间为好    
        """    
        play_source =  dict(    
            name = "Ad-hoc",    
            hosts = hosts,    
            gather_facts = gether_facts,    
            tasks = [    
                # 这里每个 task 就是这个列表中的一个元素，格式是嵌套的字典    
                # 也可以作为参数传递过来，这里就简单化了。{"action":{"module": module, "args": args}, "async": task_time, "poll": 0}])

        play = Play().load(play_source, variable_manager=self.variable_manager, loader=self.loader)

        tqm = None    
        try:    
            tqm = TaskQueueManager(    
                      inventory=self.inv_obj ,    
                      variable_manager=self.variable_manager,    
                      loader=self.loader,    
                      passwords=self.passwords,    
                      stdout_callback=self.results_callback)

            result = tqm.run(play)    
        finally:    
            if tqm is not None:    
                tqm.cleanup()    
            shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)

    def playbook(self,playbooks):    
        """    
        Keyword arguments:    
        playbooks --  需要是一个列表类型    
        """    
        from ansible.executor.playbook_executor import PlaybookExecutor

        playbook = PlaybookExecutor(playbooks=playbooks,    
                        inventory=self.inv_obj,    
                        variable_manager=self.variable_manager,    
                        loader=self.loader,    
                        passwords=self.passwords)

        # 使用回调函数    
        playbook._tqm._stdout_callback = self.results_callback

        result = playbook.run()

    
    def get_result(self):    
      result_raw = {'success':{},'failed':{},'unreachable':{}}

      # print(self.results_callback.host_ok)    
      for host,result in self.results_callback.host_ok.items():    
          result_raw['success'][host] = result._result    
      for host,result in self.results_callback.host_failed.items():    
          result_raw['failed'][host] = result._result    
      for host,result in self.results_callback.host_unreachable.items():    
          result_raw['unreachable'][host] = result._result

      # 最终打印结果，并且使用 JSON 继续格式化    
      print(json.dumps(result_raw, indent=4))    
          
      return json.dumps(result_raw)

[root@dev-technology-215l shell]# cat ansible_run_websocket-fastapi.py   
#!/usr/bin/python  
# -*- coding:utf-8 -*-

import subprocess  
import urllib.request

from fastapi import FastAPI  
import platform  
import socket,requests  
from ansible2 import *     #引用修改过的 ansible2.py 的所有模块  
import ansible_runner  
import os, sys, json, datetime, time  
import urllib.request

from fastapi import FastAPI

#check_sys = platform.system()  
#print(check_sys) // 查看系统类型

  
#project = 'sk_service_data_convert'  
#Jenvironment = 'std'

  
def project_host(project,Jenvironment):  
    js = urllib.request.urlopen('http://config.xxxx.com/index.php?apiykpjenkinsykp{tenv}ykp{tprojectname}'.format(tenv=Jenvironment, tprojectname=project))  #config.xxxx.com 是我的其他接口
    js2 = js.read()  
    hjson = json.loads(js2.decode('utf-8'))  
    #port = hjson['info']['service_port']  
    #print(port)

    host = hjson['info']['hoststr']  
    print(host)  
    hostlist = host.split(",")  
    print(hostlist)  
    return hostlist

def ansible_project_pid(project,Jenvironment):  
    sock22 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  
    sock22result = sock22.connect_ex((project_host(project, Jenvironment)[0], 22))

    if sock22result == 0:  #如果是 22 端口开着代表是 linux
        ansible3 = MyAnsiable2(inventory='/data/ansible/host/hosts', connection='smart')

        # ansible3.run(hosts= "192.168.0.94", module="shell", args='source /etc/profile && pm2 ls|grep sk_service_design_calculate_rda')  
        ansible3.run(hosts=project_host(project, Jenvironment)[0], module="shell",  
                     args='source /etc/profile && pm2 ls|grep {tproject}_{tenv}|awk {tprint}'.format(tproject=project,tenv=Jenvironment,  
                         tprint="'{print $12}'"))

        stdout_dict = json.loads(ansible3.get_result())  
        print(stdout_dict, type(stdout_dict))  
        print(stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout'])  
        pid_list = stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout'].split("\n")

        print(pid_list)

        project_pid = subprocess.getoutput('ps -ef | grep {tproject}_{tenv} | grep -v grep | awk {pri}'.format(tproject=project,tenv=Jenvironment, pri="'{print $2}'"))  
        # project_pid_int = int(project_pid)  
    else:  
        ansible3 = MyAnsiable2(inventory='/data/ansible/host/hosts', connection='smart')

        # ansible3.run(hosts= "192.168.0.94", module="shell", args='source /etc/profile && pm2 ls|grep sk_service_design_calculate_rda')  
        ansible3.run(hosts=project_host(project, Jenvironment)[0], module="raw",  
                     args="cmd /C'pm2 ls|grep {tproject}_{tenv} |grep online'".format(tproject=project,tenv=Jenvironment))

        stdout_dict = json.loads(ansible3.get_result())  
        # print(stdout_dict,type(stdout_dict))  
        # print(stdout_dict['success']['192.168.0.42']['stdout'])  
        pm2_pid_list = stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout']  
        print(pm2_pid_list)  
        project_pid = subprocess.getoutput('echo"{tline}"| tr -s"\n"| awk {tpri}'.format(tline=pm2_pid_list, tpri="'{print $12}'")).split("\n")

  
    return project_pid

def ansible_project_instance(project,Jenvironment):  
    #ansible3 = MyAnsiable2(inventory='/data/ansible/host/hosts', connection='smart')

    #ansible3.run(hosts=project_host(project,Jenvironment)[0], module="shell", args='pm2 ls|grep {tproject}_{tenv}|wc -l'.format(tproject=project,tenv=Jenvironment))

    #stdout_dict = json.loads(ansible3.get_result())  
    #print(stdout_dict, type(stdout_dict))  
    #stdout_sum = stdout_dict['success'][project_host(project,Jenvironment)[0]]['stdout']  
    js = urllib.request.urlopen('http://config.xxxx.com/index.php?apiykpjenkinsykp{tenv}ykp{tprojectname}'.format(tenv=Jenvironment,tprojectname=project))  
    js2 = js.read()  
    hjson = json.loads(js2.decode('utf-8'))  
    # port = hjson['info']['service_port']  
    # print(port)

    stdout_sum = hjson['info']['clusternumber']  
    print(stdout_sum)

    return stdout_sum

  
def ansible_project_websocket_port(project,Jenvironment):  
    sock22 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  
    sock22result = sock22.connect_ex((project_host(project, Jenvironment)[0], 22))

    if sock22result == 0:  
        ansible3 = MyAnsiable2(inventory='/data/ansible/host/hosts', connection='smart')

        # ansible3.run(hosts= "192.168.0.94", module="shell", args='source /etc/profile && pm2 ls|grep sk_service_design_calculate_rda')  
        ansible3.run(hosts=project_host(project, Jenvironment)[0], module="shell",  
                     args='source /etc/profile && pm2 ls|grep {tproject}_{tenv}|awk {tprint}'.format(tproject=project,tenv=Jenvironment,  
                         tprint="'{print $12}'"))

        stdout_dict = json.loads(ansible3.get_result())  
        #print(stdout_dict, type(stdout_dict))  
        #print(stdout_dict['success']['192.168.0.94']['stdout'])  
        pid_list = stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout'].split("\n")

        print(pid_list)

        websocket_list = []  
        for i in pid_list:  
            ansible3.run(hosts=project_host(project, Jenvironment)[0], module="shell",  
                         args='netstat -tnlp | grep {tpid} | awk {tprint}'.format(tpid=i, tprint="'{print $4}'"))

            stdout_dict = json.loads(ansible3.get_result())  
            #print(stdout_dict['success']['192.168.0.94']['stdout'])  
            websocket_list.append(stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout'])

        print(websocket_list)

    else:  
        ansible3 = MyAnsiable2(inventory='/data/ansible/host/hosts', connection='smart')

        ansible3.run(hosts=project_host(project, Jenvironment)[0], module="raw",  
                     args="cmd /C'pm2 ls|grep {tproject}_{tenv}|grep online'".format(tproject=project,tenv=Jenvironment))

        stdout_dict = json.loads(ansible3.get_result())  
        pm2_pid_list = stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout']  
        #print(pm2_pid_list)  
        pid_list = subprocess.getoutput('echo"{tline}"| tr -s"\n"| awk {tpri}'.format(tline=pm2_pid_list, tpri="'{print $12}'")).split("\n")

        print(pid_list)

        # pid_list = ['14756', '13012', '10116']

        websocket_list = []  
        for i in pid_list:  
            ansible3.run(hosts=project_host(project, Jenvironment)[0], module="win_shell",  
                         args='netstat -nao | grep 0.0.0.0 | grep {tpid}| awk {tprint}'.format(tpid=i,  
                                                                                               tprint="'{print $2}'"))

            stdout_dict = json.loads(ansible3.get_result())  
            #print(stdout_dict['success']['192.168.0.42']['stdout'])  
            websocket_list.append(stdout_dict['success'][project_host(project, Jenvironment)[0]]['stdout'])

        print(websocket_list)

    return websocket_list

  
app = FastAPI()

  
@app.get("/")  
def read_root():  
    return {"messages": "hellokugou"}

  
@app.get("/items/{item_id}")  
def read_item(item_id: int, q: str = None):  
    return {"item_id": item_id, "q": q}

  
@app.get("/websocket/{Jenvironment}/{project}")  
def read_project(project: str,Jenvironment: str):  
    print(project, '#######################project')  
    print(Jenvironment, '@@@@@@@@@@@@@@env')  
    print(ansible_project_websocket_port(project,Jenvironment),"$$$ansible_project_websocket_port-LIst")

    return {"project_name": project,"env": Jenvironment , "project_host": project_host(project,Jenvironment), "project_running-sum": ansible_project_instance(project,Jenvironment),"project_websocket_port": ansible_project_websocket_port(project,Jenvironment)}

  
if __name__ == '__main__':  
    import uvicorn

    uvicorn.run(app=app,  
                host="192.168.0.215",  
                port=9999,  
                workers=1)

运行结果：

[root@dev-technology-215l shell]# curl http://192.168.0.215:9999/websocket/fct/sk_service_design  
{"project_name":"sk_service_design","env":"fct","project_host":["192.168.0.42","192.168.0.43"],"project_running-sum":"3","project_websocket_port":["0.0.0.0:9638\r\n","0.0.0.0:9639\r\n","0.0.0.0:9640\r\n"]}[root@dev-technology-215l shell]#   
[root@dev-technology-215l shell]#

[root@dev-technology-215l shell]# python ansible_run_websocket-fastapi.py    
['192.168.0.42', '192.168.0.43']  
{  
    "success": {  
        "192.168.0.42": {  
            "start": "2020-07-07 10:05:45.305889",  
            "stdout": "0.0.0.0:9639\r\n",  
            "cmd": "netstat -nao | grep 0.0.0.0 | grep 7976| awk'{print $2}'","stderr":"",  
            "changed": true,  
            "rc": 0,  
            "delta": "0:00:00.546910",  
            "end": "2020-07-07 10:05:45.852800",  
            "stdout_lines": ["0.0.0.0:9639"],  
            "stderr_lines": [],  
            "_ansible_no_log": false  
        }  
    },  
    "failed": {},  
    "unreachable": {}}  
192.168.0.42,192.168.0.43  
['192.168.0.42', '192.168.0.43']  
192.168.0.42,192.168.0.43  
['192.168.0.42', '192.168.0.43']  
{  
    "success": {  
        "192.168.0.42": {  
            "start": "2020-07-07 10:05:47.243515",  
            "stdout": "0.0.0.0:9640\r\n",  
            "cmd": "netstat -nao | grep 0.0.0.0 | grep 13960| awk'{print $2}'","stderr":"",  
            "changed": true,  
            "rc": 0,  
            "delta": "0:00:00.562535",  
            "end": "2020-07-07 10:05:47.806051",  
            "stdout_lines": ["0.0.0.0:9640"],  
            "stderr_lines": [],  
            "_ansible_no_log": false  
        }  
    },  
    "failed": {},  
    "unreachable": {}}  
192.168.0.42,192.168.0.43  
['192.168.0.42', '192.168.0.43']  
['0.0.0.0:9638\r\n', '0.0.0.0:9639\r\n', '0.0.0.0:9640\r\n']  
INFO:     192.168.0.215:46513 - "GET /websocket/fct/sk_service_design HTTP/1.1" 200 OK

正文完