乐趣区

关于python:python日志logging配置

jiezi

python 日志 logging 配置

  • 为了不便 ELK 收集日志，将日志打印成 json 格局
  • 开发过程中，应用 json 格局不不便排查问题
  • 本文章应用 python 的 logging 模块，一步步减少配置，来阐明每个组件作用

原始日志

  • python 能够应用两种形式打印
  • python 默认打印级别为 ’WARNING’
import logging

# 间接应用 logging 打印日志
logging.info('info')  # 未打印
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 应用 logger 打印日志
logger = logging.getLogger('demo')
logger.info('info')  # 未打印
logger.warning('warning')  # WARNING:demo:warning
logger.error('error')  # ERROR:demo:error

应用字典配置

  • version 为固定字段，目前为 1
  • 将级别设置为 ’INFO’
import logging
import logging.config


def use_config():
    """配置日志行为"""
    config = {"version": 1, "root": {"level": "INFO"}}
    logging.config.dictConfig(config)


use_config()

# 间接应用 logging 打印日志
logging.info('info')  # INFO:root:info
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 应用 logger 打印日志
logger = logging.getLogger('demo')
logger.info('info')  # INFO:demo:info
logger.warning('warning')  # WARNING:demo:warning
logger.error('error')  # ERROR:demo:error

记录器

  • “loggers” 为指定记录器

  • “root” 本质上也是记录器，python 默认的记录器

    • logging 间接打印日志，相当于 logging.getLogger(‘root’)
    • 没有找到记录器名字的应用默认记录器（即 ’root’）
import logging
import logging.config


def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {"level": "WARNING"},
        "loggers": {
            "demo": {"level": "ERROR"}
        }
    }
    logging.config.dictConfig(config)


use_config()

# 间接应用 logging 打印日志
logging.info('info')  # 未打印
logging.warning('warning')  # WARNING:root:warning
logging.error('error')  # ERROR:root:error

# 应用 logger 打印日志
logger = logging.getLogger('demo')
logger.info('info')  # 未打印
logger.warning('warning')  # 未打印
logger.error('error')  # ERROR:demo:error

# 未找到记录器时，应用 'root' 记录器
logger2 = logging.getLogger('demo2')
logger2.info('info')  # 未打印
logger2.warning('warning')  # WARNING:demo2:warning
logger2.error('error')  # ERROR:demo2:error

处理器及格式化器

  • “handlers” 为处理器

    • 处理器有很多类型，能够打印屏幕、打印文件、发送邮件、发送 http 申请等，详情查看 Useful Handlers 官网文档
    • 记录器中能够指定多个处理器
    • 上面例子打印屏幕

  • “formatters” 为格式化器

    • format：应用 python 内置格式化器格式化日志信息 
import logging
import logging.config


def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            }
        }
    }
    logging.config.dictConfig(config)


use_config()

logging.info('info')  # 2022-10-25 20:16:59,073 - root - INFO - info
logging.warning('warning')  # 2022-10-25 20:16:59,073 - root - WARNING - warning
logging.error('error')  # 2022-10-25 20:16:59,074 - root - ERROR - error

自定义格式化器

  • 自定义格式化器继承 logging.Formatter，重写 format 办法

  • 配置中，字段名应用 ”class” 代表自定义格式化器

    • 填入导入该类的字符串，如下代码，在 ”log_demo.py” 文件里的 ”MyFormatter”，即为 ”log_demo.MyFormatter”
    • 类型导入会执行一遍代码，反复调用配置办法会增加多个处理器，导致打印多遍
    • record 中有很多字段，这里不展现，自行打印查看 
# log_demo.py
import logging
import logging.config


class MyFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        """重写日志格式化字符串"""
        return f"[{record.name}] [{record.levelname}] {record.msg}"


def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {"class": "log_demo.MyFormatter"}
        }
    }
    logging.config.dictConfig(config)


if __name__ == "__main__":
    """
    实例化 MyFormatter 类时，会执行一遍代码
    将代码放在__main__里，否则会打印两遍
    """
    use_config()

    logging.info('info') # [root] [INFO] info
    logging.warning('warning') # [root] [WARNING] warning
    logging.error('error') # [root] [ERROR] error

extra 额定字段

  • 日志办法中有 ”extra” 字段，丰盛日志信息
  • 打印 
import logging
import logging.config


class MyFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        """重写日志格式化字符串"""
        for i in record.__dict__:
            print(i, end=',')
        print('')
        return f"[{record.name}] [{record.levelname}] {record.msg}"


def use_config():
    """配置日志行为"""
    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {
            "console_formatter": {"class": "log_demo.MyFormatter"}
        }
    }
    logging.config.dictConfig(config)


if __name__ == "__main__":
    """
    实例化 MyFormatter 类时，会执行一遍代码
    将代码放在__main__里，否则会打印两遍
    """
    use_config()

    logging.info('info')
    # name,msg,args,levelname,levelno,pathname,filename,module,exc_info,exc_text,stack_info,lineno,funcName,created,msecs,relativeCreated,thread,threadName,processName,process,
    # [root] [INFO] info

    # extra 减少了 'a','b'，能够看到 record 对象也减少了 'a','b' 两个字段
    logging.info('extra info', extra={'a': 1, 'b': 2})
    # name,msg,args,levelname,levelno,pathname,filename,module,exc_info,exc_text,stack_info,lineno,funcName,created,msecs,relativeCreated,thread,threadName,processName,process,a,b,
    # [root] [INFO] extra info

最终代码

  • 减少环境变量 ”LOG_MODE”，用来管制打印行为

    • 为 ”json” 时，打印 json 格局字符串，提供给 ELK 收集
    • 为 ”text” 时，打印成文本模式，不便开发时查看
  • json 转换时加上 ”ensure_ascii=False”，否则中文会被转换为 ASCII

代码 

import os
import json
import logging
import logging.config
from datetime import datetime, timezone, timedelta


class JsonFormatter(logging.Formatter):
    __TZ = timezone(timedelta(hours=+8))

    def format(self, record: logging.LogRecord) -> str:
        # ELK 收集须要应用带时区的工夫戳，key 必须为 "@timestamp"
        create_time = datetime.fromtimestamp(record.created)
        kv = {'@timestamp': create_time.astimezone(self.__TZ).isoformat()}
        if len(record.args) > 0:
            kv['message'] = record.msg % record.args
        else:
            kv['message'] = record.msg
        # 报错信息
        if record.exc_info:
            kv['err_info'] = self.formatException(record.exc_info)

        for k in record.__dict__:
            v = getattr(record, k)
            # 其余类型 json 转换可能报错
            if v is None or isinstance(v, (int, float, bool, str)):
                kv[k] = v
            else:
                kv[k] = str(v)
        return json.dumps(kv, ensure_ascii=False)


def use_config():
    """配置日志行为"""
    if os.environ.get('LOG_MODE') == "json":
        format_data = {"class": "log_demo.JsonFormatter"}
    else:
        format_data = {"format": "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s"
        }

    config = {
        "version": 1,
        "root": {
            "level": "INFO",
            "handlers": ["console_handler"]
        },
        "handlers": {
            "console_handler": {
                "class": "logging.StreamHandler",
                "formatter": "console_formatter",
                "stream": "ext://sys.stdout"
            }
        },
        "formatters": {"console_formatter": format_data}
    }

    logging.config.dictConfig(config)


if __name__ == "__main__":
    use_config()

    logging.info('info')
    logging.info('extra info', extra={'a': 1, 'b': 2})

text 成果 

# linux 设置环境变量
export LOG_MODE=text
# windows 设置环境变量
set LOG_MODE=text

python log_demo.py
[2022-10-26 19:11:07,468] [root] [INFO] info
[2022-10-26 19:11:07,468] [root] [INFO] extra info

json 成果 

# linux 设置环境变量
export LOG_MODE=json
# windows 设置环境变量
set LOG_MODE=json

python log_demo.py
{"@timestamp": "2022-10-26T20:10:42.168191+08:00", "message": "info", "name": "root", "msg": "info", "args": "()", "levelname": "INFO", "levelno": 20, "pathname": "e:\\Work\\Code\\TestCode\\everything\\log_demo.py", "filename": "log_demo.py", "module": "log_demo", "exc_info": null, "exc_text": null, "stack_info": null, "lineno": 66, "funcName": "<module>", "created": 1666786242.168191, "msecs": 168.19095611572266, "relativeCreated": 35.9044075012207, "thread": 22716, "threadName": "MainThread", "processName": "MainProcess", "process": 19576}
{"@timestamp": "2022-10-26T20:10:42.169210+08:00", "message": "extra info", "name": "root", "msg": "extra info", "args": "()", "levelname": "INFO", "levelno": 20, "pathname": "e:\\Work\\Code\\TestCode\\everything\\log_demo.py", "filename": "log_demo.py", "module": "log_demo", "exc_info": null, "exc_text": null, "stack_info": null, "lineno": 67, "funcName": "<module>", "created": 1666786242.1692102, "msecs": 169.21019554138184, "relativeCreated": 36.92364692687988, "thread": 22716, "threadName": "MainThread", "processName": "MainProcess", "process": 19576, "a": 1, "b": 2}
退出移动版