Python，想要通过日志轮转和压缩进行日志记录

Question

任何人都可以建议一种在 python 中进行日志记录的方法：

• 每天
• 进行日志轮换 轮换时压缩日志
• 可选 - 删除最旧的日志文件以保留 X MB 的可用空间
• 可选 - sftp 日志文件到服务器

感谢您的任何回复， 弗雷德

Answer 1

• 每天进行日志轮换：使用TimedRotatingFileHandler
• 日志压缩：设置encoding='bz2'参数。 （请注意，此“技巧”仅适用于 Python2。“bz2”不再被视为 Python3 中的编码。）
• 可选 - 删除最旧的日志文件以保留 X MB 的可用空间。
  您可以（间接）使用 RotatingFileHandler 进行安排。通过设置 maxBytes 参数，日志文件达到一定大小时将滚动。通过设置 backupCount 参数，您可以控制保留的翻转次数。这两个参数一起允许您控制日志文件消耗的最大空间。您也可以将 TimeRotatingFileHandler 子类化以将此行为合并到其中。

---

只是为了好玩，以下是如何子类化 TimeRotatingFileHandler。当您运行下面的脚本时，它会将日志文件写入 /tmp/log_rotate*。

如果 time.sleep 值较小（例如 0.1），日志文件会很快填满，达到 maxBytes 限制，然后滚动。

如果 time.sleep 较大（例如 1.0），日志文件会慢慢填满，未达到 maxBytes 限制，但当达到定时间隔（10 秒）时，它们无论如何都会滚动。

下面的所有代码均来自 logging/handlers.py 。我只是以最直接的方式将 TimeRotatingFileHandler 与 RotatingFileHandler 结合起来。

import time
import re
import os
import stat
import logging
import logging.handlers as handlers


class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler):
    """
    Handler for logging to a set of files, which switches from one file
    to the next when the current file reaches a certain size, or at certain
    timed intervals
    """

    def __init__(self, filename, maxBytes=0, backupCount=0, encoding=None,
                 delay=0, when='h', interval=1, utc=False):
        handlers.TimedRotatingFileHandler.__init__(
            self, filename, when, interval, backupCount, encoding, delay, utc)
        self.maxBytes = maxBytes

    def shouldRollover(self, record):
        """
        Determine if rollover should occur.

        Basically, see if the supplied record would cause the file to exceed
        the size limit we have.
        """
        if self.stream is None:                 # delay was set...
            self.stream = self._open()
        if self.maxBytes > 0:                   # are we rolling over?
            msg = "%s\n" % self.format(record)
            # due to non-posix-compliant Windows feature
            self.stream.seek(0, 2)
            if self.stream.tell() + len(msg) >= self.maxBytes:
                return 1
        t = int(time.time())
        if t >= self.rolloverAt:
            return 1
        return 0


def demo_SizedTimedRotatingFileHandler():
    log_filename = '/tmp/log_rotate'
    logger = logging.getLogger('MyLogger')
    logger.setLevel(logging.DEBUG)
    handler = SizedTimedRotatingFileHandler(
        log_filename, maxBytes=100, backupCount=5,
        when='s', interval=10,
        # encoding='bz2',  # uncomment for bz2 compression
    )
    logger.addHandler(handler)
    for i in range(10000):
        time.sleep(0.1)
        logger.debug('i=%d' % i)

demo_SizedTimedRotatingFileHandler()

Answer 2

在旋转期间压缩日志文件的另一种方法（Python 3.3 中的新功能）是使用 BaseRotatingHandler（以及所有继承的）类属性 rotator 例如：

import gzip
import os
import logging
import logging.handlers

class GZipRotator:
    def __call__(self, source, dest):
        os.rename(source, dest)
        f_in = open(dest, 'rb')
        f_out = gzip.open("%s.gz" % dest, 'wb')
        f_out.writelines(f_in)
        f_out.close()
        f_in.close()
        os.remove(dest)

logformatter = logging.Formatter('%(asctime)s;%(levelname)s;%(message)s')
log = logging.handlers.TimedRotatingFileHandler('debug.log', 'midnight', 1, backupCount=5)
log.setLevel(logging.DEBUG)
log.setFormatter(logformatter)
log.rotator = GZipRotator()

logger = logging.getLogger('main')
logger.addHandler(log)    
logger.setLevel(logging.DEBUG)

....

更多内容，您可以参阅 此处。

Answer 3

除了 unutbu 的答案之外：以下是如何修改 TimedRotatingFileHandler 以使用 zip 文件进行压缩。

import logging
import logging.handlers
import zipfile
import codecs
import sys
import os
import time
import glob


class TimedCompressedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
    """
    Extended version of TimedRotatingFileHandler that compress logs on rollover.
    """
    def doRollover(self):
        """
        do a rollover; in this case, a date/time stamp is appended to the filename
        when the rollover happens.  However, you want the file to be named for the
        start of the interval, not the current time.  If there is a backup count,
        then we have to get a list of matching filenames, sort them and remove
        the one with the oldest suffix.
        """

        self.stream.close()
        # get the time that this sequence started at and make it a TimeTuple
        t = self.rolloverAt - self.interval
        timeTuple = time.localtime(t)
        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
        if os.path.exists(dfn):
            os.remove(dfn)
        os.rename(self.baseFilename, dfn)
        if self.backupCount > 0:
            # find the oldest log file and delete it
            s = glob.glob(self.baseFilename + ".20*")
            if len(s) > self.backupCount:
                s.sort()
                os.remove(s[0])
        #print "%s -> %s" % (self.baseFilename, dfn)
        if self.encoding:
            self.stream = codecs.open(self.baseFilename, 'w', self.encoding)
        else:
            self.stream = open(self.baseFilename, 'w')
        self.rolloverAt = self.rolloverAt + self.interval
        if os.path.exists(dfn + ".zip"):
            os.remove(dfn + ".zip")
        file = zipfile.ZipFile(dfn + ".zip", "w")
        file.write(dfn, os.path.basename(dfn), zipfile.ZIP_DEFLATED)
        file.close()
        os.remove(dfn)

if __name__=='__main__':
    ## Demo of using TimedCompressedRotatingFileHandler() to log every 5 seconds,
    ##     to one uncompressed file and five rotated and compressed files

    os.nice(19)   # I always nice test code

    logHandler = TimedCompressedRotatingFileHandler("mylog", when="S",
        interval=5, backupCount=5) # Total of six rotated log files, rotating every 5 secs
    logFormatter = logging.Formatter(
        fmt='%(asctime)s.%(msecs)03d %(message)s', 
        datefmt='%Y-%m-%d %H:%M:%S'
        )
    logHandler.setFormatter(logFormatter)
    mylogger = logging.getLogger('MyLogRef')
    mylogger.addHandler(logHandler)
    mylogger.setLevel(logging.DEBUG)

    # Write lines non-stop into the logger and rotate every 5 seconds
    ii = 0
    while True:
        mylogger.debug("Test {0}".format(ii))
        ii += 1

Answer 4

请注意：Python 3 中的类签名已更改。这是我的 Python 3.6 工作示例

import logging.handlers
import os
import zlib


def namer(name):
    return name + ".gz"


def rotator(source, dest):
    print(f'compressing {source} -> {dest}')
    with open(source, "rb") as sf:
        data = sf.read()
        compressed = zlib.compress(data, 9)
        with open(dest, "wb") as df:
            df.write(compressed)
    os.remove(source)


err_handler = logging.handlers.TimedRotatingFileHandler('/data/errors.log', when="M", interval=1,
                                                        encoding='utf-8', backupCount=30, utc=True)
err_handler.rotator = rotator
err_handler.namer = namer

logger = logging.getLogger("Rotating Log")
logger.setLevel(logging.ERROR)

logger.addHandler(err_handler)

Answer 5

我想现在加入派对已经太晚了，但这就是我所做的。我创建了一个继承 logging.handlers.RotatingFileHandler 类的新类，并在移动文件之前添加了几行以 gzip 文件。

https://github.com/rkreddy46/python_code_reference/blob/master/compressed_log_rotator。 py

#!/usr/bin/env python

# Import all the needed modules
import logging.handlers
import sys
import time
import gzip
import os
import shutil
import random
import string

__version__ = 1.0
__descr__ = "This logic is written keeping in mind UNIX/LINUX/OSX platforms only"


# Create a new class that inherits from RotatingFileHandler. This is where we add the new feature to compress the logs
class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def doRollover(self):
        """
        Do a rollover, as described in __init__().
        """
        if self.stream:
            self.stream.close()
        if self.backupCount > 0:
            for i in range(self.backupCount - 1, 0, -1):
                sfn = "%s.%d.gz" % (self.baseFilename, i)
                dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
                if os.path.exists(sfn):
                    # print "%s -> %s" % (sfn, dfn)
                    if os.path.exists(dfn):
                        os.remove(dfn)
                    os.rename(sfn, dfn)
            dfn = self.baseFilename + ".1.gz"
            if os.path.exists(dfn):
                os.remove(dfn)
            # These two lines below are the only new lines. I commented out the os.rename(self.baseFilename, dfn) and
            #  replaced it with these two lines.
            with open(self.baseFilename, 'rb') as f_in, gzip.open(dfn, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
            # os.rename(self.baseFilename, dfn)
            # print "%s -> %s" % (self.baseFilename, dfn)
        self.mode = 'w'
        self.stream = self._open()

# Specify which file will be used for our logs
log_filename = "/Users/myname/Downloads/test_logs/sample_log.txt"

# Create a logger instance and set the facility level
my_logger = logging.getLogger()
my_logger.setLevel(logging.DEBUG)

# Create a handler using our new class that rotates and compresses
file_handler = CompressedRotatingFileHandler(filename=log_filename, maxBytes=1000000, backupCount=10)

# Create a stream handler that shows the same log on the terminal (just for debug purposes)
view_handler = logging.StreamHandler(stream=sys.stdout)

# Add all the handlers to the logging instance
my_logger.addHandler(file_handler)
my_logger.addHandler(view_handler)

# This is optional to beef up the logs
random_huge_data = "".join(random.choice(string.ascii_letters) for _ in xrange(10000))

# All this code is user-specific, write your own code if you want to play around
count = 0
while True:
    my_logger.debug("This is the message number %s" % str(count))
    my_logger.debug(random_huge_data)
    count += 1
    if count % 100 == 0:
        count = 0
        time.sleep(2)

Answer 6

这是我的解决方案（修改自 evgenek），简单并且在 gzip 压缩巨大日志文件时不会阻止 python 代码：

class GZipRotator:
    def __call__(self, source, dest):
        os.rename(source, dest)
        subprocess.Popen(['gzip', dest])

Answer 7

我看到很多答案都集中在重写处理程序类的 doRollover 方法，该方法很好并且有效，但我认为更干净且推荐的解决方案是定义 rotator 属性在处理程序类上，该类在 rotate 方法中进行检查，而不是覆盖 doRollover。

def setup_logger():
    def bz2namer(name):
        return name + ".bz2"

    def bzip_rotator(source, dest):
        with open(source, "rb") as sf:
            data = sf.read()
            compressed = bz2.compress(data, 9)
            with open(dest, "wb") as df:
                df.write(compressed)
        os.remove(source)

    log = logging.getLogger("myapp")
    file_handler = RotatingFileHandler(filename="app.log", backupCount=10, maxBytes=10000000)
    file_handler.rotator = bzip_rotator
    file_handler.namer = bz2namer

    _log_format = "%(asctime)s [%(process)d] [%(levelname)s] [%(filename)s: %(funcName)s] - %(message)s "
    log_formatter = logging.Formatter(_log_format, datefmt="%Y-%m-%d %H:%M:%S")
    file_handler.setFormatter(log_formatter)
    file_handler.setLevel(logging.DEBUG)
    log.propagate = False
    log.addHandler(file_handler)

Answer 8

我认为最好的选择是使用 TimedRotatingFileHandler 的当前实现，并将日志文件重命名为旋转版本后只需压缩它：

import zipfile
import os
from logging.handlers import TimedRotatingFileHandler


class TimedCompressedRotatingFileHandler(TimedRotatingFileHandler):
    """
    Extended version of TimedRotatingFileHandler that compress logs on rollover.
    """
    def find_last_rotated_file(self):
        dir_name, base_name = os.path.split(self.baseFilename)
        file_names = os.listdir(dir_name)
        result = []
        prefix = '{}.20'.format(base_name)  # we want to find a rotated file with eg filename.2017-12-12... name
        for file_name in file_names:
            if file_name.startswith(prefix) and not file_name.endswith('.zip'):
                result.append(file_name)
        result.sort()
        return result[0]

    def doRollover(self):
        super(TimedCompressedRotatingFileHandler, self).doRollover()

        dfn = self.find_last_rotated_file()
        dfn_zipped = '{}.zip'.format(dfn)
        if os.path.exists(dfn_zipped):
            os.remove(dfn_zipped)
        with zipfile.ZipFile(dfn_zipped, 'w') as f:
            f.write(dfn, dfn_zipped, zipfile.ZIP_DEFLATED)
        os.remove(dfn)

Answer 9

要复制文件，请对复制的文件进行 gzip（使用纪元时间），然后以不会扰乱日志记录模块的方式清除现有文件：

import gzip
import logging
import os
from shutil import copy2
from time import time

def logRoll(logfile_name):
    log_backup_name = logfile_name + '.' + str(int(time()))
    try:
        copy2(logfile_name, log_backup_name)   
    except IOError, err:
        logging.debug(' No logfile to roll')
        return
    f_in = open(log_backup_name, 'rb')
    f_out = gzip.open(log_backup_name + '.gz', 'wb')
    f_out.writelines(f_in)
    f_out.close()
    f_in.close()
    os.remove(log_backup_name)
    f=open(logfile_name, 'w')
    f.close()

Answer 10

我在下面添加了一个解决方案，我基本上是在压缩旧的
备份日志到带有时间戳的 zip 文件。使用一个额外的变量
称为 ZipbackupCount。 # 没有要压缩的旧文件

例如我们有这样的日志。 （备份计数 = 5，ZipbackupCount = 2）

a.log.1

a.log.2

a.log.3

a.log.4

a.log.11-09-2020-11-11-11.zip

一旦备份日志计数达到 5，它就会触发将 a.log.5 和 a.log.4 压缩到以上 zip 并继续。

import os
import datetime
import gzip
import logging.handlers
import zipfile

from config.config import PROJECT_PATH, LOG_DIR, LOG_FILE_NAME, LOG_FILESIZE


class NewRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def __init__(self, filename, **kws):
        backupCount = kws.get('backupCount', 0)
        self.backup_count = backupCount
        self.ZipbackupCount = kws.pop('ZipbackupCount', 0)
        self.file_name = filename
        self.log_dir = os.path.split(self.file_name)[0]
        self.log_file_name = os.path.split(self.file_name)[-1]

        logging.handlers.RotatingFileHandler.__init__(self, filename, **kws)

    def doArchive(self, old_log):
        with open(old_log) as log:
            with gzip.open(old_log + '.gz', 'wb') as comp_log:
                comp_log.writelines(log)

        os.remove(old_log)

    def doRollover(self):
        super(NewRotatingFileHandler, self).doRollover()

        zip_file_name = self.log_file_name + "." + datetime.datetime.now().strftime("%d-%m-%Y-%H-%M-%S") + ".zip"

        if os.path.exists(self.rotation_filename("%s.%d" % (self.baseFilename, self.backupCount))) and self.ZipbackupCount > 0 and self.file_name:

            with zipfile.ZipFile(os.path.join(self.log_dir, zip_file_name), "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:

                for i in range(self.backupCount, self.backupCount - self.ZipbackupCount, -1):

                    sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i))

                    if os.path.exists(sfn):
                        zf.write(sfn, "%s.%d" % (self.log_file_name, i))
                        os.remove(sfn)
                    else:
                        continue

                zf.close()


# handler = NewRotatingFileHandler(filename=os.path.join(PROJECT_PATH, LOG_DIR, LOG_FILE_NAME),
#                                  maxBytes=LOG_FILESIZE, backupCount=5, ZipbackupCount=2)
#
# handler.doRollover()

Answer 11

从 python 3.3 开始，您可以轻松扩展 BaseRotatingFileHandler 的任何子类来实现 gzipping：

import os
from logging.handlers import TimedRotatingFileHandler
from gzip import open as gzip_open


class CompressingTimedRotatingFileHandler(TimedRotatingFileHandler):
    """TimedRotatingFileHandler with gzip compression on rotate"""

    def rotation_filename(self, default_name: str) -> str:
        """
        Extend the default filename to use gzip-ending.
        :param default_name: The default filename of this handler
        """
        return default_name + ".gz"

    def rotate(self, source: str, dest: str) -> None:
        """
        Rotate the current log
        :param source: The source filename. This is normally the base
                       filename, e.g. 'test.log'
        :param dest:   The destination filename. This is normally
                       what the source is rotated to, e.g. 'test.log.1'.
        """
        # compress source file and write to destination
        with open(source, 'rb') as f_in, gzip_open(dest, 'wb') as f_out:
            f_out.writelines(f_in)
        # delete source file
        os.remove(source)