需要有关 Python RSYNC 脚本的反馈

发布于 2024-11-02 12:11:21 字数 2435 浏览 7 评论 0原文

这个脚本看起来效果很好，但我相信你们中的一些专家可以对其进行优化！

脚本的目的：

“监视”目录中是否有具有特定文件扩展名的新文件，
确保文件未被复制
rsync 将文件同步到远程服务器
rsync 删除文件
脚本持续循环，永远
不会出现互联网/网络中断

不保留任何部分文件远程服务器

导入操作系统
导入子流程
导入时间
导入日志记录
导入日期时间
从系统导入argv

如果 len(argv) < 3：
    exit('请提供两个参数 - 源目标')


LOC_DIR = argv[1]
REM_DIR = argv[2]

POLL_INT = 10
运行中断 = 60
文件扩展 = '.mov'


# 日志记录设置
logging.basicConfig(filename='%s' % os.path.join(LOC_DIR, '%s script.log' % datetime.datetime.now()),level=logging.DEBUG)

# 制作一个简单的打印和记录功能
def printLog(字符串):
    print '%s %s' % (datetime.datetime.now(), string)
    logging.info('%s %s' % (datetime.datetime.now(), string))


# 获取绝对路径的文件
def getFiles(路径):
    返回 [os.path.join(path, entry) 用于 os.listdir(path) 中的条目]


# 检查文件是否仍在复制（文件大小在轮询间隔内已更改）
def checkSize(路径):
    相同=错误
    而相同的是 False：
        printLog("正在处理 '%s'" % os.path.basename(path))
        printLog('等待 %s 秒以进行任何文件大小更改' % POLL_INT)
        size1 = os.path.getsize(路径)
        时间.睡眠(POLL_INT)
        size2 = os.path.getsize(路径)
        如果尺寸 1 == 尺寸 2：
            相同 = 正确
            printLog('文件大小在 %s 秒内保持不变' % POLL_INT)
            返回相同
        别的：
            printLog('检测到文件大小更改。再等待 %s 秒' % POLL_INT)


# 检查文件扩展名是否正确
def checkExt(路径):
    如果路径.endswith(FILE_EXT):
        返回真


# rsync子进程
def rsyncFile(路径):
    printLog("正在同步文件'%s'" % os.path.basename(path))
    尝试：
        命令 = ['rsync', '-a', '--remove-source-files', 路径, REM_DIR]
        p = subprocess.Popen(命令, stdout=subprocess.PIPE)
        对于 p.stdout 中的行：
            printLog("rsync: '%s'" %line)
        p.wait()
        如果 p.returncode == 0:
            printLog('<<< 文件同步成功:) >>>')
        elif p.returncode == 10:
            printLog('****** 请检查您的互联网连接!! ***** Rsync 错误代码: %s' % p.returncode)
        别的：
            printLog('出现问题。错误代码：%s' % p.returncode)
    除了异常 e：
        日志记录.调试(e)


# 主要逻辑
def main():
    all_files = getFiles(LOC_DIR)
    文件=[]
    对于 all_files 中的 f：
        如果检查Ext（f）：
            文件.append(f)
    如果 len(文件) == 1:
        printLog('<<<找到 %s 匹配文件>>' % len(files))
    elif len(文件) > 1：
        printLog('<<<找到 %s 个匹配文件>>' % len(files))
    对于文件中的 f：
        如果检查大小（f）：
            rsyncFile(f)
    printLog('未找到文件。%s 秒后再次检查' % RUN_INT)
    时间.睡眠(RUN_INT)
    printLog('检查文件')
    主要的（）

如果 __name__ == "__main__":


    主要的（）

原文

This script appears to work great, but I'm sure this could be optimised by some of you gurus!

Purpose of script:

"watch" a directory for new files with particular file extension
ensure file is not still being copied
rsync the file to remote server
rsync deletes the file
script loops continuously forever
survives internet/network outage

leaves no partial file on remote server

import os
import subprocess
import time
import logging
import datetime
from sys import argv

if len(argv) < 3:
    exit('Please provide two arguments - Source Destination')


LOC_DIR = argv[1]
REM_DIR = argv[2]

POLL_INT = 10
RUN_INT = 60
FILE_EXT = '.mov'


# logging setup
logging.basicConfig(filename='%s' % os.path.join(LOC_DIR, '%s script.log' % datetime.datetime.now()),level=logging.DEBUG)

# make an easy print and logging function
def printLog(string):
    print '%s %s' % (datetime.datetime.now(), string)
    logging.info('%s %s' % (datetime.datetime.now(), string))


# get the files with absolute paths
def getFiles(path):
    return [os.path.join(path, entry) for entry in os.listdir(path)]


# check if file is still being copied (file size has changed within the poll interval)
def checkSize(path):
    same = False
    while same is False:
        printLog("Processing '%s'" % os.path.basename(path))
        printLog('Waiting %s seconds for any filesize change' % POLL_INT)
        size1 = os.path.getsize(path)
        time.sleep(POLL_INT)
        size2 = os.path.getsize(path)
        if size1 == size2:
            same = True
            printLog('File size stayed the same for %s seconds' % POLL_INT)
            return same
        else:
            printLog('File size change detected. Waiting a further %s seconds' % POLL_INT)


# check if correct file extension
def checkExt(path):
    if path.endswith(FILE_EXT):
        return True


# rsync subprocess
def rsyncFile(path):
    printLog("Syncing file '%s'" % os.path.basename(path))
    try:
        command = ['rsync', '-a', '--remove-source-files', path, REM_DIR]
        p = subprocess.Popen(command, stdout=subprocess.PIPE)
        for line in p.stdout:
            printLog("rsync: '%s'" %line)
        p.wait()
        if p.returncode == 0:
            printLog('<<< File synced successfully :) >>>')
        elif p.returncode == 10:
            printLog('****** Please check your internet connection!! ******  Rsync error code: %s' % p.returncode)
        else:
            printLog('There was a problem. Error code: %s' % p.returncode)
    except Exception as e:
        logging.debug(e)


# main logic
def main():
    all_files = getFiles(LOC_DIR)
    files = []
    for f in all_files:
        if checkExt(f):
            files.append(f)
    if len(files) == 1:
        printLog('<<< Found %s matching file >>>' % len(files))
    elif len(files) > 1:
        printLog('<<< Found %s matching files >>>' % len(files))
    for f in files:
        if checkSize(f):
            rsyncFile(f)
    printLog('No files found.  Checking again in %s seconds' % RUN_INT)
    time.sleep(RUN_INT)
    printLog('Checking for files')
    main()

if __name__ == "__main__":


    main()

分享到QQ

分享到微博

如果你对这篇内容有疑问，欢迎到本站社区发帖提问参与讨论，获取更多帮助，或者扫码二维码加入 Web 技术交流群。

发布评论

需要登录才能够评论，你可以免费注册一个本站的账号。

亽野灬性zι浪 2024-11-09 12:11:21

首先，消除无用的语句

# check if correct file extension
def checkExt(path):
    return path.endswith(FILE_EXT)

，然后变得更加Pythonic。

# rsync subprocess
def rsyncFile(path):
    printLog("Syncing file '%s'" % os.path.basename(path))
    try:
        p = subprocess.Popen(['rsync', '-a', '--remove-source-files', path, REM_DIR], stdout=subprocess.PIPE)
        for line in p.stdout:
            printLog("rsync: '%s'" %line)
        p.wait()
        printlog(
            { 
                0  : '<<< File synced successfully :) >>>',
                10 : '****** Please check your internet connection!! ******  Rsync error code: %s' % p.returncode,
            }.get(p.returncode, '****** Please check your internet connection!! ******  Rsync error code: %s' % p.returncode) # A switch statement in python !
        )
    except:
        logging.exception("An exception occured")

当使用“logging.exception”时，您将显示导致问题的异常和回溯。

然后减少 main 的

def main():
    while True:
        files = [f for f in getFiles(LOC_DIR) if checkExt(f)]
        if len(files) == 1:
            printLog('<<< Found %s matching file >>>' % len(files))
        elif len(files) > 1:
            printLog('<<< Found %s matching files >>>' % len(files))
        for f in files:
            if checkSize(f):
                rsyncFile(f)
        printLog('No files found.  Checking again in %s seconds' % RUN_INT)
        time.sleep(RUN_INT)
        printLog('Checking for files')

“while True:”语句将避免在主代码末尾调用 main() 时可以轻松达到的递归限制，

欢迎评论和评论:)

first, eliminate useless statement

# check if correct file extension
def checkExt(path):
    return path.endswith(FILE_EXT)

then be a little more pythonic

# rsync subprocess
def rsyncFile(path):
    printLog("Syncing file '%s'" % os.path.basename(path))
    try:
        p = subprocess.Popen(['rsync', '-a', '--remove-source-files', path, REM_DIR], stdout=subprocess.PIPE)
        for line in p.stdout:
            printLog("rsync: '%s'" %line)
        p.wait()
        printlog(
            { 
                0  : '<<< File synced successfully :) >>>',
                10 : '****** Please check your internet connection!! ******  Rsync error code: %s' % p.returncode,
            }.get(p.returncode, '****** Please check your internet connection!! ******  Rsync error code: %s' % p.returncode) # A switch statement in python !
        )
    except:
        logging.exception("An exception occured")

When using "logging.exception" you will display the exception and traceback that caused the problem.

then reduce main

def main():
    while True:
        files = [f for f in getFiles(LOC_DIR) if checkExt(f)]
        if len(files) == 1:
            printLog('<<< Found %s matching file >>>' % len(files))
        elif len(files) > 1:
            printLog('<<< Found %s matching files >>>' % len(files))
        for f in files:
            if checkSize(f):
                rsyncFile(f)
        printLog('No files found.  Checking again in %s seconds' % RUN_INT)
        time.sleep(RUN_INT)
        printLog('Checking for files')

the "while True:" statement will avoid recursion limit you could easily reach when calling main() at the end of the main code

comments and remarks are welcome :)

回复收藏 0 原文

~没有更多了~