在Python中使用GStreamer流式传输Opencv视频捕获帧用于网络摄像头

Question

我正在尝试通过网络传输视频捕获。 我为此使用了 fastapi 和 uvicorn，效果很好，但现在我正在转向无线网络，并且网络无法处理流，我的速度为 2-3fps，有 5 秒的延迟。 我读到 gstreamer 是流式传输帧的最佳方式，尽管我在流的接收端需要一个解码器。

这是我的发件人：

Sender.py

import time
import cv2

fps = 52
frame_width = 640
frame_height = 360
flip = 0
camSet='v4l2src device=/dev/video0 ! video/x-raw,width=640,height=360 ! nvvidconv flip-method='+str(flip)+' \
        ! video/x-raw(memory:NVMM), format=I420, width=640, height=360 ! nvvidconv ! video/x-raw, format=BGRx ! videoconvert \
        ! video/x-raw, format=BGR enable-max-performance=1 ! appsink '
cam=cv2.VideoCapture(camSet,cv2.CAP_GSTREAMER)

    gst_str_rtp = " appsrc ! videoconvert ! videoscale ! video/x-raw,format=I420,width=640,height=360,framerate=52/1 !  videoconvert !\
     x264enc tune=zerolatency bitrate=500 speed-preset=superfast ! rtph264pay ! \
udpsink host=0.0.0.0 port=8000"

if cam.isOpened() is not True:
    print("Cannot open camera. Exiting.")
    quit()
    
fourcc = cv2.VideoWriter_fourcc(*'H264')
out = cv2.VideoWriter(gst_str_rtp, fourcc, 52, (frame_width, frame_height), True)

while True:
    ret, frame = cam.read()

    cv2.imshow('webcam',frame)
    out.write(frame)
    cv2.moveWindow('webcam',0,0)
    if cv2.waitKey(1)==ord('q'):
        break

cam.release()
out.release()
cv2.destroyAllWindows()

我有一些输出管道示例，但我不确定哪个有效， 像这样：

gst_str_rtp ="appsrc ! videoconvert ! video/x-raw,width=1280,height=720 ! queue ! x264enc ! h264parse\
     ! rtph264pay ! udpsink host=127.0.0.1 port=8000"

或者

gst_str_rtp = "appsrc ! video/x-raw, format=I420 ! queue ! videoconvert ! \
      width=640,height=360,framerate=52/1 ! nvvidconv ! omxh264enc ! \
          video/x-h264, stream-format=byte-stream ! h264parse ! rtph264pay pt=96 config-interval=1 ! \
        udpsink host=127.0.0.1 port=8000"

我的接收器是这样的： receiver.py

global video_frame
video_frame = None

cv2.namedWindow('stream')

# Use locks for thread-safe viewing of frames in multiple browsers
# locks prevent the program from getting user inputs from multiple sources
# lock helps syncronize the program
global thread_lock 
thread_lock = threading.Lock()

app = FastAPI()

def main():
    global video_frame
    camSet='udpsrc host=0.0.0.0 port=8000 caps = "application/x-rtp, media=(string)video, clock-rate=(int)90000, \
        encoding-name=(string)H264,\
         payload=(int)96" ! rtph264depay ! decodebin ! videoconvert ! appsink'
    video_capture = cv2.VideoCapture(camSet ,cv2.CAP_GSTREAMER)#initalizes our video feed
    # video_capture.set(cv2.CAP_PROP_BUFFERSIZE, 0)#sets camera buffer to 0 meaning we wont get double frames
    print("reading frames")

    while True:
        
        ret, frame = video_capture.read()

        if not ret:
            print('empty frame')
            break

        cv2.imshow(frame,'stream')
        # with thread_lock:
        #     video_frame = frame.copy()



def encodeFrame():#encode frame function takes the frames and encoding them to jpg and encodes them again to bytes so we can stream them
    global thread_lock 
    while True:
        # Acquire thread_lock to access the global video_frame object
        with thread_lock:
            global video_frame
            if video_frame is None:
                continue
            return_key, encoded_image = cv2.imencode(".jpg", video_frame)
            if not return_key:
                continue

        # Output image as a byte array
        yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + 
            bytearray(encoded_image) + b'\r\n')


@app.get("/")
async def video_feed():#our stream function, streams the frames encodeFrame has made to the IP and port we chose
    return StreamingResponse(encodeFrame(), media_type="multipart/x-mixed-replace;boundary=frame")



if __name__ == '__main__':
    # Create a thread and attach the method that captures the image frames, to it
    process_thread = threading.Thread(target=main)

    # Start the thread
    process_thread.start()
    uvicorn.run(app, host="0.0.0.0", port=9000, access_log=True)

我正在使用flask和uvicorn将图像重新传输到我的java服务器可以读取它们。 我的发送器编码或接收器解码不起作用，或者两者都错误，我不确定哪个，或者即使我走的是正确的路，我对 gstreamer 和管道还相当陌生。 我将不胜感激任何对此的帮助，并且还需要不需要我重新播放的建议。

Answer 1

不确定这是否能解决您的问题，但以下内容可能会有所帮助：

1. 相机捕获中似乎存在拼写错误，其中 enable-max-performance=1 不适用于视频上限。这个项目更像是一个插件的属性（可能来自编码器）。如果您的相机驱动程序提供具有此分辨率的其他帧速率，则最好设置帧速率，否则您将面临与写入器 fps 不匹配的情况。

camSet='v4l2src device=/dev/video0 ! video/x-raw,width=640,height=360,framerate=52/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw(memory:NVMM), format=I420, width=640, height=360 ! nvvidconv ! video/x-raw, format=BGRx ! videoconvert ! video/x-raw, format=BGR ! queue ! appsink drop=1'

2. 多播可能会占用 wifi。更好地仅流式传输到您的接收器：

... ! udpsink host=<receiver_IP> port=8000 auto-multicast=0

您只需在接收器主机上接收：

'udpsrc port=8000 auto-multicast=0 ! application/x-rtp,media=video,encoding-name=H264 ! rtpjitterbuffer latency=300 ! rtph264depay ! decodebin ! videoconvert ! video/x-raw,format=BGR ! appsink drop=1'

# Variant for NVIDIA:
'udpsrc port=8000 auto-multicast=0 ! application/x-rtp,media=video,encoding-name=H264 ! rtpjitterbuffer latency=300 ! rtph264depay ! decodebin ! nvvidconv ! video/x-raw,format=BGRx ! videoconvert ! video/x-raw,format=BGR ! appsink drop=1'

3. 使用 gstreamer 后端时，请注意 4cc 代码是无用的。
   您可以使用硬件加速编码器，例如：

gst_str_rtp = "appsrc ! video/x-raw,format=BGR ! queue ! videoconvert ! video/x-raw,format=BGRx ! nvvidconv ! video/x-raw(memory:NVMM),format=NV12,width=640,height=360,framerate=52/1 ! nvv4l2h264enc insert-sps-pps=1 insert-vui=1 idrinterval=30 ! h264parse ! rtph264pay ! udpsink host=<receiver_IP> port=8000 auto-multicast=0"
out = cv2.VideoWriter(gst_str_rtp, cv2.CAP_GSTREAMER, 0, float(52), (frame_width, frame_height), True)