ESP32-串流显示

让你的ESP32试试串流吧

既然有了屏幕，又有了网络，那岂不是可以串流了！

序

上篇文章已经展示了如何使用ESP32点亮一块屏幕，那么这次我们来整个活。利用ESP32来显示电脑的画面！如果你用的是其他屏幕也没关系，只要是ESP32配合TFT_eSPI就可以实现，只是在帧率上会有所区别。
这个视频为你展示了在M5StickC上的运行效果
现在是除了游戏性以外，一无所有的原神@bilibili

实现方式

电脑作为发送端，负责发送图像数据->EPS32作为接收端，负责接收并绘制图像数据
发送端使用python编写，使用mss模块捕获屏幕画面，再使用python opencv 编码为JPG

接收端使用C++编写，TJpg_Decoder解码，TFT_eSPI绘制

发送的数据以帧为单位，为了减小帧的体积，将会对每一帧原始位图数据使用JPG编码，ESP32接收到数据以后，先对JPG进行解码，再进行绘制。

一次完整通信流程为:

精简的代码

common_macro.h

#ifndef COMMON_MACRO_H_
#define COMMON_MACRO_H_

// Debug情况下，暂时不去测试串流，也无需连接wifi
#define DEBUG


/*
======================串流相关======================
*/
// 帧数据接收完毕
#define FRAMEOK  0x01
// 头部接收完毕，接收帧
#define HEADEROK  0x02
// 准备完毕，接收头部 注:0x03无法正常发送
// https://www.cnblogs.com/young525/p/5873795.html
#define PREPAREOK  0x41


/*
======================屏幕相关======================
*/
#define SCREEN_WIDTH 240
#define SCREEN_HEIGHT 135


/*
======================WiFi相关======================
*/
#define ssid "CloseWrt_2.5G"
// WiFi 密码
#define password "have5seeds"

#endif 

StreamingComponent.h

#ifndef STREAMINGCOMPONENT_H_
#define STREAMINGCOMPONENT_H_

#define IDLES 0
#define RUNNING 1
#define EXITING 2

#include <TFT_eSPI.h>
#include <TJpg_Decoder.h>
#include <WiFi.h>

#include "utils.h"
#include "common_macro.h"

class StreamingComponent {
public:
    StreamingComponent(WiFiClient &clt, TFT_eSPI &tft);


    uint8_t status = IDLES;

    void enter();
    void exit();

    void loop();


    bool drawCallBack(int16_t x, int16_t y, uint16_t w, uint16_t h,
                      uint16_t *bitmap);
//    ~StreamingComponent() {
//        Serial.printf("~StreamingComponent\n");
//        free(wifiBuffer);
//        free(headerBuffer);
//        free(frameSizeBuffer);
//    };

private:
    // WiFiClient指针
    WiFiClient *client;
    // TFT_eSPI指针
    TFT_eSPI *Tft;
    // 帧率相关
    double fps_avg = 0.0;
    uint32_t sec{}, psec{};
    uint16_t fps = 0, frame_count = 0;
    // 帧率相关

    // 执行时间相关
    // 函数执行时间
    uint32_t cost{};
    // 一次loop执行时间
    uint32_t loopCost{};

    // 缓冲部分

    // 帧数据大小
    uint16_t size{};
    // 已经下载帧数据大小
    uint16_t bSize{};
    // DMA缓冲相关
    // 2020.12.04若出现发送端发送超过wifiFrameSize大小(32kb)，
    // 则会导致出错，而此处无法分配更大内存。
    // 暂时未找到正确开启SPIRAM方法
    // 2020.12.04将图片压缩方式从LZO改为jpg
    const int wifiFrameSize = 1024 * 32;
    // 头数据大小
    const int headerFrameSize = 10;
    // 待下载的jpg图片缓冲
    uint8_t *wifiBuffer =
            (uint8_t *) heap_caps_malloc(wifiFrameSize, MALLOC_CAP_8BIT);
    // 头数据缓冲
    uint8_t *headerBuffer =
            (uint8_t *) heap_caps_malloc(headerFrameSize, MALLOC_CAP_8BIT);
    // 帧数据大小缓冲，用于解析字符串为int
    uint8_t *frameSizeBuffer =
            (uint8_t *) heap_caps_malloc(headerFrameSize - 1, MALLOC_CAP_8BIT);
    // DMA 双缓冲模式
    uint16_t dmaBuffer1[16 * 16]{};  // Toggle buffer for 16*16 MCU block, 512bytes
    uint16_t dmaBuffer2[16 * 16]{};  // Toggle buffer for 16*16 MCU block, 512bytes
    uint16_t *dmaBufferPtr = dmaBuffer1;
    // 当前使用的DMA缓冲
    bool dmaBufferSel = 0;
    /**
     * 显示回调，用于Tjpeg
     * 2020-12-06
     */

    /**
     * 接收数据
     * 2020-12-01
     * size: 5222 bytes
     * cost: 16 ms
     */
    void onReceiveData();
};

#endif

StreamingComponent.cpp

#include "StreamingComponent.h"


StreamingComponent::StreamingComponent(WiFiClient &clt, TFT_eSPI &tft) {
  this->client = &clt;
  this->Tft = &tft;
  Serial.println("StreamingComponent Constuctor");
};


void StreamingComponent::enter() { status = RUNNING; };

void StreamingComponent::exit() { status = EXITING; };

void StreamingComponent::loop() {
  if (status == RUNNING) {
    Serial.println("StreamingComponent loop");
    loopCost = millis();
    onReceiveData();
    Serial.printf("fps_avg:%f,loop cost:%d ms\n", fps_avg, millis() - loopCost);
    Tft->drawString(String(fps_avg), 0, 0, 2);
  } else if (status == EXITING) {
    // 啥也不做
  }
};

bool StreamingComponent::drawCallBack(int16_t x, int16_t y, uint16_t w,
                                      uint16_t h, uint16_t *bitmap) {
  if (status == RUNNING) {
    if (y >= SCREEN_HEIGHT) return 0;
    if (dmaBufferSel) {
      dmaBufferPtr = dmaBuffer2;
    } else {
      dmaBufferPtr = dmaBuffer1;
    }
    dmaBufferSel = !dmaBufferSel;
    Tft->pushImageDMA(x, y, w, h, bitmap, dmaBufferPtr);
  }
  return true;
}

//    ~StreamingComponent() {
//        Serial.printf("~StreamingComponent\n");
//        free(wifiBuffer);
//        free(headerBuffer);
//        free(frameSizeBuffer);
//    };

void StreamingComponent::onReceiveData() {
  Serial.println("StreamingComponent onReceiveData");
  StreamingComponent::client->write(PREPAREOK);
  Serial.println("StreamingComponent client.write(PREPAREOK);");
  cost = millis();
  if (headerBuffer == nullptr) {
    Serial.printf("headerBuffer is null.\n");
  } else {
    client->readBytes(headerBuffer, headerFrameSize);
    Serial.printf("receive header cost:%d ms\n", millis() - cost);
  }

  int sum = checkSum((const char *)headerBuffer, 8);
  // Serial.printf("headerBuffer checkSum: %d\n", sum);
  if ((sum & 0xf) == c2i(headerBuffer[9]) &&
      (sum >> 4) == c2i(headerBuffer[8])) {
    // 有效头数据，准备接收帧数据
    strncpy((char *)frameSizeBuffer, (char *)headerBuffer, 8);
    frameSizeBuffer[9] = '\0';
    size = atoi((char *)frameSizeBuffer);
    // Serial.printf("valid header frame size: %d bytes\n", size);
  } else {
    // 无效头数据，丢弃
    // Serial.printf("invalid header\n");
    return;
  }

  client->write(HEADEROK);
  // // Serial.printf("send HEADEROK\n");
  cost = millis();
  bSize = 0;
  if (wifiBuffer == NULL) {
    Serial.printf("wifiBuffer is null.\n");
    Serial.printf("MALLOC_CAP_8BIT heap_caps_get_largest_free_block: %d.\n",
                  heap_caps_get_largest_free_block(MALLOC_CAP_8BIT));
    Serial.printf("MALLOC_CAP_32BIT heap_caps_get_largest_free_block: %d.\n",
                  heap_caps_get_largest_free_block(MALLOC_CAP_32BIT));
    Serial.printf("MALLOC_CAP_SPIRAM heap_caps_get_largest_free_block: %d.\n",
                  heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM));
    Serial.printf("MALLOC_CAP_8BIT: %d.\n",
                  heap_caps_get_free_size(MALLOC_CAP_8BIT));
    Serial.printf("MALLOC_CAP_32BIT: %d.\n",
                  heap_caps_get_free_size(MALLOC_CAP_32BIT));
    Serial.printf("MALLOC_CAP_SPIRAM: %d.\n",
                  heap_caps_get_free_size(MALLOC_CAP_SPIRAM));
  } else {
    bSize = client->readBytes(wifiBuffer, size);
    Serial.printf("frame size: %d bytes, receive frame cost:%d ms\n", bSize,
                  millis() - cost);
  }

  if (bSize > 64 && bSize == size) {
    cost = millis();
    Tft->startWrite();
    TJpgDec.drawJpg(0, 0, wifiBuffer, bSize);
    Tft->endWrite();
    frame_count++;
    sec = millis() / 1000;
    if (psec != sec) {
      psec = sec;
      fps = frame_count;
      fps_avg = (fps_avg + fps) / 2.0;
      frame_count = 0;
    }

    // 31ms
    Serial.printf("draw cost:%d ms\n", millis() - cost);
  } else {
    // 无效帧，丢弃
    // return;
  }
  client->write(FRAMEOK);
  // // Serial.printf("send FRAMEOK\n");
}

utils.h

#ifndef LIB_UTILS_H_
#define LIB_UTILS_H_

#include <stdint.h>
#include "TFT_eSPI.h"

int checkSum(const char* src, int length);
int c2i(char ch);

int getTextWidth(const char* text, TFT_eSprite &sprite);

int getTextWidth(const char* text, TFT_eSprite *sprite);
#endif

utils.cpp

#include "utils.h"

/**
 * @brief 计算16校验和计算
 * @param src 待校验内容
 * @param length 待校验内容长度
 * @retval 校验和
 * */
int checkSum(const char *src, int length) {
  int16_t sum = 0;
  for (int i = 0; i < length; i++) {
    sum += src[i];
  }
  sum = (sum & 0xff) + (sum >> 16);
  return ~sum & 0xff;
}

/**
 * @brief 16进制字符转int
 * @param ch 待转换内容
 * @retval 校验和
 * */
// https://www.cnblogs.com/lidabo/p/3995055.html
int c2i(char ch) {
  // 如果是数字，则用数字的ASCII码减去48, 如果ch = '2' ,则 '2' - 48 = 2
  if (isdigit(ch)) return ch - 48;

  // 如果是字母，但不是A~F,a~f则返回
  if (ch < 'A' || (ch > 'F' && ch < 'a') || ch > 'z') return -1;

  // 如果是大写字母，则用数字的ASCII码减去55, 如果ch = 'A' ,则 'A' - 55 = 10
  // 如果是小写字母，则用数字的ASCII码减去87, 如果ch = 'a' ,则 'a' - 87 = 10
  if (isalpha(ch)) return isupper(ch) ? ch - 55 : ch - 87;

  return -1;
}



int getTextWidth(const char* text, TFT_eSprite &sprite){
    return sprite.textWidth(text);
}

int getTextWidth(const char* text, TFT_eSprite *sprite){
    return sprite->textWidth(text);
}

tiny_monitor.cpp

#pragma GCC optimize("O3")
#include <stdint.h>

#include <TFT_eSPI.h>
#include <TJpg_Decoder.h>
#include <WiFi.h>
#include <Wire.h>
#include "StreamingComponent.h"

// 第三方基础组件
// WiFi客户端实例
WiFiClient client;
// 显示屏驱动实例
TFT_eSPI Tft = TFT_eSPI();

// 自定义对象
StreamingComponent *streaming;

bool drawCallback(int16_t x, int16_t y, uint16_t w, uint16_t h,
                  uint16_t *bitmap) {
    streaming->drawCallBack(x, y, w, h, bitmap);
    return true;
}

void main_setup() {
    // 配置串口
    Serial.begin(115200);
    // 配置显示
    Tft.init();
    Tft.setRotation(1);
    Tft.fillScreen(TFT_BLACK);
    Tft.initDMA();
    // 配置TJpeg
    TJpgDec.setJpgScale(1);
    TJpgDec.setSwapBytes(true);
    // 设置TJpg解码器回调函数
    TJpgDec.setCallback(drawCallback);
    // 配置WiFi
    client.setTimeout(1);
    WiFi.begin(ssid, password);
    delay(1000);
    if (WiFi.status() == WL_CONNECTED) {
       const int httpPort = 715;
       client.connect("192.168.10.207", httpPort);
       Serial.println("Socket Connected");
    }

    // 用户自定义对象初始化区
    streaming = new StreamingComponent(client, Tft);
    streaming->status = RUNNING;
}

void setup() {
  main_setup();
}

void loop() {
  streaming->loop();
}

tiny_monitor.py

import socket
import time
from multiprocessing import Process, Queue,Value,Manager
from multiprocessing.sharedctypes import Array
import ctypes
from mss.tools import to_png
import cv2
import lzo
import mss
import numpy as np

ip = "0.0.0.0"
port = 715
fps = 0


def main():
    global fps
    # 1. 创建套接字 socket
    if True:
        tcp_server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # 2. 绑定本地信息 bind
        tcp_server_socket.bind((ip, port))
        # 3. 让默认的套接字由主动变为被动 listen
        tcp_server_socket.listen(128)
        print("启动TCP服务器\r\n" + f'启动在{ip}:{port}上')
        # 4. 等待客户端的链接 accept
        print("等待客户端的链接\r\n")
        new_client_socket, client_addr = tcp_server_socket.accept()
        print(f'当前链接：{client_addr}')
        frame_buffer = grab_screen_to_buffer(0, 0, 1920, 1080)
        start_time = time.time()
        while True:
            s = time.time()
            recv = new_client_socket.recv(1)
            if recv == b'\x41':
                # 客户端就绪，发送头数据
                start_time = time.time()
                header = len(frame_buffer.tobytes())
                header = package_header(header)
                new_client_socket.sendall(header)
                end_time = time.time()
                cost = end_time - start_time
                # print("客户端就绪，发送头数据")
            elif recv == b'\x02':
                # 客户端准备头部接收完成，发送帧数据
                new_client_socket.sendall(frame_buffer.tobytes())
                frame_buffer = grab_screen_to_buffer(0, 0, 1920, 1080)
                # print("客户端准备头部接收完成，发送帧数据")
            elif recv == b'\x01':
                # pass
                # 客户端准备帧数据接收完成，等待客户端就绪
                # print("客户端准备帧数据接收完成，等待客户端就绪")
                end_time = time.time()
                cost = end_time - start_time
                print("Backend FPS:{:.2f}".format(1.0 / cost))


## grab screen by left top width height
def grab_screen_to_buffer(l, t, w, h):
    monitor = {"top": t, "left": l, "width": w, "height": h}
    with mss.mss() as sct:
        sct_frame = sct.grab(monitor)
        img = np.array(sct_frame)
        img = cv2.resize(img, dsize=(240, 135))
        # img = cv2.cvtColor(img, cv2.COLOR_BGR2BGR565)
        # print("bmp no comporess size:{}",len(img.tobytes()))
        # print("bmp lzo comporess size:{}",len(lzo.compress(img.tobytes(), 9, False)))
        quality = 60
        encode_params = [cv2.IMWRITE_JPEG_QUALITY,quality,cv2.IMWRITE_JPEG_PROGRESSIVE,0]
        retval, img = cv2.imencode(".jpg", img, encode_params)
        # with open("write.jpg", "wb") as f:
        #     f.write(img.tobytes())
        # img = img[..., ::-1]
        # print("jpg no comporess size:{}",len(img.tobytes()))
        # print("jpg lzo comporess size:{}",len(lzo.compress(img.tobytes(), 9, False)))
        return img


def package_header(size):
    data = num_package(size)
    header = ''.join(data).encode() + hex(check_sum(data)).encode()[2:]
    return header


# 校验和
def check_sum(value):
    s = 0
    for d in value:
        s += ord(d)
    s = (s & 0xff) + (s >> 16)
    return ~s & 0xff


# 打包数字
def num_package(num):
    if num < 100000000:
        li_str = list(str(num))
        result = ['0' for _ in range(8)]
        for index, item in zip(range(len(li_str)), li_str):
            result[-index + len(li_str) - 1] = item
        return result[::-1]
    else:
        return ['9' for _ in range(6)]


if __name__ == '__main__':
    time.sleep(1)
    main()

预览

参考资料

以下内容为实际开发中参考过的资料，有些资料中的方案已经舍弃，并未体现在上述代码中，特此列出，但俺仍要向他们表示由衷的感谢。

• windows环境安装lzo和python-lzo
• https://github.com/lzfcc/LearnArduino
• ESP32 官方文档（五）严重错误
• ESP32-CAM: 連接ILI9341液晶螢幕