Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
rzeldent committed Apr 1, 2024
1 parent c599657 commit f59fb8b
Show file tree
Hide file tree
Showing 10 changed files with 207 additions and 191 deletions.
6 changes: 3 additions & 3 deletions lib/micro-jpg/include/jpg.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ class jpg
public:
bool decode(const uint8_t *jpg, size_t size);

const struct jpg_section *quantization_table_0_;
const struct jpg_section *quantization_table_1_;
const jpg_section_t *quantization_table_luminance_;
const jpg_section_t *quantization_table_chrominance_;

const uint8_t *jpeg_data_start;
const uint8_t *jpeg_data_end;

private:
static const jpg_section *find_jpg_section(const uint8_t **ptr, const uint8_t *end, jpg_section::jpg_section_flag flag);
static const jpg_section_t *find_jpg_section(const uint8_t **ptr, const uint8_t *end, jpg_section_t::jpg_section_flag flag);
};
12 changes: 6 additions & 6 deletions lib/micro-jpg/include/jpg_section.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include <stdint.h>

struct __attribute__((packed)) jpg_section
typedef struct __attribute__((packed))
{
enum jpg_section_flag : uint8_t
{
Expand Down Expand Up @@ -82,9 +82,9 @@ struct __attribute__((packed)) jpg_section
static const char *flag_name(const jpg_section_flag flag);
uint16_t data_length() const;
uint16_t section_length() const;
};
} jpg_section_t;

struct __attribute__((packed)) jpg_section_app0 // 0xffe0
typedef struct __attribute__((packed)) // 0xffe0
{
char identifier[5] = {'J', 'F', 'I', 'F', 0}; // JFIF identifier, zero-terminated
uint8_t version_major = 1;
Expand All @@ -94,10 +94,10 @@ struct __attribute__((packed)) jpg_section_app0 // 0xffe0
uint16_t density_ver = 1; // density: 1 pixel "per pixel" horizontally and vertically
uint8_t thumbnail_hor = 0;
uint8_t thumbnail_ver = 0; // no thumbnail (size 0 x 0)
};
} jpg_section_app0_t;

struct __attribute__((packed)) jpg_section_dqt // 0xffdb
typedef struct __attribute__((packed)) // 0xffdb
{
uint8_t id; // 0= quantLuminance, 1= quantChrominance
uint8_t data[64];
};
} jpg_section_dqt_t;
22 changes: 11 additions & 11 deletions lib/micro-jpg/src/jpg.cpp
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
#include <esp32-hal-log.h>
#include "jpg.h"

const jpg_section *jpg::find_jpg_section(const uint8_t **ptr, const uint8_t *end, jpg_section::jpg_section_flag flag)
const jpg_section_t *jpg::find_jpg_section(const uint8_t **ptr, const uint8_t *end, jpg_section_t::jpg_section_flag flag)
{
log_d("find_jpeg_section 0x%02x (%s)", flag, jpg_section::flag_name(flag));
log_d("find_jpeg_section 0x%02x (%s)", flag, jpg_section_t::flag_name(flag));
while (*ptr < end)
{
// flag, len MSB, len LSB
auto section = reinterpret_cast<const jpg_section *>((*ptr));
auto section = reinterpret_cast<const jpg_section_t *>((*ptr));
if (section->framing != 0xff)
{
log_e("Expected framing 0xff but found: 0x%02x", section->framing);
break;
}

if (!jpg_section::is_valid_flag(section->flag))
if (!jpg_section_t::is_valid_flag(section->flag))
{
log_d("Unknown section 0x%02x", flag);
return nullptr;
Expand All @@ -24,11 +24,11 @@ const jpg_section *jpg::find_jpg_section(const uint8_t **ptr, const uint8_t *end
*ptr += section->section_length();
if (section->flag == flag)
{
log_d("Found section 0x%02x (%s), %d bytes", flag, jpg_section::flag_name(section->flag), section->section_length());
log_d("Found section 0x%02x (%s), %d bytes", flag, jpg_section_t::flag_name(section->flag), section->section_length());
return section;
}

log_d("Skipping section: 0x%02x (%s), %d bytes", section->flag, jpg_section::flag_name(section->flag), section->section_length());
log_d("Skipping section: 0x%02x (%s), %d bytes", section->flag, jpg_section_t::flag_name(section->flag), section->section_length());
}

// Not found
Expand All @@ -44,25 +44,25 @@ bool jpg::decode(const uint8_t *data, size_t size)
auto end = ptr + size;

// Check for SOI (start of image) 0xff, 0xd8
if (!find_jpg_section(&ptr, end, jpg_section::jpg_section_flag::SOI))
if (!find_jpg_section(&ptr, end, jpg_section_t::jpg_section_flag::SOI))
{
log_e("No valid start of image marker found");
return false;
}

// First quantization table (Luminance - black & white images)
if (!(quantization_table_0_ = find_jpg_section(&ptr, end, jpg_section::jpg_section_flag::DQT)))
if (!(quantization_table_luminance_ = find_jpg_section(&ptr, end, jpg_section_t::jpg_section_flag::DQT)))
{
log_e("No quantization table 0 section found");
return false;
}

// Second quantization table (Chrominance - color images)
if (!(quantization_table_1_ = find_jpg_section(&ptr, end, jpg_section::jpg_section_flag::DQT)))
if (!(quantization_table_chrominance_ = find_jpg_section(&ptr, end, jpg_section_t::jpg_section_flag::DQT)))
log_w("No quantization table 1 section found");

// Start of scan
if (!find_jpg_section(&ptr, end, jpg_section::jpg_section_flag::SOS))
if (!find_jpg_section(&ptr, end, jpg_section_t::jpg_section_flag::SOS))
{
log_e("No start of scan section found");
return false;
Expand All @@ -77,7 +77,7 @@ bool jpg::decode(const uint8_t *data, size_t size)
ptr++;

// Check if marker is an end of image marker
if (!find_jpg_section(&ptr, end, jpg_section::jpg_section_flag::EOI))
if (!find_jpg_section(&ptr, end, jpg_section_t::jpg_section_flag::EOI))
{
log_e("No end of image marker found");
return false;
Expand Down
12 changes: 6 additions & 6 deletions lib/micro-jpg/src/jpg_section.cpp
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
#include "jpg_section.h"

uint16_t jpg_section::data_length() const
uint16_t jpg_section_t::data_length() const
{
return (length_msb << 8) + length_lsb - sizeof(jpg_section::length_msb)- sizeof(jpg_section::length_lsb);
return (length_msb << 8) + length_lsb - sizeof(jpg_section_t::length_msb)- sizeof(jpg_section_t::length_lsb);
}

uint16_t jpg_section::section_length() const
uint16_t jpg_section_t::section_length() const
{
return flag == SOI || flag == EOI ? sizeof(jpg_section::framing) + sizeof(jpg_section::flag) : sizeof(jpg_section::framing) + sizeof(jpg_section::flag) + (length_msb << 8) + length_lsb;
return flag == SOI || flag == EOI ? sizeof(jpg_section_t::framing) + sizeof(jpg_section_t::flag) : sizeof(jpg_section_t::framing) + sizeof(jpg_section_t::flag) + (length_msb << 8) + length_lsb;
}

bool jpg_section::is_valid_flag(const jpg_section_flag flag)
bool jpg_section_t::is_valid_flag(const jpg_section_flag flag)
{
return flag >= SOF0 && flag <= COM;
}

// from: https://www.disktuna.com/list-of-jpeg-markers/
const char *jpg_section::flag_name(const jpg_section_flag flag)
const char *jpg_section_t::flag_name(const jpg_section_flag flag)
{
switch (flag)
{
Expand Down
15 changes: 7 additions & 8 deletions lib/micro-rtsp-server/include/micro_rtsp_server.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,19 @@

#include "micro_rtsp_camera.h"
#include "micro_rtsp_requests.h"
#include "micro_rtsp_streamer.h"

class micro_rtsp_server : WiFiServer
{
public:
micro_rtsp_server(const micro_rtsp_camera& source, unsigned frame_interval = 100);
micro_rtsp_server(micro_rtsp_camera &source, unsigned frame_interval = 100);
~micro_rtsp_server();

void begin(unsigned short port = 554);
void end();

unsigned get_frame_interval() { return frame_interval_; }
unsigned set_frame_interval(unsigned value) { return frame_interval_ = value; }
unsigned get_frame_interval() { return frame_interval_; }
unsigned set_frame_interval(unsigned value) { return frame_interval_ = value; }

void loop();

Expand All @@ -35,12 +36,10 @@ class micro_rtsp_server : WiFiServer
};

private:
const micro_rtsp_camera &source_;

unsigned frame_interval_;

micro_rtsp_camera &source_;
unsigned frame_interval_;
unsigned long next_frame_update_;

unsigned long next_check_client_;
micro_rtsp_streamer streamer_;
std::list<rtsp_client> clients_;
};
81 changes: 78 additions & 3 deletions lib/micro-rtsp-server/include/micro_rtsp_streamer.h
Original file line number Diff line number Diff line change
@@ -1,15 +1,90 @@
#pragma once

#include <jpg.h>
// https://en.wikipedia.org/wiki/Maximum_transmission_unit
constexpr size_t max_wifi_mtu = 2304;
// Payload JPG - https://www.ietf.org/rfc/rfc1890.txt
constexpr uint8_t RTP_PAYLOAD_JPG = 26;
// http://www.ietf.org/rfc/rfc2345.txt Each table is an array of 64 values given in zig-zag order, identical to the format used in a JFIF DQT marker segment.
constexpr size_t jpeg_luminance_table_length = 64;
constexpr size_t jpeg_chrominance_table_length = 64;

// https://www.ietf.org/rfc/rfc2326#section-10.12
typedef struct __attribute__((packed))
{
char magic; // Magic encapsulation ASCII dollar sign (24 hexadecimal)
uint8_t channel; // Channel identifier
uint16_t length; // Network order
} rtp_over_tcp_hdr_t;

class micro_rtsp_streamer
{
public:
micro_rtsp_streamer(const uint16_t width, const uint16_t height);
size_t create_jpg_packet(const uint8_t *jpg, uint8_t **jpg_current, const uint8_t *jpg_end, const uint32_t timestamp);
rtp_over_tcp_hdr_t *create_jpg_packet(const uint8_t *jpg_scan, const uint8_t *jpg_scan_end, uint8_t **jpg_offset, const uint32_t timestamp, const uint8_t* quantization_table_luminance, const uint8_t* quantization_table_chrominance);

private:
uint16_t width_, height_;
uint32_t ssrc_;
uint16_t sequenceNumber_;
uint16_t sequence_number_;

// RTP data header - http://www.ietf.org/rfc/rfc3550.txt
typedef struct __attribute__((packed))
{
uint16_t version : 2; // protocol version
uint16_t padding : 1; // padding flag
uint16_t extension : 1; // header extension flag
uint16_t cc : 4; // CSRC count
uint16_t marker : 1; // marker bit
uint16_t pt : 7; // payload type
uint16_t seq : 16; // sequence number
uint32_t ts; // timestamp
uint32_t ssrc; // synchronization source
} rtp_hdr_t;

// https://datatracker.ietf.org/doc/html/rfc2435
typedef struct __attribute__((packed))
{
uint32_t tspec : 8; // type-specific field
uint32_t off : 24; // fragment byte offset
uint8_t type; // id of jpeg decoder params
uint8_t q; // Q values 0-127 indicate the quantization tables. JPEG types 0 and 1 (and their corresponding types 64 and 65)
uint8_t width; // frame width in 8 pixel blocks
uint8_t height; // frame height in 8 pixel blocks
} jpeg_hdr_t;

typedef struct __attribute__((packed))
{
uint16_t dri;
uint16_t f : 1;
uint16_t l : 1;
uint16_t count : 14;
} jpeg_hdr_rst_t;

typedef struct __attribute__((packed))
{
uint8_t mbz;
uint8_t precision;
uint16_t length;
} jpeg_hdr_qtable_t;

// The types below will be returned, the jpeg_packet_with_quantization_t for the first packet, then the jpeg_packet_t

typedef struct __attribute__((packed))
{
rtp_over_tcp_hdr_t rtp_over_tcp_hdr;
rtp_hdr_t rtp_hdr;
jpeg_hdr_t jpeg_hdr;
jpeg_hdr_qtable_t jpeg_hdr_qtable;
uint8_t quantization_table_luminance[jpeg_luminance_table_length];
uint8_t quantization_table_chrominance[jpeg_chrominance_table_length];
uint8_t jpeg_data[];
} jpeg_packet_with_quantization_t;

typedef struct __attribute__((packed))
{
rtp_over_tcp_hdr_t rtp_over_tcp_hdr;
rtp_hdr_t rtp_hdr;
jpeg_hdr_t jpeg_hdr;
uint8_t jpeg_data[];
} jpeg_packet_t;
};
25 changes: 21 additions & 4 deletions lib/micro-rtsp-server/src/micro_rtsp_server.cpp
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
#include <micro_rtsp_server.h>
#include <jpg.h>
#include <vector>
#include <memory>

// Check client connections every 100 milliseconds
#define CHECK_CLIENT_INTERVAL 10

micro_rtsp_server::micro_rtsp_server(const micro_rtsp_camera &source, unsigned frame_interval /*= 100*/)
: source_(source)
micro_rtsp_server::micro_rtsp_server(micro_rtsp_camera &source, unsigned frame_interval /*= 100*/)
: source_(source), streamer_(source.width(), source.height())
{
log_i("starting RTSP server");
frame_interval_ = frame_interval;
Expand Down Expand Up @@ -51,10 +52,26 @@ void micro_rtsp_server::loop()
if (next_frame_update_ < now)
{
next_frame_update_ = now + frame_interval_;
for (auto client : clients_)

auto ts = time(nullptr);
// Get next jpg frame
source_.update_frame();
// Decode to get quantitation- and scan data
jpg jpg;
auto jpg_data = source_.data();
auto jpg_size = source_.size();
assert(jpg.decode(jpg_data, jpg_size));

auto jpg_scan_current = (uint8_t*)jpg.jpeg_data_start;
while (jpg_scan_current < jpg.jpeg_data_end)
{
;
auto packet = streamer_.create_jpg_packet(jpg.jpeg_data_start, jpg.jpeg_data_end, &jpg_scan_current, ts, jpg.quantization_table_luminance_->data, jpg.quantization_table_chrominance_->data);
for (auto client : clients_)
{
;
// client->session->broadcastCurrentFrame(now);
}
free(packet);
}
}
}
Expand Down
Loading

0 comments on commit f59fb8b

Please sign in to comment.