diff --git a/libflv/source/hevc-annexbtomp4.c b/libflv/source/hevc-annexbtomp4.c index 016192ee..5f55b36f 100644 --- a/libflv/source/hevc-annexbtomp4.c +++ b/libflv/source/hevc-annexbtomp4.c @@ -2,6 +2,7 @@ #include "mpeg4-avc.h" #include #include +#include #define H265_NAL_BLA_W_LP 16 #define H265_NAL_RSV_IRAP 23 @@ -362,6 +363,12 @@ static void hevc_handler(void* param, const uint8_t* nalu, size_t bytes) struct h265_annexbtomp4_handle_t* mp4; mp4 = (struct h265_annexbtomp4_handle_t*)param; + if (bytes < 2) + { + assert(0); + return; + } + nalutype = (nalu[0] >> 1) & 0x3f; #if defined(H2645_FILTER_AUD) if(H265_NAL_AUD == nalutype) diff --git a/libmpeg/include/mpeg-types.h b/libmpeg/include/mpeg-types.h index aca76294..68f82199 100644 --- a/libmpeg/include/mpeg-types.h +++ b/libmpeg/include/mpeg-types.h @@ -7,4 +7,6 @@ #define PTS_NO_VALUE INT64_MIN //(int64_t)0x8000000000000000L +enum { MPEG_VCL_NONE = 0, MPEG_VCL_IDR, MPEG_VCL_P, MPEG_VCL_CORRUPT }; + #endif /* !_mpeg_types_h_ */ diff --git a/libmpeg/include/mpeg-util.h b/libmpeg/include/mpeg-util.h index 6182159b..45e0482b 100644 --- a/libmpeg/include/mpeg-util.h +++ b/libmpeg/include/mpeg-util.h @@ -160,6 +160,7 @@ int mpeg_stream_type_video(int codecid); int mpeg_h264_find_nalu(const uint8_t* p, size_t bytes, size_t* leading); int mpeg_h264_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl); int mpeg_h265_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl); +int mpeg_h266_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl); int mpeg_h26x_verify(const uint8_t* data, size_t bytes, int* codec); int mpeg_h264_start_with_access_unit_delimiter(const uint8_t* p, size_t bytes); diff --git a/libmpeg/source/mpeg-packet.c b/libmpeg/source/mpeg-packet.c index ceb03657..34b69037 100644 --- a/libmpeg/source/mpeg-packet.c +++ b/libmpeg/source/mpeg-packet.c @@ -60,7 +60,8 @@ static int mpeg_packet_h264_h265_filter(uint16_t program, uint16_t stream, struc return handler(param, program, stream, pkt->codecid, pkt->flags, pkt->pts, pkt->dts, data + off + i, size - off - i); } -static int mpeg_packet_h26x(struct packet_t* pkt, const struct pes_t* pes, size_t size, pes_packet_handler handler, void* param) +// @param[out] consume used of new append data +static int mpeg_packet_h26x(struct packet_t* pkt, const struct pes_t* pes, size_t size, size_t* consume, pes_packet_handler handler, void* param) { int r, n; const uint8_t* p, *end, *data; @@ -83,11 +84,24 @@ static int mpeg_packet_h26x(struct packet_t* pkt, const struct pes_t* pes, size_ } // PES contain multiple packet - find = PSI_STREAM_H264 == pkt->codecid ? mpeg_h264_find_new_access_unit : mpeg_h265_find_new_access_unit; + find = PSI_STREAM_H264 == pkt->codecid ? mpeg_h264_find_new_access_unit : (PSI_STREAM_H265 == pkt->codecid ? mpeg_h265_find_new_access_unit : mpeg_h266_find_new_access_unit); n = find(p, end - p, &pkt->vcl); while (n >= 0) { assert(pkt->vcl > 0); + if (MPEG_VCL_CORRUPT == pkt->vcl) + { + // video data contain 00 00 01 BA + // maybe previous packet data lost + r = (p + n - pkt->data) - (pkt->size - *consume); + assert(r > 0 && r <= *consume); + *consume = (r <= 0 || r > *consume) ? *consume : r; + pkt->flags |= MPEG_FLAG_PACKET_CORRUPT; + pkt->size = 0; // clear + pkt->vcl = 0; + // todo: handle packet data ??? + return 0; + } p += n; pkt->flags = (pkt->flags ^ MPEG_FLAG_IDR_FRAME) | (1 == pkt->vcl ? MPEG_FLAG_IDR_FRAME : 0); // update key frame flags @@ -158,12 +172,13 @@ static void pes_packet_codec_verify(struct pes_t* pes, struct packet_t* pkt) #endif } -int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t size, int start, pes_packet_handler handler, void* param) +int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t size, size_t* consume, int start, pes_packet_handler handler, void* param) { int r; size_t total; total = size; + *consume = size; // all saved // use timestamp to split packet assert(PTS_NO_VALUE != pes->dts); if (pkt->size > 0 && (pkt->dts != pes->dts || start) @@ -196,7 +211,7 @@ int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t if (PSI_STREAM_H264 == pes->codecid || PSI_STREAM_H265 == pes->codecid || PSI_STREAM_H266 == pes->codecid) { - return mpeg_packet_h26x(pkt, pes, total, handler, param); + return mpeg_packet_h26x(pkt, pes, total, consume, handler, param); } else { @@ -216,7 +231,7 @@ int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t { pes_packet_codec_verify(pes, pkt); // verify on packet complete if (PSI_STREAM_H264 == pes->codecid || PSI_STREAM_H265 == pes->codecid || PSI_STREAM_H266 == pes->codecid) - return mpeg_packet_h26x(pkt, pes, size, handler, param); + return mpeg_packet_h26x(pkt, pes, size, consume, handler, param); assert(pes->pkt.size == pes->len || (pkt->flags & MPEG_FLAG_PACKET_CORRUPT)); // packet lost r = handler(param, pes->pn, pes->pid, pkt->codecid, pkt->flags, pkt->pts, pkt->dts, pes->pkt.data, pes->len); diff --git a/libmpeg/source/mpeg-pes-internal.h b/libmpeg/source/mpeg-pes-internal.h index 4b950c99..fb991d4e 100644 --- a/libmpeg/source/mpeg-pes-internal.h +++ b/libmpeg/source/mpeg-pes-internal.h @@ -90,7 +90,7 @@ int pes_read_mpeg1_header(struct pes_t* pes, struct mpeg_bits_t* reader); size_t pes_write_header(const struct pes_t *pes, uint8_t* data, size_t bytes); typedef int (*pes_packet_handler)(void* param, int program, int stream, int codecid, int flags, int64_t pts, int64_t dts, const void* data, size_t bytes); -int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t size, int start, pes_packet_handler handler, void* param); +int pes_packet(struct packet_t* pkt, struct pes_t* pes, const void* data, size_t size, size_t* consume, int start, pes_packet_handler handler, void* param); uint16_t mpeg_bits_read15(struct mpeg_bits_t* reader); uint32_t mpeg_bits_read30(struct mpeg_bits_t* reader); diff --git a/libmpeg/source/mpeg-ps-dec.c b/libmpeg/source/mpeg-ps-dec.c index 0d688f3f..1a5e5163 100644 --- a/libmpeg/source/mpeg-ps-dec.c +++ b/libmpeg/source/mpeg-ps-dec.c @@ -157,7 +157,7 @@ static struct pes_t* psm_fetch(struct psm_t* psm, uint8_t sid) return NULL; } -static int ps_demuxer_packet(struct ps_demuxer_t *ps, const uint8_t* data, size_t bytes) +static int ps_demuxer_packet(struct ps_demuxer_t *ps, const uint8_t* data, size_t bytes, size_t *consume) { int r; struct pes_t* pes; @@ -173,7 +173,7 @@ static int ps_demuxer_packet(struct ps_demuxer_t *ps, const uint8_t* data, size_ #endif pes->flags = pes->data_alignment_indicator ? MPEG_FLAG_IDR_FRAME : 0; - r = pes_packet(&pes->pkt, pes, data, bytes, ps->start, ps_demuxer_onpes, ps); + r = pes_packet(&pes->pkt, pes, data, bytes, consume, ps->start, ps_demuxer_onpes, ps); ps->start = 0; // clear start flags return r; } @@ -346,7 +346,7 @@ static int ps_demuxer_buffer(struct ps_demuxer_t* ps, const uint8_t* data, size_ int ps_demuxer_input(struct ps_demuxer_t* ps, const uint8_t* data, size_t bytes) { int r; - size_t i; + size_t i, consume; struct mpeg_bits_t reader; for(i = 0; i < bytes; ) @@ -391,8 +391,8 @@ int ps_demuxer_input(struct ps_demuxer_t* ps, const uint8_t* data, size_t bytes) assert(ps->pes && ps->pes_length <= ps->pes->len); if (bytes - i >= ps->pes->len - ps->pes_length) { - r = ps_demuxer_packet(ps, data + i, ps->pes->len - ps->pes_length); - i += ps->pes->len - ps->pes_length; + r = ps_demuxer_packet(ps, data + i, ps->pes->len - ps->pes_length, &consume); + i += consume; // ps->pes->len - ps->pes_length; // fix: pack start code in video data ps->pes_length = 0; ps->start = 0; // clear start flag ps->state = PS_DEMUXER_STATE_START; // next round @@ -400,9 +400,11 @@ int ps_demuxer_input(struct ps_demuxer_t* ps, const uint8_t* data, size_t bytes) else { // need more data - r = ps_demuxer_packet(ps, data + i, bytes - i); - ps->pes_length += bytes - i; - i = bytes; + r = ps_demuxer_packet(ps, data + i, bytes - i, &consume); + ps->pes_length = consume == bytes - i ? ps->pes_length + (bytes - i) : 0; + ps->start = consume == bytes - i ? ps->start : 0; + ps->state = consume == bytes - i ? ps->state : PS_DEMUXER_STATE_START; + i += consume; //i = bytes; // fix: pack start code in video data } break; diff --git a/libmpeg/source/mpeg-ts-dec.c b/libmpeg/source/mpeg-ts-dec.c index a968a47e..ee788e80 100644 --- a/libmpeg/source/mpeg-ts-dec.c +++ b/libmpeg/source/mpeg-ts-dec.c @@ -126,6 +126,7 @@ static uint32_t adaptation_filed_read(struct ts_adaptation_field_t *adp, const u int ts_demuxer_flush(struct ts_demuxer_t* ts) { uint32_t i, j; + size_t consume; for (i = 0; i < ts->pat.pmt_count; i++) { for (j = 0; j < ts->pat.pmts[i].stream_count; j++) @@ -137,22 +138,22 @@ int ts_demuxer_flush(struct ts_demuxer_t* ts) if (PSI_STREAM_H264 == pes->codecid) { const uint8_t aud[] = {0,0,0,1,0x09,0xf0}; - pes_packet(&pes->pkt, pes, aud, sizeof(aud), 0, ts->onpacket, ts->param); + pes_packet(&pes->pkt, pes, aud, sizeof(aud), &consume, 0, ts->onpacket, ts->param); } else if (PSI_STREAM_H265 == pes->codecid) { const uint8_t aud[] = {0,0,0,1,0x46,0x01,0x50}; - pes_packet(&pes->pkt, pes, aud, sizeof(aud), 0, ts->onpacket, ts->param); + pes_packet(&pes->pkt, pes, aud, sizeof(aud), &consume, 0, ts->onpacket, ts->param); } else if (PSI_STREAM_H266 == pes->codecid) { const uint8_t aud[] = { 0,0,0,1,0x00,0xA1,0x18 }; - pes_packet(&pes->pkt, pes, aud, sizeof(aud), 0, ts->onpacket, ts->param); + pes_packet(&pes->pkt, pes, aud, sizeof(aud), &consume, 0, ts->onpacket, ts->param); } else { //assert(0); - pes_packet(&pes->pkt, pes, NULL, 0, 0, ts->onpacket, ts->param); + pes_packet(&pes->pkt, pes, NULL, 0, &consume, 0, ts->onpacket, ts->param); } } } @@ -164,6 +165,7 @@ int ts_demuxer_input(struct ts_demuxer_t* ts, const uint8_t* data, size_t bytes) int r = 0; uint32_t i, j, k; uint32_t PID; + size_t consume; unsigned int count; struct mpeg_bits_t reader; struct ts_packet_header_t pkhd; @@ -269,7 +271,7 @@ int ts_demuxer_input(struct ts_demuxer_t* ts, const uint8_t* data, size_t bytes) continue; // don't have pes header yet } - r = pes_packet(&pes->pkt, pes, data + i, bytes - i, pkhd.payload_unit_start_indicator, ts->onpacket, ts->param); + r = pes_packet(&pes->pkt, pes, data + i, bytes - i, &consume, pkhd.payload_unit_start_indicator, ts->onpacket, ts->param); pes->have_pes_header = (r || (0 == pes->pkt.size && pes->len > 0)) ? 0 : 1; // packet completed break; // find stream } diff --git a/libmpeg/source/mpeg-ts-h264.c b/libmpeg/source/mpeg-ts-h264.c index 2834332b..bc6c62b9 100644 --- a/libmpeg/source/mpeg-ts-h264.c +++ b/libmpeg/source/mpeg-ts-h264.c @@ -1,5 +1,6 @@ #include "mpeg-types.h" #include "mpeg-util.h" +#include "mpeg-proto.h" #include #include @@ -140,7 +141,13 @@ int mpeg_h264_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl) } else if (nal_type > 0 && nal_type < 6) { - *vcl = H264_NAL_IDR == nal_type ? 1 : 2; + *vcl = H264_NAL_IDR == nal_type ? MPEG_VCL_IDR : MPEG_VCL_P; + } + else if (PES_SID_START == p[n]) + { + // pes data loss ??? + *vcl = MPEG_VCL_CORRUPT; + return (int)(p - data + n - leading); } else { diff --git a/libmpeg/source/mpeg-ts-h265.c b/libmpeg/source/mpeg-ts-h265.c index 215de7f7..a6666a90 100644 --- a/libmpeg/source/mpeg-ts-h265.c +++ b/libmpeg/source/mpeg-ts-h265.c @@ -1,9 +1,17 @@ #include "mpeg-types.h" #include "mpeg-util.h" +#include "mpeg-proto.h" #include #include -#define H265_NAL_AUD 35 +#define H265_NAL_BLA_W_LP 16 +#define H265_NAL_RSV_IRAP 23 +#define H265_NAL_VPS 32 +#define H265_NAL_SPS 33 +#define H265_NAL_PPS 34 +#define H265_NAL_AUD 35 +#define H265_NAL_SEI_PREFIX 39 +#define H265_NAL_SEI_SUFFIX 40 /// @param[out] leading optional leading zero bytes /// @return -1-not found, other-AUD position(include start code) @@ -63,8 +71,6 @@ static int mpeg_h265_find_keyframe(const uint8_t* p, size_t bytes) static int mpeg_h265_is_new_access_unit(const uint8_t* nalu, size_t bytes) { - enum { NAL_VPS = 32, NAL_SPS = 33, NAL_PPS = 34, NAL_AUD = 35, NAL_PREFIX_SEI = 39, }; - uint8_t nal_type; uint8_t nuh_layer_id; @@ -75,12 +81,12 @@ static int mpeg_h265_is_new_access_unit(const uint8_t* nalu, size_t bytes) nuh_layer_id = ((nalu[0] & 0x01) << 5) | ((nalu[1] >> 3) &0x1F); // 7.4.2.4.4 Order of NAL units and coded pictures and their association to access units - if(NAL_VPS == nal_type || NAL_SPS == nal_type || NAL_PPS == nal_type || - (nuh_layer_id == 0 && (NAL_AUD == nal_type || NAL_PREFIX_SEI == nal_type || (41 <= nal_type && nal_type <= 44) || (48 <= nal_type && nal_type <= 55)))) + if(H265_NAL_VPS == nal_type || H265_NAL_SPS == nal_type || H265_NAL_PPS == nal_type || + (nuh_layer_id == 0 && (H265_NAL_AUD == nal_type || H265_NAL_SEI_PREFIX == nal_type || (41 <= nal_type && nal_type <= 44) || (48 <= nal_type && nal_type <= 55)))) return 1; // 7.4.2.4.5 Order of VCL NAL units and association to coded pictures - if (nal_type <= 31) + if (nal_type < H265_NAL_VPS) { //first_slice_segment_in_pic_flag 0x80 return (nalu[2] & 0x80) ? 1 : 0; @@ -108,9 +114,15 @@ int mpeg_h265_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl) { return (int)(p - data + n - leading); } - else if (nal_type <= 31) + else if (nal_type < H265_NAL_VPS) + { + *vcl = (H265_NAL_BLA_W_LP <= nal_type && nal_type <= H265_NAL_RSV_IRAP) ? MPEG_VCL_IDR : MPEG_VCL_P; + } + else if (PES_SID_START == p[n]) { - *vcl = (16 <= nal_type && nal_type <= 23) ? 1 : 2; + // pes data loss ??? + *vcl = MPEG_VCL_CORRUPT; // for assert + return (int)(p - data + n - leading); } else { diff --git a/libmpeg/source/mpeg-ts-h266.c b/libmpeg/source/mpeg-ts-h266.c index 7a254e75..fd937552 100644 --- a/libmpeg/source/mpeg-ts-h266.c +++ b/libmpeg/source/mpeg-ts-h266.c @@ -1,9 +1,22 @@ #include "mpeg-types.h" #include "mpeg-util.h" +#include "mpeg-proto.h" #include #include -#define H266_NAL_AUD 20 +#define H266_NAL_IDR_W_RADL 7 +#define H266_NAL_RSV_IRAP 11 +#define H266_NAL_OPI 12 +#define H266_NAL_DCI 13 +#define H266_NAL_VPS 14 +#define H266_NAL_SPS 15 +#define H266_NAL_PPS 16 +#define H266_PREFIX_APS_NUT 17 +#define H266_SUFFIX_APS_NUT 18 +#define H266_PH_NUT 19 +#define H266_NAL_AUD 20 +#define H266_NAL_PREFIX_SEI 23 +#define H266_NAL_SUFFIX_SEI 24 /// @return 0-not find, 1-find ok int mpeg_h266_start_with_access_unit_delimiter(const uint8_t* p, size_t bytes) @@ -18,3 +31,68 @@ int mpeg_h266_start_with_access_unit_delimiter(const uint8_t* p, size_t bytes) nalu = (p[i + 1] >> 3) & 0x1f; return H266_NAL_AUD == nalu ? 1 : 0; } + +static int mpeg_h266_is_new_access_unit(const uint8_t* nalu, size_t bytes) +{ + uint8_t nal_type; + uint8_t nuh_layer_id; + + if (bytes < 3) + return 0; + + nal_type = (nalu[1] >> 3) & 0x1f; + nuh_layer_id = nalu[0] & 0x3F; + + // 7.4.2.4.3 Order of PUs and their association to AUs + if (H266_NAL_OPI == nal_type || H266_NAL_DCI == nal_type || H266_NAL_VPS == nal_type || H266_NAL_SPS == nal_type || H266_NAL_PPS == nal_type || + (nuh_layer_id == 0 && (H266_NAL_AUD == nal_type || H266_PREFIX_APS_NUT == nal_type || H266_PH_NUT == nal_type || H266_NAL_PREFIX_SEI == nal_type || + 26 == nal_type || (28 <= nal_type && nal_type <= 29)))) + return 1; + + // 7.4.2.4.4 Order of NAL units and coded pictures and their association to PUs + if (nal_type < H266_NAL_OPI) + { + //sh_picture_header_in_slice_header_flag == 1 + return (nalu[2] & 0x80) ? 1 : 0; + } + + return 0; +} + +int mpeg_h266_find_new_access_unit(const uint8_t* data, size_t bytes, int* vcl) +{ + int n; + size_t leading; + uint8_t nal_type; + const uint8_t* p, * end; + + end = data + bytes; + for (p = data; p && p < end; p += n) + { + n = mpeg_h264_find_nalu(p, end - p, &leading); + if (n < 1) + return -1; + + nal_type = (p[n+1] >> 3) & 0x1f; + if (*vcl > 0 && mpeg_h266_is_new_access_unit(p + n, end - p - n)) + { + return (int)(p - data + n - leading); + } + else if (nal_type < H266_NAL_OPI) + { + *vcl = (H266_NAL_IDR_W_RADL <= nal_type && nal_type <= H266_NAL_RSV_IRAP) ? MPEG_VCL_IDR : MPEG_VCL_P; + } + else if (PES_SID_START == p[n]) + { + // pes data loss ??? + *vcl = MPEG_VCL_CORRUPT; // for assert + return (int)(p - data + n - leading); + } + else + { + // nothing to do + } + } + + return -1; +}