想要看一下 这个 flv 的格式主要因素为 rtsp视频服务 转换为 rtmp服务 转换为前端可用的服务 , 然后 里面有 flv.js 的代码, 因为之前出现了一些问题 flvjs 播放 ws 服务代理的不存在的 rtsp 连接, Cannot read properties of null (reading ‘flushStashedSamples‘)

然后看了一下 flvjs 的代码, 并大致了解了一下 flv 的格式 以及 相关约束 

然后 后面有点时间, 可以 解析一下 flv 的文件, 以及 websocket 交互的 flv 数据流, 来用一下 

flv 格式的官方文档 http://www.adobe.com/devnet/flv/

参考文章 FLV格式详解_狗蛋儿l的博客-CSDN博客_flv格式

flv 解析

主要是基于 HXCodec 来做基础的 resolve 

这里编辑一下抽象的脉络 

FlvFile = FlvHeader + FlvBody 
FlvBody = prevTagSize0 + FlvTag1 + prevTagSize1 + FlvTag2 + prevTagSize2 + FlvTag3 + prevTagSize3 + ... +  FlvTagN + prevTagSizeN
FlvTag = tagType + dataSize + timeStamp + timeStampExt + streamId + FlvTagData
FlvTagData = FlvTagScriptData/FlvTagAudioData/FlvTagVideoData
FlvTagScriptData = amfData1 + amfData2 
FlvTagAudioData = metadata + audioData
FlvTagVideoData = metadata + videoData

相关实体

大多数的实体 描述了各个字段的编码解码情况, 一部分特殊的实体使用 特殊的 Codec 单独编码解码处理 

FlvFile 

/*** FlvFile** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 20:36*/
@Data
public class FlvFile implements Serializable {private FlvHeader header;private FlvBody body;public FlvFile() {}@Field(sort = 0, name = "header", dataType = DataType.GENERIC_BEAN, desc = "header", version = {1})public FlvHeader getHeader() {return header;}@Field(sort = 10, name = "body", dataType = DataType.GENERIC_BEAN, desc = "body", version = {1})public FlvBody getBody() {return body;}}

FlvHeader

@Data
public class FlvHeader implements Serializable {private String magic;private Integer version;private Integer typeFlag;private Integer dataOffset;@Field(sort = 0, name = "magic", dataType = DataType.CHARSET_ENCODING_WITH_FIXED_LEN_STRING, lengthInBytes = 3, desc = "magic", version = {1})public String getMagic() {return magic;}@Field(sort = 10, name = "version", dataType = DataType.BYTE, desc = "version", version = {1})public Integer getVersion() {return version;}@Field(sort = 20, name = "typeFlag", dataType = DataType.BYTE, desc = "typeFlag", version = {1})public Integer getTypeFlag() {return typeFlag;}@Field(sort = 30, name = "dataOffset", dataType = DataType.DWORD, desc = "dataOffset", version = {1})public Integer getDataOffset() {return dataOffset;}}

FlvBody

/*** FlvHeader** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 20:37*/
@Data
public class FlvBody implements Serializable {private Integer prevTagSize0;private List tagAndSizeList;@Field(sort = 0, name = "prevTagSize0", dataType = DataType.DWORD, desc = "prevTagSize0", version = {1})public Integer getPrevTagSize0() {return prevTagSize0;}@Field(sort = 10, name = "tagAndSizeList", dataType = DataType.GENERIC_BEAN_COLLECTION, desc = "tagAndSizeList", version = {1})public List getTagAndSizeList() {return tagAndSizeList;}}

FlvTagAndSize

/*** FlvHeader** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 20:37*/
@Data
public class FlvTagAndSize implements Serializable {private FlvTag tag;private Integer prevTagSize;@Field(sort = 0, name = "tag", dataType = DataType.GENERIC_BEAN, desc = "tag", codecFactoryClazz = FlvTagCodecFactory.class, version = {1})public FlvTag getTag() {return tag;}@Field(sort = 10, name = "prevTagSize", dataType = DataType.DWORD, desc = "prevTagSize", version = {1})public Integer getPrevTagSize() {return prevTagSize;}}

FlvTag 

/*** FlvHeader** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 20:37*/
@Data
public class FlvTag implements Serializable {private Integer tagType;private Integer[] dataSize;private Integer[] timeStamp;private Integer timeStampExt;private Integer[] streamId;private FlvTagData tagData;private long offsetInStream;@Field(sort = 0, name = "tagType", dataType = DataType.BYTE, desc = "tagType", version = {1})public Integer getTagType() {return tagType;}@Field(sort = 10, name = "dataSize", dataType = DataType.BYTE_ARRAY_WITH_EXACTLY_LEN, eleLength = 3, desc = "dataSize", version = {1})public Integer[] getDataSize() {return dataSize;}@Field(sort = 20, name = "timeStamp", dataType = DataType.BYTE_ARRAY_WITH_EXACTLY_LEN, eleLength = 3, desc = "timeStamp", version = {1})public Integer[] getTimeStamp() {return timeStamp;}@Field(sort = 30, name = "timeStampExt", dataType = DataType.BYTE, desc = "timeStampExt", version = {1})public Integer getTimeStampExt() {return timeStampExt;}@Field(sort = 40, name = "streamId", dataType = DataType.BYTE_ARRAY_WITH_EXACTLY_LEN, eleLength = 3, desc = "streamId", version = {1})public Integer[] getStreamId() {return streamId;}public FlvTagData getTagData() {return tagData;}public Integer getDataSizeInBytes() {return (usnignedSz(dataSize[0]) << 16) | (usnignedSz(dataSize[1]) << 8) | (usnignedSz(dataSize[2]));}public static Integer usnignedSz(int result) {if (result < 0) {result += (1 << 8);}return result;}}

FlvTagScriptData

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagScriptData extends FlvTagData {private FlvTagScriptDataAMF amfData1;private FlvTagScriptDataAMF amfData2;@Field(sort = 0, name = "amfData1", dataType = DataType.GENERIC_BEAN, desc = "amfData1", codecFactoryClazz = FlvTagScriptDataAMFCodecFactory.class, version = {1})public FlvTagScriptDataAMF getAmfData1() {return amfData1;}@Field(sort = 10, name = "amfData2", dataType = DataType.GENERIC_BEAN, desc = "amfData2", codecFactoryClazz = FlvTagScriptDataAMFCodecFactory.class, version = {1})public FlvTagScriptDataAMF getAmfData2() {return amfData2;}}

FlvTagScriptDataAMFData1

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagScriptDataAMFData1 extends FlvTagScriptDataAMFData {private String data;@Field(sort = 0, name = "data", dataType = DataType.CHARSET_ENCODING_WITH_LEN_STRING, lengthByteType = ByteType.WORD, desc = "data", version = {1})public String getData() {return data;}
}

FlvTagScriptDataAMFData2

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagScriptDataAMFData2 extends FlvTagScriptDataAMFData {private List data;@Field(sort = 0, name = "data", dataType = DataType.GENERIC_BEAN_COLLECTION_WITH_LEN, lengthByteType = ByteType.DWORD, desc = "data", codecFactoryClazz = FlvTagScriptDataAMFData2Data2CodecFactory.class, version = {1})public List getData() {return data;}}

FlvTagScriptDataAMFDataAttr

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagScriptDataAMFDataAttr implements Serializable {private String key;private Object value;@Field(sort = 0, name = "key", dataType = DataType.CHARSET_ENCODING_WITH_LEN_STRING, lengthByteType = ByteType.WORD, desc = "key", version = {1})public String getKey() {return key;}@Field(sort = 10, name = "value", dataType = DataType.CHARSET_ENCODING_WITH_LEN_STRING, lengthByteType = ByteType.WORD, desc = "value", version = {1})public Object getValue() {return value;}
}

FlvTagAudioData

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagAudioData extends FlvTagData {private Integer metadata;private Integer[] data;@Field(sort = 0, name = "metadata", dataType = DataType.UNSIGNED_BYTE, desc = "metadata", version = {1})public Integer getMetadata() {return metadata;}@Field(sort = 10, name = "data", dataType = DataType.BYTE_ARRAY, desc = "data", version = {1})public Integer[] getData() {return data;}// resolved by metadatapublic Integer getSoundFormat() {return (metadata >> 4) & 0xf;}public Integer getSoundRate() {return (metadata >> 2) & 0x3;}public Integer getSoundSize() {return (metadata >> 1) & 0x1;}public Integer getSoundType() {return (metadata) & 0x1;}}

FlvTagVideoData

/*** FlvTagData** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:45*/
@Data
public class FlvTagVideoData extends FlvTagData {private Integer metadata;private Integer[] data;@Field(sort = 0, name = "metadata", dataType = DataType.UNSIGNED_BYTE, desc = "metadata", version = {1})public Integer getMetadata() {return metadata;}@Field(sort = 10, name = "data", dataType = DataType.BYTE_ARRAY, desc = "data", version = {1})public Integer[] getData() {return data;}// resolved by metadatapublic Integer getVideoFrameType() {return (metadata >> 4) & 0xf;}public Integer getVideoCodecId() {return metadata & 0xf;}}

特殊的Codec

FlvTagCodec

/*** FlvTagCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:43*/
public class FlvTagCodec extends AbstractCodec {private AbstractCodec byteCodec = new ByteCodec();private AbstractCodec byteArray3Codec = new ByteArrayWithExactlyLenCodec(3);private AbstractCodec scriptCodec = CodecUtils.createCodecForClazz(FlvTagScriptData.class, 1);// 3 codec@Overridepublic void encode(FlvTag entity, ByteBuf buf) {byteCodec.encode(entity.getTagType(), buf);byteArray3Codec.encode(entity.getDataSize(), buf);byteArray3Codec.encode(entity.getTimeStamp(), buf);byteCodec.encode(entity.getTimeStampExt(), buf);byteArray3Codec.encode(entity.getStreamId(), buf);int tagType = entity.getTagType();Integer dataSizeInBytes = entity.getDataSizeInBytes();if (tagType == 0x12) {scriptCodec.encode((FlvTagScriptData) entity.getTagData(), buf);} else if (tagType == 0x08) {FlvTagAudioDataCodec codec = new FlvTagAudioDataCodec(dataSizeInBytes - 1);codec.encode((FlvTagAudioData) entity.getTagData(), buf);} else if (tagType == 0x09) {FlvTagVideoDataCodec codec = new FlvTagVideoDataCodec(dataSizeInBytes - 1);codec.encode((FlvTagVideoData) entity.getTagData(), buf);}}@Overridepublic FlvTag decode(ByteBuf buf) {long offsetInStream = buf.readerIndex();Integer tagType = byteCodec.decode(buf);Integer[] dataSize = byteArray3Codec.decode(buf);Integer[] timestamp = byteArray3Codec.decode(buf);Integer timestampExt = byteCodec.decode(buf);Integer[] streamId = byteArray3Codec.decode(buf);FlvTag result = new FlvTag();result.setOffsetInStream(offsetInStream);result.setTagType(tagType);result.setDataSize(dataSize);result.setTimeStamp(timestamp);result.setTimeStampExt(timestampExt);result.setStreamId(streamId);FlvTagData tagData = null;Integer dataSizeInBytes = result.getDataSizeInBytes();if (tagType == 0x12) {// consume 00 00 09tagData = scriptCodec.decode(buf);} else if (tagType == 0x08) {FlvTagAudioDataCodec codec = new FlvTagAudioDataCodec(dataSizeInBytes - 1);tagData = codec.decode(buf);} else if (tagType == 0x09) {FlvTagVideoDataCodec codec = new FlvTagVideoDataCodec(dataSizeInBytes - 1);tagData = codec.decode(buf);}result.setTagData(tagData);return result;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}}

FlvTagScriptDataAMFCodec

/*** FlvTagScriptDataAMFScriptDataAMFCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 16:01*/
public class FlvTagScriptDataAMFCodec extends AbstractCodec {private AbstractCodec byteCodec = new ByteCodec();private AbstractCodec data1Codec = CodecUtils.createCodecForClazz(FlvTagScriptDataAMFData1.class, 1);private AbstractCodec data2Codec = CodecUtils.createCodecForClazz(FlvTagScriptDataAMFData2.class, 1);@Overridepublic void encode(FlvTagScriptDataAMF entity, ByteBuf buf) {byteCodec.encode(entity.getAmfType(), buf);FlvTagScriptDataAMFData data = entity.getData();if (data instanceof FlvTagScriptDataAMFData1) {data1Codec.encode((FlvTagScriptDataAMFData1) data, buf);} else if (data instanceof FlvTagScriptDataAMFData2) {data2Codec.encode((FlvTagScriptDataAMFData2) data, buf);}}@Overridepublic FlvTagScriptDataAMF decode(ByteBuf buf) {Integer amfType = byteCodec.decode(buf);FlvTagScriptDataAMFData data = null;if (amfType == 0x02) {data = data1Codec.decode(buf);} else if (amfType == 0x08) {data = data2Codec.decode(buf);Integer last1 = byteCodec.decode(buf);Integer last2 = byteCodec.decode(buf);Integer last3 = byteCodec.decode(buf);AssertUtils.assert0(last1 == 0, " ex ");AssertUtils.assert0(last2 == 0, " ex ");AssertUtils.assert0(last3 == 9, " ex ");}FlvTagScriptDataAMF result = new FlvTagScriptDataAMF();result.setAmfType(amfType);result.setData(data);return result;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}
}

FlvTagScriptDataAMFData2Codec

/*** FlvTagScriptDataAMFDataAttrCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 17:28*/
public class FlvTagScriptDataAMFData2Codec extends AbstractCodec, List> {private DWordCodec dWordCodec = new DWordCodec();private FlvTagScriptDataAMFDataAttrCodec keyCodec = new FlvTagScriptDataAMFDataAttrCodec();@Overridepublic void encode(List entity, ByteBuf buf) {dWordCodec.encode(entity.size(), buf);for (FlvTagScriptDataAMFDataAttr attr : entity) {keyCodec.encode(attr, buf);}}@Overridepublic List decode(ByteBuf buf) {Integer length = dWordCodec.decode(buf);List list = new ArrayList<>();for (int i = 0; i < length; i++) {FlvTagScriptDataAMFDataAttr attr = keyCodec.decode(buf);list.add(attr);}return list;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}}

FlvTagScriptDataAMFDataAttrCodec

/*** FlvTagScriptDataAMFDataAttrCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 17:28*/
public class FlvTagScriptDataAMFDataAttrCodec extends AbstractCodec {private CharsetEncodingStringWithLenCodec keyCodec = new CharsetEncodingStringWithLenCodec(ByteType.WORD);private ByteCodec byteCodec = new ByteCodec();private WordCodec wordCodec = new WordCodec();private DWordCodec dwordCodec = new DWordCodec();private QWordCodec qwordCodec = new QWordCodec();private ByteArrayWithExactlyLenCodec eightByteCodec = new ByteArrayWithExactlyLenCodec(8);@Overridepublic void encode(FlvTagScriptDataAMFDataAttr entity, ByteBuf buf) {keyCodec.encode(entity.getKey(), buf);Object value = entity.getValue();if (value instanceof Number) {byteCodec.encode(0, buf);qwordCodec.encode((Long) value, buf);} else if (value instanceof Boolean) {byteCodec.encode(1, buf);byteCodec.encode((Boolean) value ? 1 : 0, buf);} else if (value instanceof String) {byteCodec.encode(2, buf);keyCodec.encode((String) value, buf);} else if (value instanceof JSONObject) {byteCodec.encode(3, buf);// impl// ECMA script Array} else if (value instanceof JSONArray) {byteCodec.encode(8, buf);// impl// ScriptDataObjectEnd} else if (value instanceof JSONArray) {byteCodec.encode(9, buf);// impl// Strict array type} else if (value instanceof JSONArray) {byteCodec.encode(10, buf);// impl// Date type} else if (value instanceof JSONArray) {byteCodec.encode(11, buf);// impl// Long string type} else if (value instanceof JSONArray) {byteCodec.encode(12, buf);// impl}}@Overridepublic FlvTagScriptDataAMFDataAttr decode(ByteBuf buf) {String key = keyCodec.decode(buf);Integer type = byteCodec.decode(buf);
//        System.out.println(key + " -> " + type + " -> " + buf.readerIndex());Object value = null;if (type == 0) {value = eightByteCodec.decode(buf);} else if (type == 1) {Integer tmp = byteCodec.decode(buf);value = (tmp == 1);} else if (type == 2) {value = keyCodec.decode(buf);} else if (type == 3) {// impl// ECMA script Array} else if (type == 8) {// impl// ScriptDataObjectEnd} else if (type == 9) {// impl// Strict array type} else if (type == 10) {// impl// Date type} else if (type == 11) {// impl// Long string type} else if (type == 12) {// impl}FlvTagScriptDataAMFDataAttr result = new FlvTagScriptDataAMFDataAttr();result.setKey(key);result.setValue(value);return result;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}}

FlvTagAudioDataCodec

/*** FlvTagAudioDataCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:43*/
public class FlvTagAudioDataCodec extends AbstractCodec {private int lengthInBytes;private AbstractCodec byteCodec = new UnsignedByteCodec();public FlvTagAudioDataCodec(int lengthInBytes) {this.lengthInBytes = lengthInBytes;}@Overridepublic void encode(FlvTagAudioData entity, ByteBuf buf) {ByteArrayWithExactlyLenCodec dataCodec = new ByteArrayWithExactlyLenCodec(lengthInBytes);byteCodec.encode(entity.getMetadata(), buf);dataCodec.encode(entity.getData(), buf);}@Overridepublic FlvTagAudioData decode(ByteBuf buf) {ByteArrayWithExactlyLenCodec dataCodec = new ByteArrayWithExactlyLenCodec(lengthInBytes);Integer metadata = byteCodec.decode(buf);Integer[] data = dataCodec.decode(buf);FlvTagAudioData result = new FlvTagAudioData();result.setMetadata(metadata);result.setData(data);return result;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}}

FlvTagVideoDataCodec

/*** FlvTagVideoDataCodec** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-13 15:43*/
public class FlvTagVideoDataCodec extends AbstractCodec {private int lengthInBytes;private AbstractCodec byteCodec = new UnsignedByteCodec();public FlvTagVideoDataCodec(int lengthInBytes) {this.lengthInBytes = lengthInBytes;}@Overridepublic void encode(FlvTagVideoData entity, ByteBuf buf) {ByteArrayWithExactlyLenCodec dataCodec = new ByteArrayWithExactlyLenCodec(lengthInBytes);byteCodec.encode(entity.getMetadata(), buf);dataCodec.encode(entity.getData(), buf);}@Overridepublic FlvTagVideoData decode(ByteBuf buf) {ByteArrayWithExactlyLenCodec dataCodec = new ByteArrayWithExactlyLenCodec(lengthInBytes);Integer metadata = byteCodec.decode(buf);Integer[] data = dataCodec.decode(buf);FlvTagVideoData result = new FlvTagVideoData();result.setMetadata(metadata);result.setData(data);return result;}@Overridepublic boolean isFixedLength() {return false;}@Overridepublic int length() {return 0;}}

解析结果展示

解析 flv 文件 

/*** Test17ResolveFlv** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 16:52*/
public class Test17ResolveFlv {// Test17ResolveFlvpublic static void main(String[] args) throws Exception {String path = "/Users/jerry/Jobs/12_flvResolve/dump.flv";byte[] bytes = IOUtils.toByteArray(new FileInputStream(path));AbstractCodec flvCodec = CodecUtils.createCodecForClazz(FlvFile.class, 1);ByteBuf buf = Unpooled.wrappedBuffer(bytes);FlvFile flvFile = flvCodec.decode(buf);for (FlvTagAndSize tagAndSize : flvFile.getBody().getTagAndSizeList()) {AssertUtils.assert0(tagAndSize.getPrevTagSize() == tagAndSize.getTag().getDataSizeInBytes() + 11, " ex data size ");}int x = 0;}}

解析效果如下 

解析 websocket 传输的 flv 视频数据 

这里从 websocket 请求吧传输的数据拿过来, 然后 这里仅仅保留了两个 videoData 的数据 

/*** Test17ResolveFlv** @author Jerry.X.He <970655147@qq.com>* @version 1.0* @date 2022-09-12 16:52*/
public class Test17ResolveWebsocketFlv {// Test17ResolveFlvpublic static void main(String[] args) throws Exception {String path = "/Users/jerry/Tmp/09_flv_from_websocket/test_rtps.flv.txt";List lines = Tools.getContentWithList(path);byte[] fullBytes = websocketHex2Bytes(lines);byte[] bytes = retainTwoVideoTag(fullBytes);AbstractCodec flvCodec = CodecUtils.createCodecForClazz(FlvFile.class, 1);ByteBuf buf = Unpooled.wrappedBuffer(bytes);FlvFile flvFile = flvCodec.decode(buf);for (FlvTagAndSize tagAndSize : flvFile.getBody().getTagAndSizeList()) {AssertUtils.assert0(tagAndSize.getPrevTagSize() == tagAndSize.getTag().getDataSizeInBytes() + 11, " ex data size ");}int x = 0;}// websocketHex2Bytespublic static byte[] websocketHex2Bytes(List lines) throws Exception {ByteArrayOutputStream baos = new ByteArrayOutputStream();for (String line : lines) {String[] towByteSplits = line.split("\\s+");for (String twoByteStr : towByteSplits) {byte[] twoByte = HexUtils.decodeHex(twoByteStr);baos.write(twoByte);}}return baos.toByteArray();}// retainTwoVideoTagpublic static byte[] retainTwoVideoTag(byte[] bytes) {int secondVideoTagEnd = 46701;ByteArrayOutputStream baos = new ByteArrayOutputStream(secondVideoTagEnd);baos.write(bytes, 0, secondVideoTagEnd);return baos.toByteArray();}}

解析效果如下 

本文更加侧重于是对于 flv 的结构理解 

并不涉及具体的 音频数据, 视频数据 的处理 

完