From 68dc1e5196c44761c4a3b861ebcd337ffdaf8e4d Mon Sep 17 00:00:00 2001 From: Cacsjep Date: Wed, 24 Jan 2024 16:53:45 +0100 Subject: [PATCH] Review changes add HardwareConfigs to codec create codec_hardware_config create codec_hardware_config_method_flag update example update flags better error handling in TransferHardwareData update CreateHardwareDeviceContext, better null handling remove hw ctx test because it fails when no nvidia is present some reordering of conss in hardware_device_type --- README.md | 2 +- codec.go | 25 ++++++++ codec_hardware_config.go | 22 +++++++ codec_hardware_config_method_flag.go | 15 +++++ examples/hardware_decoding/main.go | 85 +++++++++++++++------------- flags.go | 20 +++++++ flags_test.go | 9 +++ frame.go | 6 +- hardware_device_context.go | 30 +++------- hardware_device_context_test.go | 14 ----- hardware_device_type.go | 14 ++--- internal/cmd/flags/main.go | 1 + 12 files changed, 153 insertions(+), 90 deletions(-) create mode 100644 codec_hardware_config.go create mode 100644 codec_hardware_config_method_flag.go delete mode 100644 hardware_device_context_test.go diff --git a/README.md b/README.md index aa187df..c43e743 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ Examples are located in the [examples](examples) directory and mirror as much as |---|---|---| |Demuxing/Decoding|[see](examples/demuxing_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/demuxing_decoding.c) |Filtering|[see](examples/filtering/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/filtering_video.c) +|Hardware Decoding|[see](examples/hardware_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/hw_decode.c) |Remuxing|[see](examples/remuxing/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/remuxing.c) |Transcoding|[see](examples/transcoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/transcoding.c) -|Hardware Decoding|[see](examples/hardware_decoding/main.go)|[see](https://github.com/FFmpeg/FFmpeg/blob/n5.1.2/doc/examples/hw_decode.c) *Tip: you can use the video sample located in the `testdata` directory for your tests* diff --git a/codec.go b/codec.go index d1d75c2..08fc17f 100644 --- a/codec.go +++ b/codec.go @@ -5,6 +5,7 @@ package astiav //#include import "C" import ( + "fmt" "unsafe" ) @@ -100,3 +101,27 @@ func FindEncoderByName(n string) *Codec { defer C.free(unsafe.Pointer(cn)) return newCodecFromC(C.avcodec_find_encoder_by_name(cn)) } + +func (c *Codec) HardwareConfigs(dt HardwareDeviceType) ([]CodecHardwareConfig, error) { + var configs []CodecHardwareConfig + var i int + + for { + config := C.avcodec_get_hw_config(c.c, C.int(i)) + if config == nil { + break + } + + if HardwareDeviceType(config.device_type) == dt { + configs = append(configs, CodecHardwareConfig{c: config}) + } + + i++ + } + + if len(configs) == 0 { + return nil, fmt.Errorf("Decoder does not support device type %s", dt) + } + + return configs, nil +} diff --git a/codec_hardware_config.go b/codec_hardware_config.go new file mode 100644 index 0000000..9b89ace --- /dev/null +++ b/codec_hardware_config.go @@ -0,0 +1,22 @@ +package astiav + +//#cgo pkg-config: libavcodec +//#include +import "C" + +// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavcodec/codec.h#L460 +type CodecHardwareConfig struct { + c *C.AVCodecHWConfig +} + +func (chc CodecHardwareConfig) HardwareDeviceType() HardwareDeviceType { + return HardwareDeviceType(chc.c.device_type) +} + +func (chc CodecHardwareConfig) MethodFlags() CodecHardwareConfigMethodFlags { + return CodecHardwareConfigMethodFlags(chc.c.methods) +} + +func (chc CodecHardwareConfig) PixelFormat() PixelFormat { + return PixelFormat(chc.c.pix_fmt) +} diff --git a/codec_hardware_config_method_flag.go b/codec_hardware_config_method_flag.go new file mode 100644 index 0000000..561bd3d --- /dev/null +++ b/codec_hardware_config_method_flag.go @@ -0,0 +1,15 @@ +package astiav + +//#cgo pkg-config: libavcodec +//#include +import "C" + +// https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavcodec/codec.h#L420 +type CodecHardwareConfigMethodFlag int + +const ( + CodecHardwareConfigMethodHwDeviceCtx = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) + CodecHardwareConfigMethodHwFramesCtx = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) + CodecHardwareConfigMethodInternal = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_INTERNAL) + CodecHardwareConfigMethodAdHoc = CodecHardwareConfigMethodFlag(C.AV_CODEC_HW_CONFIG_METHOD_AD_HOC) +) diff --git a/examples/hardware_decoding/main.go b/examples/hardware_decoding/main.go index d1a20ef..88baf3c 100644 --- a/examples/hardware_decoding/main.go +++ b/examples/hardware_decoding/main.go @@ -11,8 +11,8 @@ import ( ) var ( - input = flag.String("i", "", "the input path") - device_type = flag.String("d", "", "the hardware device type like: cuda") + hardwareDeviceTypeName = flag.String("d", "", "the hardware device type like: cuda") + input = flag.String("i", "", "the input path") ) type stream struct { @@ -22,15 +22,6 @@ type stream struct { inputStream *astiav.Stream } -func hw_decoder_init(ctx *astiav.CodecContext, t astiav.HardwareDeviceType) (*astiav.HardwareDeviceContext, error) { - hdc, err := astiav.CreateHardwareDeviceContext(t, "", &astiav.Dictionary{}) - if err != nil { - return nil, fmt.Errorf("Unable to create hardware device context: %w", err) - } - ctx.SetHardwareDeviceContext(hdc) - return hdc, nil -} - func main() { // Handle ffmpeg logs astiav.SetLogLevel(astiav.LogLevelDebug) @@ -42,13 +33,14 @@ func main() { flag.Parse() // Usage - if *input == "" || *device_type == "" { + if *input == "" || *hardwareDeviceTypeName == "" { log.Println("Usage: -d -i ") return } - hw_device := astiav.FindHardwareDeviceTypeByName(*device_type) - if hw_device == astiav.HardwareDeviceTypeNone { + // Get hardware device type + hardwareDeviceType := astiav.FindHardwareDeviceTypeByName(*hardwareDeviceTypeName) + if hardwareDeviceType == astiav.HardwareDeviceTypeNone { log.Fatal(errors.New("main: hardware device not found")) } @@ -56,11 +48,13 @@ func main() { pkt := astiav.AllocPacket() defer pkt.Free() - gpu_f := astiav.AllocFrame() - defer gpu_f.Free() + // Alloc hardware frame + hardwareFrame := astiav.AllocFrame() + defer hardwareFrame.Free() - sw_f := astiav.AllocFrame() - defer sw_f.Free() + // Alloc software frame + softwareFrame := astiav.AllocFrame() + defer softwareFrame.Free() // Alloc input format context inputFormatContext := astiav.AllocFormatContext() @@ -80,9 +74,8 @@ func main() { log.Fatal(fmt.Errorf("main: finding stream info failed: %w", err)) } - var hw_pix_fmt astiav.PixelFormat - // Loop through streams + hardwarePixelFormat := astiav.PixelFormatNone streams := make(map[int]*stream) // Indexed by input stream index for _, is := range inputFormatContext.Streams() { var err error @@ -106,11 +99,20 @@ func main() { } defer s.decCodecContext.Free() - hw_pix_fmt, err = astiav.FindSuitableHardwareFormat(s.decCodec, hw_device) + codec_hardware_configs, err := s.decCodec.HardwareConfigs(hardwareDeviceType) if err != nil { - log.Fatal(fmt.Errorf("main: find decoder hw format fails: %w", err)) - } else { - log.Printf("Using hw_pix_fmt: %s", hw_pix_fmt.Name()) + log.Fatal(fmt.Errorf("main: no codec hardware config found: %w", err)) + } + + for _, p := range codec_hardware_configs { + if p.MethodFlags().Has(astiav.CodecHardwareConfigMethodHwDeviceCtx) && p.HardwareDeviceType() == hardwareDeviceType { + hardwarePixelFormat = p.PixelFormat() + break + } + } + + if hardwarePixelFormat == astiav.PixelFormatNone { + log.Fatal(fmt.Errorf("main: Decoder %s does not support device type", hardwareDeviceType.String())) } // Update codec context @@ -118,9 +120,12 @@ func main() { log.Fatal(fmt.Errorf("main: updating codec context failed: %w", err)) } - if s.hwDeviceContext, err = hw_decoder_init(s.decCodecContext, hw_device); err != nil { - log.Fatal(fmt.Errorf("main: init hardware device failed: %w", err)) + // Create hardware device context + hdc, err := astiav.CreateHardwareDeviceContext(hardwareDeviceType, "", nil) + if err != nil { + log.Fatal(fmt.Errorf("main: creating hardware device context failed: %w", err)) } + s.decCodecContext.SetHardwareDeviceContext(hdc) // Open codec context if err := s.decCodecContext.Open(s.decCodec, nil); err != nil { @@ -155,33 +160,33 @@ func main() { // Loop for { // Receive frame - if err := s.decCodecContext.ReceiveFrame(gpu_f); err != nil { + if err := s.decCodecContext.ReceiveFrame(hardwareFrame); err != nil { if errors.Is(err, astiav.ErrEof) || errors.Is(err, astiav.ErrEagain) { break } log.Fatal(fmt.Errorf("main: receiving frame failed: %w", err)) } - if gpu_f.PixelFormat() == hw_pix_fmt { - err := gpu_f.TransferHardwareData(sw_f) - if err != nil { + // Get frame + var finalFrame *astiav.Frame + if hardwareFrame.PixelFormat() == hardwarePixelFormat { + // Transfer hardware data + if err := hardwareFrame.TransferHardwareData(softwareFrame); err != nil { log.Fatal(fmt.Errorf("main: Unable to transfer frame from gpu: %w", err)) } - data, err := sw_f.Data().Bytes(1) - - if err != nil { - log.Fatal(fmt.Errorf("main: Unable to get frame bytes: %w", err)) - } - - sw_f.SetPts(gpu_f.Pts()) + // Update pts + softwareFrame.SetPts(hardwareFrame.Pts()) - // Do something with decoded frame - log.Printf("new frame: pts: %d gpu frame pix fmt: %s, sw frame (transfered) pix fmt: %s, size: %d", sw_f.Pts(), gpu_f.PixelFormat().Name(), sw_f.PixelFormat().Name(), len(data)) + // Update final frame + finalFrame = softwareFrame } else { - log.Fatal(fmt.Errorf("main: Mismatch in pixel format: gpu: %s sw (transfered): %s", gpu_f.PixelFormat().Name(), sw_f.PixelFormat().Name())) + // Update final frame + finalFrame = hardwareFrame } + // Do something with decoded frame + log.Printf("new frame: stream %d - pts: %d - data transferred from hardware: %v", pkt.StreamIndex(), finalFrame.Pts(), finalFrame.PixelFormat() == hardwarePixelFormat) } } diff --git a/flags.go b/flags.go index cabbc4f..2761e17 100644 --- a/flags.go +++ b/flags.go @@ -85,6 +85,26 @@ func (fs CodecContextFlags2) Del(f CodecContextFlag2) CodecContextFlags2 { func (fs CodecContextFlags2) Has(f CodecContextFlag2) bool { return astikit.BitFlags(fs).Has(uint64(f)) } +type CodecHardwareConfigMethodFlags astikit.BitFlags + +func NewCodecHardwareConfigMethodFlags(fs ...CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags { + o := CodecHardwareConfigMethodFlags(0) + for _, f := range fs { + o = o.Add(f) + } + return o +} + +func (fs CodecHardwareConfigMethodFlags) Add(f CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags { + return CodecHardwareConfigMethodFlags(astikit.BitFlags(fs).Add(uint64(f))) +} + +func (fs CodecHardwareConfigMethodFlags) Del(f CodecHardwareConfigMethodFlag) CodecHardwareConfigMethodFlags { + return CodecHardwareConfigMethodFlags(astikit.BitFlags(fs).Del(uint64(f))) +} + +func (fs CodecHardwareConfigMethodFlags) Has(f CodecHardwareConfigMethodFlag) bool { return astikit.BitFlags(fs).Has(uint64(f)) } + type DictionaryFlags astikit.BitFlags func NewDictionaryFlags(fs ...DictionaryFlag) DictionaryFlags { diff --git a/flags_test.go b/flags_test.go index a05dbb0..c133e68 100644 --- a/flags_test.go +++ b/flags_test.go @@ -43,6 +43,15 @@ func TestCodecContextFlags2(t *testing.T) { require.False(t, fs.Has(astiav.CodecContextFlag2(2))) } +func TestCodecHardwareConfigMethodFlags(t *testing.T) { + fs := astiav.NewCodecHardwareConfigMethodFlags(astiav.CodecHardwareConfigMethodFlag(1)) + require.True(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(1))) + fs = fs.Add(astiav.CodecHardwareConfigMethodFlag(2)) + require.True(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(2))) + fs = fs.Del(astiav.CodecHardwareConfigMethodFlag(2)) + require.False(t, fs.Has(astiav.CodecHardwareConfigMethodFlag(2))) +} + func TestDictionaryFlags(t *testing.T) { fs := astiav.NewDictionaryFlags(astiav.DictionaryFlag(1)) require.True(t, fs.Has(astiav.DictionaryFlag(1))) diff --git a/frame.go b/frame.go index 117159b..06a6cad 100644 --- a/frame.go +++ b/frame.go @@ -191,11 +191,7 @@ func (f *Frame) SetWidth(w int) { } func (f *Frame) TransferHardwareData(dst *Frame) error { - ret := C.av_hwframe_transfer_data(dst.c, f.c, 0) - if ret < 0 { - return newError(ret) - } - return nil + return newError(C.av_hwframe_transfer_data(dst.c, f.c, 0)) } func (f *Frame) Free() { diff --git a/hardware_device_context.go b/hardware_device_context.go index 4f26b32..d96a19d 100644 --- a/hardware_device_context.go +++ b/hardware_device_context.go @@ -5,7 +5,6 @@ package astiav //#include import "C" import ( - "fmt" "unsafe" ) @@ -16,34 +15,19 @@ type HardwareDeviceContext struct { func CreateHardwareDeviceContext(t HardwareDeviceType, device string, options *Dictionary) (*HardwareDeviceContext, error) { hdc := HardwareDeviceContext{} - - // Check for an emtpy string and pass NULL to av_hwdevice_ctx_create if its emtpy - var deviceC *C.char + deviceC := (*C.char)(nil) if device != "" { deviceC = C.CString(device) defer C.free(unsafe.Pointer(deviceC)) - } else { - deviceC = (*C.char)(nil) } - errorCode := C.av_hwdevice_ctx_create(&hdc.c, (C.enum_AVHWDeviceType)(t), deviceC, options.c, 0) - if errorCode < 0 { - return nil, newError(errorCode) + var optionsC *C.struct_AVDictionary + if options != nil { + optionsC = options.c } - return &hdc, nil -} - -func FindSuitableHardwareFormat(decoder *Codec, deviceType HardwareDeviceType) (PixelFormat, error) { - var i int - for { - config := C.avcodec_get_hw_config(decoder.c, C.int(i)) - if config == nil { - return 0, fmt.Errorf("Decoder %s does not support device type %s", decoder.Name(), deviceType.String()) - } - if config.methods&C.AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX != 0 && config.device_type == C.enum_AVHWDeviceType(deviceType) { - return PixelFormat(config.pix_fmt), nil - } - i++ + if err := newError(C.av_hwdevice_ctx_create(&hdc.c, (C.enum_AVHWDeviceType)(t), deviceC, optionsC, 0)); err != nil { + return nil, err } + return &hdc, nil } func (hdc *HardwareDeviceContext) Free() { diff --git a/hardware_device_context_test.go b/hardware_device_context_test.go deleted file mode 100644 index 60c9b59..0000000 --- a/hardware_device_context_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package astiav_test - -import ( - "testing" - - "github.com/asticode/go-astiav" - "github.com/stretchr/testify/require" -) - -func TestHardwareDeviceContext(t *testing.T) { - hcd, err := astiav.CreateHardwareDeviceContext(astiav.HardwareDeviceTypeCUDA, "", astiav.NewDictionary()) - require.NoError(t, err) - defer hcd.Free() -} diff --git a/hardware_device_type.go b/hardware_device_type.go index 8e97063..9a422a1 100644 --- a/hardware_device_type.go +++ b/hardware_device_type.go @@ -11,17 +11,17 @@ import ( type HardwareDeviceType C.enum_AVHWDeviceType const ( - HardwareDeviceTypeNone = HardwareDeviceType(C.AV_HWDEVICE_TYPE_NONE) - HardwareDeviceTypeVDPAU = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VDPAU) HardwareDeviceTypeCUDA = HardwareDeviceType(C.AV_HWDEVICE_TYPE_CUDA) - HardwareDeviceTypeVAAPI = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VAAPI) - HardwareDeviceTypeDXVA2 = HardwareDeviceType(C.AV_HWDEVICE_TYPE_DXVA2) - HardwareDeviceTypeQSV = HardwareDeviceType(C.AV_HWDEVICE_TYPE_QSV) - HardwareDeviceTypeVideoToolbox = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VIDEOTOOLBOX) HardwareDeviceTypeD3D11VA = HardwareDeviceType(C.AV_HWDEVICE_TYPE_D3D11VA) HardwareDeviceTypeDRM = HardwareDeviceType(C.AV_HWDEVICE_TYPE_DRM) - HardwareDeviceTypeOpenCL = HardwareDeviceType(C.AV_HWDEVICE_TYPE_OPENCL) + HardwareDeviceTypeDXVA2 = HardwareDeviceType(C.AV_HWDEVICE_TYPE_DXVA2) HardwareDeviceTypeMediaCodec = HardwareDeviceType(C.AV_HWDEVICE_TYPE_MEDIACODEC) + HardwareDeviceTypeNone = HardwareDeviceType(C.AV_HWDEVICE_TYPE_NONE) + HardwareDeviceTypeOpenCL = HardwareDeviceType(C.AV_HWDEVICE_TYPE_OPENCL) + HardwareDeviceTypeQSV = HardwareDeviceType(C.AV_HWDEVICE_TYPE_QSV) + HardwareDeviceTypeVAAPI = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VAAPI) + HardwareDeviceTypeVDPAU = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VDPAU) + HardwareDeviceTypeVideoToolbox = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VIDEOTOOLBOX) HardwareDeviceTypeVulkan = HardwareDeviceType(C.AV_HWDEVICE_TYPE_VULKAN) ) diff --git a/internal/cmd/flags/main.go b/internal/cmd/flags/main.go index e7a66be..1e5c1a1 100644 --- a/internal/cmd/flags/main.go +++ b/internal/cmd/flags/main.go @@ -18,6 +18,7 @@ var list = []listItem{ {Name: "Buffersrc"}, {Name: "CodecContext"}, {Name: "CodecContext", Suffix: "2"}, + {Name: "CodecHardwareConfigMethod"}, {Name: "Dictionary"}, {Name: "FilterCommand"}, {Name: "FormatContextCtx"},