FFMPEGMedia：虚幻FFMPEG插件，支持更多视频格式和Alpha视频

// Copyright 1998-2018 Epic Games, Inc. All Rights Reserved. #include "FFMPEGMediaTracks.h" #include "FFMPEGMediaPrivate.h" #include "LambdaFunctionRunnable.h" #include "FFMPEGMediaSettings.h" #include "FFMPEGDecoder.h" #include "FFMPEGFrame.h" #include "IMediaOptions.h" #include "MediaHelpers.h" #include "Misc/ScopeLock.h" #include "UObject/Class.h" #include "IMediaBinarySample.h" #include "IMediaEventSink.h" #if WITH_ENGINE #include "Engine/Engine.h" #endif #include "FFMPEGMediaBinarySample.h" #include "FFMPEGMediaOverlaySample.h" #include "FFMPEGMediaAudioSample.h" #include "FFMPEGMediaTextureSample.h" extern "C" { #include "libavformat/avformat.h" #include "libavcodec/avcodec.h" #include "libavutil/imgutils.h" #include "libavutil/hwcontext.h" #include "libavutil/time.h" #include "libswscale/swscale.h" #include "libswresample/swresample.h" } #define MAX_QUEUE_SIZE (15 * 1024 * 1024) #define MIN_FRAMES 30 /* polls for possible required screen refresh at least this often, should be less than 1/fps */ #define REFRESH_RATE 0.01 /* AV sync correction is done if above the maximum AV sync threshold */ #define AV_SYNC_THRESHOLD_MAX 0.1 /* no AV sync correction is done if below the minimum AV sync threshold */ #define AV_SYNC_THRESHOLD_MIN 0.04 /* AV sync correction is done if above the maximum AV sync threshold */ #define AV_SYNC_THRESHOLD_MAX 0.1 /* If a frame duration is longer than this, it will not be duplicated to compensate AV sync */ #define AV_SYNC_FRAMEDUP_THRESHOLD 0.1 #define EXTERNAL_CLOCK_MIN_FRAMES 2 #define EXTERNAL_CLOCK_MAX_FRAMES 10 /* external clock speed adjustment constants for realtime sources based on buffer fullness */ #define EXTERNAL_CLOCK_SPEED_MIN 0.900 #define EXTERNAL_CLOCK_SPEED_MAX 1.010 #define EXTERNAL_CLOCK_SPEED_STEP 0.001 /* Minimum audio buffer size, in samples. */ #define AUDIO_MIN_BUFFER_SIZE 512 /* Calculate actual buffer size keeping in mind not cause too frequent audio callbacks */ #define AUDIO_MAX_CALLBACKS_PER_SEC 30 /* no AV correction is done if too big error */ #define AV_NOSYNC_THRESHOLD 10.0 /* we use about AUDIO_DIFF_AVG_NB A-V differences to make the average */ #define AUDIO_DIFF_AVG_NB 20 /* maximum audio speed change to get correct sync */ #define SAMPLE_CORRECTION_PERCENT_MAX 10 #define LOCTEXT_NAMESPACE "FFMPEGMediaTracks" /* FFFMPEGMediaTracks structors *****************************************************************************/ FFFMPEGMediaTracks::FFFMPEGMediaTracks() : AudioSamplePool(new FFFMPEGMediaAudioSamplePool) , FormatContext(NULL) , MediaSourceChanged(false) , SelectedAudioTrack(INDEX_NONE) , SelectedCaptionTrack(INDEX_NONE) , SelectedMetadataTrack(INDEX_NONE) , SelectedVideoTrack(INDEX_NONE) , SelectionChanged(false) , VideoSamplePool(new FFFMPEGMediaTextureSamplePool) , CurrentRate(0.0f) , CurrentState (EMediaState::Closed) , LastState (EMediaState::Closed) , CurrentTime(FTimespan::Zero()) , Duration(FTimespan::Zero()) , ShouldLoop(false) , bPrerolled(false) , imgConvertCtx(NULL) , readThread(nullptr) , audioThread(nullptr) , videoThread(nullptr) , subtitleThread(nullptr) , displayThread(nullptr) , audioStream(NULL) , videoStream(NULL) , subTitleStream(NULL) , video_ctx(NULL) , hw_device_ctx(NULL) , hw_frames_ctx(NULL) , swrContext(NULL) , aborted(false) , displayRunning(false) , step(false) , seekReq(false) , seekPos(0) , seekRel(0) , seekFlags(0) , queueAttachmentsReq(false) , videoStreamIdx(-1) , audioStreamIdx(-1) , subtitleStreamIdx(-1) , forceRefresh(false) , frameDropsLate(0) , frameDropsEarly(0) , frameTimer(0.0) , maxFrameDuration(0.0) , realtime(false) , audioBuf(NULL) , audioBuf1(NULL) , audioBufSize(0) , audioBuf1Size(0) , audioClockSerial(0) , audioClock(0) , audioCallbackTime(0) , audioDiffAvgCoef(0) , audioDiffThreshold(0) , audioDiffAvgCount(0) , audioDiffCum(0) , totalStreams(0) , currentStreams(0) , hwAccelPixFmt(AV_PIX_FMT_NONE) , hwAccelDeviceType(AV_HWDEVICE_TYPE_NONE){ } FFFMPEGMediaTracks::~FFFMPEGMediaTracks() { Shutdown(); delete AudioSamplePool; AudioSamplePool = nullptr; delete VideoSamplePool; VideoSamplePool = nullptr; } /* FFFMPEGMediaTracks interface *****************************************************************************/ void FFFMPEGMediaTracks::AppendStats(FString &OutStats) const { FScopeLock Lock(&CriticalSection); // audio tracks OutStats += TEXT("Audio Tracks\n"); if (AudioTracks.Num() == 0) { OutStats += TEXT("\tnone\n"); } else { for (const FTrack& Track : AudioTracks) { OutStats += FString::Printf(TEXT("\t%s\n"), *Track.DisplayName.ToString()); OutStats += TEXT("\t\tNot implemented yet"); } } // video tracks OutStats += TEXT("Video Tracks\n"); if (VideoTracks.Num() == 0) { OutStats += TEXT("\tnone\n"); } else { for (const FTrack& Track : VideoTracks) { OutStats += FString::Printf(TEXT("\t%s\n"), *Track.DisplayName.ToString()); OutStats += TEXT("\t\tNot implemented yet"); } } } void FFFMPEGMediaTracks::ClearFlags() { FScopeLock Lock(&CriticalSection); MediaSourceChanged = false; SelectionChanged = false; } void FFFMPEGMediaTracks::GetEvents(TArray<EMediaEvent>& OutEvents) { EMediaEvent Event; while (DeferredEvents.Dequeue(Event)) { OutEvents.Add(Event); } } void FFFMPEGMediaTracks::GetFlags(bool& OutMediaSourceChanged, bool& OutSelectionChanged) const { FScopeLock Lock(&CriticalSection); OutMediaSourceChanged = MediaSourceChanged; OutSelectionChanged = SelectionChanged; } void FFFMPEGMediaTracks::Initialize(AVFormatContext* ic, const FString& Url) { Shutdown(); UE_LOG(LogFFMPEGMedia, Verbose, TEXT("Tracks: %p: Initializing (media source %p)"), this, ic); FScopeLock Lock(&CriticalSection); MediaSourceChanged = true; SelectionChanged = true; if (!ic) { CurrentState = EMediaState::Error; return; } if (pictq.Init(&videoq, VIDEO_PICTURE_QUEUE_SIZE, 1) < 0) { Shutdown(); CurrentState = EMediaState::Error; return ; } realtime = IsRealtime(ic) != 0; if (subpq.Init(&subtitleq, SUBPICTURE_QUEUE_SIZE, 0) < 0) { Shutdown(); CurrentState = EMediaState::Error; return; } if (sampq.Init(&audioq, SAMPLE_QUEUE_SIZE, 1) < 0) { Shutdown(); CurrentState = EMediaState::Error; return; } vidclk.Init(&videoq); audclk.Init(&audioq); extclk.Init(&extclk); this->FormatContext = ic; audioClockSerial = -1; CurrentState = EMediaState::Preparing; // bool AllStreamsAdded = true; for (int i = 0; i < (int)ic->nb_streams; i++) { AVStream *st = ic->streams[i]; bool streamAdded = AddStreamToTracks(i, false, Info); AllStreamsAdded &= streamAdded; if ( streamAdded) { totalStreams++; } enum AVMediaType type = st->codecpar->codec_type; st->discard = AVDISCARD_ALL; } if (!AllStreamsAdded) { UE_LOG(LogFFMPEGMedia, Verbose, TEXT("Tracks %p: Not all available streams were added to the track collection"), this); } int64_t duration = ic->duration + (ic->duration <= INT64_MAX - 5000 ? 5000 : 0); Duration = duration * 10; maxFrameDuration = (ic->iformat->flags & AVFMT_TS_DISCONT) ? 10.0 : 3600.0; SetRate(0.0f); DeferredEvents.Enqueue(EMediaEvent::MediaOpened); //Start the read thread aborted = false; const auto Settings = GetDefault<UFFMPEGMediaSettings>(); sychronizationType = Settings->SyncType; readThread = LambdaFunctionRun