OpenShot Library | libopenshot  0.2.5
FFmpegReader.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #ifndef OPENSHOT_FFMPEG_READER_H
35 #define OPENSHOT_FFMPEG_READER_H
36 
37 #include "ReaderBase.h"
38 
39 // Include FFmpeg headers and macros
40 #include "FFmpegUtilities.h"
41 
42 #include <cmath>
43 #include <ctime>
44 #include <iostream>
45 #include <stdio.h>
46 #include <memory>
47 #include "CacheMemory.h"
48 #include "Clip.h"
49 #include "Exceptions.h"
50 #include "OpenMPUtilities.h"
51 #include "Settings.h"
52 
53 
54 namespace openshot {
55  /**
56  * @brief This struct holds the associated video frame and starting sample # for an audio packet.
57  *
58  * Because audio packets do not match up with video frames, this helps determine exactly
59  * where the audio packet's samples belong.
60  */
61  struct AudioLocation {
62  int64_t frame;
64 
65  bool is_near(AudioLocation location, int samples_per_frame, int64_t amount);
66  };
67 
68  /**
69  * @brief This class uses the FFmpeg libraries, to open video files and audio files, and return
70  * openshot::Frame objects for any frame in the file.
71  *
72  * All seeking and caching is handled internally, and the primary public interface is the GetFrame()
73  * method. To use this reader, simply create an instance of this class, and call the GetFrame method
74  * to start retrieving frames. Use the <b>info</b> struct to obtain information on the file, such as the length
75  * (# of frames), height, width, bit rate, frames per second (fps), etc...
76  *
77  * @code
78  * // Create a reader for a video
79  * openshot::FFmpegReader r("MyAwesomeVideo.webm");
80  * r.Open(); // Open the reader
81  *
82  * // Get frame number 1 from the video
83  * std::shared_ptr<openshot::Frame> f = r.GetFrame(1);
84  *
85  * // Now that we have an openshot::Frame object, lets have some fun!
86  * f->Display(); // Display the frame on the screen
87  * f->DisplayWaveform(); // Display the audio waveform as an image
88  * f->Play(); // Play the audio through your speaker
89  *
90  * // Close the reader
91  * r.Close();
92  * @endcode
93  */
94  class FFmpegReader : public ReaderBase {
95  private:
96  std::string path;
97 
98  AVFormatContext *pFormatCtx;
99  int i, videoStream, audioStream;
100  AVCodecContext *pCodecCtx, *aCodecCtx;
101 #if HAVE_HW_ACCEL
102  AVBufferRef *hw_device_ctx = NULL; //PM
103 #endif
104  AVStream *pStream, *aStream;
105  AVPacket *packet;
106  AVFrame *pFrame;
107  bool is_open;
108  bool is_duration_known;
109  bool check_interlace;
110  bool check_fps;
111  bool has_missing_frames;
112 
113  CacheMemory working_cache;
114  CacheMemory missing_frames;
115  std::map<int64_t, int64_t> processing_video_frames;
116  std::multimap<int64_t, int64_t> processing_audio_frames;
117  std::map<int64_t, int64_t> processed_video_frames;
118  std::map<int64_t, int64_t> processed_audio_frames;
119  std::multimap<int64_t, int64_t> missing_video_frames;
120  std::multimap<int64_t, int64_t> missing_video_frames_source;
121  std::multimap<int64_t, int64_t> missing_audio_frames;
122  std::multimap<int64_t, int64_t> missing_audio_frames_source;
123  std::map<int64_t, int> checked_frames;
124  AudioLocation previous_packet_location;
125 
126  // DEBUG VARIABLES (FOR AUDIO ISSUES)
127  int prev_samples;
128  int64_t prev_pts;
129  int64_t pts_total;
130  int64_t pts_counter;
131  int64_t num_packets_since_video_frame;
132  int64_t num_checks_since_final;
133  std::shared_ptr<openshot::Frame> last_video_frame;
134 
135  bool is_seeking;
136  int64_t seeking_pts;
137  int64_t seeking_frame;
138  bool is_video_seek;
139  int seek_count;
140  int64_t seek_audio_frame_found;
141  int64_t seek_video_frame_found;
142 
143  int64_t audio_pts_offset;
144  int64_t video_pts_offset;
145  int64_t last_frame;
146  int64_t largest_frame_processed;
147  int64_t current_video_frame; // can't reliably use PTS of video to determine this
148 
149  int hw_de_supported = 0; // Is set by FFmpegReader
150 #if HAVE_HW_ACCEL
151  AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
152  AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
153  int IsHardwareDecodeSupported(int codecid);
154 #endif
155 
156  /// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
157  void CheckFPS();
158 
159  /// Check the current seek position and determine if we need to seek again
160  bool CheckSeek(bool is_video);
161 
162  /// Check if a frame is missing and attempt to replace its frame image (and
163  bool CheckMissingFrame(int64_t requested_frame);
164 
165  /// Check the working queue, and move finished frames to the finished queue
166  void CheckWorkingFrames(bool end_of_stream, int64_t requested_frame);
167 
168  /// Convert Frame Number into Audio PTS
169  int64_t ConvertFrameToAudioPTS(int64_t frame_number);
170 
171  /// Convert Frame Number into Video PTS
172  int64_t ConvertFrameToVideoPTS(int64_t frame_number);
173 
174  /// Convert Video PTS into Frame Number
175  int64_t ConvertVideoPTStoFrame(int64_t pts);
176 
177  /// Create a new Frame (or return an existing one) and add it to the working queue.
178  std::shared_ptr<openshot::Frame> CreateFrame(int64_t requested_frame);
179 
180  /// Calculate Starting video frame and sample # for an audio PTS
181  AudioLocation GetAudioPTSLocation(int64_t pts);
182 
183  /// Get an AVFrame (if any)
184  bool GetAVFrame();
185 
186  /// Get the next packet (if any)
187  int GetNextPacket();
188 
189  /// Get the smallest video frame that is still being processed
190  int64_t GetSmallestVideoFrame();
191 
192  /// Get the smallest audio frame that is still being processed
193  int64_t GetSmallestAudioFrame();
194 
195  /// Get the PTS for the current video packet
196  int64_t GetVideoPTS();
197 
198  /// Remove partial frames due to seek
199  bool IsPartialFrame(int64_t requested_frame);
200 
201  /// Process a video packet
202  void ProcessVideoPacket(int64_t requested_frame);
203 
204  /// Process an audio packet
205  void ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample);
206 
207  /// Read the stream until we find the requested Frame
208  std::shared_ptr<openshot::Frame> ReadStream(int64_t requested_frame);
209 
210  /// Remove AVFrame from cache (and deallocate its memory)
211  void RemoveAVFrame(AVFrame *);
212 
213  /// Remove AVPacket from cache (and deallocate its memory)
214  void RemoveAVPacket(AVPacket *);
215 
216  /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs.
217  void Seek(int64_t requested_frame);
218 
219  /// Update PTS Offset (if any)
220  void UpdatePTSOffset(bool is_video);
221 
222  /// Update File Info for audio streams
223  void UpdateAudioInfo();
224 
225  /// Update File Info for video streams
226  void UpdateVideoInfo();
227 
228  public:
229  /// Final cache object used to hold final frames
231 
232  /// Enable or disable seeking. Seeking can more quickly locate the requested frame, but some
233  /// codecs have trouble seeking, and can introduce artifacts or blank images into the video.
235 
236  /// Constructor for FFmpegReader. This automatically opens the media file and loads
237  /// frame 1, or it throws one of the following exceptions.
238  FFmpegReader(std::string path);
239 
240  /// Constructor for FFmpegReader. This only opens the media file to inspect its properties
241  /// if inspect_reader=true. When not inspecting the media file, it's much faster, and useful
242  /// when you are inflating the object using JSON after instantiating it.
243  FFmpegReader(std::string path, bool inspect_reader);
244 
245  /// Destructor
246  virtual ~FFmpegReader();
247 
248  /// Close File
249  void Close();
250 
251  /// Get the cache object used by this reader
253 
254  /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader.
255  ///
256  /// @returns The requested frame of video
257  /// @param requested_frame The frame number that is requested.
258  std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame);
259 
260  /// Determine if reader is open or closed
261  bool IsOpen() { return is_open; };
262 
263  /// Return the type name of the class
264  std::string Name() { return "FFmpegReader"; };
265 
266  /// Get and Set JSON methods
267  std::string Json() const override; ///< Generate JSON string of this object
268  void SetJson(const std::string value); ///< Load JSON string into this object
269  Json::Value JsonValue() const override; ///< Generate Json::Value for this object
270  void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
271 
272  /// Open File - which is called by the constructor automatically
273  void Open();
274  };
275 
276 }
277 
278 #endif
Header file for CacheMemory class.
Header file for Clip class.
Header file for all Exception classes.
Header file for FFmpegUtilities.
Header file for OpenMPUtilities (set some common macros)
Header file for ReaderBase class.
Header file for global Settings class.
This class is a memory-based cache manager for Frame objects.
Definition: CacheMemory.h:51
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:94
FFmpegReader(std::string path)
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame)
CacheMemory * GetCache()
Get the cache object used by this reader.
Definition: FFmpegReader.h:252
Json::Value JsonValue() const override
Generate Json::Value for this object.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:230
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
virtual ~FFmpegReader()
Destructor.
std::string Json() const override
Get and Set JSON methods.
void Open()
Open File - which is called by the constructor automatically.
void SetJson(const std::string value)
Load JSON string into this object.
bool IsOpen()
Determine if reader is open or closed.
Definition: FFmpegReader.h:261
void Close()
Close File.
std::string Name()
Return the type name of the class.
Definition: FFmpegReader.h:264
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:98
This namespace is the default namespace for all code in the openshot library.
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:61
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)