OpenShot Library | libopenshot  0.2.7
CVTracker.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Track an object selected by the user
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35 
36 #include <google/protobuf/util/time_util.h>
37 
38 #include "OpenCVUtilities.h"
39 #include "CVTracker.h"
40 
41 using namespace openshot;
42 using google::protobuf::util::TimeUtil;
43 
44 // Constructor
45 CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController)
46 : processingController(&processingController), json_interval(false){
47  SetJson(processInfoJson);
48  start = 1;
49  end = 1;
50 }
51 
52 // Set desirable tracker method
53 cv::Ptr<OPENCV_TRACKER_TYPE> CVTracker::selectTracker(std::string trackerType){
54 
55  if (trackerType == "BOOSTING")
56  return OPENCV_TRACKER_NS::TrackerBoosting::create();
57  if (trackerType == "MIL")
58  return OPENCV_TRACKER_NS::TrackerMIL::create();
59  if (trackerType == "KCF")
60  return OPENCV_TRACKER_NS::TrackerKCF::create();
61  if (trackerType == "TLD")
62  return OPENCV_TRACKER_NS::TrackerTLD::create();
63  if (trackerType == "MEDIANFLOW")
64  return OPENCV_TRACKER_NS::TrackerMedianFlow::create();
65  if (trackerType == "MOSSE")
66  return OPENCV_TRACKER_NS::TrackerMOSSE::create();
67  if (trackerType == "CSRT")
68  return OPENCV_TRACKER_NS::TrackerCSRT::create();
69 
70  return nullptr;
71 }
72 
73 // Track object in the hole clip or in a given interval
74 void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
75 
76  video.Open();
77  if(!json_interval){
78  start = _start; end = _end;
79 
80  if(!process_interval || end <= 1 || end-start == 0){
81  // Get total number of frames in video
82  start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
83  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
84  }
85  }
86  else{
87  start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
88  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
89  }
90 
91  if(error){
92  return;
93  }
94 
95  processingController->SetError(false, "");
96  bool trackerInit = false;
97 
98  size_t frame;
99  // Loop through video
100  for (frame = start; frame <= end; frame++)
101  {
102 
103  // Stop the feature tracker process
104  if(processingController->ShouldStop()){
105  return;
106  }
107 
108  size_t frame_number = frame;
109  // Get current frame
110  std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
111 
112  // Grab OpenCV Mat image
113  cv::Mat cvimage = f->GetImageCV();
114 
115  if(frame == start){
116  // Take the normalized inital bounding box and multiply to the current video shape
117  bbox = cv::Rect2d(bbox.x*cvimage.cols,bbox.y*cvimage.rows,bbox.width*cvimage.cols,
118  bbox.height*cvimage.rows);
119  }
120 
121  // Pass the first frame to initialize the tracker
122  if(!trackerInit){
123 
124  // Initialize the tracker
125  initTracker(cvimage, frame_number);
126 
127  trackerInit = true;
128  }
129  else{
130  // Update the object tracker according to frame
131  trackerInit = trackFrame(cvimage, frame_number);
132 
133  // Draw box on image
134  FrameData fd = GetTrackedData(frame_number);
135 
136  }
137  // Update progress
138  processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
139  }
140 }
141 
142 // Initialize the tracker
143 bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
144 
145  // Create new tracker object
146  tracker = selectTracker(trackerType);
147 
148  // Correct if bounding box contains negative proportions (width and/or height < 0)
149  if(bbox.width < 0){
150  bbox.x = bbox.x - abs(bbox.width);
151  bbox.width = abs(bbox.width);
152  }
153  if(bbox.height < 0){
154  bbox.y = bbox.y - abs(bbox.height);
155  bbox.height = abs(bbox.height);
156  }
157 
158  // Initialize tracker
159  tracker->init(frame, bbox);
160 
161  float fw = frame.size().width;
162  float fh = frame.size().height;
163 
164  // Add new frame data
165  trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw,
166  (bbox.y)/fh,
167  (bbox.x+bbox.width)/fw,
168  (bbox.y+bbox.height)/fh);
169 
170  return true;
171 }
172 
173 // Update the object tracker according to frame
174 bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
175  // Update the tracking result
176  bool ok = tracker->update(frame, bbox);
177 
178  // Add frame number and box coords if tracker finds the object
179  // Otherwise add only frame number
180  if (ok)
181  {
182  float fw = frame.size().width;
183  float fh = frame.size().height;
184 
185  cv::Rect2d filtered_box = filter_box_jitter(frameId);
186  // Add new frame data
187  trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw,
188  (filtered_box.y)/fh,
189  (filtered_box.x+filtered_box.width)/fw,
190  (filtered_box.y+filtered_box.height)/fh);
191  }
192  else
193  {
194  // Copy the last frame data if the tracker get lost
195  trackedDataById[frameId] = trackedDataById[frameId-1];
196  }
197 
198  return ok;
199 }
200 
201 cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){
202  // get tracked data for the previous frame
203  float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1;
204  float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1;
205 
206  float curr_box_width = bbox.width;
207  float curr_box_height = bbox.height;
208  // keep the last width and height if the difference is less than 1%
209  float threshold = 0.01;
210 
211  cv::Rect2d filtered_box = bbox;
212  if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){
213  filtered_box.width = last_box_width;
214  }
215  if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){
216  filtered_box.height = last_box_height;
217  }
218  return filtered_box;
219 }
220 
222  using std::ios;
223 
224  // Create tracker message
225  pb_tracker::Tracker trackerMessage;
226 
227  // Iterate over all frames data and save in protobuf message
228  for(std::map<size_t,FrameData>::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){
229  FrameData fData = it->second;
230  pb_tracker::Frame* pbFrameData;
231  AddFrameDataToProto(trackerMessage.add_frame(), fData);
232  }
233 
234  // Add timestamp
235  *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
236 
237  {
238  // Write the new message to disk.
239  std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
240  if (!trackerMessage.SerializeToOstream(&output)) {
241  std::cerr << "Failed to write protobuf message." << std::endl;
242  return false;
243  }
244  }
245 
246  // Delete all global objects allocated by libprotobuf.
247  google::protobuf::ShutdownProtobufLibrary();
248 
249  return true;
250 
251 }
252 
253 // Add frame tracked data into protobuf message.
254 void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) {
255 
256  // Save frame number and rotation
257  pbFrameData->set_id(fData.frame_id);
258  pbFrameData->set_rotation(0);
259 
260  pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
261  // Save bounding box data
262  box->set_x1(fData.x1);
263  box->set_y1(fData.y1);
264  box->set_x2(fData.x2);
265  box->set_y2(fData.y2);
266 }
267 
268 // Get tracker info for the desired frame
270 
271  // Check if the tracker info for the requested frame exists
272  if ( trackedDataById.find(frameId) == trackedDataById.end() ) {
273 
274  return FrameData();
275  } else {
276 
277  return trackedDataById[frameId];
278  }
279 
280 }
281 
282 // Load JSON string into this object
283 void CVTracker::SetJson(const std::string value) {
284  // Parse JSON string into JSON objects
285  try
286  {
287  const Json::Value root = openshot::stringToJson(value);
288  // Set all values that match
289 
290  SetJsonValue(root);
291  }
292  catch (const std::exception& e)
293  {
294  // Error parsing JSON (or missing keys)
295  throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
296  }
297 }
298 
299 // Load Json::Value into this object
300 void CVTracker::SetJsonValue(const Json::Value root) {
301 
302  // Set data from Json (if key is found)
303  if (!root["protobuf_data_path"].isNull()){
304  protobuf_data_path = (root["protobuf_data_path"].asString());
305  }
306  if (!root["tracker-type"].isNull()){
307  trackerType = (root["tracker-type"].asString());
308  }
309 
310  if (!root["region"].isNull()){
311  double x = root["region"]["normalized_x"].asDouble();
312  double y = root["region"]["normalized_y"].asDouble();
313  double w = root["region"]["normalized_width"].asDouble();
314  double h = root["region"]["normalized_height"].asDouble();
315  cv::Rect2d prev_bbox(x,y,w,h);
316  bbox = prev_bbox;
317  }
318  else{
319  processingController->SetError(true, "No initial bounding box selected");
320  error = true;
321  }
322 
323  if (!root["region"]["first-frame"].isNull()){
324  start = root["region"]["first-frame"].asInt64();
325  json_interval = true;
326  }
327  else{
328  processingController->SetError(true, "No first-frame");
329  error = true;
330  }
331 }
332 
333 /*
334 ||||||||||||||||||||||||||||||||||||||||||||||||||
335  ONLY FOR MAKE TEST
336 ||||||||||||||||||||||||||||||||||||||||||||||||||
337 */
338 
339 // Load protobuf data file
341  using std::ios;
342 
343  // Create tracker message
344  pb_tracker::Tracker trackerMessage;
345 
346  {
347  // Read the existing tracker message.
348  std::fstream input(protobuf_data_path, ios::in | ios::binary);
349  if (!trackerMessage.ParseFromIstream(&input)) {
350  std::cerr << "Failed to parse protobuf message." << std::endl;
351  return false;
352  }
353  }
354 
355  // Make sure the trackedData is empty
356  trackedDataById.clear();
357 
358  // Iterate over all frames of the saved message
359  for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
360  const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i);
361 
362  // Load frame and rotation data
363  size_t id = pbFrameData.id();
364  float rotation = pbFrameData.rotation();
365 
366  // Load bounding box data
367  const pb_tracker::Frame::Box& box = pbFrameData.bounding_box();
368  float x1 = box.x1();
369  float y1 = box.y1();
370  float x2 = box.x2();
371  float y2 = box.y2();
372 
373  // Assign data to tracker map
374  trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
375  }
376 
377  // Delete all global objects allocated by libprotobuf.
378  google::protobuf::ShutdownProtobufLibrary();
379 
380  return true;
381 }
Track an object selected by the user.
Header file for OpenCVUtilities (set some common macros)
void SetError(bool err, std::string message)
void trackClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Definition: CVTracker.cpp:74
CVTracker(std::string processInfoJson, ProcessingController &processingController)
Definition: CVTracker.cpp:45
bool SaveTrackedData()
Save protobuf file.
Definition: CVTracker.cpp:221
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CVTracker.cpp:300
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CVTracker.cpp:283
cv::Rect2d filter_box_jitter(size_t frameId)
Filter current bounding box jitter.
Definition: CVTracker.cpp:201
void AddFrameDataToProto(pb_tracker::Frame *pbFrameData, FrameData &fData)
Add frame tracked data into protobuf message.
Definition: CVTracker.cpp:254
FrameData GetTrackedData(size_t frameId)
Get tracked data for a given frame.
Definition: CVTracker.cpp:269
cv::Ptr< OPENCV_TRACKER_TYPE > selectTracker(std::string trackerType)
Definition: CVTracker.cpp:53
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
void Open() override
Open the internal reader.
Definition: Clip.cpp:302
float End() const
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve.
Definition: Clip.cpp:338
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition: Clip.cpp:279
Exception for invalid JSON.
Definition: Exceptions.h:206
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34