48 ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
51 init_effect_details();
54 LoadObjDetectdData(clipObDetectDataPath);
57 selectedObjectIndex = trackedObjects.begin()->first;
61 ObjectDetection::ObjectDetection()
64 init_effect_details();
67 selectedObjectIndex = trackedObjects.begin()->first;
71 void ObjectDetection::init_effect_details()
77 info.class_name =
"Object Detector";
78 info.name =
"Object Detector";
79 info.description =
"Detect objects through the video.";
80 info.has_audio =
false;
81 info.has_video =
true;
82 info.has_tracked_object =
true;
87 std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
90 cv::Mat cv_image = frame->GetImageCV();
98 std::vector<QRectF> boxRects;
100 std::vector<std::shared_ptr<QImage>> childClipImages;
103 if (detectionsData.find(frame_number) != detectionsData.end()) {
104 float fw = cv_image.size().width;
105 float fh = cv_image.size().height;
108 for(
int i = 0; i<detections.
boxes.size(); i++){
111 if(detections.
confidences.at(i) < confidence_threshold){
115 if( display_classes.size() > 0 &&
116 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
121 int objectId = detections.
objectIds.at(i);
124 auto trackedObject_it = trackedObjects.find(objectId);
127 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
130 if (trackedObject->Contains(frame_number) &&
131 trackedObject->visible.GetValue(frame_number) == 1)
134 BBox trackedBox = trackedObject->GetBox(frame_number);
135 bool draw_text = !display_box_text.GetValue(frame_number);
136 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
137 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
138 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
139 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
140 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
152 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
153 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
154 (
int)( trackedBox.
width*fw),
155 (
int)( trackedBox.
height*fh)
159 if (trackedObject->draw_box.GetValue(frame_number) == 0)
166 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
168 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
172 if (trackedObject->ChildClipId() !=
""){
177 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
180 std::shared_ptr<Frame> f(
new Frame(1, frame->GetWidth(), frame->GetHeight(),
"#00000000"));
182 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(f, frame_number);
183 childClipImages.push_back(childClipFrame->GetImage());
187 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
188 (
int)((trackedBox.
cy - trackedBox.
height/2)*fh),
189 (
int)(trackedBox.
width*fw),
190 (
int)(trackedBox.
height*fh));
191 boxRects.push_back(boxRect);
200 frame->SetImageCV(cv_image);
203 if(boxRects.size() > 0){
205 QImage frameImage = *(frame->GetImage());
206 for(
int i; i < boxRects.size();i++){
208 QPainter painter(&frameImage);
210 painter.drawImage(boxRects[i], *childClipImages[i], QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
213 frame->AddImage(std::make_shared<QImage>(frameImage));
219 void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
220 int thickness,
bool is_background){
222 cv::Point2f vertices2f[4];
223 box.points(vertices2f);
231 cv::Mat overlayFrame;
232 frame_image.copyTo(overlayFrame);
235 cv::Point vertices[4];
236 for(
int i = 0; i < 4; ++i){
237 vertices[i] = vertices2f[i];}
239 cv::Rect rect = box.boundingRect();
240 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
242 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
245 cv::Mat overlayFrame;
246 frame_image.copyTo(overlayFrame);
249 for (
int i = 0; i < 4; i++)
251 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
252 thickness, cv::LINE_AA);
256 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
260 void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
261 float alpha,
int thickness,
bool is_background,
bool display_text)
265 cv::Mat overlayFrame;
266 frame.copyTo(overlayFrame);
269 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
272 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
275 cv::Mat overlayFrame;
276 frame.copyTo(overlayFrame);
279 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
283 std::string label = cv::format(
"%.2f", conf);
284 if (!classNames.empty())
286 CV_Assert(classId < (
int)classNames.size());
287 label = classNames[classId] +
":" + label;
292 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
295 double top = std::max((
int)box.y, labelSize.height);
297 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
298 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
299 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
302 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
307 bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
309 pb_objdetect::ObjDetect objMessage;
312 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
313 if (!objMessage.ParseFromIstream(&input)) {
314 std::cerr <<
"Failed to parse protobuf message." << std::endl;
320 detectionsData.clear();
321 trackedObjects.clear();
326 for(
int i = 0; i < objMessage.classnames_size(); i++)
328 classNames.push_back(objMessage.classnames(i));
329 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
333 for (
size_t i = 0; i < objMessage.frame_size(); i++)
336 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
339 size_t id = pbFrameData.id();
342 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
345 std::vector<int> classIds;
346 std::vector<float> confidences;
347 std::vector<cv::Rect_<float>> boxes;
348 std::vector<int> objectIds;
351 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
354 float x = pBox.Get(i).x();
355 float y = pBox.Get(i).y();
356 float w = pBox.Get(i).w();
357 float h = pBox.Get(i).h();
359 int classId = pBox.Get(i).classid();
361 float confidence = pBox.Get(i).confidence();
364 int objectId = pBox.Get(i).objectid();
367 auto trackedObject = trackedObjects.find(objectId);
369 if (trackedObject != trackedObjects.end())
372 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
377 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
378 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
380 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
381 ClipBase* parentClip = this->ParentClip();
382 trackedObjPtr->ParentClip(parentClip);
386 trackedObjPtr->Id(std::to_string(objectId));
387 trackedObjects.insert({objectId, trackedObjPtr});
391 cv::Rect_<float> box(x, y, w, h);
394 boxes.push_back(box);
395 classIds.push_back(classId);
396 confidences.push_back(confidence);
397 objectIds.push_back(objectId);
401 detectionsData[id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
405 google::protobuf::ShutdownProtobufLibrary();
411 std::string ObjectDetection::GetVisibleObjects(int64_t frame_number)
const{
415 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
416 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
419 if (detectionsData.find(frame_number) == detectionsData.end()){
420 return root.toStyledString();
425 for(
int i = 0; i<detections.
boxes.size(); i++){
427 if(detections.
confidences.at(i) < confidence_threshold){
432 if( display_classes.size() > 0 &&
433 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
437 int objectId = detections.
objectIds.at(i);
439 auto trackedObject = trackedObjects.find(objectId);
442 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
444 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
445 trackedObject->second->ExactlyContains(frame_number)){
447 root[
"visible_objects_index"].append(trackedObject->first);
448 root[
"visible_objects_id"].append(trackedObject->second->Id());
452 return root.toStyledString();
456 std::string ObjectDetection::Json()
const {
459 return JsonValue().toStyledString();
463 Json::Value ObjectDetection::JsonValue()
const {
466 Json::Value root = EffectBase::JsonValue();
467 root[
"type"] = info.class_name;
468 root[
"protobuf_data_path"] = protobuf_data_path;
469 root[
"selected_object_index"] = selectedObjectIndex;
470 root[
"confidence_threshold"] = confidence_threshold;
471 root[
"display_box_text"] = display_box_text.JsonValue();
475 for (
auto const& trackedObject : trackedObjects){
476 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
478 objects[trackedObject.second->Id()] = trackedObjectJSON;
480 root[
"objects"] = objects;
487 void ObjectDetection::SetJson(
const std::string value) {
496 catch (
const std::exception& e)
499 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
504 void ObjectDetection::SetJsonValue(
const Json::Value root) {
506 EffectBase::SetJsonValue(root);
509 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
510 protobuf_data_path = root[
"protobuf_data_path"].asString();
512 if(!LoadObjDetectdData(protobuf_data_path)){
513 throw InvalidFile(
"Invalid protobuf data path",
"");
514 protobuf_data_path =
"";
519 if (!root[
"selected_object_index"].isNull())
520 selectedObjectIndex = root[
"selected_object_index"].asInt();
522 if (!root[
"confidence_threshold"].isNull())
523 confidence_threshold = root[
"confidence_threshold"].asFloat();
525 if (!root[
"display_box_text"].isNull())
526 display_box_text.SetJsonValue(root[
"display_box_text"]);
528 if (!root[
"class_filter"].isNull()){
529 class_filter = root[
"class_filter"].asString();
530 std::stringstream ss(class_filter);
531 display_classes.clear();
536 std::getline( ss, substr,
',' );
537 display_classes.push_back( substr );
541 if (!root[
"objects"].isNull()){
542 for (
auto const& trackedObject : trackedObjects){
543 std::string obj_id = std::to_string(trackedObject.first);
544 if(!root[
"objects"][obj_id].isNull()){
545 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
551 if (!root[
"objects_id"].isNull()){
552 for (
auto const& trackedObject : trackedObjects){
553 Json::Value trackedObjectJSON;
554 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
555 trackedObject.second->SetJsonValue(trackedObjectJSON);
561 std::string ObjectDetection::PropertiesJSON(int64_t requested_frame)
const {
567 if(trackedObjects.count(selectedObjectIndex) != 0){
568 auto selectedObject = trackedObjects.at(selectedObjectIndex);
570 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
572 objects[selectedObject->Id()] = trackedObjectJSON;
575 root[
"objects"] = objects;
577 root[
"selected_object_index"] = add_property_json(
"Selected Object", selectedObjectIndex,
"int",
"", NULL, 0, 200,
false, requested_frame);
578 root[
"id"] = add_property_json(
"ID", 0.0,
"string", Id(), NULL, -1, -1,
true, requested_frame);
579 root[
"position"] = add_property_json(
"Position", Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
580 root[
"layer"] = add_property_json(
"Track", Layer(),
"int",
"", NULL, 0, 20,
false, requested_frame);
581 root[
"start"] = add_property_json(
"Start", Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
582 root[
"end"] = add_property_json(
"End", End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
583 root[
"duration"] = add_property_json(
"Duration", Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
584 root[
"confidence_threshold"] = add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
585 root[
"class_filter"] = add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
587 root[
"display_box_text"] = add_property_json(
"Draw Box Text", display_box_text.GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
588 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"Off", 1, display_box_text.GetValue(requested_frame)));
589 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"On", 0, display_box_text.GetValue(requested_frame)));
592 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
This class represents a clip (used to arrange readers on the timeline)
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
This class represents a single frame of video (i.e. image & audio data)
Exception for files that can not be found or opened.
Exception for invalid JSON.
This class represents a timeline.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width