32 #include <OpenShotAudio.h>
34 #include <QApplication>
42 #include <QHBoxLayout>
55 Frame::Frame(int64_t number,
int width,
int height, std::string color,
int samples,
int channels)
56 : audio(std::make_shared<juce::AudioSampleBuffer>(channels, samples)),
57 number(number), width(width), height(height),
58 pixel_ratio(1,1), color(color), qbuffer(NULL),
61 has_audio_data(false), has_image_data(false),
72 Frame::Frame(int64_t number,
int width,
int height, std::string color)
73 :
Frame::
Frame(number, width, height, color, 0, 2) {};
77 :
Frame::
Frame(number, 1, 1,
"#000000", samples, channels) {};
100 channels = other.channels;
102 height = other.height;
103 channel_layout = other.channel_layout;
106 sample_rate = other.sample_rate;
107 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
109 max_audio_sample = other.max_audio_sample;
112 image = std::make_shared<QImage>(*(other.image));
114 audio = std::make_shared<juce::AudioSampleBuffer>(*(other.
audio));
115 if (other.wave_image)
116 wave_image = std::make_shared<QImage>(*(other.wave_image));
132 if (!QApplication::instance()) {
135 static char* argv[1] = {NULL};
136 previewApp = std::make_shared<QApplication>(argc, argv);
140 std::shared_ptr<QImage> previewImage =
GetImage();
143 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
146 int new_width = previewImage->size().width();
150 previewImage = std::make_shared<QImage>(previewImage->scaled(
151 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
155 QWidget previewWindow;
156 previewWindow.setStyleSheet(
"background-color: #000000;");
161 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163 layout.addWidget(&previewLabel);
166 previewWindow.setLayout(&layout);
167 previewWindow.show();
172 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
178 QVector<QPointF> lines;
179 QVector<QPointF> labels;
183 if (total_samples > 0)
186 int new_height = 200 *
audio->getNumChannels();
187 int height_padding = 20 * (
audio->getNumChannels() - 1);
188 int total_height = new_height + height_padding;
193 for (
int channel = 0; channel <
audio->getNumChannels(); channel++)
198 const float *samples =
audio->getReadPointer(channel);
203 float value = samples[sample] * 100.0;
206 lines.push_back(QPointF(X,Y+1.0));
207 lines.push_back(QPointF(X,(Y-value)+1.0));
211 labels.push_back(QPointF(5.0, Y - 5.0));
214 Y += (200 + height_padding);
219 wave_image = std::make_shared<QImage>(
220 total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
221 wave_image->fill(QColor(0,0,0,0));
224 QPainter painter(wave_image.get());
228 pen.setColor(QColor(Red, Green, Blue, Alpha));
230 pen.setStyle(Qt::SolidLine);
234 painter.drawLines(lines);
238 if (width != total_width || height != total_height) {
239 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
240 wave_image = std::make_shared<QImage>(scaled_wave_image);
246 wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
247 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
265 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
268 return wave_image->constBits();
277 if (!QApplication::instance()) {
280 static char* argv[1] = {NULL};
281 previewApp = std::make_shared<QApplication>(argc, argv);
285 QWidget previewWindow;
286 previewWindow.setStyleSheet(
"background-color: #000000;");
291 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
292 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
293 layout.addWidget(&previewLabel);
296 previewWindow.setLayout(&layout);
297 previewWindow.show();
309 return audio->getMagnitude(channel, sample, magnitude_range);
313 return audio->getMagnitude(sample, magnitude_range);
321 return audio->getWritePointer(channel);
327 float *output = NULL;
328 juce::AudioSampleBuffer *buffer(
audio.get());
329 int num_of_channels =
audio->getNumChannels();
333 if (new_sample_rate != sample_rate)
342 num_of_samples = buffer->getNumSamples();
346 output =
new float[num_of_channels * num_of_samples];
350 for (
int channel = 0; channel < num_of_channels; channel++)
352 for (
int sample = 0; sample < num_of_samples; sample++)
355 output[position] = buffer->getReadPointer(channel)[sample];
363 *sample_count = num_of_samples;
373 float *output = NULL;
374 juce::AudioSampleBuffer *buffer(
audio.get());
375 int num_of_channels =
audio->getNumChannels();
379 if (new_sample_rate != sample_rate && resampler)
388 num_of_samples = buffer->getNumSamples();
392 output =
new float[num_of_channels * num_of_samples];
396 for (
int sample = 0; sample < num_of_samples; sample++)
398 for (
int channel = 0; channel < num_of_channels; channel++)
401 output[position] = buffer->getReadPointer(channel)[sample];
409 *sample_count = num_of_samples;
418 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
420 return audio->getNumChannels();
428 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
429 return max_audio_sample;
440 int64_t total_bytes = 0;
442 total_bytes += (width * height *
sizeof(char) * 4);
445 total_bytes += (sample_rate / 24.0) *
sizeof(
float);
461 return image->constBits();
473 return image->constScanLine(row);
477 bool Frame::CheckPixel(
int row,
int col,
int red,
int green,
int blue,
int alpha,
int threshold) {
478 int col_pos = col * 4;
479 if (!image || row < 0 || row >= (height - 1) ||
480 col_pos < 0 || col_pos >= (width - 1) ) {
485 const unsigned char* pixels =
GetPixels(row);
486 if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
487 pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
488 pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
489 pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
501 pixel_ratio.
num = num;
502 pixel_ratio.
den = den;
518 double previous_samples = (sample_rate * fps_rate) * (
number - 1);
519 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
520 previous_samples -= previous_samples_remainder;
523 double total_samples = (sample_rate * fps_rate) *
number;
524 double total_samples_remainder = fmod(total_samples, (
double)channels);
525 total_samples -= total_samples_remainder;
529 int samples_per_frame = round(total_samples - previous_samples);
530 if (samples_per_frame < 0)
531 samples_per_frame = 0;
532 return samples_per_frame;
562 return channel_layout;
570 std::shared_ptr<QImage> previewImage =
GetImage();
573 if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
575 int new_width = width;
576 int new_height = height;
579 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
582 int new_width = previewImage->size().width();
586 previewImage = std::make_shared<QImage>(previewImage->scaled(
587 new_width, new_height,
588 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
592 previewImage = std::make_shared<QImage>(previewImage->scaled(
593 new_width * scale, new_height * scale,
594 Qt::KeepAspectRatio, Qt::SmoothTransformation));
598 previewImage->save(QString::fromStdString(
path), format.c_str(), quality);
602 void Frame::Thumbnail(std::string
path,
int new_width,
int new_height, std::string mask_path, std::string overlay_path,
603 std::string background_color,
bool ignore_aspect, std::string format,
int quality,
float rotate) {
606 auto thumbnail = std::make_shared<QImage>(
607 new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
608 thumbnail->fill(QColor(QString::fromStdString(background_color)));
611 QPainter painter(thumbnail.get());
612 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
615 std::shared_ptr<QImage> previewImage =
GetImage();
618 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
621 int aspect_width = previewImage->size().width();
622 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
625 previewImage = std::make_shared<QImage>(previewImage->scaled(
626 aspect_width, aspect_height,
627 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
633 previewImage = std::make_shared<QImage>(previewImage->scaled(
634 new_width, new_height,
635 Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
638 previewImage = std::make_shared<QImage>(previewImage->scaled(
639 new_width, new_height,
640 Qt::KeepAspectRatio, Qt::SmoothTransformation));
643 int x = (new_width - previewImage->size().width()) / 2.0;
644 int y = (new_height - previewImage->size().height()) / 2.0;
645 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
649 QTransform transform;
650 float origin_x = previewImage->width() / 2.0;
651 float origin_y = previewImage->height() / 2.0;
652 transform.translate(origin_x, origin_y);
653 transform.rotate(rotate);
654 transform.translate(-origin_x,-origin_y);
655 painter.setTransform(transform);
658 painter.drawImage(x, y, *previewImage);
662 if (overlay_path !=
"") {
664 auto overlay = std::make_shared<QImage>();
665 overlay->load(QString::fromStdString(overlay_path));
668 overlay = std::make_shared<QImage>(
669 overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
672 overlay = std::make_shared<QImage>(overlay->scaled(
673 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
676 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
677 painter.drawImage(0, 0, *overlay);
682 if (mask_path !=
"") {
684 auto mask = std::make_shared<QImage>();
685 mask->load(QString::fromStdString(mask_path));
688 mask = std::make_shared<QImage>(
689 mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
692 mask = std::make_shared<QImage>(mask->scaled(
693 new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
696 mask->invertPixels();
699 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
700 const unsigned char *mask_pixels = (
const unsigned char *) mask->constBits();
704 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
707 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
708 int Frame_Alpha = pixels[byte_index + 3];
709 int Mask_Value = constrain(Frame_Alpha - gray_value);
712 pixels[byte_index + 3] = Mask_Value;
721 thumbnail->save(QString::fromStdString(
path), format.c_str(), quality);
725 int Frame::constrain(
int color_value)
730 else if (color_value > 255)
743 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
744 image = std::make_shared<QImage>(new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
747 image->fill(QColor(QString::fromStdString(color)));
750 width = image->width();
751 height = image->height();
757 int new_width,
int new_height,
int bytes_per_pixel,
758 QImage::Format type,
const unsigned char *pixels_)
762 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
767 auto new_image = std::make_shared<QImage>(
769 new_width, new_height,
770 new_width * bytes_per_pixel,
786 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
790 if (image->format() != QImage::Format_RGBA8888_Premultiplied)
791 *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
794 width = image->width();
795 height = image->height();
814 if (image == new_image || image->size() != new_image->size()) {
817 else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
818 new_image = std::make_shared<QImage>(
819 new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
826 const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
827 unsigned char *pixels = image->bits();
828 const unsigned char *new_pixels = new_image->constBits();
835 for (
int row = start; row < image->height(); row += 2) {
836 int offset = row * image->bytesPerLine();
837 memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
841 height = image->height();
842 width = image->width();
851 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
854 audio->setSize(channels, length,
true,
true,
false);
855 channel_layout = layout;
859 max_audio_sample = length;
863 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
864 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
867 int destStartSampleAdjusted = max(destStartSample, 0);
870 int new_length = destStartSampleAdjusted + numSamples;
871 int new_channel_length =
audio->getNumChannels();
872 if (destChannel >= new_channel_length)
873 new_channel_length = destChannel + 1;
874 if (new_length >
audio->getNumSamples() || new_channel_length >
audio->getNumChannels())
875 audio->setSize(new_channel_length, new_length,
true,
true,
false);
879 audio->clear(destChannel, destStartSampleAdjusted, numSamples);
882 audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
886 if (new_length > max_audio_sample)
887 max_audio_sample = new_length;
891 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
893 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
896 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
915 cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
916 cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
917 int from_to[] = { 0,0, 1,1, 2,2 };
918 cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
919 cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
939 cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
940 QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
942 std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
945 if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
946 *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
959 #ifdef USE_IMAGEMAGICK
969 const QRgb *tmpBits = (
const QRgb*)image->constBits();
972 auto magick_image = std::make_shared<Magick::Image>(
973 image->width(), image->height(),
"RGBA", Magick::CharPixel, tmpBits);
976 magick_image->backgroundColor(Magick::Color(
"none"));
977 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
984 #ifdef USE_IMAGEMAGICK
989 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
994 qbuffer =
new unsigned char[bufferSize]();
995 unsigned char *buffer = (
unsigned char*)qbuffer;
997 MagickCore::ExceptionInfo exception;
999 MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(),
"RGBA", Magick::CharPixel, buffer, &exception);
1002 image = std::make_shared<QImage>(
1003 qbuffer, width, height, width * BPP, QImage::Format_RGBA8888_Premultiplied,
1007 width = image->width();
1008 height = image->height();
1020 juce::AudioDeviceManager deviceManager;
1021 juce::String error = deviceManager.initialise (
1028 if (error.isNotEmpty()) {
1029 cout <<
"Error on initialise(): " << error << endl;
1032 juce::AudioSourcePlayer audioSourcePlayer;
1033 deviceManager.addAudioCallback (&audioSourcePlayer);
1035 std::unique_ptr<AudioBufferSource> my_source;
1039 juce::TimeSliceThread my_thread(
"Audio buffer thread");
1042 my_thread.startThread();
1044 juce::AudioTransportSource transport1;
1045 transport1.setSource (my_source.get(),
1048 (
double) sample_rate,
1049 audio->getNumChannels());
1050 transport1.setPosition (0);
1051 transport1.setGain(1.0);
1055 juce::MixerAudioSource mixer;
1056 mixer.addInputSource(&transport1,
false);
1057 audioSourcePlayer.setSource (&mixer);
1062 while (transport1.isPlaying())
1064 cout <<
"playing" << endl;
1065 std::this_thread::sleep_for(std::chrono::seconds(1));
1068 cout <<
"DONE!!!" << endl;
1071 transport1.setSource (0);
1072 audioSourcePlayer.setSource (0);
1073 my_thread.stopThread(500);
1074 deviceManager.removeAudioCallback (&audioSourcePlayer);
1075 deviceManager.closeAudioDevice();
1076 deviceManager.removeAllChangeListeners();
1077 deviceManager.dispatchPendingMessages();
1079 cout <<
"End of Play()" << endl;
1090 unsigned char* ptr_to_qbuffer = (
unsigned char*) info;
1091 delete[] ptr_to_qbuffer;
1098 const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1101 audio->setSize(channels, numSamples,
false,
true,
false);
1106 if (numSamples > max_audio_sample)
1107 max_audio_sample = numSamples;
Header file for Frame class.
#define MAGICK_IMAGE_ALPHA(im, a)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
This class is used to resample audio data for many sequential frames.
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
This class represents a fraction.
int num
Numerator for the fraction.
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
int den
Denominator for the fraction.
This class represents a single frame of video (i.e. image & audio data)
Frame & operator=(const Frame &other)
Assignment operator.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
const unsigned char * GetPixels()
Get pixel data (as packets)
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
int GetAudioChannelsCount()
Get number of audio channels.
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
bool has_image_data
This frame has been loaded with pixel data.
int GetWidth()
Get height of image.
int SampleRate()
Get the original sample rate of this frame's audio data.
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
void Play()
Play audio samples for this frame.
std::shared_ptr< juce::AudioSampleBuffer > audio
bool has_audio_data
This frame has been loaded with audio data.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
openshot::ChannelLayout ChannelsLayout()
void AddAudioSilence(int numSamples)
Add audio silence.
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
void DisplayWaveform()
Display the wave form.
juce::AudioSampleBuffer * GetAudioSampleBuffer()
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
float * GetAudioSamples(int channel)
Get an array of sample data.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
virtual ~Frame()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Frame()
Constructor - blank frame.
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
int GetAudioSamplesCount()
Get number of audio samples.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
int GetHeight()
Get height of image.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
void SetFrameNumber(int64_t number)
Set frame number.
int64_t number
This is the frame number (starting at 1)
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
This namespace is the default namespace for all code in the openshot library.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...