OpenShot Library | libopenshot  0.2.5
CacheDisk.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for CacheDisk class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/CacheDisk.h"
32 
33 using namespace std;
34 using namespace openshot;
35 
36 // Default constructor, no max bytes
37 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
38  // Set cache type name
39  cache_type = "CacheDisk";
40  range_version = 0;
41  needs_range_processing = false;
42  frame_size_bytes = 0;
43  image_format = format;
44  image_quality = quality;
45  image_scale = scale;
46  max_bytes = 0;
47 
48  // Init path directory
49  InitPath(cache_path);
50 }
51 
52 // Constructor that sets the max bytes to cache
53 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
54  // Set cache type name
55  cache_type = "CacheDisk";
56  range_version = 0;
57  needs_range_processing = false;
58  frame_size_bytes = 0;
59  image_format = format;
60  image_quality = quality;
61  image_scale = scale;
62 
63  // Init path directory
64  InitPath(cache_path);
65 }
66 
67 // Initialize cache directory
68 void CacheDisk::InitPath(std::string cache_path) {
69  QString qpath;
70 
71  if (!cache_path.empty()) {
72  // Init QDir with cache directory
73  qpath = QString(cache_path.c_str());
74 
75  } else {
76  // Init QDir with user's temp directory
77  qpath = QDir::tempPath() + QString("/preview-cache/");
78  }
79 
80  // Init QDir with cache directory
81  path = QDir(qpath);
82 
83  // Check if cache directory exists
84  if (!path.exists())
85  // Create
86  path.mkpath(qpath);
87 }
88 
89 // Calculate ranges of frames
90 void CacheDisk::CalculateRanges() {
91  // Only calculate when something has changed
92  if (needs_range_processing) {
93 
94  // Create a scoped lock, to protect the cache from multiple threads
95  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
96 
97  // Sort ordered frame #s, and calculate JSON ranges
98  std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
99 
100  // Clear existing JSON variable
101  Json::Value ranges = Json::Value(Json::arrayValue);
102 
103  // Increment range version
104  range_version++;
105 
106  int64_t starting_frame = *ordered_frame_numbers.begin();
107  int64_t ending_frame = starting_frame;
108 
109  // Loop through all known frames (in sequential order)
110  for (const auto frame_number : ordered_frame_numbers) {
111  if (frame_number - ending_frame > 1) {
112  // End of range detected
113  Json::Value range;
114 
115  // Add JSON object with start/end attributes
116  // Use strings, since int64_ts are supported in JSON
117  range["start"] = std::to_string(starting_frame);
118  range["end"] = std::to_string(ending_frame);
119  ranges.append(range);
120 
121  // Set new starting range
122  starting_frame = frame_number;
123  }
124 
125  // Set current frame as end of range, and keep looping
126  ending_frame = frame_number;
127  }
128 
129  // APPEND FINAL VALUE
130  Json::Value range;
131 
132  // Add JSON object with start/end attributes
133  // Use strings, since int64_ts are supported in JSON
134  range["start"] = std::to_string(starting_frame);
135  range["end"] = std::to_string(ending_frame);
136  ranges.append(range);
137 
138  // Cache range JSON as string
139  json_ranges = ranges.toStyledString();
140 
141  // Reset needs_range_processing
142  needs_range_processing = false;
143  }
144 }
145 
146 // Default destructor
148 {
149  frames.clear();
150  frame_numbers.clear();
151  ordered_frame_numbers.clear();
152 
153  // remove critical section
154  delete cacheCriticalSection;
155  cacheCriticalSection = NULL;
156 }
157 
158 // Add a Frame to the cache
159 void CacheDisk::Add(std::shared_ptr<Frame> frame)
160 {
161  // Create a scoped lock, to protect the cache from multiple threads
162  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
163  int64_t frame_number = frame->number;
164 
165  // Freshen frame if it already exists
166  if (frames.count(frame_number))
167  // Move frame to front of queue
168  MoveToFront(frame_number);
169 
170  else
171  {
172  // Add frame to queue and map
173  frames[frame_number] = frame_number;
174  frame_numbers.push_front(frame_number);
175  ordered_frame_numbers.push_back(frame_number);
176  needs_range_processing = true;
177 
178  // Save image to disk (if needed)
179  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
180  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
181  if (frame_size_bytes == 0) {
182  // Get compressed size of frame image (to correctly apply max size against)
183  QFile image_file(frame_path);
184  frame_size_bytes = image_file.size();
185  }
186 
187  // Save audio data (if needed)
188  if (frame->has_audio_data) {
189  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
190  QFile audio_file(audio_path);
191 
192  if (audio_file.open(QIODevice::WriteOnly)) {
193  QTextStream audio_stream(&audio_file);
194  audio_stream << frame->SampleRate() << endl;
195  audio_stream << frame->GetAudioChannelsCount() << endl;
196  audio_stream << frame->GetAudioSamplesCount() << endl;
197  audio_stream << frame->ChannelsLayout() << endl;
198 
199  // Loop through all samples
200  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
201  {
202  // Get audio for this channel
203  float *samples = frame->GetAudioSamples(channel);
204  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
205  audio_stream << samples[sample] << endl;
206  }
207 
208  }
209 
210  }
211 
212  // Clean up old frames
213  CleanUp();
214  }
215 }
216 
217 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
218 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
219 {
220  // Create a scoped lock, to protect the cache from multiple threads
221  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
222 
223  // Does frame exists in cache?
224  if (frames.count(frame_number)) {
225  // Does frame exist on disk
226  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
227  if (path.exists(frame_path)) {
228 
229  // Load image file
230  std::shared_ptr<QImage> image = std::shared_ptr<QImage>(new QImage());
231  bool success = image->load(QString::fromStdString(frame_path.toStdString()));
232 
233  // Set pixel formatimage->
234  image = std::shared_ptr<QImage>(new QImage(image->convertToFormat(QImage::Format_RGBA8888)));
235 
236  // Create frame object
237  std::shared_ptr<Frame> frame(new Frame());
238  frame->number = frame_number;
239  frame->AddImage(image);
240 
241  // Get audio data (if found)
242  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
243  QFile audio_file(audio_path);
244  if (audio_file.exists()) {
245  // Open audio file
246  QTextStream in(&audio_file);
247  if (audio_file.open(QIODevice::ReadOnly)) {
248  int sample_rate = in.readLine().toInt();
249  int channels = in.readLine().toInt();
250  int sample_count = in.readLine().toInt();
251  int channel_layout = in.readLine().toInt();
252 
253  // Set basic audio properties
254  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
255 
256  // Loop through audio samples and add to frame
257  int current_channel = 0;
258  int current_sample = 0;
259  float *channel_samples = new float[sample_count];
260  while (!in.atEnd()) {
261  // Add sample to channel array
262  channel_samples[current_sample] = in.readLine().toFloat();
263  current_sample++;
264 
265  if (current_sample == sample_count) {
266  // Add audio to frame
267  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
268 
269  // Increment channel, and reset sample position
270  current_channel++;
271  current_sample = 0;
272  }
273 
274  }
275  }
276  }
277 
278  // return the Frame object
279  return frame;
280  }
281  }
282 
283  // no Frame found
284  return std::shared_ptr<Frame>();
285 }
286 
287 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
288 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
289 {
290  // Create a scoped lock, to protect the cache from multiple threads
291  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
292  std::shared_ptr<openshot::Frame> f;
293 
294  // Loop through frame numbers
295  std::deque<int64_t>::iterator itr;
296  int64_t smallest_frame = -1;
297  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
298  {
299  if (*itr < smallest_frame || smallest_frame == -1)
300  smallest_frame = *itr;
301  }
302 
303  // Return frame
304  f = GetFrame(smallest_frame);
305 
306  return f;
307 }
308 
309 // Gets the maximum bytes value
311 {
312  // Create a scoped lock, to protect the cache from multiple threads
313  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
314 
315  int64_t total_bytes = 0;
316 
317  // Loop through frames, and calculate total bytes
318  std::deque<int64_t>::reverse_iterator itr;
319  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
320  total_bytes += frame_size_bytes;
321 
322  return total_bytes;
323 }
324 
325 // Remove a specific frame
326 void CacheDisk::Remove(int64_t frame_number)
327 {
328  Remove(frame_number, frame_number);
329 }
330 
331 // Remove range of frames
332 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
333 {
334  // Create a scoped lock, to protect the cache from multiple threads
335  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
336 
337  // Loop through frame numbers
338  std::deque<int64_t>::iterator itr;
339  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
340  {
341  //deque<int64_t>::iterator current = itr++;
342  if (*itr >= start_frame_number && *itr <= end_frame_number)
343  {
344  // erase frame number
345  itr = frame_numbers.erase(itr);
346  } else
347  itr++;
348  }
349 
350  // Loop through ordered frame numbers
351  std::vector<int64_t>::iterator itr_ordered;
352  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
353  {
354  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
355  {
356  // erase frame number
357  frames.erase(*itr_ordered);
358 
359  // Remove the image file (if it exists)
360  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
361  QFile image_file(frame_path);
362  if (image_file.exists())
363  image_file.remove();
364 
365  // Remove audio file (if it exists)
366  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
367  QFile audio_file(audio_path);
368  if (audio_file.exists())
369  audio_file.remove();
370 
371  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
372  } else
373  itr_ordered++;
374  }
375 
376  // Needs range processing (since cache has changed)
377  needs_range_processing = true;
378 }
379 
380 // Move frame to front of queue (so it lasts longer)
381 void CacheDisk::MoveToFront(int64_t frame_number)
382 {
383  // Does frame exists in cache?
384  if (frames.count(frame_number))
385  {
386  // Create a scoped lock, to protect the cache from multiple threads
387  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
388 
389  // Loop through frame numbers
390  std::deque<int64_t>::iterator itr;
391  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
392  {
393  if (*itr == frame_number)
394  {
395  // erase frame number
396  frame_numbers.erase(itr);
397 
398  // add frame number to 'front' of queue
399  frame_numbers.push_front(frame_number);
400  break;
401  }
402  }
403  }
404 }
405 
406 // Clear the cache of all frames
408 {
409  // Create a scoped lock, to protect the cache from multiple threads
410  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
411 
412  // Clear all containers
413  frames.clear();
414  frame_numbers.clear();
415  ordered_frame_numbers.clear();
416  needs_range_processing = true;
417  frame_size_bytes = 0;
418 
419  // Delete cache directory, and recreate it
420  QString current_path = path.path();
421  path.removeRecursively();
422 
423  // Re-init folder
424  InitPath(current_path.toStdString());
425 }
426 
427 // Count the frames in the queue
429 {
430  // Create a scoped lock, to protect the cache from multiple threads
431  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
432 
433  // Return the number of frames in the cache
434  return frames.size();
435 }
436 
437 // Clean up cached frames that exceed the number in our max_bytes variable
438 void CacheDisk::CleanUp()
439 {
440  // Do we auto clean up?
441  if (max_bytes > 0)
442  {
443  // Create a scoped lock, to protect the cache from multiple threads
444  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
445 
446  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
447  {
448  // Get the oldest frame number.
449  int64_t frame_to_remove = frame_numbers.back();
450 
451  // Remove frame_number and frame
452  Remove(frame_to_remove);
453  }
454  }
455 }
456 
457 // Generate JSON string of this object
458 std::string CacheDisk::Json() {
459 
460  // Return formatted string
461  return JsonValue().toStyledString();
462 }
463 
464 // Generate Json::Value for this object
465 Json::Value CacheDisk::JsonValue() {
466 
467  // Process range data (if anything has changed)
468  CalculateRanges();
469 
470  // Create root json object
471  Json::Value root = CacheBase::JsonValue(); // get parent properties
472  root["type"] = cache_type;
473  root["path"] = path.path().toStdString();
474 
475  Json::Value version;
476  std::stringstream range_version_str;
477  range_version_str << range_version;
478  root["version"] = range_version_str.str();
479 
480  // Parse and append range data (if any)
481  // Parse and append range data (if any)
482  try {
483  const Json::Value ranges = openshot::stringToJson(json_ranges);
484  root["ranges"] = ranges;
485  } catch (...) { }
486 
487  // return JsonValue
488  return root;
489 }
490 
491 // Load JSON string into this object
492 void CacheDisk::SetJson(const std::string value) {
493 
494  // Parse JSON string into JSON objects
495  try
496  {
497  const Json::Value root = openshot::stringToJson(value);
498  // Set all values that match
499  SetJsonValue(root);
500  }
501  catch (const std::exception& e)
502  {
503  // Error parsing JSON (or missing keys)
504  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
505  }
506 }
507 
508 // Load Json::Value into this object
509 void CacheDisk::SetJsonValue(const Json::Value root) {
510 
511  // Close timeline before we do anything (this also removes all open and closing clips)
512  Clear();
513 
514  // Set parent data
516 
517  if (!root["type"].isNull())
518  cache_type = root["type"].asString();
519  if (!root["path"].isNull())
520  // Update duration of timeline
521  InitPath(root["path"].asString());
522 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:33
openshot::Frame::SampleRate
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:565
openshot::Frame::GetAudioSamples
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:334
openshot::CacheBase::max_bytes
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:53
openshot::Frame::has_audio_data
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:130
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AudioBufferSource.h:38
openshot::CacheBase::cacheCriticalSection
juce::CriticalSection * cacheCriticalSection
Section lock for multiple threads.
Definition: CacheBase.h:56
openshot::Frame
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:106
openshot::CacheDisk::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:326
openshot::CacheDisk::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:407
openshot::CacheBase
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:49
openshot::Frame::ResizeAudio
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:860
openshot::CacheBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: CacheBase.cpp:70
openshot::CacheDisk::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CacheDisk.cpp:509
openshot::CacheDisk::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:428
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:205
openshot::Frame::AddImage
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:754
openshot::CacheDisk::Json
std::string Json()
Get and Set JSON methods.
Definition: CacheDisk.cpp:458
openshot::CacheDisk::CacheDisk
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:37
openshot::CacheDisk::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:218
openshot::CacheBase::JsonValue
virtual Json::Value JsonValue()=0
Generate Json::Value for this object.
Definition: CacheBase.cpp:57
openshot::Frame::GetAudioChannelsCount
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:432
openshot::CacheDisk::GetBytes
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:310
openshot::CacheBase::cache_type
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:52
openshot::Frame::AddAudio
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:874
openshot::Frame::Save
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:578
openshot::CacheDisk::JsonValue
Json::Value JsonValue()
Generate Json::Value for this object.
Definition: CacheDisk.cpp:465
openshot::CacheDisk::GetSmallestFrame
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:288
openshot::Frame::ChannelsLayout
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:571
openshot::CacheDisk::SetJson
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:492
openshot::CacheDisk::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:159
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:46
openshot::CacheDisk::MoveToFront
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:381
openshot::CacheDisk::~CacheDisk
~CacheDisk()
Definition: CacheDisk.cpp:147
openshot::Frame::number
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:129
openshot::Frame::GetAudioSamplesCount
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:442