Libosmium  2.15.4
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  }; // class Compressor
86 
87  class Decompressor {
88 
89  std::atomic<std::size_t> m_file_size{0};
90  std::atomic<std::size_t> m_offset{0};
91 
92  public:
93 
94  enum {
95  input_buffer_size = 1024U * 1024U
96  };
97 
98  Decompressor() = default;
99 
100  Decompressor(const Decompressor&) = delete;
101  Decompressor& operator=(const Decompressor&) = delete;
102 
103  Decompressor(Decompressor&&) = delete;
104  Decompressor& operator=(Decompressor&&) = delete;
105 
106  virtual ~Decompressor() noexcept = default;
107 
108  virtual std::string read() = 0;
109 
110  virtual void close() = 0;
111 
112  std::size_t file_size() const noexcept {
113  return m_file_size;
114  }
115 
116  void set_file_size(const std::size_t size) noexcept {
117  m_file_size = size;
118  }
119 
120  std::size_t offset() const noexcept {
121  return m_offset;
122  }
123 
124  void set_offset(const std::size_t offset) noexcept {
125  m_offset = offset;
126  }
127 
128  }; // class Decompressor
129 
138 
139  public:
140 
141  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
142  using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>;
143  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
144 
145  private:
146 
147  using callbacks_type = std::tuple<create_compressor_type,
150 
151  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
152 
154 
155  CompressionFactory() = default;
156 
158  const auto it = m_callbacks.find(compression);
159 
160  if (it != m_callbacks.end()) {
161  return it->second;
162  }
163 
164  std::string error_message{"Support for compression '"};
165  error_message += as_string(compression);
166  error_message += "' not compiled into this binary";
167  throw unsupported_file_format_error{error_message};
168  }
169 
170  public:
171 
172  CompressionFactory(const CompressionFactory&) = delete;
174 
177 
178  ~CompressionFactory() noexcept = default;
179 
181  static CompressionFactory factory;
182  return factory;
183  }
184 
186  osmium::io::file_compression compression,
187  const create_compressor_type& create_compressor,
188  const create_decompressor_type_fd& create_decompressor_fd,
189  const create_decompressor_type_buffer& create_decompressor_buffer) {
190 
191  compression_map_type::value_type cc{compression,
192  std::make_tuple(create_compressor,
193  create_decompressor_fd,
194  create_decompressor_buffer)};
195 
196  return m_callbacks.insert(cc).second;
197  }
198 
199  template <typename... TArgs>
200  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
201  const auto callbacks = find_callbacks(compression);
202  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
203  }
204 
205  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
206  const auto callbacks = find_callbacks(compression);
207  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
208  p->set_file_size(osmium::file_size(fd));
209  return p;
210  }
211 
212  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
213  const auto callbacks = find_callbacks(compression);
214  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
215  }
216 
217  }; // class CompressionFactory
218 
219  class NoCompressor : public Compressor {
220 
221  int m_fd;
222 
223  public:
224 
225  NoCompressor(const int fd, const fsync sync) :
226  Compressor(sync),
227  m_fd(fd) {
228  }
229 
230  NoCompressor(const NoCompressor&) = delete;
231  NoCompressor& operator=(const NoCompressor&) = delete;
232 
233  NoCompressor(NoCompressor&&) = delete;
234  NoCompressor& operator=(NoCompressor&&) = delete;
235 
236  ~NoCompressor() noexcept final {
237  try {
238  close();
239  } catch (...) {
240  // Ignore any exceptions because destructor must not throw.
241  }
242  }
243 
244  void write(const std::string& data) final {
245  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
246  }
247 
248  void close() final {
249  if (m_fd >= 0) {
250  const int fd = m_fd;
251  m_fd = -1;
252  if (do_fsync()) {
253  osmium::io::detail::reliable_fsync(fd);
254  }
255  osmium::io::detail::reliable_close(fd);
256  }
257  }
258 
259  }; // class NoCompressor
260 
261  class NoDecompressor : public Decompressor {
262 
263  int m_fd = -1;
264  const char* m_buffer = nullptr;
265  std::size_t m_buffer_size = 0;
266  std::size_t m_offset = 0;
267 
268  public:
269 
270  explicit NoDecompressor(const int fd) :
271  m_fd(fd) {
272  }
273 
274  NoDecompressor(const char* buffer, const std::size_t size) :
275  m_buffer(buffer),
276  m_buffer_size(size) {
277  }
278 
279  NoDecompressor(const NoDecompressor&) = delete;
280  NoDecompressor& operator=(const NoDecompressor&) = delete;
281 
282  NoDecompressor(NoDecompressor&&) = delete;
284 
285  ~NoDecompressor() noexcept final {
286  try {
287  close();
288  } catch (...) {
289  // Ignore any exceptions because destructor must not throw.
290  }
291  }
292 
293  std::string read() final {
294  std::string buffer;
295 
296  if (m_buffer) {
297  if (m_buffer_size != 0) {
298  const std::size_t size = m_buffer_size;
299  m_buffer_size = 0;
300  buffer.append(m_buffer, size);
301  }
302  } else {
304  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
305  buffer.resize(std::string::size_type(nread));
306  }
307 
308  m_offset += buffer.size();
309  set_offset(m_offset);
310 
311  return buffer;
312  }
313 
314  void close() final {
315  if (m_fd >= 0) {
316  const int fd = m_fd;
317  m_fd = -1;
318  osmium::io::detail::reliable_close(fd);
319  }
320  }
321 
322  }; // class NoDecompressor
323 
324  namespace detail {
325 
326  // we want the register_compression() function to run, setting
327  // the variable is only a side-effect, it will never be used
329  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
330  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
331  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
332  );
333 
334  // dummy function to silence the unused variable warning from above
335  inline bool get_registered_no_compression() noexcept {
336  return registered_no_compression;
337  }
338 
339  } // namespace detail
340 
341  } // namespace io
342 
343 } // namespace osmium
344 
345 #endif // OSMIUM_IO_COMPRESSION_HPP
Definition: compression.hpp:95
void close() final
Definition: compression.hpp:314
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:274
std::size_t file_size(int fd)
Definition: file.hpp:109
std::string read() final
Definition: compression.hpp:293
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:200
static CompressionFactory & instance()
Definition: compression.hpp:180
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
void close() final
Definition: compression.hpp:248
Definition: location.hpp:550
void write(const std::string &data) final
Definition: compression.hpp:244
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:149
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:185
virtual ~Compressor() noexcept=default
Definition: compression.hpp:87
fsync m_fsync
Definition: compression.hpp:59
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:124
void set_file_size(const std::size_t size) noexcept
Definition: compression.hpp:116
bool do_fsync() const noexcept
Definition: compression.hpp:63
virtual void write(const std::string &data)=0
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:212
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: attr.hpp:333
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:157
fsync
Definition: writer_options.hpp:51
virtual void close()=0
std::size_t offset() const noexcept
Definition: compression.hpp:120
~NoCompressor() noexcept final
Definition: compression.hpp:236
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:143
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:151
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:141
Compressor & operator=(const Compressor &)=default
Definition: compression.hpp:219
Definition: compression.hpp:57
int m_fd
Definition: compression.hpp:221
NoDecompressor(const int fd)
Definition: compression.hpp:270
compression_map_type m_callbacks
Definition: compression.hpp:153
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:142
Definition: compression.hpp:261
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:205
Definition: compression.hpp:137
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:225
~NoDecompressor() noexcept final
Definition: compression.hpp:285