Libosmium  2.15.0
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
37 #include <osmium/util/file.hpp>
38 
39 #include <cassert>
40 #include <cerrno>
41 #include <cstddef>
42 #include <stdexcept>
43 #include <system_error>
44 
45 #ifndef _WIN32
46 # include <sys/mman.h>
47 #else
48 # include <fcntl.h>
49 # include <io.h>
50 # include <windows.h>
51 # include <sys/types.h>
52 #endif
53 
54 namespace osmium {
55 
56  inline namespace util {
57 
95  class MemoryMapping {
96 
97  public:
98 
99  enum class mapping_mode {
100  readonly = 0,
101  write_private = 1,
102  write_shared = 2
103  };
104 
105  private:
106 
108  std::size_t m_size;
109 
111  off_t m_offset;
112 
114  int m_fd;
115 
118 
119 #ifdef _WIN32
120  HANDLE m_handle;
121 #endif
122 
124  void* m_addr;
125 
126  bool is_valid() const noexcept;
127 
128  void make_invalid() noexcept;
129 
130 #ifdef _WIN32
131  using flag_type = DWORD;
132 #else
133  using flag_type = int;
134 #endif
135 
136  flag_type get_protection() const noexcept;
137 
138  flag_type get_flags() const noexcept;
139 
140  static std::size_t check_size(std::size_t size) {
141  if (size == 0) {
142  return osmium::get_pagesize();
143  }
144  return size;
145  }
146 
147 #ifdef _WIN32
148  HANDLE get_handle() const noexcept;
149  HANDLE create_file_mapping() const noexcept;
150  void* map_view_of_file() const noexcept;
151 #endif
152 
153  int resize_fd(int fd) {
154  // Anonymous mapping doesn't need resizing.
155  if (fd == -1) {
156  return -1;
157  }
158 
159  // Make sure the file backing this mapping is large enough.
160  if (osmium::file_size(fd) < m_size + m_offset) {
161  osmium::resize_file(fd, m_size + m_offset);
162  }
163  return fd;
164  }
165 
166  public:
167 
184  MemoryMapping(std::size_t size, mapping_mode mode, int fd = -1, off_t offset = 0);
185 
191  OSMIUM_DEPRECATED MemoryMapping(std::size_t size, bool writable = true, int fd = -1, off_t offset = 0) : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
192  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
193  }
194 
196  MemoryMapping(const MemoryMapping&) = delete;
197 
199  MemoryMapping& operator=(const MemoryMapping&) = delete;
200 
205  MemoryMapping(MemoryMapping&& other) noexcept;
206 
210  MemoryMapping& operator=(MemoryMapping&& other) noexcept;
211 
216  ~MemoryMapping() noexcept {
217  try {
218  unmap();
219  } catch (const std::system_error&) {
220  // Ignore any exceptions because destructor must not throw.
221  }
222  }
223 
230  void unmap();
231 
243  void resize(std::size_t new_size);
244 
249  explicit operator bool() const noexcept {
250  return is_valid();
251  }
252 
258  std::size_t size() const noexcept {
259  return m_size;
260  }
261 
267  int fd() const noexcept {
268  return m_fd;
269  }
270 
274  bool writable() const noexcept {
275  return m_mapping_mode != mapping_mode::readonly;
276  }
277 
283  template <typename T = void>
284  T* get_addr() const {
285  if (is_valid()) {
286  return reinterpret_cast<T*>(m_addr);
287  }
288  throw std::runtime_error{"invalid memory mapping"};
289  }
290 
291  }; // class MemoryMapping
292 
304 
305  public:
306 
307  explicit AnonymousMemoryMapping(std::size_t size) :
309  }
310 
311 #ifndef __linux__
312 
316  void resize(std::size_t) = delete;
317 #endif
318 
319  }; // class AnonymousMemoryMapping
320 
330  template <typename T>
332 
334 
335  public:
336 
343  explicit TypedMemoryMapping(std::size_t size) :
344  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
345  }
346 
357  TypedMemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
358  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
359  }
360 
366  OSMIUM_DEPRECATED TypedMemoryMapping(std::size_t size, bool writable, int fd, off_t offset = 0) :
367  m_mapping(sizeof(T) * size,
369  fd,
370  sizeof(T) * offset) {
371  }
372 
374  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
375 
378 
383  TypedMemoryMapping(TypedMemoryMapping&& other) noexcept = default;
384 
388  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) noexcept = default;
389 
394  ~TypedMemoryMapping() noexcept = default;
395 
402  void unmap() {
403  m_mapping.unmap();
404  }
405 
416  void resize(std::size_t new_size) {
417  m_mapping.resize(sizeof(T) * new_size);
418  }
419 
424  explicit operator bool() const noexcept {
425  return !!m_mapping;
426  }
427 
433  std::size_t size() const noexcept {
434  assert(m_mapping.size() % sizeof(T) == 0);
435  return m_mapping.size() / sizeof(T);
436  }
437 
443  int fd() const noexcept {
444  return m_mapping.fd();
445  }
446 
450  bool writable() const noexcept {
451  return m_mapping.writable();
452  }
453 
459  T* begin() {
460  return m_mapping.get_addr<T>();
461  }
462 
468  T* end() {
469  return m_mapping.get_addr<T>() + size();
470  }
471 
472  const T* cbegin() const {
473  return m_mapping.get_addr<T>();
474  }
475 
476  const T* cend() const {
477  return m_mapping.get_addr<T>() + size();
478  }
479 
480  const T* begin() const {
481  return m_mapping.get_addr<T>();
482  }
483 
484  const T* end() const {
485  return m_mapping.get_addr<T>() + size();
486  }
487 
488  }; // class TypedMemoryMapping
489 
490  template <typename T>
492 
493  public:
494 
495  explicit AnonymousTypedMemoryMapping(std::size_t size) :
496  TypedMemoryMapping<T>(size) {
497  }
498 
499 #ifndef __linux__
500 
504  void resize(std::size_t) = delete;
505 #endif
506 
507  }; // class AnonymousTypedMemoryMapping
508 
509  } // namespace util
510 
511 } // namespace osmium
512 
513 #ifndef _WIN32
514 
515 // =========== Unix implementation =============
516 
517 // MAP_FAILED is often a macro containing an old style cast
518 #pragma GCC diagnostic push
519 #pragma GCC diagnostic ignored "-Wold-style-cast"
520 
521 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
522  return m_addr != MAP_FAILED; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast)
523 }
524 
526  m_addr = MAP_FAILED; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast)
527 }
528 
529 #pragma GCC diagnostic pop
530 
531 // for BSD systems
532 #ifndef MAP_ANONYMOUS
533 # define MAP_ANONYMOUS MAP_ANON
534 #endif
535 
536 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
538  return PROT_READ;
539  }
540  return PROT_READ | PROT_WRITE; // NOLINT(hicpp-signed-bitwise)
541 }
542 
543 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
544  if (m_fd == -1) {
545  return MAP_PRIVATE | MAP_ANONYMOUS; // NOLINT(hicpp-signed-bitwise)
546  }
548  return MAP_SHARED;
549  }
550  return MAP_PRIVATE;
551 }
552 
553 inline osmium::util::MemoryMapping::MemoryMapping(std::size_t size, mapping_mode mode, int fd, off_t offset) :
554  m_size(check_size(size)),
555  m_offset(offset),
556  m_fd(resize_fd(fd)),
557  m_mapping_mode(mode),
558  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
559  assert(!(fd == -1 && mode == mapping_mode::readonly));
560  if (!is_valid()) {
561  throw std::system_error{errno, std::system_category(), "mmap failed"};
562  }
563 }
564 
566  m_size(other.m_size),
567  m_offset(other.m_offset),
568  m_fd(other.m_fd),
569  m_mapping_mode(other.m_mapping_mode),
570  m_addr(other.m_addr) {
571  other.make_invalid();
572 }
573 
575  unmap();
576  m_size = other.m_size;
577  m_offset = other.m_offset;
578  m_fd = other.m_fd;
579  m_mapping_mode = other.m_mapping_mode;
580  m_addr = other.m_addr;
581  other.make_invalid();
582  return *this;
583 }
584 
586  if (is_valid()) {
587  if (::munmap(m_addr, m_size) != 0) {
588  throw std::system_error{errno, std::system_category(), "munmap failed"};
589  }
590  make_invalid();
591  }
592 }
593 
594 inline void osmium::util::MemoryMapping::resize(std::size_t new_size) {
595  assert(new_size > 0 && "can not resize to zero size");
596  if (m_fd == -1) { // anonymous mapping
597 #ifdef __linux__
598  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
599  if (!is_valid()) {
600  throw std::system_error{errno, std::system_category(), "mremap failed"};
601  }
602  m_size = new_size;
603 #else
604  assert(false && "can't resize anonymous mappings on non-linux systems");
605 #endif
606  } else { // file-based mapping
607  unmap();
608  m_size = new_size;
609  resize_fd(m_fd);
610  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
611  if (!is_valid()) {
612  throw std::system_error{errno, std::system_category(), "mmap (remap) failed"};
613  }
614  }
615 }
616 
617 #else
618 
619 // =========== Windows implementation =============
620 
621 /* References:
622  * CreateFileMapping: https://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
623  * CloseHandle: https://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
624  * MapViewOfFile: https://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
625  * UnmapViewOfFile: https://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
626  */
627 
628 namespace osmium {
629 
630  inline namespace util {
631 
632  inline DWORD dword_hi(uint64_t x) {
633  return static_cast<DWORD>(x >> 32);
634  }
635 
636  inline DWORD dword_lo(uint64_t x) {
637  return static_cast<DWORD>(x & 0xffffffff);
638  }
639 
640  } // namespace util
641 
642 } // namespace osmium
643 
644 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
645  switch (m_mapping_mode) {
647  return PAGE_READONLY;
649  return PAGE_WRITECOPY;
650  default: // mapping_mode::write_shared
651  break;
652  }
653  return PAGE_READWRITE;
654 }
655 
656 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
657  switch (m_mapping_mode) {
659  return FILE_MAP_READ;
661  return FILE_MAP_COPY;
662  default: // mapping_mode::write_shared
663  break;
664  }
665  return FILE_MAP_WRITE;
666 }
667 
668 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
669  if (m_fd == -1) {
670  return INVALID_HANDLE_VALUE;
671  }
672  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
673 }
674 
675 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
676  if (m_fd != -1) {
677  _setmode(m_fd, _O_BINARY);
678  }
679  return CreateFileMapping(get_handle(),
680  nullptr,
681  get_protection(),
682  osmium::dword_hi(static_cast<uint64_t>(m_size) + m_offset),
683  osmium::dword_lo(static_cast<uint64_t>(m_size) + m_offset),
684  nullptr);
685 }
686 
687 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
688  return MapViewOfFile(m_handle,
689  get_flags(),
690  osmium::dword_hi(m_offset),
691  osmium::dword_lo(m_offset),
692  m_size);
693 }
694 
695 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
696  return m_addr != nullptr;
697 }
698 
699 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
700  m_addr = nullptr;
701 }
702 
703 // GetLastError() returns a DWORD (A 32-bit unsigned integer), but the error
704 // code for std::system_error is an int. So we convert this here and hope
705 // it all works.
706 inline int last_error() noexcept {
707  return static_cast<int>(GetLastError());
708 }
709 
710 inline osmium::util::MemoryMapping::MemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
711  m_size(check_size(size)),
712  m_offset(offset),
713  m_fd(resize_fd(fd)),
714  m_mapping_mode(mode),
715  m_handle(create_file_mapping()),
716  m_addr(nullptr) {
717 
718  if (!m_handle) {
719  throw std::system_error{last_error(), std::system_category(), "CreateFileMapping failed"};
720  }
721 
722  m_addr = map_view_of_file();
723  if (!is_valid()) {
724  throw std::system_error{last_error(), std::system_category(), "MapViewOfFile failed"};
725  }
726 }
727 
729  m_size(other.m_size),
730  m_offset(other.m_offset),
731  m_fd(other.m_fd),
732  m_mapping_mode(other.m_mapping_mode),
733  m_handle(std::move(other.m_handle)),
734  m_addr(other.m_addr) {
735  other.make_invalid();
736  other.m_handle = nullptr;
737 }
738 
740  unmap();
741  m_size = other.m_size;
742  m_offset = other.m_offset;
743  m_fd = other.m_fd;
744  m_mapping_mode = other.m_mapping_mode;
745  m_handle = std::move(other.m_handle);
746  m_addr = other.m_addr;
747  other.make_invalid();
748  other.m_handle = nullptr;
749  return *this;
750 }
751 
753  if (is_valid()) {
754  if (!UnmapViewOfFile(m_addr)) {
755  throw std::system_error{last_error(), std::system_category(), "UnmapViewOfFile failed"};
756  }
757  make_invalid();
758  }
759 
760  if (m_handle) {
761  if (!CloseHandle(m_handle)) {
762  throw std::system_error{last_error(), std::system_category(), "CloseHandle failed"};
763  }
764  m_handle = nullptr;
765  }
766 }
767 
768 inline void osmium::util::MemoryMapping::resize(std::size_t new_size) {
769  unmap();
770 
771  m_size = new_size;
772  resize_fd(m_fd);
773 
774  m_handle = create_file_mapping();
775  if (!m_handle) {
776  throw std::system_error{last_error(), std::system_category(), "CreateFileMapping failed"};
777  }
778 
779  m_addr = map_view_of_file();
780  if (!is_valid()) {
781  throw std::system_error{last_error(), std::system_category(), "MapViewOfFile failed"};
782  }
783 }
784 
785 #endif
786 
787 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:216
bool is_valid() const noexcept
Definition: memory_mapping.hpp:521
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
OSMIUM_DEPRECATED TypedMemoryMapping(std::size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:366
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:536
MemoryMapping m_mapping
Definition: memory_mapping.hpp:333
int flag_type
Definition: memory_mapping.hpp:133
OSMIUM_DEPRECATED MemoryMapping(std::size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:191
std::size_t file_size(int fd)
Definition: file.hpp:109
const T * begin() const
Definition: memory_mapping.hpp:480
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:543
int fd() const noexcept
Definition: memory_mapping.hpp:267
void unmap()
Definition: memory_mapping.hpp:585
Definition: memory_mapping.hpp:95
Definition: location.hpp:554
mapping_mode
Definition: memory_mapping.hpp:99
int resize_fd(int fd)
Definition: memory_mapping.hpp:153
std::size_t size() const noexcept
Definition: memory_mapping.hpp:433
T * end()
Definition: memory_mapping.hpp:468
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:124
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:111
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:533
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:114
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:117
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: memory_mapping.hpp:491
std::size_t get_pagesize()
Definition: file.hpp:194
T * begin()
Definition: memory_mapping.hpp:459
Definition: memory_mapping.hpp:303
TypedMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:343
void make_invalid() noexcept
Definition: memory_mapping.hpp:525
std::size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:108
const T * cbegin() const
Definition: memory_mapping.hpp:472
int fd() const noexcept
Definition: memory_mapping.hpp:443
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:331
const T * end() const
Definition: memory_mapping.hpp:484
void unmap()
Definition: memory_mapping.hpp:402
MemoryMapping(std::size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:553
bool writable() const noexcept
Definition: memory_mapping.hpp:274
AnonymousTypedMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:495
AnonymousMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:307
void resize_file(int fd, std::size_t new_size)
Definition: file.hpp:177
std::size_t size() const noexcept
Definition: memory_mapping.hpp:258
void resize(std::size_t new_size)
Definition: memory_mapping.hpp:594
static std::size_t check_size(std::size_t size)
Definition: memory_mapping.hpp:140
TypedMemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:357
T * get_addr() const
Definition: memory_mapping.hpp:284
const T * cend() const
Definition: memory_mapping.hpp:476
void resize(std::size_t new_size)
Definition: memory_mapping.hpp:416
bool writable() const noexcept
Definition: memory_mapping.hpp:450