Libosmium  2.19.0
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2023 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  virtual std::size_t file_size() const {
86  return 0;
87  }
88 
89  }; // class Compressor
90 
91  class Decompressor {
92 
93  std::atomic<std::size_t>* m_offset_ptr{nullptr};
94 
95  std::atomic_bool m_want_buffered_pages_removed{false};
96 
97  public:
98 
99  enum {
100  input_buffer_size = 1024U * 1024U
101  };
102 
103  Decompressor() = default;
104 
105  Decompressor(const Decompressor&) = delete;
107 
110 
111  virtual ~Decompressor() noexcept = default;
112 
113  virtual std::string read() = 0;
114 
115  virtual void close() = 0;
116 
117  virtual bool is_real() const noexcept {
118  return true;
119  }
120 
121  void set_offset_ptr(std::atomic<std::size_t>* offset_ptr) noexcept {
122  m_offset_ptr = offset_ptr;
123  }
124 
125  void set_offset(const std::size_t offset) noexcept {
126  if (m_offset_ptr) {
127  *m_offset_ptr = offset;
128  }
129  }
130 
131  bool want_buffered_pages_removed() const noexcept {
133  }
134 
135  void set_want_buffered_pages_removed(bool value) noexcept {
137  }
138 
139  }; // class Decompressor
140 
149 
150  public:
151 
152  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
154  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
155 
156  private:
157 
161 
162  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
163 
165 
166  CompressionFactory() = default;
167 
169  const auto it = m_callbacks.find(compression);
170 
171  if (it != m_callbacks.end()) {
172  return it->second;
173  }
174 
175  std::string error_message{"Support for compression '"};
176  error_message += as_string(compression);
177  error_message += "' not compiled into this binary";
178  throw unsupported_file_format_error{error_message};
179  }
180 
181  public:
182 
185 
188 
189  ~CompressionFactory() noexcept = default;
190 
192  static CompressionFactory factory;
193  return factory;
194  }
195 
197  osmium::io::file_compression compression,
199  const create_decompressor_type_fd& create_decompressor_fd,
200  const create_decompressor_type_buffer& create_decompressor_buffer) {
201 
202  const compression_map_type::value_type cc{compression,
203  std::make_tuple(create_compressor,
204  create_decompressor_fd,
205  create_decompressor_buffer)};
206 
207  return m_callbacks.insert(cc).second;
208  }
209 
210  template <typename... TArgs>
211  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
212  const auto callbacks = find_callbacks(compression);
213  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
214  }
215 
216  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
217  const auto callbacks = find_callbacks(compression);
218  return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
219  }
220 
221  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
222  const auto callbacks = find_callbacks(compression);
223  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
224  }
225 
226  }; // class CompressionFactory
227 
228  class NoCompressor final : public Compressor {
229 
230  std::size_t m_file_size = 0;
231  int m_fd;
232 
233  public:
234 
235  NoCompressor(const int fd, const fsync sync) :
236  Compressor(sync),
237  m_fd(fd) {
238  }
239 
240  NoCompressor(const NoCompressor&) = delete;
242 
245 
246  ~NoCompressor() noexcept override {
247  try {
248  close();
249  } catch (...) {
250  // Ignore any exceptions because destructor must not throw.
251  }
252  }
253 
254  void write(const std::string& data) override {
255  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
256  m_file_size += data.size();
257  }
258 
259  void close() override {
260  if (m_fd >= 0) {
261  const int fd = m_fd;
262  m_fd = -1;
263 
264  // Do not sync or close stdout
265  if (fd == 1) {
266  return;
267  }
268 
269  if (do_fsync()) {
270  osmium::io::detail::reliable_fsync(fd);
271  }
272  osmium::io::detail::reliable_close(fd);
273  }
274  }
275 
276  std::size_t file_size() const override {
277  return m_file_size;
278  }
279 
280  }; // class NoCompressor
281 
287  class DummyDecompressor final : public Decompressor {
288  public:
289 
290  DummyDecompressor() = default;
291 
294 
297 
298  ~DummyDecompressor() noexcept override = default;
299 
300  std::string read() override {
301  return {};
302  }
303 
304  void close() override {
305  }
306 
307  bool is_real() const noexcept override {
308  return false;
309  }
310 
311  }; // class DummyDecompressor
312 
313  class NoDecompressor final : public Decompressor {
314 
315  int m_fd = -1;
316  const char* m_buffer = nullptr;
317  std::size_t m_buffer_size = 0;
318  std::size_t m_offset = 0;
319 
320  public:
321 
322  explicit NoDecompressor(const int fd) :
323  m_fd(fd) {
324  }
325 
326  NoDecompressor(const char* buffer, const std::size_t size) :
327  m_buffer(buffer),
328  m_buffer_size(size) {
329  }
330 
331  NoDecompressor(const NoDecompressor&) = delete;
333 
336 
337  ~NoDecompressor() noexcept override {
338  try {
339  close();
340  } catch (...) {
341  // Ignore any exceptions because destructor must not throw.
342  }
343  }
344 
345  std::string read() override {
346  std::string buffer;
347 
348  if (m_buffer) {
349  if (m_buffer_size != 0) {
350  const std::size_t size = m_buffer_size;
351  m_buffer_size = 0;
352  buffer.append(m_buffer, size);
353  }
354  } else {
357  osmium::io::detail::remove_buffered_pages(m_fd, m_offset);
358  }
359  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
360  buffer.resize(std::string::size_type(nread));
361  }
362 
363  m_offset += buffer.size();
365 
366  return buffer;
367  }
368 
369  void close() override {
370  if (m_fd >= 0) {
372  osmium::io::detail::remove_buffered_pages(m_fd);
373  }
374  const int fd = m_fd;
375  m_fd = -1;
376  osmium::io::detail::reliable_close(fd);
377  }
378  }
379 
380  }; // class NoDecompressor
381 
382  namespace detail {
383 
384  // we want the register_compression() function to run, setting
385  // the variable is only a side-effect, it will never be used
387  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
388  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
389  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
390  );
391 
392  // dummy function to silence the unused variable warning from above
393  inline bool get_registered_no_compression() noexcept {
394  return registered_no_compression;
395  }
396 
397  } // namespace detail
398 
399  } // namespace io
400 
401 } // namespace osmium
402 
403 #endif // OSMIUM_IO_COMPRESSION_HPP
Definition: compression.hpp:148
~CompressionFactory() noexcept=default
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:196
compression_map_type m_callbacks
Definition: compression.hpp:164
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:216
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:152
CompressionFactory & operator=(CompressionFactory &&)=delete
CompressionFactory & operator=(const CompressionFactory &)=delete
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:162
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:153
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:221
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:211
CompressionFactory(const CompressionFactory &)=delete
static CompressionFactory & instance()
Definition: compression.hpp:191
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:154
CompressionFactory(CompressionFactory &&)=delete
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:168
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:160
Definition: compression.hpp:57
bool do_fsync() const noexcept
Definition: compression.hpp:63
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
Compressor(const Compressor &)=default
Compressor(Compressor &&) noexcept=default
fsync m_fsync
Definition: compression.hpp:59
virtual void write(const std::string &data)=0
virtual void close()=0
Compressor & operator=(const Compressor &)=default
virtual std::size_t file_size() const
Definition: compression.hpp:85
Definition: compression.hpp:91
std::atomic_bool m_want_buffered_pages_removed
Definition: compression.hpp:95
void set_offset_ptr(std::atomic< std::size_t > *offset_ptr) noexcept
Definition: compression.hpp:121
@ input_buffer_size
Definition: compression.hpp:100
virtual bool is_real() const noexcept
Definition: compression.hpp:117
Decompressor & operator=(const Decompressor &)=delete
Decompressor & operator=(Decompressor &&)=delete
virtual std::string read()=0
void set_want_buffered_pages_removed(bool value) noexcept
Definition: compression.hpp:135
virtual void close()=0
virtual ~Decompressor() noexcept=default
std::atomic< std::size_t > * m_offset_ptr
Definition: compression.hpp:93
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:125
bool want_buffered_pages_removed() const noexcept
Definition: compression.hpp:131
Decompressor(const Decompressor &)=delete
Decompressor(Decompressor &&)=delete
Definition: compression.hpp:287
~DummyDecompressor() noexcept override=default
void close() override
Definition: compression.hpp:304
bool is_real() const noexcept override
Definition: compression.hpp:307
std::string read() override
Definition: compression.hpp:300
DummyDecompressor(DummyDecompressor &&)=delete
DummyDecompressor(const DummyDecompressor &)=delete
DummyDecompressor & operator=(const DummyDecompressor &)=delete
DummyDecompressor & operator=(DummyDecompressor &&)=delete
Definition: compression.hpp:228
void close() override
Definition: compression.hpp:259
void write(const std::string &data) override
Definition: compression.hpp:254
NoCompressor(const NoCompressor &)=delete
~NoCompressor() noexcept override
Definition: compression.hpp:246
std::size_t m_file_size
Definition: compression.hpp:230
NoCompressor & operator=(const NoCompressor &)=delete
NoCompressor & operator=(NoCompressor &&)=delete
std::size_t file_size() const override
Definition: compression.hpp:276
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:235
NoCompressor(NoCompressor &&)=delete
int m_fd
Definition: compression.hpp:231
Definition: compression.hpp:313
const char * m_buffer
Definition: compression.hpp:316
NoDecompressor(const NoDecompressor &)=delete
int m_fd
Definition: compression.hpp:315
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:326
std::size_t m_buffer_size
Definition: compression.hpp:317
std::size_t m_offset
Definition: compression.hpp:318
NoDecompressor & operator=(NoDecompressor &&)=delete
NoDecompressor & operator=(const NoDecompressor &)=delete
std::string read() override
Definition: compression.hpp:345
NoDecompressor(NoDecompressor &&)=delete
~NoDecompressor() noexcept override
Definition: compression.hpp:337
NoDecompressor(const int fd)
Definition: compression.hpp:322
void close() override
Definition: compression.hpp:369
Definition: attr.hpp:342
file_compression
Definition: file_compression.hpp:42
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
fsync
Definition: writer_options.hpp:51
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
Definition: location.hpp:555