GRASS GIS 8 Programmer's Manual 8.2.0(2022)-exported
compress.c
Go to the documentation of this file.
1/*
2 ****************************************************************************
3 * -- GRASS Development Team --
4 *
5 * MODULE: GRASS gis library
6 * FILENAME: compress.c
7 * AUTHOR(S): Markus Metz
8 * PURPOSE: To provide an interface for compressing and
9 * decompressing data using various methods. Its primary
10 * use is in the storage and reading of GRASS rasters.
11 *
12 * DATE CREATED: Dec 17 2015
13 * COPYRIGHT: (C) 2015 by the GRASS Development Team
14 *
15 * This program is free software under the GNU General Public
16 * License (version 2 or greater). Read the file COPYING that
17 * comes with GRASS for details.
18 *
19 *****************************************************************************/
20
21/********************************************************************
22 * Compression methods: *
23 * 1 : RLE (generic Run-Length Encoding of single bytes) *
24 * 2 : ZLIB's DEFLATE (good speed and compression) *
25 * 3 : LZ4 (fastest, low compression) *
26 * 4 : BZIP2 (slowest, high compression) *
27 * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB) *
28 * *
29 * int *
30 * G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
31 * int fd, rbytes, nbytes; *
32 * unsigned char *dst; *
33 * ---------------------------------------------------------------- *
34 * This is the basic function for reading a compressed chunk of a *
35 * data file. The file descriptor should be in the proper location *
36 * and the 'dst' array should have enough space for the data. *
37 * 'nbytes' is the size of 'dst'. The 'rbytes' parameter is the *
38 * number of bytes to read (knowable from the offsets index). For *
39 * best results, 'nbytes' should be the exact amount of space *
40 * needed for the expansion. Too large a value of nbytes may cause *
41 * more data to be expanded than is desired. *
42 * Returns: The number of bytes decompressed into dst, or an error. *
43 * *
44 * Errors include: *
45 * -1 -- Error Reading or Decompressing data. *
46 * -2 -- Not enough space in dst. You must make dst larger *
47 * and then call the function again (remembering to *
48 * reset the file descriptor to it's proper location. *
49 * *
50 * ================================================================ *
51 * int *
52 * G_write_compressed (fd, src, nbytes, compression_type) *
53 * int fd, nbytes; *
54 * unsigned char *src; *
55 * ---------------------------------------------------------------- *
56 * This is the basic function for writing and compressing a data *
57 * chunk to a file. The file descriptor should be in the correct *
58 * location prior to this call. The function will compress 'nbytes' *
59 * of 'src' and write it to the file 'fd'. Returns the number of *
60 * bytes written or an error code: *
61 * *
62 * Errors include: *
63 * -1 -- Compression Failed. *
64 * -2 -- Unable to write to file. *
65 * *
66 * ================================================================ *
67 * int *
68 * G_write_uncompressed (fd, src, nbytes) *
69 * int fd, nbytes; *
70 * unsigned char *src; *
71 * ---------------------------------------------------------------- *
72 * Works similar to G_write_compressed() except no attempt at *
73 * compression is made. This is quicker, but may result in larger *
74 * files. *
75 * Returns the number of bytes written, or -1 for an error. It will *
76 * return an error if it fails to write nbytes. Otherwise, the *
77 * return value will always be nbytes + 1 (for compression flag). *
78 * *
79 ********************************************************************
80 */
81
82#include <grass/config.h>
83
84#include <stdio.h>
85#include <stdlib.h>
86#include <string.h>
87#include <errno.h>
88#include <unistd.h>
89#include <grass/gis.h>
90#include <grass/glocale.h>
91
92#include "compress.h"
93
94#define G_COMPRESSED_NO (unsigned char)'0'
95#define G_COMPRESSED_YES (unsigned char)'1'
96
97/* get compressor number
98 * return -1 on error
99 * return number >= 0 for known processor */
101{
102 int i;
103
104 if (!name)
105 return -1;
106
107 for (i = 0; compressor[i].name ; i++) {
108 if (G_strcasecmp(name, compressor[i].name) == 0)
109 return i;
110 }
111
112 return -1;
113}
114
115/* get compressor name
116 * return NULL on error
117 * return string (name) of known processor */
118char *G_compressor_name(int number)
119{
120 if (number < 0 || number >= n_compressors)
121 return NULL;
122
123 return compressor[number].name;
124}
125
127{
128#ifdef HAVE_ZSTD_H
129 /* ZSTD */
130 return 5;
131#endif
132 /* ZLIB */
133 return 2;
134}
135
136/* check compressor number
137 * return -1 on error
138 * return 0 known but not available
139 * return 1 known and available */
140int G_check_compressor(int number)
141{
142 if (number < 0 || number >= n_compressors) {
143 G_warning(_("Request for unsupported compressor"));
144 return -1;
145 }
146
147 return compressor[number].available;
148}
149
150int G_no_compress_bound(int src_sz)
151{
152 return src_sz;
153}
154
155int
156G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
157 int dst_sz)
158{
159 /* Catch errors early */
160 if (src == NULL || dst == NULL)
161 return -1;
162
163 /* Don't do anything if src is empty */
164 if (src_sz <= 0)
165 return 0;
166
167 /* dst too small */
168 if (dst_sz < src_sz)
169 return -2;
170
171 /* Copy the data from src to dst */
172 memcpy(dst, src, src_sz);
173
174 return src_sz;
175}
176
177int
178G_no_expand(unsigned char *src, int src_sz, unsigned char *dst,
179 int dst_sz)
180{
181 /* Catch errors early */
182 if (src == NULL || dst == NULL)
183 return -1;
184
185 /* Don't do anything if src is empty */
186 if (src_sz <= 0)
187 return 0;
188
189 /* dst too small */
190 if (dst_sz < src_sz)
191 return -2;
192
193 /* Copy the data from src to dst */
194 memcpy(dst, src, src_sz);
195
196 return src_sz;
197}
198
199/* G_*_compress_bound() returns an upper bound on the compressed size
200 * which can be larger than the input size
201 * some compressors are a bit faster if the size of the destination
202 * is at least the upper bound (no need to test for buffer overlflow)
203 * read comments on the specific compressor interfaces
204 */
205int G_compress_bound(int src_sz, int number)
206{
207 if (number < 0 || number >= n_compressors) {
208 G_fatal_error(_("Request for unsupported compressor"));
209 return -1;
210 }
211
212 return compressor[number].bound(src_sz);
213}
214
215/* G_*_compress() returns
216 * > 0: number of bytes in dst
217 * 0: nothing done
218 * -1: error
219 * -2: dst too small
220 */
221int G_compress(unsigned char *src, int src_sz, unsigned char *dst,
222 int dst_sz, int number)
223{
224 if (number < 0 || number >= n_compressors) {
225 G_fatal_error(_("Request for unsupported compressor"));
226 return -1;
227 }
228
229 return compressor[number].compress(src, src_sz, dst, dst_sz);
230}
231
232/* G_*_expand() returns
233 * > 0: number of bytes in dst
234 * -1: error
235 */
236int G_expand(unsigned char *src, int src_sz, unsigned char *dst,
237 int dst_sz, int number)
238{
239 if (number < 0 || number >= n_compressors) {
240 G_fatal_error(_("Request for unsupported compressor"));
241 return -1;
242 }
243
244 return compressor[number].expand(src, src_sz, dst, dst_sz);
245}
246
247int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
248 int number)
249{
250 int bsize, nread, err;
251 unsigned char *b;
252
253 if (dst == NULL || nbytes <= 0) {
254 if (dst == NULL)
255 G_warning(_("No destination buffer allocated"));
256 if (nbytes <= 0)
257 G_warning(_("Invalid destination buffer size %d"), nbytes);
258 return -2;
259 }
260
261 if (rbytes <= 0) {
262 G_warning(_("Invalid read size %d"), nbytes);
263 return -2;
264 }
265
266 bsize = rbytes;
267
268 /* Our temporary input buffer for read */
269 if (NULL == (b = (unsigned char *)
270 G_calloc(bsize, sizeof(unsigned char))))
271 return -1;
272
273 /* Read from the file until we get our bsize or an error */
274 nread = 0;
275 do {
276 err = read(fd, b + nread, bsize - nread);
277 if (err >= 0)
278 nread += err;
279 } while (err > 0 && nread < bsize);
280
281 if (err <= 0) {
282 if (err == 0)
283 G_warning(_("Unable to read %d bytes: end of file"), rbytes);
284 else
285 G_warning(_("Unable to read %d bytes: %s"), rbytes, strerror(errno));
286 return -1;
287 }
288
289 /* If the bsize if less than rbytes and we didn't get an error.. */
290 if (nread < rbytes) {
291 G_free(b);
292 G_warning("Unable to read %d bytes, got %d bytes", rbytes, nread);
293 return -1;
294 }
295
296 /* Test if row is compressed */
297 if (b[0] == G_COMPRESSED_NO) {
298 /* Then just copy it to dst */
299 for (err = 0; err < nread - 1 && err < nbytes; err++)
300 dst[err] = b[err + 1];
301
302 G_free(b);
303 return (nread - 1);
304 }
305 else if (b[0] != G_COMPRESSED_YES) {
306 /* We're not at the start of a row */
307 G_free(b);
308 G_warning("Read error: We're not at the start of a row");
309 return -1;
310 }
311 /* Okay it's a compressed row */
312
313 /* Just call G_expand() with the buffer we read,
314 * Account for first byte being a flag
315 */
316 err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
317
318 /* We're done with b */
319 G_free(b);
320
321 /* Return whatever G_expand() returned */
322 return err;
323
324} /* G_read_compressed() */
325
326int G_write_compressed(int fd, unsigned char *src, int nbytes,
327 int number)
328{
329 int dst_sz, nwritten, err;
330 unsigned char *dst, compressed;
331
332 /* Catch errors */
333 if (src == NULL || nbytes < 0) {
334 if (src == NULL)
335 G_warning(_("No source buffer"));
336 if (nbytes <= 0)
337 G_warning(_("Invalid source buffer size %d"), nbytes);
338 return -1;
339 }
340
341 /* get upper bound of compressed size */
342 dst_sz = G_compress_bound(nbytes, number);
343 if (NULL == (dst = (unsigned char *)
344 G_calloc(dst_sz, sizeof(unsigned char))))
345 return -1;
346
347 /* Now just call G_compress() */
348 err = G_compress(src, nbytes, dst, dst_sz, number);
349
350 /* If compression succeeded write compressed row,
351 * otherwise write uncompressed row. Compression will fail
352 * if dst is too small (i.e. compressed data is larger)
353 */
354 if (err > 0 && err < nbytes) {
355 dst_sz = err;
356 /* Write the compression flag */
357 compressed = G_COMPRESSED_YES;
358 if (write(fd, &compressed, 1) != 1) {
359 G_free(dst);
360 G_warning(_("Unable to write compression flag"));
361 return -1;
362 }
363 nwritten = 0;
364 do {
365 err = write(fd, dst + nwritten, dst_sz - nwritten);
366 if (err >= 0)
367 nwritten += err;
368 } while (err > 0 && nwritten < dst_sz);
369 if (err <= 0) {
370 if (err == 0)
371 G_warning(_("Unable to write %d bytes: nothing written"), dst_sz);
372 else
373 G_warning(_("Unable to write %d bytes: %s"), dst_sz, strerror(errno));
374 }
375 /* Account for extra byte */
376 nwritten++;
377 }
378 else {
379 /* Write compression flag */
380 compressed = G_COMPRESSED_NO;
381 if (write(fd, &compressed, 1) != 1) {
382 G_free(dst);
383 G_warning(_("Unable to write compression flag"));
384 return -1;
385 }
386 nwritten = 0;
387 do {
388 err = write(fd, src + nwritten, nbytes - nwritten);
389 if (err >= 0)
390 nwritten += err;
391 } while (err > 0 && nwritten < nbytes);
392 if (err <= 0) {
393 if (err == 0)
394 G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
395 else
396 G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
397 }
398 /* Account for extra byte */
399 nwritten++;
400 } /* if (err > 0) */
401
402 /* Done with the dst buffer */
403 G_free(dst);
404
405 /* If we didn't write all the data return an error */
406 if (err < 0)
407 return -2;
408
409 return nwritten;
410} /* G_write_compressed() */
411
412int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
413{
414 int err, nwritten;
415 unsigned char compressed;
416
417 /* Catch errors */
418 if (src == NULL || nbytes < 0)
419 return -1;
420
421 /* Write the compression flag */
422 compressed = G_COMPRESSED_NO;
423 if (write(fd, &compressed, 1) != 1) {
424 G_warning(_("Unable to write compression flag"));
425 return -1;
426 }
427
428 /* Now write the data */
429 nwritten = 0;
430 do {
431 err = write(fd, src + nwritten, nbytes - nwritten);
432 if (err > 0)
433 nwritten += err;
434 } while (err > 0 && nwritten < nbytes);
435 if (err <= 0) {
436 if (err == 0)
437 G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
438 else
439 G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
440 }
441
442 if (err < 0 || nwritten != nbytes)
443 return -1;
444
445 /* Account for extra compressed flag */
446 nwritten++;
447
448 /* That's all */
449 return nwritten;
450
451} /* G_write_uncompressed() */
452
453
454/* vim: set softtabstop=4 shiftwidth=4 expandtab: */
void G_free(void *buf)
Free allocated memory.
Definition: alloc.c:149
#define NULL
Definition: ccmath.h:32
int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:221
char * G_compressor_name(int number)
Definition: compress.c:118
int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:236
int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
Definition: compress.c:326
int G_default_compressor(void)
Definition: compress.c:126
int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
Definition: compress.c:412
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes, int number)
Definition: compress.c:247
int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:156
#define G_COMPRESSED_YES
Definition: compress.c:95
int G_compressor_number(char *name)
Definition: compress.c:100
#define G_COMPRESSED_NO
Definition: compress.c:94
int G_check_compressor(int number)
Definition: compress.c:140
int G_no_compress_bound(int src_sz)
Definition: compress.c:150
int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:178
int G_compress_bound(int src_sz, int number)
Definition: compress.c:205
struct compressor_list compressor[]
Definition: compress.h:54
double b
void G_fatal_error(const char *msg,...)
Print a fatal error message to stderr.
Definition: gis/error.c:160
void G_warning(const char *msg,...)
Print a warning message to stderr.
Definition: gis/error.c:204
char * dst
Definition: lz4.h:599
const char * name
Definition: named_colr.c:7
int G_strcasecmp(const char *x, const char *y)
String compare ignoring case (upper or lower)
Definition: strings.c:47
bound_fn * bound
Definition: compress.h:39
compress_fn * compress
Definition: compress.h:37
expand_fn * expand
Definition: compress.h:38
char * name
Definition: compress.h:40
SYMBOL * err(FILE *fp, SYMBOL *s, char *msg)
Definition: symbol/read.c:220