GRASS GIS 7 Programmer's Manual  7.4.2(2018)-exported
compress.c
Go to the documentation of this file.
1 /*
2  ****************************************************************************
3  * -- GRASS Development Team --
4  *
5  * MODULE: GRASS gis library
6  * FILENAME: compress.c
7  * AUTHOR(S): Markus Metz
8  * PURPOSE: To provide an interface for compressing and
9  * decompressing data using various methods. Its primary
10  * use is in the storage and reading of GRASS rasters.
11  *
12  * DATE CREATED: Dec 17 2015
13  * COPYRIGHT: (C) 2015 by the GRASS Development Team
14  *
15  * This program is free software under the GNU General Public
16  * License (version 2 or greater). Read the file COPYING that
17  * comes with GRASS for details.
18  *
19  *****************************************************************************/
20 
21 /********************************************************************
22  * Compression methods: *
23  * 1 : RLE (generic Run-Length Encoding of single bytes) *
24  * 2 : ZLIB's DEFLATE (good speed and compression) *
25  * 3 : LZ4 (fastest, low compression) *
26  * 4 : BZIP2 (slowest, high compression) *
27  * *
28  * int *
29  * G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
30  * int fd, rbytes, nbytes; *
31  * unsigned char *dst; *
32  * ---------------------------------------------------------------- *
33  * This is the basic function for reading a compressed chunk of a *
34  * data file. The file descriptor should be in the proper location *
35  * and the 'dst' array should have enough space for the data. *
36  * 'nbytes' is the size of 'dst'. The 'rbytes' parameter is the *
37  * number of bytes to read (knowable from the offsets index). For *
38  * best results, 'nbytes' should be the exact amount of space *
39  * needed for the expansion. Too large a value of nbytes may cause *
40  * more data to be expanded than is desired. *
41  * Returns: The number of bytes decompressed into dst, or an error. *
42  * *
43  * Errors include: *
44  * -1 -- Error Reading or Decompressing data. *
45  * -2 -- Not enough space in dst. You must make dst larger *
46  * and then call the function again (remembering to *
47  * reset the file descriptor to it's proper location. *
48  * *
49  * ================================================================ *
50  * int *
51  * G_write_compressed (fd, src, nbytes, compression_type) *
52  * int fd, nbytes; *
53  * unsigned char *src; *
54  * ---------------------------------------------------------------- *
55  * This is the basic function for writing and compressing a data *
56  * chunk to a file. The file descriptor should be in the correct *
57  * location prior to this call. The function will compress 'nbytes' *
58  * of 'src' and write it to the file 'fd'. Returns the number of *
59  * bytes written or an error code: *
60  * *
61  * Errors include: *
62  * -1 -- Compression Failed. *
63  * -2 -- Unable to write to file. *
64  * *
65  * ================================================================ *
66  * int *
67  * G_write_uncompressed (fd, src, nbytes) *
68  * int fd, nbytes; *
69  * unsigned char *src; *
70  * ---------------------------------------------------------------- *
71  * Works similar to G_write_compressed() except no attempt at *
72  * compression is made. This is quicker, but may result in larger *
73  * files. *
74  * Returns the number of bytes written, or -1 for an error. It will *
75  * return an error if it fails to write nbytes. Otherwise, the *
76  * return value will always be nbytes + 1 (for compression flag). *
77  * *
78  ********************************************************************
79  */
80 
81 #include <grass/config.h>
82 
83 #include <stdio.h>
84 #include <stdlib.h>
85 #include <string.h>
86 #include <errno.h>
87 #include <unistd.h>
88 #include <grass/gis.h>
89 #include <grass/glocale.h>
90 
91 #include "compress.h"
92 
93 #define G_COMPRESSED_NO (unsigned char)'0'
94 #define G_COMPRESSED_YES (unsigned char)'1'
95 
96 /* get compressor number
97  * return -1 on error
98  * return number >= 0 for known processor */
100 {
101  int i;
102 
103  if (!name)
104  return -1;
105 
106  for (i = 0; compressor[i].name ; i++) {
107  if (G_strcasecmp(name, compressor[i].name) == 0)
108  return i;
109  }
110 
111  return -1;
112 }
113 
114 /* get compressor name
115  * return NULL on error
116  * return string (name) of known processor */
117 char *G_compressor_name(int number)
118 {
119  if (number < 0 || number >= n_compressors)
120  return NULL;
121 
122  return compressor[number].name;
123 }
124 
125 /* check compressor number
126  * return -1 on error
127  * return 0 known but not available
128  * return 1 known and available */
129 int G_check_compressor(int number)
130 {
131  if (number < 0 || number >= n_compressors) {
132  G_warning(_("Request for unsupported compressor"));
133  return -1;
134  }
135 
136  return compressor[number].available;
137 }
138 
139 int
140 G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
141  int dst_sz)
142 {
143  /* Catch errors early */
144  if (src == NULL || dst == NULL)
145  return -1;
146 
147  /* Don't do anything if src is empty */
148  if (src_sz <= 0)
149  return 0;
150 
151  /* dst too small */
152  if (dst_sz < src_sz)
153  return -2;
154 
155  /* Copy the data from src to dst */
156  memcpy(dst, src, src_sz);
157 
158  return src_sz;
159 }
160 
161 int
162 G_no_expand(unsigned char *src, int src_sz, unsigned char *dst,
163  int dst_sz)
164 {
165  /* Catch errors early */
166  if (src == NULL || dst == NULL)
167  return -1;
168 
169  /* Don't do anything if src is empty */
170  if (src_sz <= 0)
171  return 0;
172 
173  /* dst too small */
174  if (dst_sz < src_sz)
175  return -2;
176 
177  /* Copy the data from src to dst */
178  memcpy(dst, src, src_sz);
179 
180  return src_sz;
181 }
182 
183 /* G_*_compress() returns
184  * > 0: number of bytes in dst
185  * 0: nothing done
186  * -1: error
187  * -2: dst too small
188  */
189 int G_compress(unsigned char *src, int src_sz, unsigned char *dst,
190  int dst_sz, int number)
191 {
192  if (number < 0 || number >= n_compressors) {
193  G_fatal_error(_("Request for unsupported compressor"));
194  return -1;
195  }
196 
197  return compressor[number].compress(src, src_sz, dst, dst_sz);
198 }
199 
200 /* G_*_expand() returns
201  * > 0: number of bytes in dst
202  * -1: error
203  */
204 int G_expand(unsigned char *src, int src_sz, unsigned char *dst,
205  int dst_sz, int number)
206 {
207  if (number < 0 || number >= n_compressors) {
208  G_fatal_error(_("Request for unsupported compressor"));
209  return -1;
210  }
211 
212  return compressor[number].expand(src, src_sz, dst, dst_sz);
213 }
214 
215 int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
216  int number)
217 {
218  int bsize, nread, err;
219  unsigned char *b;
220 
221  if (dst == NULL || nbytes <= 0) {
222  if (dst == NULL)
223  G_warning(_("No destination buffer allocated"));
224  if (nbytes <= 0)
225  G_warning(_("Invalid destination buffer size %d"), nbytes);
226  return -2;
227  }
228 
229  if (rbytes <= 0) {
230  G_warning(_("Invalid read size %d"), nbytes);
231  return -2;
232  }
233 
234  bsize = rbytes;
235 
236  /* Our temporary input buffer for read */
237  if (NULL == (b = (unsigned char *)
238  G_calloc(bsize, sizeof(unsigned char))))
239  return -1;
240 
241  /* Read from the file until we get our bsize or an error */
242  nread = 0;
243  do {
244  err = read(fd, b + nread, bsize - nread);
245  if (err >= 0)
246  nread += err;
247  } while (err > 0 && nread < bsize);
248 
249  if (err <= 0) {
250  if (err == 0)
251  G_warning(_("Unable to read %d bytes: end of file"), rbytes);
252  else
253  G_warning(_("Unable to read %d bytes: %s"), rbytes, strerror(errno));
254  return -1;
255  }
256 
257  /* If the bsize if less than rbytes and we didn't get an error.. */
258  if (nread < rbytes) {
259  G_free(b);
260  G_warning("Unable to read %d bytes, got %d bytes", rbytes, nread);
261  return -1;
262  }
263 
264  /* Test if row is compressed */
265  if (b[0] == G_COMPRESSED_NO) {
266  /* Then just copy it to dst */
267  for (err = 0; err < nread - 1 && err < nbytes; err++)
268  dst[err] = b[err + 1];
269 
270  G_free(b);
271  return (nread - 1);
272  }
273  else if (b[0] != G_COMPRESSED_YES) {
274  /* We're not at the start of a row */
275  G_free(b);
276  G_warning("Read error: We're not at the start of a row");
277  return -1;
278  }
279  /* Okay it's a compressed row */
280 
281  /* Just call G_expand() with the buffer we read,
282  * Account for first byte being a flag
283  */
284  err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
285 
286  /* We're done with b */
287  G_free(b);
288 
289  /* Return whatever G_expand() returned */
290  return err;
291 
292 } /* G_read_compressed() */
293 
294 int G_write_compressed(int fd, unsigned char *src, int nbytes,
295  int number)
296 {
297  int dst_sz, nwritten, err;
298  unsigned char *dst, compressed;
299 
300  /* Catch errors */
301  if (src == NULL || nbytes < 0) {
302  if (src == NULL)
303  G_warning(_("No source buffer"));
304  if (nbytes <= 0)
305  G_warning(_("Invalid source buffer size %d"), nbytes);
306  return -1;
307  }
308 
309  dst_sz = nbytes;
310  if (NULL == (dst = (unsigned char *)
311  G_calloc(dst_sz, sizeof(unsigned char))))
312  return -1;
313 
314  /* Now just call G_compress() */
315  err = G_compress(src, nbytes, dst, dst_sz, number);
316 
317  /* If compression succeeded write compressed row,
318  * otherwise write uncompressed row. Compression will fail
319  * if dst is too small (i.e. compressed data is larger)
320  */
321  if (err > 0 && err < nbytes) {
322  dst_sz = err;
323  /* Write the compression flag */
324  compressed = G_COMPRESSED_YES;
325  if (write(fd, &compressed, 1) != 1) {
326  G_free(dst);
327  G_warning(_("Unable to write compression flag"));
328  return -1;
329  }
330  nwritten = 0;
331  do {
332  err = write(fd, dst + nwritten, dst_sz - nwritten);
333  if (err >= 0)
334  nwritten += err;
335  } while (err > 0 && nwritten < dst_sz);
336  if (err <= 0) {
337  if (err == 0)
338  G_warning(_("Unable to write %d bytes: nothing written"), dst_sz);
339  else
340  G_warning(_("Unable to write %d bytes: %s"), dst_sz, strerror(errno));
341  }
342  /* Account for extra byte */
343  nwritten++;
344  }
345  else {
346  /* Write compression flag */
347  compressed = G_COMPRESSED_NO;
348  if (write(fd, &compressed, 1) != 1) {
349  G_free(dst);
350  G_warning(_("Unable to write compression flag"));
351  return -1;
352  }
353  nwritten = 0;
354  do {
355  err = write(fd, src + nwritten, nbytes - nwritten);
356  if (err >= 0)
357  nwritten += err;
358  } while (err > 0 && nwritten < nbytes);
359  if (err <= 0) {
360  if (err == 0)
361  G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
362  else
363  G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
364  }
365  /* Account for extra byte */
366  nwritten++;
367  } /* if (err > 0) */
368 
369  /* Done with the dst buffer */
370  G_free(dst);
371 
372  /* If we didn't write all the data return an error */
373  if (err < 0)
374  return -2;
375 
376  return nwritten;
377 } /* G_write_compressed() */
378 
379 int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
380 {
381  int err, nwritten;
382  unsigned char compressed;
383 
384  /* Catch errors */
385  if (src == NULL || nbytes < 0)
386  return -1;
387 
388  /* Write the compression flag */
389  compressed = G_COMPRESSED_NO;
390  if (write(fd, &compressed, 1) != 1) {
391  G_warning(_("Unable to write compression flag"));
392  return -1;
393  }
394 
395  /* Now write the data */
396  nwritten = 0;
397  do {
398  err = write(fd, src + nwritten, nbytes - nwritten);
399  if (err > 0)
400  nwritten += err;
401  } while (err > 0 && nwritten < nbytes);
402  if (err <= 0) {
403  if (err == 0)
404  G_warning(_("Unable to write %d bytes: nothing written"), nbytes);
405  else
406  G_warning(_("Unable to write %d bytes: %s"), nbytes, strerror(errno));
407  }
408 
409  if (err < 0 || nwritten != nbytes)
410  return -1;
411 
412  /* Account for extra compressed flag */
413  nwritten++;
414 
415  /* That's all */
416  return nwritten;
417 
418 } /* G_write_uncompressed() */
419 
420 
421 /* vim: set softtabstop=4 shiftwidth=4 expandtab: */
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes, int number)
Definition: compress.c:215
int G_strcasecmp(const char *x, const char *y)
String compare ignoring case (upper or lower)
Definition: strings.c:46
struct compressor_list compressor[]
Definition: compress.h:41
int G_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:189
compress_fn * compress
Definition: compress.h:27
char * G_compressor_name(int number)
Definition: compress.c:117
expand_fn * expand
Definition: compress.h:28
int G_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz, int number)
Definition: compress.c:204
char * dst
Definition: lz4.h:354
#define NULL
Definition: ccmath.h:32
void G_fatal_error(const char *msg,...)
Print a fatal error message to stderr.
Definition: gis/error.c:160
SYMBOL * err(FILE *fp, SYMBOL *s, char *msg)
Definition: symbol/read.c:220
int G_no_expand(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:162
double b
int G_write_uncompressed(int fd, const unsigned char *src, int nbytes)
Definition: compress.c:379
char * name
Definition: compress.h:29
#define G_COMPRESSED_NO
Definition: compress.c:93
int G_no_compress(unsigned char *src, int src_sz, unsigned char *dst, int dst_sz)
Definition: compress.c:140
#define G_COMPRESSED_YES
Definition: compress.c:94
int G_check_compressor(int number)
Definition: compress.c:129
int G_compressor_number(char *name)
Definition: compress.c:99
int G_write_compressed(int fd, unsigned char *src, int nbytes, int number)
Definition: compress.c:294
const char * name
Definition: named_colr.c:7
void G_free(void *buf)
Free allocated memory.
Definition: alloc.c:149
void G_warning(const char *msg,...)
Print a warning message to stderr.
Definition: gis/error.c:204