src/serialization.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "serialization.h"
  21
  22 #include "util/serialize.h"
  23
  24 #include "zlib.h"
  25
  26 /* report a zlib or i/o error */
  27 void zerr(int ret)
  28 {
  29     dstream<<"zerr: ";
  30     switch (ret) {
  31     case Z_ERRNO:
  32         if (ferror(stdin))
  33             dstream<<"error reading stdin"<<std::endl;
  34         if (ferror(stdout))
  35             dstream<<"error writing stdout"<<std::endl;
  36         break;
  37     case Z_STREAM_ERROR:
  38         dstream<<"invalid compression level"<<std::endl;
  39         break;
  40     case Z_DATA_ERROR:
  41         dstream<<"invalid or incomplete deflate data"<<std::endl;
  42         break;
  43     case Z_MEM_ERROR:
  44         dstream<<"out of memory"<<std::endl;
  45         break;
  46     case Z_VERSION_ERROR:
  47         dstream<<"zlib version mismatch!"<<std::endl;
  48                 break;
  49         default:
  50                 dstream<<"return value = "<<ret<<std::endl;
  51     }
  52 }
  53
  54 void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
  55 {
  56         z_stream z;
  57         const s32 bufsize = 16384;
  58         char output_buffer[bufsize];
  59         int status = 0;
  60         int ret;
  61
  62         z.zalloc = Z_NULL;
  63         z.zfree = Z_NULL;
  64         z.opaque = Z_NULL;
  65
  66         ret = deflateInit(&z, level);
  67         if(ret != Z_OK)
  68                 throw SerializationError("compressZlib: deflateInit failed");
  69
  70         // Point zlib to our input buffer
  71         z.next_in = (Bytef*)&data[0];
  72         z.avail_in = data_size;
  73         // And get all output
  74         for(;;)
  75         {
  76                 z.next_out = (Bytef*)output_buffer;
  77                 z.avail_out = bufsize;
  78
  79                 status = deflate(&z, Z_FINISH);
  80                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
  81                                 || status == Z_MEM_ERROR)
  82                 {
  83                         zerr(status);
  84                         throw SerializationError("compressZlib: deflate failed");
  85                 }
  86                 int count = bufsize - z.avail_out;
  87                 if(count)
  88                         os.write(output_buffer, count);
  89                 // This determines zlib has given all output
  90                 if(status == Z_STREAM_END)
  91                         break;
  92         }
  93
  94         deflateEnd(&z);
  95 }
  96
  97 void compressZlib(const std::string &data, std::ostream &os, int level)
  98 {
  99         compressZlib((u8*)data.c_str(), data.size(), os, level);
 100 }
 101
 102 void decompressZlib(std::istream &is, std::ostream &os, size_t limit)
 103 {
 104         z_stream z;
 105         const s32 bufsize = 16384;
 106         char input_buffer[bufsize];
 107         char output_buffer[bufsize];
 108         int status = 0;
 109         int ret;
 110         int bytes_read = 0;
 111         int bytes_written = 0;
 112         int input_buffer_len = 0;
 113
 114         z.zalloc = Z_NULL;
 115         z.zfree = Z_NULL;
 116         z.opaque = Z_NULL;
 117
 118         ret = inflateInit(&z);
 119         if(ret != Z_OK)
 120                 throw SerializationError("dcompressZlib: inflateInit failed");
 121
 122         z.avail_in = 0;
 123
 124         //dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 125
 126         for(;;)
 127         {
 128                 int output_size = bufsize;
 129                 z.next_out = (Bytef*)output_buffer;
 130                 z.avail_out = output_size;
 131
 132                 if (limit) {
 133                         int limit_remaining = limit - bytes_written;
 134                         if (limit_remaining <= 0) {
 135                                 // we're aborting ahead of time - throw an error?
 136                                 break;
 137                         }
 138                         if (limit_remaining < output_size) {
 139                                 z.avail_out = output_size = limit_remaining;
 140                         }
 141                 }
 142
 143                 if(z.avail_in == 0)
 144                 {
 145                         z.next_in = (Bytef*)input_buffer;
 146                         is.read(input_buffer, bufsize);
 147                         input_buffer_len = is.gcount();
 148                         z.avail_in = input_buffer_len;
 149                         //dstream<<"read fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 150                 }
 151                 if(z.avail_in == 0)
 152                 {
 153                         //dstream<<"z.avail_in == 0"<<std::endl;
 154                         break;
 155                 }
 156
 157                 //dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
 158                 status = inflate(&z, Z_NO_FLUSH);
 159                 //dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
 160                 bytes_read += is.gcount() - z.avail_in;
 161                 //dstream<<"bytes_read="<<bytes_read<<std::endl;
 162
 163                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
 164                                 || status == Z_MEM_ERROR)
 165                 {
 166                         zerr(status);
 167                         throw SerializationError("decompressZlib: inflate failed");
 168                 }
 169                 int count = output_size - z.avail_out;
 170                 //dstream<<"count="<<count<<std::endl;
 171                 if(count)
 172                         os.write(output_buffer, count);
 173                 bytes_written += count;
 174                 if(status == Z_STREAM_END)
 175                 {
 176                         //dstream<<"Z_STREAM_END"<<std::endl;
 177
 178                         //dstream<<"z.avail_in="<<z.avail_in<<std::endl;
 179                         //dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 180                         // Unget all the data that inflate didn't take
 181                         is.clear(); // Just in case EOF is set
 182                         for(u32 i=0; i < z.avail_in; i++)
 183                         {
 184                                 is.unget();
 185                                 if(is.fail() || is.bad())
 186                                 {
 187                                         dstream<<"unget #"<<i<<" failed"<<std::endl;
 188                                         dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 189                                         throw SerializationError("decompressZlib: unget failed");
 190                                 }
 191                         }
 192
 193                         break;
 194                 }
 195         }
 196
 197         inflateEnd(&z);
 198 }
 199
 200 void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
 201 {
 202         if(version >= 11)
 203         {
 204                 compressZlib(*data ,data.getSize(), os);
 205                 return;
 206         }
 207
 208         if(data.getSize() == 0)
 209                 return;
 210
 211         // Write length (u32)
 212
 213         u8 tmp[4];
 214         writeU32(tmp, data.getSize());
 215         os.write((char*)tmp, 4);
 216
 217         // We will be writing 8-bit pairs of more_count and byte
 218         u8 more_count = 0;
 219         u8 current_byte = data[0];
 220         for(u32 i=1; i<data.getSize(); i++)
 221         {
 222                 if(
 223                         data[i] != current_byte
 224                         || more_count == 255
 225                 )
 226                 {
 227                         // write count and byte
 228                         os.write((char*)&more_count, 1);
 229                         os.write((char*)&current_byte, 1);
 230                         more_count = 0;
 231                         current_byte = data[i];
 232                 }
 233                 else
 234                 {
 235                         more_count++;
 236                 }
 237         }
 238         // write count and byte
 239         os.write((char*)&more_count, 1);
 240         os.write((char*)&current_byte, 1);
 241 }
 242
 243 void decompress(std::istream &is, std::ostream &os, u8 version)
 244 {
 245         if(version >= 11)
 246         {
 247                 decompressZlib(is, os);
 248                 return;
 249         }
 250
 251         // Read length (u32)
 252
 253         u8 tmp[4];
 254         is.read((char*)tmp, 4);
 255         u32 len = readU32(tmp);
 256
 257         // We will be reading 8-bit pairs of more_count and byte
 258         u32 count = 0;
 259         for(;;)
 260         {
 261                 u8 more_count=0;
 262                 u8 byte=0;
 263
 264                 is.read((char*)&more_count, 1);
 265
 266                 is.read((char*)&byte, 1);
 267
 268                 if(is.eof())
 269                         throw SerializationError("decompress: stream ended halfway");
 270
 271                 for(s32 i=0; i<(u16)more_count+1; i++)
 272                         os.write((char*)&byte, 1);
 273
 274                 count += (u16)more_count+1;
 275
 276                 if(count == len)
 277                         break;
 278         }
 279 }
 280
 281