1 /** 2 * Compress/decompress data using the zlib library. 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.utils.zlib; 15 16 import etc.c.zlib; 17 import std.conv; 18 import std.exception; 19 20 import ae.sys.data; 21 22 /// Thrown on zlib errors. 23 class ZlibException : Exception 24 { 25 private static string getmsg(int err) nothrow @nogc pure @safe 26 { 27 switch (err) 28 { 29 case Z_STREAM_END: return "stream end"; 30 case Z_NEED_DICT: return "need dict"; 31 case Z_ERRNO: return "errno"; 32 case Z_STREAM_ERROR: return "stream error"; 33 case Z_DATA_ERROR: return "data error"; 34 case Z_MEM_ERROR: return "mem error"; 35 case Z_BUF_ERROR: return "buf error"; 36 case Z_VERSION_ERROR: return "version error"; 37 default: return "unknown error"; 38 } 39 } 40 41 this(int err, z_stream* zs) 42 { 43 if (zs.msg) 44 super(to!string(zs.msg)); 45 else 46 super(getmsg(err)); 47 } /// 48 49 this(string msg) { super(msg); } /// 50 } 51 52 /// File format. 53 enum ZlibMode 54 { 55 normal, /// Normal deflate stream. 56 raw, /// Raw deflate stream. 57 gzipOnly, /// gzip deflate stream. Require gzip input. 58 gzipAuto, /// Output and detect gzip, but do not require it. 59 } 60 61 /// Compression/decompression options. 62 struct ZlibOptions 63 { 64 int deflateLevel = Z_DEFAULT_COMPRESSION; /// Compression level. 65 int windowBits = 15; /// Window size (8..15) - actual windowBits, without additional meaning 66 ZlibMode mode; /// File format. 67 68 invariant() 69 { 70 assert(deflateLevel == Z_DEFAULT_COMPRESSION || (deflateLevel >= 0 && deflateLevel <= 9)); 71 assert(windowBits >= 8 && windowBits <= 15); 72 } 73 74 private: 75 @property 76 int zwindowBits() 77 { 78 final switch (mode) 79 { 80 case ZlibMode.normal: 81 return windowBits; 82 case ZlibMode.raw: 83 return -windowBits; 84 case ZlibMode.gzipOnly: 85 return 16+windowBits; 86 case ZlibMode.gzipAuto: 87 return 32+windowBits; 88 } 89 } 90 } 91 92 /// Implements a zlib compression or decompression process. 93 struct ZlibProcess(bool COMPRESSING) 94 { 95 /// Initialize zlib. 96 void init(ZlibOptions options = ZlibOptions.init) 97 { 98 static if (COMPRESSING) 99 //zenforce(deflateInit(&zs, options.deflateLevel)); 100 zenforce(deflateInit2(&zs, options.deflateLevel, Z_DEFLATED, options.zwindowBits, 8, Z_DEFAULT_STRATEGY)); 101 else 102 //zenforce(inflateInit(&zs)); 103 zenforce(inflateInit2(&zs, options.zwindowBits)); 104 } 105 106 /// Process one chunk of data. 107 void processChunk(Data chunk) 108 { 109 if (!chunk.length) 110 return; 111 112 assert(zs.avail_in == 0); 113 zs.next_in = cast(ubyte*) chunk.ptr; 114 zs.avail_in = to!uint(chunk.length); 115 116 do 117 { 118 if (zs.avail_out == 0) 119 allocChunk(adjustSize(zs.avail_in)); 120 121 assert(zs.avail_in && zs.next_in ); 122 assert(zs.avail_out && zs.next_out); 123 if (zend(processFunc(&zs, Z_NO_FLUSH))) 124 enforce(zs.avail_in==0, new ZlibException("Trailing data")); 125 } while (zs.avail_in); 126 } 127 128 /// Signal end of input and flush. 129 Data[] flush() 130 { 131 if (zs.avail_out == 0) 132 allocChunk(adjustSize(zs.avail_in)); 133 134 while (!zend(processFunc(&zs, Z_FINISH))) 135 allocChunk(zs.avail_out*2+1); 136 137 saveChunk(); 138 return outputChunks; 139 } 140 141 /// Process all input. 142 static Data[] process(Data[] input, ZlibOptions options = ZlibOptions.init) 143 { 144 typeof(this) zp; 145 zp.init(options); 146 foreach (ref chunk; input) 147 zp.processChunk(chunk); 148 return zp.flush(); 149 } 150 151 /// Process input and return output as a single contiguous `Data`. 152 static Data process(Data input, ZlibOptions options = ZlibOptions.init) 153 { 154 return process([input], options).joinData(); 155 } 156 157 ~this() 158 { 159 zenforce(endFunc(&zs)); 160 } 161 162 private: 163 z_stream zs; 164 Data currentChunk; 165 Data[] outputChunks; 166 167 static if (COMPRESSING) 168 { 169 alias deflate processFunc; 170 alias deflateEnd endFunc; 171 172 size_t adjustSize(size_t sz) { return sz / 4 + 1; } 173 } 174 else 175 { 176 alias inflate processFunc; 177 alias inflateEnd endFunc; 178 179 size_t adjustSize(size_t sz) { return sz * 4 + 1; } 180 } 181 182 void zenforce(int ret) 183 { 184 if (ret != Z_OK) 185 throw new ZlibException(ret, &zs); 186 } 187 188 bool zend(int ret) 189 { 190 if (ret == Z_STREAM_END) 191 return true; 192 zenforce(ret); 193 return false; 194 } 195 196 void saveChunk() 197 { 198 if (zs.next_out && zs.next_out != currentChunk.ptr) 199 { 200 outputChunks ~= currentChunk[0..zs.next_out-cast(ubyte*)currentChunk.ptr]; 201 currentChunk = Data(); 202 } 203 zs.next_out = null; 204 } 205 206 void allocChunk(size_t sz) 207 { 208 saveChunk(); 209 currentChunk = Data(sz); 210 currentChunk.length = currentChunk.capacity; 211 zs.next_out = cast(ubyte*)currentChunk.mptr; 212 zs.avail_out = to!uint(currentChunk.length); 213 } 214 } 215 216 alias ZlibProcess!true ZlibDeflater; /// ditto 217 alias ZlibProcess!false ZlibInflater; /// ditto 218 219 alias ZlibDeflater.process compress; /// 220 alias ZlibInflater.process uncompress; /// 221 222 /// Shorthand for compressing at a certain level. 223 Data compress(Data input, int level) 224 { 225 return compress(input, ZlibOptions(level)); 226 } 227 228 unittest 229 { 230 void testRoundtrip(ubyte[] src) 231 { 232 ubyte[] def = cast(ubyte[]) compress(Data(src)).toHeap; 233 ubyte[] res = cast(ubyte[])uncompress(Data(def)).toHeap; 234 assert(res == src); 235 } 236 237 testRoundtrip(cast(ubyte[]) 238 "the quick brown fox jumps over the lazy dog\r 239 the quick brown fox jumps over the lazy dog\r 240 "); 241 testRoundtrip([0]); 242 testRoundtrip(null); 243 }