1 /** 2 * Compress/decompress data using the zlib library. 3 * 4 * License: 5 * This Source Code Form is subject to the terms of 6 * the Mozilla Public License, v. 2.0. If a copy of 7 * the MPL was not distributed with this file, You 8 * can obtain one at http://mozilla.org/MPL/2.0/. 9 * 10 * Authors: 11 * Vladimir Panteleev <ae@cy.md> 12 */ 13 14 module ae.utils.zlib; 15 16 import etc.c.zlib; 17 import std.algorithm.mutation : move; 18 import std.conv; 19 import std.exception; 20 21 import ae.sys.data; 22 import ae.utils.array; 23 24 /// Thrown on zlib errors. 25 class ZlibException : Exception 26 { 27 private static string getmsg(int err) nothrow @nogc pure @safe 28 { 29 switch (err) 30 { 31 case Z_STREAM_END: return "stream end"; 32 case Z_NEED_DICT: return "need dict"; 33 case Z_ERRNO: return "errno"; 34 case Z_STREAM_ERROR: return "stream error"; 35 case Z_DATA_ERROR: return "data error"; 36 case Z_MEM_ERROR: return "mem error"; 37 case Z_BUF_ERROR: return "buf error"; 38 case Z_VERSION_ERROR: return "version error"; 39 default: return "unknown error"; 40 } 41 } 42 43 this(int err, z_stream* zs) 44 { 45 if (zs.msg) 46 super(to!string(zs.msg)); 47 else 48 super(getmsg(err)); 49 } /// 50 51 this(string msg) { super(msg); } /// 52 } 53 54 /// File format. 55 enum ZlibMode 56 { 57 normal, /// Normal deflate stream. 58 raw, /// Raw deflate stream. 59 gzipOnly, /// gzip deflate stream. Require gzip input. 60 gzipAuto, /// Output and detect gzip, but do not require it. 61 } 62 63 /// Compression/decompression options. 64 struct ZlibOptions 65 { 66 int deflateLevel = Z_DEFAULT_COMPRESSION; /// Compression level. 67 int windowBits = 15; /// Window size (8..15) - actual windowBits, without additional meaning 68 ZlibMode mode; /// File format. 69 70 invariant() 71 { 72 assert(deflateLevel == Z_DEFAULT_COMPRESSION || (deflateLevel >= 0 && deflateLevel <= 9)); 73 assert(windowBits >= 8 && windowBits <= 15); 74 } 75 76 private: 77 @property 78 int zwindowBits() 79 { 80 final switch (mode) 81 { 82 case ZlibMode.normal: 83 return windowBits; 84 case ZlibMode.raw: 85 return -windowBits; 86 case ZlibMode.gzipOnly: 87 return 16+windowBits; 88 case ZlibMode.gzipAuto: 89 return 32+windowBits; 90 } 91 } 92 } 93 94 /// Implements a zlib compression or decompression process. 95 struct ZlibProcess(bool COMPRESSING) 96 { 97 /// Initialize zlib. 98 void init(ZlibOptions options = ZlibOptions.init) 99 { 100 static if (COMPRESSING) 101 //zenforce(deflateInit(&zs, options.deflateLevel)); 102 zenforce(deflateInit2(&zs, options.deflateLevel, Z_DEFLATED, options.zwindowBits, 8, Z_DEFAULT_STRATEGY)); 103 else 104 //zenforce(inflateInit(&zs)); 105 zenforce(inflateInit2(&zs, options.zwindowBits)); 106 } 107 108 /// Process one chunk of data. 109 void processChunk(const Data chunk) 110 { 111 if (!chunk.length) 112 return; 113 114 assert(zs.avail_in == 0); 115 zs.next_in = cast(ubyte*) chunk.ptr; 116 zs.avail_in = to!uint(chunk.length); 117 118 do 119 { 120 if (zs.avail_out == 0) 121 allocChunk(adjustSize(zs.avail_in)); 122 123 assert(zs.avail_in && zs.next_in ); 124 assert(zs.avail_out && zs.next_out); 125 if (zend(processFunc(&zs, Z_NO_FLUSH))) 126 enforce(zs.avail_in==0, new ZlibException("Trailing data")); 127 } while (zs.avail_in); 128 } 129 130 /// Signal end of input and flush. 131 DataVec flush() 132 { 133 if (zs.avail_out == 0) 134 allocChunk(adjustSize(zs.avail_in)); 135 136 while (!zend(processFunc(&zs, Z_FINISH))) 137 allocChunk(zs.avail_out*2+1); 138 139 saveChunk(); 140 return move(outputChunks); 141 } 142 143 /// Process all input. 144 static DataVec process(scope const(Data)[] input, ZlibOptions options = ZlibOptions.init) 145 { 146 typeof(this) zp; 147 zp.init(options); 148 foreach (ref chunk; input) 149 zp.processChunk(chunk); 150 return zp.flush(); 151 } 152 153 /// Process input and return output as a single contiguous `Data`. 154 static Data process(Data input, ZlibOptions options = ZlibOptions.init) 155 { 156 return process(input.toArray, options).joinData(); 157 } 158 159 ~this() 160 { 161 zenforce(endFunc(&zs)); 162 } 163 164 private: 165 z_stream zs; 166 Data currentChunk; 167 DataVec outputChunks; 168 169 static if (COMPRESSING) 170 { 171 alias deflate processFunc; 172 alias deflateEnd endFunc; 173 174 size_t adjustSize(size_t sz) { return sz / 4 + 1; } 175 } 176 else 177 { 178 alias inflate processFunc; 179 alias inflateEnd endFunc; 180 181 size_t adjustSize(size_t sz) { return sz * 4 + 1; } 182 } 183 184 void zenforce(int ret) 185 { 186 if (ret != Z_OK) 187 throw new ZlibException(ret, &zs); 188 } 189 190 bool zend(int ret) 191 { 192 if (ret == Z_STREAM_END) 193 return true; 194 zenforce(ret); 195 return false; 196 } 197 198 void saveChunk() 199 { 200 if (zs.next_out && zs.next_out != currentChunk.ptr) 201 { 202 outputChunks ~= currentChunk[0..zs.next_out-cast(ubyte*)currentChunk.ptr]; 203 currentChunk = Data(); 204 } 205 zs.next_out = null; 206 } 207 208 void allocChunk(size_t sz) 209 { 210 saveChunk(); 211 currentChunk = Data(sz); 212 currentChunk.length = currentChunk.capacity; 213 zs.next_out = cast(ubyte*)currentChunk.mptr; 214 zs.avail_out = to!uint(currentChunk.length); 215 } 216 } 217 218 alias ZlibProcess!true ZlibDeflater; /// ditto 219 alias ZlibProcess!false ZlibInflater; /// ditto 220 221 alias ZlibDeflater.process compress; /// 222 alias ZlibInflater.process uncompress; /// 223 224 /// Shorthand for compressing at a certain level. 225 Data compress(Data input, int level) 226 { 227 return compress(input, ZlibOptions(level)); 228 } 229 230 unittest 231 { 232 void testRoundtrip(ubyte[] src) 233 { 234 ubyte[] def = cast(ubyte[]) compress(Data(src)).toHeap; 235 ubyte[] res = cast(ubyte[])uncompress(Data(def)).toHeap; 236 assert(res == src); 237 } 238 239 testRoundtrip(cast(ubyte[]) 240 "the quick brown fox jumps over the lazy dog\r 241 the quick brown fox jumps over the lazy dog\r 242 "); 243 testRoundtrip([0]); 244 testRoundtrip(null); 245 }