1 /**
2  * Compress/decompress data using the zlib library.
3  *
4  * License:
5  *   This Source Code Form is subject to the terms of
6  *   the Mozilla Public License, v. 2.0. If a copy of
7  *   the MPL was not distributed with this file, You
8  *   can obtain one at http://mozilla.org/MPL/2.0/.
9  *
10  * Authors:
11  *   Vladimir Panteleev <ae@cy.md>
12  */
13 
14 module ae.utils.zlib;
15 
16 import etc.c.zlib;
17 import std.algorithm.mutation : move;
18 import std.conv;
19 import std.exception;
20 
21 import ae.sys.data;
22 import ae.utils.array;
23 
24 /// Thrown on zlib errors.
25 class ZlibException : Exception
26 {
27 	private static string getmsg(int err) nothrow @nogc pure @safe
28 	{
29 		switch (err)
30 		{
31 			case Z_STREAM_END:      return "stream end";
32 			case Z_NEED_DICT:       return "need dict";
33 			case Z_ERRNO:           return "errno";
34 			case Z_STREAM_ERROR:    return "stream error";
35 			case Z_DATA_ERROR:      return "data error";
36 			case Z_MEM_ERROR:       return "mem error";
37 			case Z_BUF_ERROR:       return "buf error";
38 			case Z_VERSION_ERROR:   return "version error";
39 			default:                return "unknown error";
40 		}
41 	}
42 
43 	this(int err, z_stream* zs)
44 	{
45 		if (zs.msg)
46 			super(to!string(zs.msg));
47 		else
48 			super(getmsg(err));
49 	} ///
50 
51 	this(string msg) { super(msg); } ///
52 }
53 
54 /// File format.
55 enum ZlibMode
56 {
57 	normal,   /// Normal deflate stream.
58 	raw,      /// Raw deflate stream.
59 	gzipOnly, /// gzip deflate stream. Require gzip input.
60 	gzipAuto, /// Output and detect gzip, but do not require it.
61 }
62 
63 /// Compression/decompression options.
64 struct ZlibOptions
65 {
66 	int deflateLevel = Z_DEFAULT_COMPRESSION; /// Compression level.
67 	int windowBits = 15; /// Window size (8..15) - actual windowBits, without additional meaning
68 	ZlibMode mode; /// File format.
69 
70 	invariant()
71 	{
72 		assert(deflateLevel == Z_DEFAULT_COMPRESSION || (deflateLevel >= 0 && deflateLevel <= 9));
73 		assert(windowBits >= 8 && windowBits <= 15);
74 	}
75 
76 private:
77 	@property
78 	int zwindowBits()
79 	{
80 		final switch (mode)
81 		{
82 		case ZlibMode.normal:
83 			return windowBits;
84 		case ZlibMode.raw:
85 			return -windowBits;
86 		case ZlibMode.gzipOnly:
87 			return 16+windowBits;
88 		case ZlibMode.gzipAuto:
89 			return 32+windowBits;
90 		}
91 	}
92 }
93 
94 /// Implements a zlib compression or decompression process.
95 struct ZlibProcess(bool COMPRESSING)
96 {
97 	/// Initialize zlib.
98 	void init(ZlibOptions options = ZlibOptions.init)
99 	{
100 		static if (COMPRESSING)
101 			//zenforce(deflateInit(&zs, options.deflateLevel));
102 			zenforce(deflateInit2(&zs, options.deflateLevel, Z_DEFLATED, options.zwindowBits, 8, Z_DEFAULT_STRATEGY));
103 		else
104 			//zenforce(inflateInit(&zs));
105 			zenforce(inflateInit2(&zs, options.zwindowBits));
106 	}
107 
108 	/// Process one chunk of data.
109 	void processChunk(const Data chunk)
110 	{
111 		if (!chunk.length)
112 			return;
113 
114 		assert(zs.avail_in == 0);
115 		zs.next_in  = cast(ubyte*) chunk.ptr;
116 		zs.avail_in = to!uint(chunk.length);
117 
118 		do
119 		{
120 			if (zs.avail_out == 0)
121 				allocChunk(adjustSize(zs.avail_in));
122 
123 			assert(zs.avail_in  && zs.next_in );
124 			assert(zs.avail_out && zs.next_out);
125 			if (zend(processFunc(&zs, Z_NO_FLUSH)))
126 				enforce(zs.avail_in==0, new ZlibException("Trailing data"));
127 		} while (zs.avail_in);
128 	}
129 
130 	/// Signal end of input and flush.
131 	DataVec flush()
132 	{
133 		if (zs.avail_out == 0)
134 			allocChunk(adjustSize(zs.avail_in));
135 
136 		while (!zend(processFunc(&zs, Z_FINISH)))
137 			allocChunk(zs.avail_out*2+1);
138 
139 		saveChunk();
140 		return move(outputChunks);
141 	}
142 
143 	/// Process all input.
144 	static DataVec process(scope const(Data)[] input, ZlibOptions options = ZlibOptions.init)
145 	{
146 		typeof(this) zp;
147 		zp.init(options);
148 		foreach (ref chunk; input)
149 			zp.processChunk(chunk);
150 		return zp.flush();
151 	}
152 
153 	/// Process input and return output as a single contiguous `Data`.
154 	static Data process(Data input, ZlibOptions options = ZlibOptions.init)
155 	{
156 		return process(input.toArray, options).joinData();
157 	}
158 
159 	~this()
160 	{
161 		zenforce(endFunc(&zs));
162 	}
163 
164 private:
165 	z_stream zs;
166 	Data currentChunk;
167 	DataVec outputChunks;
168 
169 	static if (COMPRESSING)
170 	{
171 		alias deflate processFunc;
172 		alias deflateEnd endFunc;
173 
174 		size_t adjustSize(size_t sz) { return sz / 4 + 1; }
175 	}
176 	else
177 	{
178 		alias inflate processFunc;
179 		alias inflateEnd endFunc;
180 
181 		size_t adjustSize(size_t sz) { return sz * 4 + 1; }
182 	}
183 
184 	void zenforce(int ret)
185 	{
186 		if (ret != Z_OK)
187 			throw new ZlibException(ret, &zs);
188 	}
189 
190 	bool zend(int ret)
191 	{
192 		if (ret == Z_STREAM_END)
193 			return true;
194 		zenforce(ret);
195 		return false;
196 	}
197 
198 	void saveChunk()
199 	{
200 		if (zs.next_out && zs.next_out != currentChunk.ptr)
201 		{
202 			outputChunks ~= currentChunk[0..zs.next_out-cast(ubyte*)currentChunk.ptr];
203 			currentChunk = Data();
204 		}
205 		zs.next_out = null;
206 	}
207 
208 	void allocChunk(size_t sz)
209 	{
210 		saveChunk();
211 		currentChunk = Data(sz);
212 		currentChunk.length = currentChunk.capacity;
213 		zs.next_out  = cast(ubyte*)currentChunk.mptr;
214 		zs.avail_out = to!uint(currentChunk.length);
215 	}
216 }
217 
218 alias ZlibProcess!true  ZlibDeflater; /// ditto
219 alias ZlibProcess!false ZlibInflater; /// ditto
220 
221 alias ZlibDeflater.process compress;   ///
222 alias ZlibInflater.process uncompress; ///
223 
224 /// Shorthand for compressing at a certain level.
225 Data compress(Data input, int level)
226 {
227 	return compress(input, ZlibOptions(level));
228 }
229 
230 unittest
231 {
232 	void testRoundtrip(ubyte[] src)
233 	{
234 		ubyte[] def = cast(ubyte[])  compress(Data(src)).toHeap;
235 		ubyte[] res = cast(ubyte[])uncompress(Data(def)).toHeap;
236 		assert(res == src);
237 	}
238 
239 	testRoundtrip(cast(ubyte[])
240 "the quick brown fox jumps over the lazy dog\r
241 the quick brown fox jumps over the lazy dog\r
242 ");
243 	testRoundtrip([0]);
244 	testRoundtrip(null);
245 }