Nilorea Library
C utilities for networking, threading, graphics
Loading...
Searching...
No Matches
n_zip.c
Go to the documentation of this file.
1/*
2 * Nilorea Library
3 * Copyright (C) 2005-2026 Castagnier Mickael
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14 * implied. See the License for the specific language governing
15 * permissions and limitations under the License.
16 *
17 * SPDX-License-Identifier: Apache-2.0
18 */
19
27
28#include <stdio.h>
29#include <stdlib.h>
30#include <stdint.h>
31#include <string.h>
32
33#include <zlib.h>
34
35#include "nilorea/n_log.h"
36#include "nilorea/n_zip.h"
37
40#define N_ZIP_TAIL (22 + 65535)
42#define N_ZIP_CHUNK 262144
43
45static unsigned le16(const unsigned char* p) {
46 return (unsigned)p[0] | ((unsigned)p[1] << 8);
47}
48
49static uint32_t le32(const unsigned char* p) {
50 return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
51}
52
54static int seek_to(FILE* f, uint64_t at) {
55 if (at > (uint64_t)0x7fffffff) return -1;
56 return fseek(f, (long)at, SEEK_SET);
57}
58
67static int zip_scan(const char* path, const char* name, int ending, N_ZIP_MEMBER* member, char* found_name, size_t found_len) {
68 FILE* f = NULL;
69 unsigned char* tail = NULL;
70 long size = 0, start = 0, i = 0;
71 size_t got = 0, name_len = 0;
72 uint32_t dir_size = 0, dir_at = 0;
73 unsigned entries = 0, k = 0;
74 int found = -1;
75
76 if (!path || !name) return -1;
77 name_len = strlen(name);
78 f = fopen(path, "rb");
79 if (!f) return -1;
80 if (fseek(f, 0, SEEK_END) != 0 || (size = ftell(f)) < 22) {
81 fclose(f);
82 return -1;
83 }
84 start = size > N_ZIP_TAIL ? size - N_ZIP_TAIL : 0;
85 tail = (unsigned char*)malloc((size_t)(size - start));
86 if (!tail || fseek(f, start, SEEK_SET) != 0 || (got = fread(tail, 1, (size_t)(size - start), f)) < 22) {
87 free(tail);
88 fclose(f);
89 return -1;
90 }
91 /* The end of central directory record, the last one in the file. */
92 for (i = (long)got - 22; i >= 0; i--) {
93 if (le32(tail + i) == 0x06054b50u) break;
94 }
95 if (i < 0) {
96 free(tail);
97 fclose(f);
98 return -1;
99 }
100 entries = le16(tail + i + 10);
101 dir_size = le32(tail + i + 12);
102 dir_at = le32(tail + i + 16);
103 free(tail);
104 tail = NULL;
105 if (dir_at == 0xffffffffu || dir_size == 0xffffffffu || (long)dir_at + (long)dir_size > size) {
106 n_log(LOG_ERR, "zip: %s is ZIP64 or damaged, not read", path);
107 fclose(f);
108 return -1;
109 }
110 tail = (unsigned char*)malloc(dir_size ? dir_size : 1);
111 if (!tail || seek_to(f, dir_at) != 0 || fread(tail, 1, dir_size, f) != dir_size) {
112 free(tail);
113 fclose(f);
114 return -1;
115 }
116 fclose(f);
117 /* The central directory: one record a member, each 46 bytes and then
118 * its name, extra field and comment. */
119 for (k = 0, i = 0; k < entries && (uint32_t)i + 46 <= dir_size; k++) {
120 const unsigned char* r = tail + i;
121 unsigned n = 0, extra = 0, comment = 0;
122
123 if (le32(r) != 0x02014b50u) break;
124 n = le16(r + 28);
125 extra = le16(r + 30);
126 comment = le16(r + 32);
127 if ((uint32_t)i + 46 + n > dir_size) break;
128 if ((!ending && n == name_len && memcmp(r + 46, name, n) == 0) ||
129 (ending && n >= name_len && memcmp(r + 46 + n - name_len, name, name_len) == 0)) {
130 if (found_name && found_len) {
131 size_t copy = n < found_len - 1 ? n : found_len - 1;
132
133 memcpy(found_name, r + 46, copy);
134 found_name[copy] = '\0';
135 }
136 if (member) {
137 member->method = le16(r + 10);
138 member->crc = le32(r + 16);
139 member->compressed = le32(r + 20);
140 member->size = le32(r + 24);
141 member->offset = le32(r + 42);
142 }
143 found = (le32(r + 20) == 0xffffffffu || le32(r + 24) == 0xffffffffu) ? -1 : 0;
144 break;
145 }
146 i += 46 + (long)n + (long)extra + (long)comment;
147 }
148 free(tail);
149 return found;
150}
151
152int n_zip_find(const char* path, const char* name, N_ZIP_MEMBER* member) {
153 return zip_scan(path, name, 0, member, NULL, 0);
154}
155
156int n_zip_find_ending(const char* path, const char* ending, char* name, size_t name_len) {
157 if (!name || !name_len) return -1;
158 return zip_scan(path, ending, 1, NULL, name, name_len);
159}
160
162typedef struct {
164 FILE* file;
166 unsigned char* buffer;
168 size_t used;
170 size_t size;
171} ZIP_SINK;
172
178static int sink_put(ZIP_SINK* sink, const unsigned char* data, size_t len) {
179 if (sink->file) return fwrite(data, 1, len, sink->file) == len ? 0 : -1;
180 if (len > sink->size - sink->used) return -1;
181 memcpy(sink->buffer + sink->used, data, len);
182 sink->used += len;
183 return 0;
184}
185
192static int zip_unpack(const char* path, const char* name, const N_ZIP_MEMBER* member, ZIP_SINK* sink) {
193 unsigned char local[30];
194 unsigned char *in_buf = NULL, *out_buf = NULL;
195 FILE* f = NULL;
196 uint64_t left = 0, written = 0;
197 uLong crc = crc32(0L, Z_NULL, 0);
198 z_stream zs;
199 int zs_ready = 0, result = -1;
200
201 if (member->method != 0 && member->method != 8) {
202 n_log(LOG_ERR, "zip: %s in %s is compressed by method %u, not read", name, path, member->method);
203 return -1;
204 }
205 f = fopen(path, "rb");
206 if (!f) return -1;
207 /* The local header's own name and extra lengths, which may differ from
208 * the central directory's, say where the data begins. */
209 if (seek_to(f, member->offset) != 0 || fread(local, 1, 30, f) != 30 || le32(local) != 0x04034b50u ||
210 seek_to(f, member->offset + 30 + le16(local + 26) + le16(local + 28)) != 0) {
211 fclose(f);
212 return -1;
213 }
214 in_buf = (unsigned char*)malloc(N_ZIP_CHUNK);
215 out_buf = (unsigned char*)malloc(N_ZIP_CHUNK);
216 if (!in_buf || !out_buf) goto done;
217 memset(&zs, 0, sizeof(zs));
218 /* Raw deflate: a ZIP member has no zlib header. */
219 if (member->method == 8) {
220 if (inflateInit2(&zs, -MAX_WBITS) != Z_OK) goto done;
221 zs_ready = 1;
222 }
223 left = member->compressed;
224 while (left > 0) {
225 size_t take = left < N_ZIP_CHUNK ? (size_t)left : N_ZIP_CHUNK;
226
227 if (fread(in_buf, 1, take, f) != take) goto done;
228 left -= take;
229 if (member->method == 0) {
230 if (sink_put(sink, in_buf, take) != 0) goto done;
231 crc = crc32(crc, in_buf, (uInt)take);
232 written += take;
233 continue;
234 }
235 zs.next_in = in_buf;
236 zs.avail_in = (uInt)take;
237 do {
238 int rc = 0;
239 size_t have = 0;
240
241 zs.next_out = out_buf;
242 zs.avail_out = N_ZIP_CHUNK;
243 rc = inflate(&zs, Z_NO_FLUSH);
244 if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) goto done;
245 have = N_ZIP_CHUNK - zs.avail_out;
246 if (have && sink_put(sink, out_buf, have) != 0) goto done;
247 crc = crc32(crc, out_buf, (uInt)have);
248 written += have;
249 if (rc == Z_STREAM_END) break;
250 } while (zs.avail_in > 0 || zs.avail_out == 0);
251 }
252 if (written != member->size || (uint32_t)crc != member->crc) {
253 n_log(LOG_ERR, "zip: %s in %s came out %llu bytes, crc %08lx; %llu, %08x expected", name, path,
254 (unsigned long long)written, (unsigned long)crc, (unsigned long long)member->size, member->crc);
255 goto done;
256 }
257 result = 0;
258
259done:
260 if (zs_ready) inflateEnd(&zs);
261 free(in_buf);
262 free(out_buf);
263 fclose(f);
264 return result;
265}
266
267int n_zip_extract(const char* path, const char* name, const char* out) {
268 N_ZIP_MEMBER member;
269 ZIP_SINK sink;
270 int result = -1;
271
272 if (!out || n_zip_find(path, name, &member) != 0) return -1;
273 memset(&sink, 0, sizeof(sink));
274 sink.file = fopen(out, "wb");
275 if (!sink.file) return -1;
276 result = zip_unpack(path, name, &member, &sink);
277 if (fclose(sink.file) != 0) result = -1;
278 if (result != 0) remove(out);
279 return result;
280}
281
282int n_zip_read(const char* path, const char* name, size_t max_size, unsigned char** data, size_t* len) {
283 N_ZIP_MEMBER member;
284 ZIP_SINK sink;
285
286 if (!data || !len) return -1;
287 *data = NULL;
288 *len = 0;
289 if (n_zip_find(path, name, &member) != 0) return -1;
290 if (member.size > max_size || member.size > (uint64_t)SIZE_MAX - 1) {
291 n_log(LOG_ERR, "zip: %s in %s is %llu bytes, more than the %zu allowed", name, path,
292 (unsigned long long)member.size, max_size);
293 return -1;
294 }
295 memset(&sink, 0, sizeof(sink));
296 sink.size = (size_t)member.size;
297 /* One more byte, so a text member can be ended with a zero. */
298 sink.buffer = (unsigned char*)malloc(sink.size + 1);
299 if (!sink.buffer) return -1;
300 if (zip_unpack(path, name, &member, &sink) != 0 || sink.used != sink.size) {
301 free(sink.buffer);
302 return -1;
303 }
304 sink.buffer[sink.size] = 0;
305 *data = sink.buffer;
306 *len = sink.size;
307 return 0;
308}
volatile sig_atomic_t done
#define n_log(__LEVEL__,...)
Logging function wrapper to get line and func.
Definition n_log.h:113
#define LOG_ERR
error conditions
Definition n_log.h:100
uint64_t size
its size once out
Definition n_zip.h:58
uint64_t compressed
its size in the archive
Definition n_zip.h:56
uint32_t crc
its CRC-32
Definition n_zip.h:60
uint64_t offset
where its local header is
Definition n_zip.h:62
unsigned method
0 stored, 8 deflated
Definition n_zip.h:54
int n_zip_extract(const char *path, const char *name, const char *out)
write a member out to a file, checking its CRC-32
Definition n_zip.c:267
int n_zip_find(const char *path, const char *name, N_ZIP_MEMBER *member)
find a member by its full name in an archive's central directory
Definition n_zip.c:152
int n_zip_find_ending(const char *path, const char *ending, char *name, size_t name_len)
find the first member whose name ends with a given text, as a PyTorch checkpoint's "archive/data....
Definition n_zip.c:156
int n_zip_read(const char *path, const char *name, size_t max_size, unsigned char **data, size_t *len)
read a member into memory, checking its CRC-32
Definition n_zip.c:282
What n_zip_find() says about a member.
Definition n_zip.h:52
Generic log system.
FILE * file
the file, or NULL
Definition n_zip.c:164
static uint32_t le32(const unsigned char *p)
helper: a little endian 32 bit value
Definition n_zip.c:49
size_t size
its size
Definition n_zip.c:170
unsigned char * buffer
the buffer, allocated to the member's size
Definition n_zip.c:166
static unsigned le16(const unsigned char *p)
helper: a little endian 16 bit value
Definition n_zip.c:45
static int seek_to(FILE *f, uint64_t at)
helper: seek to a 64 bit offset where the platform's fseek takes a long
Definition n_zip.c:54
static int zip_scan(const char *path, const char *name, int ending, N_ZIP_MEMBER *member, char *found_name, size_t found_len)
find a member by its whole name, or by how its name ends
Definition n_zip.c:67
#define N_ZIP_CHUNK
the pieces a member is read and written in
Definition n_zip.c:42
#define N_ZIP_TAIL
how much of an archive's end is searched for its end record: the record and the longest comment it ma...
Definition n_zip.c:40
static int zip_unpack(const char *path, const char *name, const N_ZIP_MEMBER *member, ZIP_SINK *sink)
unpack a member into a sink, checking its size and CRC-32
Definition n_zip.c:192
static int sink_put(ZIP_SINK *sink, const unsigned char *data, size_t len)
put unpacked bytes where they go
Definition n_zip.c:178
size_t used
how much of it is written
Definition n_zip.c:168
where unpacked bytes go: a file, or a buffer in memory
Definition n_zip.c:162
Reading members out of a ZIP archive, to a file or into memory.