40#define N_ZIP_TAIL (22 + 65535)
42#define N_ZIP_CHUNK 262144
45static unsigned le16(
const unsigned char* p) {
46 return (
unsigned)p[0] | ((unsigned)p[1] << 8);
49static uint32_t
le32(
const unsigned char* p) {
50 return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
55 if (at > (uint64_t)0x7fffffff)
return -1;
56 return fseek(f, (
long)at, SEEK_SET);
67static int zip_scan(
const char* path,
const char* name,
int ending,
N_ZIP_MEMBER* member,
char* found_name,
size_t found_len) {
69 unsigned char* tail = NULL;
70 long size = 0, start = 0, i = 0;
71 size_t got = 0, name_len = 0;
72 uint32_t dir_size = 0, dir_at = 0;
73 unsigned entries = 0, k = 0;
76 if (!path || !name)
return -1;
77 name_len = strlen(name);
78 f = fopen(path,
"rb");
80 if (fseek(f, 0, SEEK_END) != 0 || (size = ftell(f)) < 22) {
85 tail = (
unsigned char*)malloc((
size_t)(size - start));
86 if (!tail || fseek(f, start, SEEK_SET) != 0 || (got = fread(tail, 1, (
size_t)(size - start), f)) < 22) {
92 for (i = (
long)got - 22; i >= 0; i--) {
93 if (
le32(tail + i) == 0x06054b50u)
break;
100 entries =
le16(tail + i + 10);
101 dir_size =
le32(tail + i + 12);
102 dir_at =
le32(tail + i + 16);
105 if (dir_at == 0xffffffffu || dir_size == 0xffffffffu || (
long)dir_at + (
long)dir_size > size) {
106 n_log(
LOG_ERR,
"zip: %s is ZIP64 or damaged, not read", path);
110 tail = (
unsigned char*)malloc(dir_size ? dir_size : 1);
111 if (!tail ||
seek_to(f, dir_at) != 0 || fread(tail, 1, dir_size, f) != dir_size) {
119 for (k = 0, i = 0; k < entries && (uint32_t)i + 46 <= dir_size; k++) {
120 const unsigned char* r = tail + i;
121 unsigned n = 0, extra = 0, comment = 0;
123 if (
le32(r) != 0x02014b50u)
break;
125 extra =
le16(r + 30);
126 comment =
le16(r + 32);
127 if ((uint32_t)i + 46 + n > dir_size)
break;
128 if ((!ending && n == name_len && memcmp(r + 46, name, n) == 0) ||
129 (ending && n >= name_len && memcmp(r + 46 + n - name_len, name, name_len) == 0)) {
130 if (found_name && found_len) {
131 size_t copy = n < found_len - 1 ? n : found_len - 1;
133 memcpy(found_name, r + 46, copy);
134 found_name[copy] =
'\0';
143 found = (
le32(r + 20) == 0xffffffffu ||
le32(r + 24) == 0xffffffffu) ? -1 : 0;
146 i += 46 + (long)n + (
long)extra + (long)comment;
153 return zip_scan(path, name, 0, member, NULL, 0);
157 if (!name || !name_len)
return -1;
158 return zip_scan(path, ending, 1, NULL, name, name_len);
179 if (sink->
file)
return fwrite(data, 1, len, sink->
file) == len ? 0 : -1;
180 if (len > sink->
size - sink->
used)
return -1;
193 unsigned char local[30];
194 unsigned char *in_buf = NULL, *out_buf = NULL;
196 uint64_t left = 0, written = 0;
197 uLong crc = crc32(0L, Z_NULL, 0);
199 int zs_ready = 0, result = -1;
202 n_log(
LOG_ERR,
"zip: %s in %s is compressed by method %u, not read", name, path, member->
method);
205 f = fopen(path,
"rb");
209 if (
seek_to(f, member->
offset) != 0 || fread(local, 1, 30, f) != 30 ||
le32(local) != 0x04034b50u ||
216 if (!in_buf || !out_buf)
goto done;
217 memset(&zs, 0,
sizeof(zs));
219 if (member->
method == 8) {
220 if (inflateInit2(&zs, -MAX_WBITS) != Z_OK)
goto done;
227 if (fread(in_buf, 1, take, f) != take)
goto done;
229 if (member->
method == 0) {
231 crc = crc32(crc, in_buf, (uInt)take);
236 zs.avail_in = (uInt)take;
241 zs.next_out = out_buf;
243 rc = inflate(&zs, Z_NO_FLUSH);
244 if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR)
goto done;
246 if (have &&
sink_put(sink, out_buf, have) != 0)
goto done;
247 crc = crc32(crc, out_buf, (uInt)have);
249 if (rc == Z_STREAM_END)
break;
250 }
while (zs.avail_in > 0 || zs.avail_out == 0);
252 if (written != member->
size || (uint32_t)crc != member->
crc) {
253 n_log(
LOG_ERR,
"zip: %s in %s came out %llu bytes, crc %08lx; %llu, %08x expected", name, path,
254 (
unsigned long long)written, (
unsigned long)crc, (
unsigned long long)member->
size, member->
crc);
260 if (zs_ready) inflateEnd(&zs);
272 if (!out ||
n_zip_find(path, name, &member) != 0)
return -1;
273 memset(&sink, 0,
sizeof(sink));
274 sink.
file = fopen(out,
"wb");
275 if (!sink.
file)
return -1;
276 result =
zip_unpack(path, name, &member, &sink);
277 if (fclose(sink.
file) != 0) result = -1;
278 if (result != 0) remove(out);
282int n_zip_read(
const char* path,
const char* name,
size_t max_size,
unsigned char** data,
size_t* len) {
286 if (!data || !len)
return -1;
289 if (
n_zip_find(path, name, &member) != 0)
return -1;
290 if (member.
size > max_size || member.
size > (uint64_t)SIZE_MAX - 1) {
291 n_log(
LOG_ERR,
"zip: %s in %s is %llu bytes, more than the %zu allowed", name, path,
292 (
unsigned long long)member.
size, max_size);
295 memset(&sink, 0,
sizeof(sink));
298 sink.
buffer = (
unsigned char*)malloc(sink.
size + 1);
299 if (!sink.
buffer)
return -1;
volatile sig_atomic_t done
#define n_log(__LEVEL__,...)
Logging function wrapper to get line and func.
#define LOG_ERR
error conditions
uint64_t size
its size once out
uint64_t compressed
its size in the archive
uint64_t offset
where its local header is
unsigned method
0 stored, 8 deflated
int n_zip_extract(const char *path, const char *name, const char *out)
write a member out to a file, checking its CRC-32
int n_zip_find(const char *path, const char *name, N_ZIP_MEMBER *member)
find a member by its full name in an archive's central directory
int n_zip_find_ending(const char *path, const char *ending, char *name, size_t name_len)
find the first member whose name ends with a given text, as a PyTorch checkpoint's "archive/data....
int n_zip_read(const char *path, const char *name, size_t max_size, unsigned char **data, size_t *len)
read a member into memory, checking its CRC-32
What n_zip_find() says about a member.
FILE * file
the file, or NULL
static uint32_t le32(const unsigned char *p)
helper: a little endian 32 bit value
unsigned char * buffer
the buffer, allocated to the member's size
static unsigned le16(const unsigned char *p)
helper: a little endian 16 bit value
static int seek_to(FILE *f, uint64_t at)
helper: seek to a 64 bit offset where the platform's fseek takes a long
static int zip_scan(const char *path, const char *name, int ending, N_ZIP_MEMBER *member, char *found_name, size_t found_len)
find a member by its whole name, or by how its name ends
#define N_ZIP_CHUNK
the pieces a member is read and written in
#define N_ZIP_TAIL
how much of an archive's end is searched for its end record: the record and the longest comment it ma...
static int zip_unpack(const char *path, const char *name, const N_ZIP_MEMBER *member, ZIP_SINK *sink)
unpack a member into a sink, checking its size and CRC-32
static int sink_put(ZIP_SINK *sink, const unsigned char *data, size_t len)
put unpacked bytes where they go
size_t used
how much of it is written
where unpacked bytes go: a file, or a buffer in memory
Reading members out of a ZIP archive, to a file or into memory.