#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <zlib.h>
#include "nilorea/n_common.h"
#include "nilorea/n_log.h"
#include "nilorea/n_pth.h"

Include dependency graph for ex_pth.c:

Data Structures
struct	MEMBER
	one stored member More...

Functions
static void	expect_true (const char *what, int ok)
	count a failed check
int	main (int argc, char **argv)
	run the checks
static int	process_args (int argc, char **argv)
	read -V
static void	put_le (unsigned char *p, unsigned long v, int n)
	put a little endian value
static int	values_are (const N_PTH_TENSOR t, const float want, size_t n)
	are a tensor's values these
static int	write_zip (const char path, const MEMBER m, int count)
	write a ZIP of stored members, as torch.save() does

Variables
static const unsigned char	checkpoint_pickle []
	What torch.save() writes as data.pkl for {"meta": {...}, "state_dict": OrderedDict(layer.weight [2,3], layer.weight_t its transpose, layer.bias the last two values, half [4] float16)}, protocol 2, made by Python's own pickler with the same names torch uses.
static int	failures = 0
	failed checks
static int	log_level = 3
	log level from -V

Data Structure Documentation

◆ MEMBER

struct MEMBER

one stored member

one member to write

Examples: ex_pth.c, and ex_zip.c.

Definition at line 130 of file ex_pth.c.

Collaboration diagram for MEMBER:

Data Fields
const unsigned char *	data	its bytes
size_t	len	how many
unsigned	method	0 stored, 8 deflated
const char *	name	its name

Function Documentation

◆ expect_true()

void expect_true	(	const char *	what,
		int	ok )

static

count a failed check

Parameters

what	the check
ok	its result

Definition at line 93 of file ex_pth.c.

References failures, LOG_ERR, LOG_INFO, and n_log.

◆ main()

int main	(	int	argc,
		char **	argv )

run the checks

Parameters

argc	argument count
argv	arguments

Returns: 0 on success

Definition at line 215 of file ex_pth.c.

References checkpoint_pickle, N_PTH::count, MEMBER::data, N_PTH_TENSOR::dims, expect_true(), failures, MEMBER::len, LOG_ERR, log_level, n_pth_find(), n_pth_free(), n_pth_load(), MEMBER::name, process_args(), put_le(), set_log_level(), N_PTH_TENSOR::shape, values_are(), and write_zip().

Here is the call graph for this function:

◆ process_args()

int process_args	(	int	argc,
		char **	argv )

static

read -V

Parameters

argc	argument count
argv	arguments

Returns: 0, or -1 on a bad argument

Definition at line 106 of file ex_pth.c.

References log_level.

◆ put_le()

void put_le	(	unsigned char *	p,
		unsigned long	v,
		int	n )

static

put a little endian value

Parameters

p	where
v	the value
n	its size, 2 or 4

Examples: ex_pth.c.

Definition at line 123 of file ex_pth.c.

Referenced by main(), and write_zip().

Here is the caller graph for this function:

◆ values_are()

int values_are	(	const N_PTH_TENSOR *	t,
		const float *	want,
		size_t	n )

static

are a tensor's values these

Parameters

t	the tensor, may be NULL
want	the values
n	how many

Returns: non-zero when they are

Examples: ex_pth.c.

Definition at line 202 of file ex_pth.c.

References N_PTH_TENSOR::count, and N_PTH_TENSOR::data.

Referenced by main().

Here is the caller graph for this function:

◆ write_zip()

int write_zip	(	const char *	path,
		const MEMBER *	m,
		int	count )

static

write a ZIP of stored members, as torch.save() does

Parameters

path	where
m	the members
count	how many, up to 8

Returns: 0, or -1

Examples: ex_pth.c, and ex_zip.c.

Definition at line 144 of file ex_pth.c.

References MEMBER::len, and put_le().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ checkpoint_pickle

const unsigned char checkpoint_pickle[]

static

Initial value:

= {
    0x80, 0x02, 0x7d, 0x71, 0x00, 0x28, 0x58, 0x04, 0x00, 0x00, 0x00, 0x6d,
    0x65, 0x74, 0x61, 0x71, 0x01, 0x7d, 0x71, 0x02, 0x28, 0x58, 0x05, 0x00,
    0x00, 0x00, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x71, 0x03, 0x4b, 0x0c, 0x58,
    0x04, 0x00, 0x00, 0x00, 0x6e, 0x6f, 0x74, 0x65, 0x71, 0x04, 0x58, 0x01,
    0x00, 0x00, 0x00, 0x78, 0x71, 0x05, 0x58, 0x02, 0x00, 0x00, 0x00, 0x6c,
    0x72, 0x71, 0x06, 0x47, 0x3f, 0x50, 0x62, 0x4d, 0xd2, 0xf1, 0xa9, 0xfc,
    0x75, 0x58, 0x0a, 0x00, 0x00, 0x00, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f,
    0x64, 0x69, 0x63, 0x74, 0x71, 0x07, 0x63, 0x63, 0x6f, 0x6c, 0x6c, 0x65,
    0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x0a, 0x4f, 0x72, 0x64, 0x65, 0x72,
    0x65, 0x64, 0x44, 0x69, 0x63, 0x74, 0x0a, 0x71, 0x08, 0x29, 0x52, 0x71,
    0x09, 0x28, 0x58, 0x0c, 0x00, 0x00, 0x00, 0x6c, 0x61, 0x79, 0x65, 0x72,
    0x2e, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x71, 0x0a, 0x63, 0x74, 0x6f,
    0x72, 0x63, 0x68, 0x2e, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x73, 0x0a, 0x5f,
    0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x74, 0x65, 0x6e, 0x73,
    0x6f, 0x72, 0x5f, 0x76, 0x32, 0x0a, 0x71, 0x0b, 0x28, 0x28, 0x58, 0x07,
    0x00, 0x00, 0x00, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x71, 0x0c,
    0x63, 0x74, 0x6f, 0x72, 0x63, 0x68, 0x0a, 0x46, 0x6c, 0x6f, 0x61, 0x74,
    0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x0a, 0x71, 0x0d, 0x58, 0x01,
    0x00, 0x00, 0x00, 0x30, 0x71, 0x0e, 0x58, 0x03, 0x00, 0x00, 0x00, 0x63,
    0x70, 0x75, 0x71, 0x0f, 0x4b, 0x06, 0x74, 0x71, 0x10, 0x51, 0x4b, 0x00,
    0x4b, 0x02, 0x4b, 0x03, 0x86, 0x71, 0x11, 0x4b, 0x03, 0x4b, 0x01, 0x86,
    0x71, 0x12, 0x89, 0x68, 0x08, 0x29, 0x52, 0x71, 0x13, 0x74, 0x71, 0x14,
    0x52, 0x71, 0x15, 0x58, 0x0e, 0x00, 0x00, 0x00, 0x6c, 0x61, 0x79, 0x65,
    0x72, 0x2e, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x5f, 0x74, 0x71, 0x16,
    0x68, 0x0b, 0x28, 0x68, 0x10, 0x51, 0x4b, 0x00, 0x4b, 0x03, 0x4b, 0x02,
    0x86, 0x71, 0x17, 0x4b, 0x01, 0x4b, 0x03, 0x86, 0x71, 0x18, 0x89, 0x68,
    0x08, 0x29, 0x52, 0x71, 0x19, 0x74, 0x71, 0x1a, 0x52, 0x71, 0x1b, 0x58,
    0x0a, 0x00, 0x00, 0x00, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x2e, 0x62, 0x69,
    0x61, 0x73, 0x71, 0x1c, 0x68, 0x0b, 0x28, 0x68, 0x10, 0x51, 0x4b, 0x04,
    0x4b, 0x02, 0x85, 0x71, 0x1d, 0x4b, 0x01, 0x85, 0x71, 0x1e, 0x89, 0x68,
    0x08, 0x29, 0x52, 0x71, 0x1f, 0x74, 0x71, 0x20, 0x52, 0x71, 0x21, 0x58,
    0x04, 0x00, 0x00, 0x00, 0x68, 0x61, 0x6c, 0x66, 0x71, 0x22, 0x68, 0x0b,
    0x28, 0x28, 0x68, 0x0c, 0x63, 0x74, 0x6f, 0x72, 0x63, 0x68, 0x0a, 0x48,
    0x61, 0x6c, 0x66, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x0a, 0x71,
    0x23, 0x58, 0x01, 0x00, 0x00, 0x00, 0x31, 0x71, 0x24, 0x68, 0x0f, 0x4b,
    0x04, 0x74, 0x71, 0x25, 0x51, 0x4b, 0x00, 0x4b, 0x04, 0x85, 0x71, 0x26,
    0x68, 0x1e, 0x89, 0x68, 0x08, 0x29, 0x52, 0x71, 0x27, 0x74, 0x71, 0x28,
    0x52, 0x71, 0x29, 0x75, 0x75, 0x2e}

What torch.save() writes as data.pkl for {"meta": {...}, "state_dict": OrderedDict(layer.weight [2,3], layer.weight_t its transpose, layer.bias the last two values, half [4] float16)}, protocol 2, made by Python's own pickler with the same names torch uses.

Examples: ex_pth.c.

Definition at line 50 of file ex_pth.c.

Referenced by main().

◆ failures

int failures = 0

static

failed checks

Definition at line 44 of file ex_pth.c.

◆ log_level

int log_level = 3

static

log level from -V

Definition at line 42 of file ex_pth.c.

Data Structures

Functions

Variables

Data Structure Documentation

◆ MEMBER

Function Documentation

◆ expect_true()

◆ main()

◆ process_args()

◆ put_le()

◆ values_are()

◆ write_zip()

Variable Documentation

◆ checkpoint_pickle

◆ failures

◆ log_level