ex_network_ssl_nonblock.c

Non-blocking (resumable) TLS: both peers drive netw_ssl_handshake_once / netw_ssl_server_handshake_once + the *_once movers from their own poll() loop, never the thread engine.

Non-blocking (resumable) TLS: both peers drive netw_ssl_handshake_once / netw_ssl_server_handshake_once + the *_once movers from their own poll() loop, never the thread engine.

Author

Castagnier Mickael

Version

1.0

/*
 * Nilorea Library
 * Copyright (C) 2005-2026 Castagnier Mickael
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */

 
#include "nilorea/n_common.h"
#include "nilorea/n_log.h"
#include "nilorea/n_network.h"
#include "nilorea/n_str.h"
#include "nilorea/n_x509.h"
 
#ifndef __windows__
#include <poll.h>
#endif
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
 
#include <openssl/bio.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/x509.h>

#define MSG "nonblock-client->server"
#define REPLY "nonblock-server->client"
#define MSG_LEN 24

static void leave_stale_openssl_error(void) {
    static const char garbage[] = "not a certificate";
    BIO* b = BIO_new_mem_buf(garbage, (int)(sizeof(garbage) - 1));
    if (b) {
        X509* x = PEM_read_bio_X509(b, NULL, NULL, NULL); /* fails: queues */
        if (x) X509_free(x);
        BIO_free(b);
    }
}

typedef struct SRV {
    NETWORK* listen; 
    SSL_CTX* ctx;    
    int ok;          
} SRV;
 
/* Parse the -V LOG_LEVEL verbosity argument. */
static void process_args(int argc, char** argv) {
    int opt = 0;
    while ((opt = getopt(argc, argv, "hvV:")) != EOF) {
        switch (opt) {
            case 'V':
                if (!strncmp("LOG_NULL", optarg, 8))
                    set_log_level(LOG_NULL);
                else if (!strncmp("LOG_NOTICE", optarg, 10))
                    set_log_level(LOG_NOTICE);
                else if (!strncmp("LOG_INFO", optarg, 8))
                    set_log_level(LOG_INFO);
                else if (!strncmp("LOG_ERR", optarg, 7))
                    set_log_level(LOG_ERR);
                else if (!strncmp("LOG_DEBUG", optarg, 9))
                    set_log_level(LOG_DEBUG);
                else {
                    fprintf(stderr, "Unknown log level %s\n", optarg);
                    exit(1);
                }
                break;
            case 'v':
                fprintf(stderr, "ex_network_ssl_nonblock\n");
                exit(1);
            case 'h':
            default:
                fprintf(stderr, "usage: %s [-V LOG_LEVEL]\n", argv[0]);
                break;
        }
    }
}
 
/* Build a server ctx from an in-memory PEM cert + key, pinned to the newest TLS
 * version the linked TLS library can negotiate. That is TLS 1.3 from OpenSSL
 * 1.1.1 (LibreSSL 3.5) on; on OpenSSL 1.0.2 neither the version-flexible
 * TLS_server_method() nor the min/max_proto_version setters exist, and
 * TLSv1_2_server_method() already pins the context to the one version that
 * build speaks. What this example
 * exercises is the non-blocking handshake and I/O drive loop, which is the
 * same either way. */
static SSL_CTX* make_server_ctx(const N_STR* cert_pem, const N_STR* key_pem) {
#if N_OSSL_HAVE_PROTO_VERSION_SETTERS
    SSL_CTX* ctx = SSL_CTX_new(TLS_server_method());
    if (!ctx)
        return NULL;
    SSL_CTX_set_min_proto_version(ctx, NETW_TLS_MAX_VERSION);
    SSL_CTX_set_max_proto_version(ctx, NETW_TLS_MAX_VERSION);
#else
    SSL_CTX* ctx = SSL_CTX_new(TLSv1_2_server_method());
    if (!ctx)
        return NULL;
#endif
    SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
 
    BIO* cb = BIO_new_mem_buf(cert_pem->data, (int)cert_pem->written);
    BIO* kb = BIO_new_mem_buf(key_pem->data, (int)key_pem->written);
    X509* x = cb ? PEM_read_bio_X509(cb, NULL, NULL, NULL) : NULL;
    EVP_PKEY* pk = kb ? PEM_read_bio_PrivateKey(kb, NULL, NULL, NULL) : NULL;
    int ok = x && pk && SSL_CTX_use_certificate(ctx, x) == 1 &&
             SSL_CTX_use_PrivateKey(ctx, pk) == 1;
    if (x)
        X509_free(x);
    if (pk)
        EVP_PKEY_free(pk);
    if (cb)
        BIO_free(cb);
    if (kb)
        BIO_free(kb);
    if (!ok) {
        SSL_CTX_free(ctx);
        return NULL;
    }
    return ctx;
}
 
/* Wait for a socket to become readable (want_write==0) or writable.
 * poll() is POSIX only. Winsock has no <poll.h>, and its WSAPoll() equivalent
 * needs _WIN32_WINNT >= 0x0600 headers while n_windows.h targets 0x0501, so the
 * Windows branch waits with select(), which Winsock always provides. A failed
 * non-blocking connect is signalled through exceptfds there (not writefds), so
 * that set is watched too: the caller learns the real outcome from its next
 * netw_connect_check() / SSL attempt. */
static int poll_fd(SOCKET fd, int want_write, int timeout_ms) {
#ifdef __windows__
    fd_set rset, wset, eset;
    struct timeval tv;
 
    FD_ZERO(&rset);
    FD_ZERO(&wset);
    FD_ZERO(&eset);
    if (want_write) {
        FD_SET(fd, &wset);
        FD_SET(fd, &eset);
    } else {
        FD_SET(fd, &rset);
    }
    tv.tv_sec = timeout_ms / 1000;
    tv.tv_usec = (timeout_ms % 1000) * 1000;
 
    return select((int)fd + 1, want_write ? NULL : &rset,
                  want_write ? &wset : NULL, want_write ? &eset : NULL, &tv);
#else
    struct pollfd p;
    p.fd = fd;
    p.events = (short)(want_write ? POLLOUT : POLLIN);
    p.revents = 0;
    return poll(&p, 1, timeout_ms);
#endif
}
 
/* Drive a resumable handshake to completion from a poll() loop. */
static int drive_handshake(NETWORK* netw, int is_server, SSL_CTX* sctx) {
    int guard;
    for (guard = 0; guard < 10000; guard++) {
        int r = is_server ? netw_ssl_server_handshake_once(netw, sctx)
                          : netw_ssl_handshake_once(netw, NULL);
        if (r == TRUE)
            return TRUE;
        if (r == NETW_IO_WANT_READ) {
            if (poll_fd(netw->link.sock, 0, 2000) <= 0)
                return FALSE;
            continue;
        }
        if (r == NETW_IO_WANT_WRITE) {
            if (poll_fd(netw->link.sock, 1, 2000) <= 0)
                return FALSE;
            continue;
        }
        return FALSE; /* fatal */
    }
    return FALSE;
}
 
/* Drive a non-blocking connect to completion from a poll() loop. */
static int drive_connect(NETWORK* netw) {
    int guard;
    for (guard = 0; guard < 10000; guard++) {
        int r = netw_connect_check(netw);
        if (r == TRUE)
            return TRUE;
        if (r == NETW_IO_WANT_WRITE) {
            if (poll_fd(netw->link.sock, 1, 2000) <= 0)
                return FALSE;
            continue;
        }
        return FALSE; /* all addresses failed */
    }
    return FALSE;
}
 
/* Move exactly n bytes with the single-attempt movers, polling on WANT. */
static int io_full(NETWORK* netw, char* buf, uint32_t n, int is_send) {
    uint32_t done = 0;
    int guard;
    for (guard = 0; guard < 100000 && done < n; guard++) {
        ssize_t r = is_send ? send_ssl_data_once(netw, buf + done, n - done)
                            : recv_ssl_data_once(netw, buf + done, n - done);
        if (r > 0) {
            done += (uint32_t)r;
            continue;
        }
        if (r == NETW_IO_WANT_READ) {
            if (poll_fd(netw->link.sock, 0, 2000) <= 0)
                return FALSE;
            continue;
        }
        if (r == NETW_IO_WANT_WRITE) {
            if (poll_fd(netw->link.sock, 1, 2000) <= 0)
                return FALSE;
            continue;
        }
        return FALSE; /* disconnect or error */
    }
    return done == n ? TRUE : FALSE;
}
 
/* Server thread: accept one connection (non-blocking), complete the resumable
 * handshake, read the probe, echo the reply. */
static void* server_fn(void* p) {
    SRV* s = (SRV*)p;
    NETWORK* c = NULL;
    int guard;
    for (guard = 0; guard < 10000 && !c; guard++) {
        if (poll_fd(s->listen->link.sock, 0, 2000) <= 0)
            break;
        c = netw_accept_nonblock_from(s->listen, 0);
    }
    if (!c)
        return NULL;
    if (drive_handshake(c, 1, s->ctx) == TRUE) {
        char buf[MSG_LEN] = "";
        char reply[MSG_LEN];
        memcpy(reply, REPLY, MSG_LEN);
        if (io_full(c, buf, MSG_LEN, 0) == TRUE && memcmp(buf, MSG, MSG_LEN) == 0) {
            n_log(LOG_NOTICE, "server received: %s", buf);
            if (io_full(c, reply, MSG_LEN, 1) == TRUE)
                s->ok = 1;
        }
    }
    netw_close(&c);
    return NULL;
}
 
/* Bind a loopback listening socket on the first free port in a range. */
static NETWORK* bind_free(char* port_out, size_t port_out_len) {
    int p;
    for (p = 18500; p < 18600; p++) {
        NETWORK* l = NULL;
        char ps[16];
        snprintf(ps, sizeof(ps), "%d", p);
        if (netw_make_listening(&l, "127.0.0.1", ps, 5, NETWORK_IPALL) == TRUE && l) {
            snprintf(port_out, port_out_len, "%s", ps);
            return l;
        }
        if (l)
            netw_close(&l);
    }
    return NULL;
}
 
int main(int argc, char** argv) {
    N_STR* cert = NULL;
    N_STR* key = NULL;
    SSL_CTX* sctx = NULL;
    NETWORK* listen = NULL;
    NETWORK* cli = NULL;
    SRV srv;
    pthread_t th;
    char port[16] = "";
    int failures = 0;
 
    set_log_level(LOG_NOTICE);
    process_args(argc, argv);
 
    /* OpenSSL is initialised once in the main thread: the server and client
     * threads both use TLS and the lazy init guard is not synchronized. */
    netw_init_openssl();
 
    /* A private self-signed CA cert doubles as the server leaf here; the client
     * turns verification off (it is not in any trust store). */
    if (n_x509_generate_ca("Nilorea NonBlock TLS Test", 3650, &cert, &key) != 0 || !cert || !key) {
        n_log(LOG_ERR, "certificate generation failed");
        netw_unload_openssl();
        return 1;
    }
    sctx = make_server_ctx(cert, key);
    if (!sctx) {
        n_log(LOG_ERR, "server ctx setup failed");
        free_nstr(&cert);
        free_nstr(&key);
        netw_unload_openssl();
        return 1;
    }
 
    listen = bind_free(port, sizeof(port));
    if (!listen) {
        n_log(LOG_ERR, "could not bind a loopback port");
        SSL_CTX_free(sctx);
        free_nstr(&cert);
        free_nstr(&key);
        netw_unload_openssl();
        return 1;
    }
 
    srv.listen = listen;
    srv.ctx = sctx;
    srv.ok = 0;
    pthread_create(&th, NULL, server_fn, &srv);
 
    /* The whole client path is non-blocking + poll-driven: start the TCP
     * connect, drive it to completion, set up the client TLS ctx, then run the
     * TLS handshake -- none of it blocks a shared thread. */
    if (netw_connect_nonblock(&cli, "127.0.0.1", port, NETWORK_IPALL) != TRUE || !cli) {
        n_log(LOG_ERR, "client connect start failed");
        failures++;
    } else if (drive_connect(cli) != TRUE) {
        n_log(LOG_ERR, "client non-blocking connect failed");
        failures++;
    } else if (netw_ssl_start_client(cli) != TRUE) {
        n_log(LOG_ERR, "client TLS ctx setup failed");
        failures++;
    } else {
        netw_ssl_set_verify(cli, 0);
        if (netw_ssl_set_versions(cli, NETW_TLS_MAX_VERSION, NETW_TLS_MAX_VERSION) != TRUE) {
            n_log(LOG_ERR, "client TLS version pin failed");
            failures++;
        }
        if (drive_handshake(cli, 0, NULL) != TRUE) {
            n_log(LOG_ERR, "client non-blocking handshake failed");
            failures++;
        } else {
            char msg[MSG_LEN];
            char buf[MSG_LEN] = "";
            /* A stale entry on the thread's error queue -- left by some
             * OTHER connection, e.g. one shut down mid-handshake -- must not
             * fail this healthy one: OpenSSL's SSL_get_error() reports
             * SSL_ERROR_SSL whenever the queue is non-empty, so the *_once
             * movers must clear it first. The server is waiting for our
             * message, so the idle read can only be WANT_READ. (A
             * single-threaded DarkChan server closed an innocent client this
             * way and lost its first frames, 2026-10-08.) */
            leave_stale_openssl_error();
            ssize_t idle = recv_ssl_data_once(cli, buf, MSG_LEN);
            if (idle != NETW_IO_WANT_READ) {
                n_log(LOG_ERR, "stale error queue: idle recv returned %zd, not WANT_READ", idle);
                failures++;
            }
            leave_stale_openssl_error(); /* and the send that follows */
            memcpy(msg, MSG, MSG_LEN);
            if (io_full(cli, msg, MSG_LEN, 1) != TRUE) {
                n_log(LOG_ERR, "client send failed");
                failures++;
            } else if (io_full(cli, buf, MSG_LEN, 0) != TRUE) {
                n_log(LOG_ERR, "client recv failed");
                failures++;
            } else if (memcmp(buf, REPLY, MSG_LEN) != 0) {
                n_log(LOG_ERR, "client got an unexpected reply");
                failures++;
            } else {
                n_log(LOG_NOTICE, "client received: %s", buf);
            }
        }
    }
 
    netw_close(&cli);
    pthread_join(th, NULL);
    if (!srv.ok) {
        n_log(LOG_ERR, "server did not complete the exchange");
        failures++;
    }
 
    netw_close(&listen);
    SSL_CTX_free(sctx);
    free_nstr(&cert);
    free_nstr(&key);
    netw_unload_openssl();
 
    if (failures) {
        n_log(LOG_ERR, "ex_network_ssl_nonblock: %d failure(s)", failures);
        return 1;
    }
    n_log(LOG_NOTICE, "ex_network_ssl_nonblock: all checks passed");
    return 0;
}