Version 0.0.9

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.65])
-AC_INIT([glorytun], [0.0.2], [https://github.com/angt/glorytun/issues],
+AC_INIT([glorytun], [0.0.9], [https://github.com/angt/glorytun/issues],
         [glorytun], [https://github.com/angt/glorytun])
 AC_CONFIG_SRCDIR([src/common.h])
 AC_CONFIG_AUX_DIR([build-aux])

src/common.c

@@ -3,6 +3,19 @@
 #include <stdio.h>
 #include <stdarg.h>
 
+int gt_print (const char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    int ret = vfprintf(stdout, fmt, ap);
+    va_end(ap);
+
+    if (ret<0)
+        return 0;
+
+    return ret;
+}
+
 void gt_log (const char *fmt, ...)
 {
     va_list ap;

src/common.h

@@ -14,9 +14,13 @@
 #define PALIGN(x)      ((void *)ALIGN((size_t)(x)))
 #define PALIGN_DOWN(x) ((void *)ALIGN_DOWN((size_t)(x)))
 
+#define _1_(x)         (__builtin_expect((x), 1))
+#define _0_(x)         (__builtin_expect((x), 0))
+
 #define _printf_(A,B)  __attribute__((format(printf,A,B)))
 #define _noreturn_     __attribute__((noreturn))
 #define _unused_       __attribute__((unused))
+#define _align_(...)   __attribute__((aligned(__VA_ARGS__)))
 
 typedef struct buffer buffer_t;
 
@@ -27,6 +31,7 @@ struct buffer {
     uint8_t *end;
 };
 
+int  gt_print (const char *, ...) _printf_(1,2);
 void gt_log   (const char *, ...) _printf_(1,2);
 void gt_fatal (const char *, ...) _printf_(1,2) _noreturn_;
 void gt_na    (const char *);

src/ip-static.h

@@ -28,3 +28,12 @@ static inline ssize_t ip_get_size (const uint8_t *data, size_t size)
     return 0;
 }
 
+static inline int ip_get_dscp (const uint8_t *data, size_t size)
+{
+    switch (ip_get_version(data, size)) {
+    case 4:
+        return data[1]>>2;
+    }
+
+    return 0;
+}

src/main.c

@@ -1,8 +1,8 @@
 #include <inttypes.h>
+#include <limits.h>
 #include <stdio.h>
 #include <signal.h>
 #include <poll.h>
-
 #include <sys/time.h>
 #include <sys/fcntl.h>
 #include <sys/socket.h>
@@ -24,12 +24,19 @@
 
 #define GT_BUFFER_SIZE  (4*1024*1024)
 #define GT_TIMEOUT      (1000)
+#define GT_MTU_MAX      (1500)
+#define GT_TUNR_SIZE    (0x7FFF-16)
+#define GT_TUNW_SIZE    (0x7FFF)
 
-struct netio {
+struct fdbuf {
     int fd;
-    struct {
-        buffer_t buf;
-    } write, read;
+    buffer_t read;
+    buffer_t write;
+};
+
+struct blk {
+    size_t size;
+    uint8_t data[GT_MTU_MAX] _align_(16);
 };
 
 struct crypto_ctx {
@@ -41,6 +48,7 @@ struct crypto_ctx {
 };
 
 volatile sig_atomic_t gt_close = 0;
+volatile sig_atomic_t gt_info = 0;
 
 static int64_t dt_ms (struct timeval *ta, struct timeval *tb)
 {
@@ -67,51 +75,81 @@ static void fd_set_nonblock (int fd)
         perror("fcntl O_NONBLOCK");
 }
 
-static void sk_set_nodelay (int fd)
+enum sk_opt {
+    sk_nodelay,
+    sk_reuseaddr,
+    sk_keepalive,
+    sk_keepcnt,
+    sk_keepidle,
+    sk_keepintvl,
+    sk_congestion,
+    sk_defer_accept,
+    sk_quickack,
+};
+
+static void sk_set (int fd, enum sk_opt opt, const void *val, socklen_t len)
 {
-    int val = 1;
-
-    if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY , &val, sizeof(val))==-1)
-        perror("setsockopt TCP_NODELAY");
-}
-
-static void sk_set_reuseaddr (int fd)
-{
-    int val = 1;
-
-    if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val))==-1)
-        perror("setsockopt SO_REUSEADDR");
-}
-
-static void sk_set_keepalive (int fd)
-{
-    int val = 1;
-
-    if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val))==-1)
-        perror("setsockopt SO_KEEPALIVE");
-}
-
-#ifdef TCP_CONGESTION
-static void sk_set_congestion (int fd, const char *name)
-{
-    size_t len = str_len(name);
-
-    if (!len)
+    if (!val || len<=0)
         return;
 
-    if (setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, name, len+1)==-1)
-        perror("setsockopt TCP_CONGESTION");
-}
-#else
-static void sk_set_congestion (_unused_ int fd, _unused_ const char *name)
-{
-    gt_na("TCP_CONGESTION");
-}
+    struct {
+        const char *name;
+        const int present;
+        const int level;
+        const int option;
+    } opts[] = {
+        [sk_nodelay] = { "TCP_NODELAY", 1, IPPROTO_TCP, TCP_NODELAY, },
+        [sk_reuseaddr] = { "SO_REUSEADDR", 1, SOL_SOCKET, SO_REUSEADDR, },
+        [sk_keepalive] = { "SO_KEEPALIVE", 1, SOL_SOCKET, SO_KEEPALIVE, },
+        [sk_keepcnt] = { "TCP_KEEPCNT",
+#ifdef TCP_KEEPCNT
+            1, IPPROTO_TCP, TCP_KEEPCNT,
 #endif
+        },
+        [sk_keepidle] = { "TCP_KEEPIDLE",
+#ifdef TCP_KEEPIDLE
+            1, IPPROTO_TCP, TCP_KEEPIDLE,
+#endif
+        },
+        [sk_keepintvl] = { "TCP_KEEPINTVL",
+#ifdef TCP_KEEPINTVL
+            1, IPPROTO_TCP, TCP_KEEPINTVL,
+#endif
+        },
+        [sk_congestion] = { "TCP_CONGESTION",
+#ifdef TCP_CONGESTION
+            1, IPPROTO_TCP, TCP_CONGESTION,
+#endif
+        },
+        [sk_defer_accept] = { "TCP_DEFER_ACCEPT",
+#ifdef TCP_DEFER_ACCEPT
+            1, IPPROTO_TCP, TCP_DEFER_ACCEPT,
+#endif
+        },
+        [sk_quickack] = { "TCP_QUICKACK",
+#ifdef TCP_QUICKACK
+            1, IPPROTO_TCP, TCP_QUICKACK,
+#endif
+        },
+    };
+
+    if (!opts[opt].present) {
+        gt_na(opts[opt].name);
+        return;
+    }
+
+    if (setsockopt(fd, opts[opt].level, opts[opt].option, val, len)==-1)
+        gt_log("couldn't set socket option `%s'\n", opts[opt].name);
+}
+
+static void sk_set_int (int fd, enum sk_opt opt, int val)
+{
+    return sk_set(fd, opt, &val, sizeof(val));
+}
 
 static int sk_listen (int fd, struct addrinfo *ai)
 {
-    sk_set_reuseaddr(fd);
+    sk_set_int(fd, sk_reuseaddr, 1);
 
     int ret = bind(fd, ai->ai_addr, ai->ai_addrlen);
 
@@ -120,13 +158,15 @@ static int sk_listen (int fd, struct addrinfo *ai)
         return -1;
     }
 
-    ret = listen(fd, 1);
+    ret = listen(fd, 8);
 
     if (ret==-1) {
         perror("listen");
         return -1;
     }
 
+    sk_set_int(fd, sk_defer_accept, GT_TIMEOUT/1000);
+
     return 0;
 }
 
@@ -280,20 +320,29 @@ static void gt_sa_stop (int sig)
 {
     switch (sig) {
     case SIGINT:
+    case SIGQUIT:
     case SIGTERM:
         gt_close = 1;
+        break;
+    case SIGUSR1:
+        gt_info = 1;
+        break;
     }
 }
 
 static void gt_set_signal (void)
 {
-    struct sigaction sa;
+    struct sigaction sa = {
+        .sa_flags = 0,
+    };
 
-    byte_set(&sa, 0, sizeof(sa));
+    sigemptyset(&sa.sa_mask);
 
     sa.sa_handler = gt_sa_stop;
     sigaction(SIGINT,  &sa, NULL);
+    sigaction(SIGQUIT, &sa, NULL);
     sigaction(SIGTERM, &sa, NULL);
+    sigaction(SIGUSR1, &sa, NULL);
 
     sa.sa_handler = SIG_IGN;
     sigaction(SIGHUP,  &sa, NULL);
@@ -303,7 +352,7 @@ static void gt_set_signal (void)
 static ssize_t fd_read (int fd, void *data, size_t size)
 {
     if (!size)
-        return -2;
+        return -1;
 
     ssize_t ret = read(fd, data, size);
 
@@ -323,7 +372,7 @@ static ssize_t fd_read (int fd, void *data, size_t size)
 static ssize_t fd_write (int fd, const void *data, size_t size)
 {
     if (!size)
-        return -2;
+        return -1;
 
     ssize_t ret = write(fd, data, size);
 
@@ -394,52 +443,69 @@ static ssize_t fd_write_all (int fd, const void *data, size_t size)
     return done;
 }
 
-static int encrypt_packet (struct crypto_ctx *ctx, uint8_t *packet, size_t size, buffer_t *buffer)
+static int gt_encrypt (struct crypto_ctx *ctx, buffer_t *dst, buffer_t *src)
 {
-    const size_t ws = size + crypto_aead_aes256gcm_ABYTES;
+    size_t rs = buffer_read_size(src);
+    size_t ws = buffer_write_size(dst);
 
-    if (buffer_write_size(buffer) < ws)
-        return 1;
+    if (!rs || !ws)
+        return 0;
 
-    const int hs = 4;
+    size_t size = rs+crypto_aead_aes256gcm_ABYTES;
 
-    byte_cpy(buffer->write, packet, size);
+    if (size+2>ws)
+        return 0;
+
+    dst->write[0] = 0xFF&(size>>8);
+    dst->write[1] = 0xFF&(size);
 
     crypto_aead_aes256gcm_encrypt_afternm(
-            buffer->write + hs, NULL,
-            packet + hs, size - hs,
-            packet, hs,
+            dst->write+2, NULL,
+            src->read, rs,
+            dst->write, 2,
             NULL, ctx->write.nonce,
             (const crypto_aead_aes256gcm_state *)&ctx->write.state);
 
     sodium_increment(ctx->write.nonce, crypto_aead_aes256gcm_NPUBBYTES);
-    buffer->write += ws;
+
+    src->read += rs;
+    dst->write += size+2;
 
     return 0;
 }
 
-static int decrypt_packet (struct crypto_ctx *ctx, uint8_t *packet, size_t size, buffer_t *buffer)
+static int gt_decrypt (struct crypto_ctx *ctx, buffer_t *dst, buffer_t *src)
 {
-    const size_t rs = size + crypto_aead_aes256gcm_ABYTES;
+    size_t rs = buffer_read_size(src);
+    size_t ws = buffer_write_size(dst);
 
-    if (buffer_read_size(buffer) < rs)
-        return 1;
+    if (!rs || !ws)
+        return 0;
 
-    const int hs = 4;
+    if (rs<=2+16)
+        return 0;
 
-    byte_cpy(packet, buffer->read, hs);
+    size_t size = (src->read[0]<<8)|src->read[1];
+
+    if (size-crypto_aead_aes256gcm_ABYTES>ws)
+        return 0;
+
+    if (size+2>rs)
+        return 0;
 
     if (crypto_aead_aes256gcm_decrypt_afternm(
-                packet + hs, NULL,
+                dst->write, NULL,
                 NULL,
-                buffer->read + hs, rs - hs,
-                packet, hs,
+                src->read+2, size,
+                src->read, 2,
                 ctx->read.nonce,
                 (const crypto_aead_aes256gcm_state *)&ctx->read.state))
         return -1;
 
     sodium_increment(ctx->read.nonce, crypto_aead_aes256gcm_NPUBBYTES);
-    buffer->read += rs;
+
+    src->read += size+2;
+    dst->write += size-crypto_aead_aes256gcm_ABYTES;
 
     return 0;
 }
@@ -471,9 +537,13 @@ static int gt_setup_secretkey (struct crypto_ctx *ctx, char *keyfile)
     if (!keyfile)
         return 0;
 
-    int fd = open(keyfile, O_RDONLY|O_CLOEXEC);
+    int fd;
 
-    if (fd<0) {
+    do {
+        fd = open(keyfile, O_RDONLY|O_CLOEXEC);
+    } while (fd==-1 && errno==EINTR);
+
+    if (fd==-1) {
         perror("open keyfile");
         return -1;
     }
@@ -574,48 +644,58 @@ int main (int argc, char **argv)
 {
     gt_set_signal();
 
-    int listener = 0;
     char *host = NULL;
     char *port = "5000";
     char *dev = PACKAGE_NAME;
     char *keyfile = NULL;
     char *congestion = NULL;
-    int delay = 0;
-    int multiqueue = 0;
-    int keepalive = 0;
-    int version = 0;
-    int debug = 0;
+    long buffer_size = GT_BUFFER_SIZE;
+    long ka_count = -1;
+    long ka_idle = -1;
+    long ka_interval = -1;
 
-#ifdef TCP_INFO
-    struct {
-        struct timeval time;
-        struct tcp_info info;
-    } tcpinfo = {0};
-#endif
+    struct option ka_opts[] = {
+        { "count",    &ka_count,    option_long },
+        { "idle",     &ka_idle,     option_long },
+        { "interval", &ka_interval, option_long },
+        { NULL },
+    };
 
     struct option opts[] = {
-        { "listener",   &listener,   option_flag },
+        { "listener",    NULL,         option_option },
         { "host",        &host,        option_str    },
         { "port",        &port,        option_str    },
         { "dev",         &dev,         option_str    },
         { "keyfile",     &keyfile,     option_str    },
         { "congestion",  &congestion,  option_str    },
-        { "delay",      &delay,      option_flag },
-        { "multiqueue", &multiqueue, option_flag },
-        { "keepalive",  &keepalive,  option_flag },
-        { "debug",      &debug,      option_flag },
-        { "version",    &version,    option_flag },
+        { "delay",       NULL,         option_option },
+        { "multiqueue",  NULL,         option_option },
+        { "keepalive",   ka_opts,      option_option },
+        { "buffer-size", &buffer_size, option_long   },
+        { "noquickack",  NULL,         option_option },
+        { "daemon",      NULL,         option_option },
+        { "version",     NULL,         option_option },
         { NULL },
     };
 
     if (option(opts, argc, argv))
         return 1;
 
-    if (version) {
-        printf(PACKAGE_STRING"\n");
+    if (option_is_set(opts, "version")) {
+        gt_print(PACKAGE_STRING"\n");
         return 0;
     }
 
+    int listener = option_is_set(opts, "listener");
+    int delay = option_is_set(opts, "delay");
+    int keepalive = option_is_set(opts, "keepalive");
+    int noquickack = option_is_set(opts, "noquickack");
+
+    if (buffer_size < 2048) {
+        buffer_size = 2048;
+        gt_log("buffer size must be greater than 2048!\n");
+    }
+
     if (sodium_init()==-1) {
         gt_log("libsodium initialization has failed!\n");
         return 1;
@@ -636,18 +716,29 @@ int main (int argc, char **argv)
     if (!ai)
         return 1;
 
-    struct netio tun  = { .fd = -1 };
-    struct netio sock = { .fd = -1 };
+    struct fdbuf tun  = { .fd = -1 };
+    struct fdbuf sock = { .fd = -1 };
 
-    tun.fd = tun_create(dev, multiqueue);
+    tun.fd = tun_create(dev, option_is_set(opts, "multiqueue"));
 
     if (tun.fd==-1)
         return 1;
 
+    struct blk *blks = calloc(256, sizeof(struct blk));
+    size_t blk_count = 0;
+    uint8_t blk_read = 0;
+    uint8_t blk_write = 0;
+
+    if (!blks)
+        return 1;
+
     fd_set_nonblock(tun.fd);
 
-    buffer_setup(&sock.write.buf, NULL, GT_BUFFER_SIZE);
-    buffer_setup(&sock.read.buf, NULL, GT_BUFFER_SIZE);
+    buffer_setup(&tun.write, NULL, GT_TUNW_SIZE);
+    buffer_setup(&tun.read,  NULL, GT_TUNR_SIZE);
+
+    buffer_setup(&sock.write, NULL, buffer_size);
+    buffer_setup(&sock.read,  NULL, buffer_size);
 
     int fd = -1;
 
@@ -658,30 +749,54 @@ int main (int argc, char **argv)
             return 1;
     }
 
-    while (1) {
+    if (option_is_set(opts, "daemon")) {
+        switch (fork()) {
+        case -1:
+            perror("fork");
+            return 1;
+        case 0:
+            if (setsid()==-1)
+                perror("setsid");
+            break;
+        default:
+            _exit(0);
+        }
+    }
+
+    while (!gt_close) {
         sock.fd = listener?sk_accept(fd):sk_create(ai, sk_connect);
 
         if (sock.fd==-1) {
             usleep(100000);
-            goto restart;
+            continue;
         }
 
         char *sockname = sk_get_name(sock.fd);
 
-        if (!sockname)
-            goto restart;
+        if (!sockname) {
+            close(sock.fd);
+            continue;
+        }
 
         gt_log("%s: connected\n", sockname);
 
-        if (!delay)
-            sk_set_nodelay(sock.fd);
-
         fd_set_nonblock(sock.fd);
 
-        if (keepalive)
-            sk_set_keepalive(sock.fd);
+        sk_set_int(sock.fd, sk_nodelay, !delay);
+        sk_set_int(sock.fd, sk_keepalive, keepalive);
 
-        sk_set_congestion(sock.fd, congestion);
+        if (keepalive) {
+            if (ka_count>=0 && ka_count<=INT_MAX)
+                sk_set_int(sock.fd, sk_keepcnt, ka_count);
+
+            if (ka_idle>=0 && ka_idle<=INT_MAX)
+                sk_set_int(sock.fd, sk_keepidle, ka_idle);
+
+            if (ka_interval>=0 && ka_interval<=INT_MAX)
+                sk_set_int(sock.fd, sk_keepintvl, ka_interval);
+        }
+
+        sk_set(sock.fd, sk_congestion, congestion, str_len(congestion));
 
         switch (gt_setup_crypto(&ctx, sock.fd, listener)) {
             case -2: gt_log("%s: key exchange could not be verified!\n", sockname);
@@ -689,28 +804,22 @@ int main (int argc, char **argv)
             default: break;
         }
 
-        struct {
-            uint8_t buf[2048];
-            size_t size;
-        } tunr, tunw;
-
-        tunr.size = 0;
-        tunw.size = 0;
-
         fd_set rfds, wfds;
         FD_ZERO(&rfds);
         FD_ZERO(&wfds);
 
         int stop_loop = 0;
 
+        buffer_format(&sock.write);
+        buffer_format(&sock.read);
+
         while (1) {
             if (gt_close)
-                stop_loop = 1;
+                stop_loop |= 1;
 
             if (stop_loop) {
-                if (((stop_loop>>1)==3) &&
-                    (buffer_read_size(&sock.write.buf)==0) &&
-                    (buffer_read_size(&sock.read.buf)==0))
+                if (((stop_loop&(1<<2)) || !buffer_read_size(&sock.write)) &&
+                    ((stop_loop&(1<<1)) || !buffer_read_size(&sock.read)))
                     goto restart;
                 FD_CLR(tun.fd, &rfds);
             } else {
@@ -719,130 +828,154 @@ int main (int argc, char **argv)
 
             FD_SET(sock.fd, &rfds);
 
-            if (select(sock.fd+1, &rfds, &wfds, NULL, NULL)==-1 && errno!=EINTR) {
+            if (select(sock.fd+1, &rfds, &wfds, NULL, NULL)==-1) {
+                if (errno==EINTR)
+                    continue;
                 perror("select");
                 return 1;
             }
 
-#ifdef TCP_INFO
-            struct timeval now;
-            gettimeofday(&now, NULL);
+            FD_CLR(sock.fd, &wfds);
+            FD_CLR(tun.fd, &wfds);
 
-            if (debug && dt_ms(&now, &tcpinfo.time)>1000LL) {
-                tcpinfo.time = now;
-                if (sk_get_info(sock.fd, &tcpinfo.info))
-                    print_tcp_info(sockname, &tcpinfo.info);
+         // TODO
+         // struct timeval now;
+         // gettimeofday(&now, NULL);
+
+#ifdef TCP_INFO
+            if (gt_info) {
+                struct tcp_info ti;
+
+                if (sk_get_info(sock.fd, &ti))
+                    print_tcp_info(sockname, &ti);
+
+                gt_info = 0;
             }
 #endif
 
-            buffer_shift(&sock.write.buf);
-
             if (FD_ISSET(tun.fd, &rfds)) {
-                while (1) {
-                    if (buffer_write_size(&sock.write.buf)<sizeof(tunr.buf)+16)
+                while (!blks[blk_write].size) {
+                    uint8_t *data = blks[blk_write].data;
+                    ssize_t r = tun_read(tun.fd, data, GT_MTU_MAX);
+
+                    if (r<=0) {
+                        gt_close |= !r;
                         break;
+                    }
 
-                    ssize_t r = tun_read(tun.fd, tunr.buf, sizeof(tunr.buf));
-
-                    if (!r)
-                        return 2;
-
-                    if (r<0)
-                        break;
-
-                    ssize_t ip_size = ip_get_size(tunr.buf, sizeof(tunr.buf));
+                    ssize_t ip_size = ip_get_size(data, GT_MTU_MAX);
 
                     if (ip_size<=0)
                         continue;
 
                     if (ip_size!=r) {
-                        dump_ip_header(tunr.buf, r);
+                        dump_ip_header(data, r);
 
                         if (r<ip_size) {
-                            ip_set_size(tunr.buf, r);
+                            ip_set_size(data, r);
                         } else {
                             continue;
                         }
                     }
 
-                    encrypt_packet(&ctx, tunr.buf, r, &sock.write.buf);
+                    blks[blk_write++].size = r;
+                    blk_count++;
                 }
             }
 
-            if (FD_ISSET(sock.fd, &wfds))
-                FD_CLR(sock.fd, &wfds);
-
-            if (buffer_read_size(&sock.write.buf)) {
-                ssize_t r = fd_write(sock.fd, sock.write.buf.read, buffer_read_size(&sock.write.buf));
-
-                if (r==-1)
-                    FD_SET(sock.fd, &wfds);
-
-                if (!r) {
-                    stop_loop |= (1<<2);
-                    buffer_format(&sock.write.buf);
-                }
-
-                if (r>0)
-                    sock.write.buf.read += r;
-            } else {
-                if (stop_loop) {
-                    stop_loop |= (1<<2);
-                    shutdown(sock.fd, SHUT_WR);
-                }
-            }
-
-            buffer_shift(&sock.read.buf);
-
-            if (FD_ISSET(sock.fd, &rfds)) {
-                ssize_t r = fd_read(sock.fd, sock.read.buf.write, buffer_write_size(&sock.read.buf));
-
-                if (!r)
-                    stop_loop |= (1<<1);
-
-                if (r>0)
-                    sock.read.buf.write += r;
-            }
-
-            if (FD_ISSET(tun.fd, &wfds))
-                FD_CLR(tun.fd, &wfds);
-
             while (1) {
-                if (!tunw.size) {
-                    size_t size = buffer_read_size(&sock.read.buf);
-                    ssize_t ip_size = ip_get_size(sock.read.buf.read, size);
+                buffer_shift(&tun.read);
 
-                    if (!ip_size)
-                        goto restart;
-
-                    if (ip_size<0 || (size_t)ip_size+16>size) {
-                        if (stop_loop&(1<<1))
-                            buffer_format(&sock.read.buf);
+                if (!stop_loop) {
+                    for (; blk_count; blk_read++) {
+                        if (!blks[blk_read].size)
                             break;
+
+                        if (buffer_write_size(&tun.read)<blks[blk_read].size)
+                            break;
+
+                        byte_cpy(tun.read.write, blks[blk_read].data, blks[blk_read].size);
+                        tun.read.write += blks[blk_read].size;
+
+                        blks[blk_read].size = 0;
+                        blk_count--;
                     }
 
-                    if (decrypt_packet(&ctx, tunw.buf, ip_size, &sock.read.buf)) {
+                    gt_encrypt(&ctx, &sock.write, &tun.read);
+                }
+
+                if (!buffer_read_size(&sock.write))
+                    break;
+
+                ssize_t r = fd_write(sock.fd, sock.write.read,
+                                     buffer_read_size(&sock.write));
+
+                if (r>0) {
+                    sock.write.read += r;
+                } else {
+                    if (!r) {
+                        stop_loop |= (1<<2);
+                    } else {
+                        FD_SET(sock.fd, &wfds);
+                    }
+                    break;
+                }
+            }
+
+            if (stop_loop && !buffer_read_size(&sock.write)) {
+                if (!(stop_loop&(1<<2))) {
+                    stop_loop |= (1<<2);
+                    shutdown(sock.fd, SHUT_WR);
+                    gt_log("%s: shutdown\n", sockname);
+                }
+            }
+
+            buffer_shift(&sock.write);
+            buffer_shift(&sock.read);
+
+            if (FD_ISSET(sock.fd, &rfds)) {
+                if (noquickack)
+                    sk_set_int(sock.fd, sk_quickack, 0);
+
+                ssize_t r = fd_read(sock.fd, sock.read.write,
+                                    buffer_write_size(&sock.read));
+
+                if (r>0) {
+                    sock.read.write += r;
+                } else if (!r) {
+                    stop_loop |= (1<<1);
+                }
+            }
+
+            while (1) {
+                if (gt_decrypt(&ctx, &tun.write, &sock.read)) {
                     gt_log("%s: message could not be verified!\n", sockname);
                     goto restart;
                 }
 
-                    tunw.size = ip_size;
+                size_t size = buffer_read_size(&tun.write);
+                ssize_t ip_size = ip_get_size(tun.write.read, size);
+
+                if (!ip_size) {
+                    gt_log("%s: bad packet!\n", sockname);
+                    goto restart;
                 }
-                if (tunw.size) {
-                    ssize_t r = tun_write(tun.fd, tunw.buf, tunw.size);
 
-                    if (!r)
-                        return 2;
-
-                    if (r==-1)
-                        FD_SET(tun.fd, &wfds);
-
-                    if (r<0)
+                if (ip_size<0 || (size_t)ip_size>size)
                     break;
 
-                    tunw.size = 0;
+                ssize_t r = tun_write(tun.fd, tun.write.read, ip_size);
+
+                if (r>0) {
+                    tun.write.read += r;
+                } else {
+                    gt_close |= !r;
+                    FD_SET(tun.fd, &wfds);
+                    break;
                 }
             }
+
+            buffer_shift(&tun.write);
         }
 
     restart:
@@ -855,15 +988,17 @@ int main (int argc, char **argv)
             close(sock.fd);
             sock.fd = -1;
         }
-
-        if (gt_close)
-            break;
     }
 
     freeaddrinfo(ai);
 
-    free(sock.write.buf.data);
-    free(sock.read.buf.data);
+    free(blks);
+
+    free(sock.write.data);
+    free(sock.read.data);
+
+    free(tun.write.data);
+    free(tun.read.data);
 
     return 0;
 }

src/option.c

@@ -5,18 +5,10 @@
 
 #include "option.h"
 
-int option_flag (void *data, _unused_ int argc, _unused_ char **argv)
-{
-    const int one = 1;
-    byte_cpy(data, &one, sizeof(one));
-
-    return 0;
-}
-
 int option_str (void *data, int argc, char **argv)
 {
     if (argc<2 || !argv[1]) {
-        printf("option `%s' need a string argument\n", argv[0]);
+        gt_print("option `%s' need a string argument\n", argv[0]);
         return -1;
     }
 
@@ -28,7 +20,7 @@ int option_str (void *data, int argc, char **argv)
 int option_long (void *data, int argc, char **argv)
 {
     if (argc<2 || !argv[1]) {
-        printf("option `%s' need an integer argument\n", argv[0]);
+        gt_print("option `%s' need an integer argument\n", argv[0]);
         return -1;
     }
 
@@ -37,7 +29,7 @@ int option_long (void *data, int argc, char **argv)
     long val = strtol(argv[1], &end, 0);
 
     if (errno || argv[1]==end) {
-        printf("argument `%s' is not a valid integer\n", argv[1]);
+        gt_print("argument `%s' is not a valid integer\n", argv[1]);
         return -1;
     }
 
@@ -46,8 +38,21 @@ int option_long (void *data, int argc, char **argv)
     return 1;
 }
 
+int option_is_set (struct option *opts, const char *name)
+{
+    for (int k=0; opts[k].name; k++) {
+        if (!str_cmp(opts[k].name, name))
+            return opts[k].set;
+    }
+
+    return 0;
+}
+
 int option_option (void *data, int argc, char **argv)
 {
+    if (!data)
+        return 0;
+
     struct option *opts = (struct option *)data;
 
     for (int k=0; opts[k].name; k++)
@@ -61,7 +66,7 @@ int option_option (void *data, int argc, char **argv)
                 continue;
 
             if (opts[k].set) {
-                printf("option `%s' is already set\n", opts[k].name);
+                gt_print("option `%s' is already set\n", opts[k].name);
                 return -1;
             }
 
@@ -84,31 +89,31 @@ int option_option (void *data, int argc, char **argv)
     return argc;
 }
 
-static void option_usage (struct option *opts, char *name)
+static int option_usage (struct option *opts, int slen)
 {
-    char *usage = "usage: ";
-    size_t slen = str_len(usage)+str_len(name);
-    size_t len = slen;
+    if (!opts)
+        return 0;
 
-    printf("%s%s", usage, name);
-
-    if (slen>40)
-        slen = 12;
+    int len = slen;
 
     for (int k=0; opts[k].name; k++) {
-        char *arg = (opts[k].call==option_flag)?"":" ARG";
-        size_t inc = str_len(opts[k].name)+str_len(arg)+3;
-
-        if (len+inc>72) {
-            printf("\n%*s", (int)slen, "");
+        if (len>slen+40) {
+            gt_print("\n%*s", (int)slen, "");
             len = slen;
         }
 
-        printf(" [%s%s]", opts[k].name, arg);
-        len += inc;
+        len += gt_print(" [%s", opts[k].name);
+
+        if (opts[k].call==option_option) {
+            len += option_usage((struct option *)opts[k].data, len);
+        } else {
+            len += gt_print(" ARG");
         }
 
-    printf("\n");
+        len += gt_print("]");
+    }
+
+    return len;
 }
 
 int option (struct option *opts, int argc, char **argv)
@@ -121,8 +126,18 @@ int option (struct option *opts, int argc, char **argv)
     if (ret<0 || ret+1>=argc)
         return 1;
 
-    printf("option `%s' is unknown\n", argv[ret+1]);
-    option_usage(opts, argv[0]);
+    gt_print("option `%s' is unknown\n", argv[ret+1]);
+
+    int slen = gt_print("usage: %s", argv[0]);
+
+    if (slen>40) {
+        slen = 12;
+        gt_print("\n%*s", (int)slen, "");
+    }
+
+    option_usage(opts, slen);
+
+    printf("\n");
 
     return 1;
 }

src/option.h

@@ -7,9 +7,9 @@ struct option {
     int set;
 };
 
-int option_flag   (void *, int, char **);
+int option_option (void *, int, char **);
 int option_str    (void *, int, char **);
 int option_long   (void *, int, char **);
-int option_option (void *, int, char **);
 
+int option_is_set (struct option *, const char *);
 int option        (struct option *, int, char **);

src/tun.c

@@ -56,42 +56,43 @@ int tun_create (char *name, int multiqueue)
         return -1;
     }
 
-    printf("tun name: %s\n", ifr.ifr_name);
+    gt_log("tun name: %s\n", ifr.ifr_name);
 
     return fd;
 }
 #elif defined(__APPLE__)
 int tun_create (_unused_ char *name, _unused_ int mq)
 {
-    struct ctl_info ctlInfo;
-    struct sockaddr_ctl sc;
-    int fd;
+    for (unsigned dev_id = 0; dev_id < 32; dev_id++) {
+        struct ctl_info ci;
 
-    for (unsigned dev_id = 0U; dev_id<32U; dev_id++) {
-        byte_set(&ctlInfo, 0, sizeof(ctlInfo));
-        str_cpy(ctlInfo.ctl_name, UTUN_CONTROL_NAME, sizeof(ctlInfo.ctl_name));
-        fd = socket(PF_SYSTEM, SOCK_DGRAM, SYSPROTO_CONTROL);
+        byte_set(&ci, 0, sizeof(ci));
+        str_cpy(ci.ctl_name, UTUN_CONTROL_NAME, sizeof(ci.ctl_name)-1);
+
+        int fd = socket(PF_SYSTEM, SOCK_DGRAM, SYSPROTO_CONTROL);
 
         if (fd==-1)
             return -1;
 
-        if (ioctl(fd, CTLIOCGINFO, &ctlInfo)==-1) {
+        if (ioctl(fd, CTLIOCGINFO, &ci)==-1) {
             close(fd);
             continue;
         }
 
-        sc.sc_id = ctlInfo.ctl_id;
-        sc.sc_len = sizeof(sc);
-        sc.sc_family = AF_SYSTEM;
-        sc.ss_sysaddr = AF_SYS_CONTROL;
-        sc.sc_unit = dev_id+1;
+        struct sockaddr_ctl sc = {
+            .sc_id = ci.ctl_id,
+            .sc_len = sizeof(sc),
+            .sc_family = AF_SYSTEM,
+            .ss_sysaddr = AF_SYS_CONTROL,
+            .sc_unit = dev_id+1,
+        };
 
-        if (connect(fd, (struct sockaddr *) &sc, sizeof(sc))==-1) {
+        if (connect(fd, (struct sockaddr *)&sc, sizeof(sc))==-1) {
             close(fd);
             continue;
         }
 
-        printf("tun name: /dev/utun%u\n", dev_id);
+        gt_log("tun name: /dev/utun%u\n", dev_id);
 
         return fd;
     }
@@ -101,18 +102,20 @@ int tun_create (_unused_ char *name, _unused_ int mq)
 #else
 int tun_create (_unused_ char *name, _unused_ int mq)
 {
-    for (unsigned dev_id = 0U; dev_id<32U; dev_id++) {
-        char dev_path[11U];
+    for (unsigned dev_id = 0; dev_id < 32; dev_id++) {
+        char dev_path[11];
 
         snprintf(dev_path, sizeof(dev_path), "/dev/tun%u", dev_id);
 
         int fd = open(dev_path, O_RDWR);
 
-        if (fd!=-1) {
-            printf("tun name: /dev/tun%u\n", dev_id);
+        if (fd==-1)
+            continue;
+
+        gt_log("tun name: %s\n", dev_path);
+
         return fd;
     }
-    }
 
     return -1;
 }
@@ -121,10 +124,11 @@ int tun_create (_unused_ char *name, _unused_ int mq)
 ssize_t tun_read (int fd, void *data, size_t size)
 {
     if (!size)
-        return -2;
+        return -1;
 
 #ifdef GT_BSD_TUN
     uint32_t family;
+
     struct iovec iov[2] = {
         { .iov_base = &family, .iov_len = sizeof(family) },
         { .iov_base = data, .iov_len = size }
@@ -140,7 +144,7 @@ ssize_t tun_read (int fd, void *data, size_t size)
             return -1;
 
         if (errno)
-            perror("readv");
+            perror("tun read");
 
         return 0;
     }
@@ -158,7 +162,7 @@ ssize_t tun_read (int fd, void *data, size_t size)
 ssize_t tun_write (int fd, const void *data, size_t size)
 {
     if (!size)
-        return -2;
+        return -1;
 
 #ifdef GT_BSD_TUN
     uint32_t family;
@@ -189,7 +193,7 @@ ssize_t tun_write (int fd, const void *data, size_t size)
             return -1;
 
         if (errno)
-            perror("write");
+            perror("tun write");
 
         return 0;
     }