/* * Copyright (c) 2015 Kazuho Oku, DeNA Co., Ltd. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__linux__) #include #include #include #elif defined(__APPLE__) #include #elif defined(__FreeBSD__) #include #elif defined(__sun) #include #endif /* to maximize code-reuse between different stacks, we intentionally use API declared by OpenSSL as legacy */ #define OPENSSL_SUPPRESS_DEPRECATED #include #include #if defined(LIBRESSL_VERSION_NUMBER) ? LIBRESSL_VERSION_NUMBER >= 0x3050000fL : OPENSSL_VERSION_NUMBER >= 0x1010000fL /* RSA_METHOD is opaque, so RSA_meth* are used. */ #define NEVERBLEED_OPAQUE_RSA_METHOD #endif #if OPENSSL_VERSION_NUMBER >= 0x1010000fL && !defined(OPENSSL_NO_EC) && \ (!defined(LIBRESSL_VERSION_NUMBER) || LIBRESSL_VERSION_NUMBER >= 0x2090100fL) /* EC_KEY_METHOD and related APIs are avaliable, so ECDSA is enabled. */ #define NEVERBLEED_ECDSA #endif #include #ifdef NEVERBLEED_ECDSA #include #endif #include #include #include #ifdef __linux #if OPENSSL_VERSION_NUMBER >= 0x1010000fL && !defined(LIBRESSL_VERSION_NUMBER) && !defined(OPENSSL_IS_BORINGSSL) #define USE_OFFLOAD 1 #endif #if defined(OPENSSL_IS_BORINGSSL) && defined(NEVERBLEED_BORINGSSL_USE_QAT) #include "qat_bssl.h" /* the mapping seems to be missing */ #ifndef ASYNC_WAIT_CTX_get_all_fds extern int bssl_async_wait_ctx_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd, size_t *numfds); #define ASYNC_WAIT_CTX_get_all_fds bssl_async_wait_ctx_get_all_fds #endif #define USE_OFFLOAD 1 #endif #endif #if OPENSSL_VERSION_NUMBER < 0x1010000fL || (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x2070000fL) static void RSA_get0_key(const RSA *rsa, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d) { if (n) { *n = rsa->n; } if (e) { *e = rsa->e; } if (d) { *d = rsa->d; } } static int RSA_set0_key(RSA *rsa, BIGNUM *n, BIGNUM *e, BIGNUM *d) { if (n == NULL || e == NULL) { return 0; } BN_free(rsa->n); BN_free(rsa->e); BN_free(rsa->d); rsa->n = n; rsa->e = e; rsa->d = d; return 1; } static void RSA_set_flags(RSA *r, int flags) { r->flags |= flags; } #define EVP_PKEY_up_ref(p) CRYPTO_add(&(p)->references, 1, CRYPTO_LOCK_EVP_PKEY) #endif #include "neverbleed.h" enum neverbleed_type { NEVERBLEED_TYPE_ERROR, NEVERBLEED_TYPE_RSA, NEVERBLEED_TYPE_ECDSA }; struct st_neverbleed_rsa_exdata_t { neverbleed_t *nb; size_t key_index; }; struct st_neverbleed_thread_data_t { pid_t self_pid; int fd; }; /** * a variant of pthread_once, that does not require you to declare a callback, nor have a global variable */ #define NEVERBLEED_MULTITHREAD_ONCE(block) \ do { \ static volatile int lock = 0; \ int lock_loaded = lock; \ __sync_synchronize(); \ if (!lock_loaded) { \ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; \ pthread_mutex_lock(&mutex); \ if (!lock) { \ do { \ block \ } while (0); \ __sync_synchronize(); \ lock = 1; \ } \ pthread_mutex_unlock(&mutex); \ } \ } while (0) static void warnvf(const char *fmt, va_list args) { char errbuf[256]; if (errno != 0) { strerror_r(errno, errbuf, sizeof(errbuf)); } else { errbuf[0] = '\0'; } fprintf(stderr, "[openssl-privsep] "); vfprintf(stderr, fmt, args); if (errbuf[0] != '\0') fputs(errbuf, stderr); fputc('\n', stderr); } __attribute__((format(printf, 1, 2))) static void warnf(const char *fmt, ...) { va_list args; va_start(args, fmt); warnvf(fmt, args); va_end(args); } __attribute__((format(printf, 1, 2), noreturn)) static void dief(const char *fmt, ...) { va_list args; va_start(args, fmt); warnvf(fmt, args); va_end(args); abort(); } static char *dirname(const char *path) { const char *last_slash = strrchr(path, '/'); char *ret; if (last_slash == NULL) { errno = 0; dief("dirname: no slash in given path:%s", path); } if ((ret = malloc(last_slash + 1 - path)) == NULL) dief("no memory"); memcpy(ret, path, last_slash - path); ret[last_slash - path] = '\0'; return ret; } static void set_cloexec(int fd) { if (fcntl(fd, F_SETFD, O_CLOEXEC) == -1) dief("failed to set O_CLOEXEC to fd %d", fd); } static int read_nbytes(int fd, void *p, size_t sz) { while (sz != 0) { ssize_t r; while ((r = read(fd, p, sz)) == -1 && errno == EINTR) ; if (r == -1) { return -1; } else if (r == 0) { errno = 0; return -1; } p = (char *)p + r; sz -= r; } return 0; } /** * This function disposes of the memory allocated for `neverbleed_iobuf_t`, but retains the value of `next` and `processing` so that * the buffer can be "cleared" while in use by worker threads. */ static void iobuf_dispose(neverbleed_iobuf_t *buf) { if (buf->capacity != 0) OPENSSL_cleanse(buf->buf, buf->capacity); free(buf->buf); buf->buf = NULL; buf->start = NULL; buf->end = NULL; buf->capacity = 0; } static void iobuf_reserve(neverbleed_iobuf_t *buf, size_t extra) { size_t start_off, end_off; if (extra <= buf->buf - buf->end + buf->capacity) return; if (buf->capacity == 0) buf->capacity = 4096; while (buf->buf - buf->end + buf->capacity < extra) buf->capacity *= 2; if (buf->buf != NULL) { start_off = buf->start - buf->buf; end_off = buf->end - buf->buf; } else { /* C99 forbids us doing `buf->start - buf->buf` when both are NULL (undefined behavior) */ start_off = 0; end_off = 0; } if ((buf->buf = realloc(buf->buf, buf->capacity)) == NULL) dief("realloc failed"); buf->start = buf->buf + start_off; buf->end = buf->buf + end_off; } static void iobuf_push_num(neverbleed_iobuf_t *buf, size_t v) { iobuf_reserve(buf, sizeof(v)); memcpy(buf->end, &v, sizeof(v)); buf->end += sizeof(v); } static void iobuf_push_str(neverbleed_iobuf_t *buf, const char *s) { size_t l = strlen(s) + 1; iobuf_reserve(buf, l); memcpy(buf->end, s, l); buf->end += l; } static void iobuf_push_bytes(neverbleed_iobuf_t *buf, const void *p, size_t l) { iobuf_push_num(buf, l); iobuf_reserve(buf, l); memcpy(buf->end, p, l); buf->end += l; } static int iobuf_shift_num(neverbleed_iobuf_t *buf, size_t *v) { if (neverbleed_iobuf_size(buf) < sizeof(*v)) return -1; memcpy(v, buf->start, sizeof(*v)); buf->start += sizeof(*v); return 0; } static char *iobuf_shift_str(neverbleed_iobuf_t *buf) { char *nul = memchr(buf->start, '\0', neverbleed_iobuf_size(buf)), *ret; if (nul == NULL) return NULL; ret = buf->start; buf->start = nul + 1; return ret; } static void *iobuf_shift_bytes(neverbleed_iobuf_t *buf, size_t *l) { void *ret; if (iobuf_shift_num(buf, l) != 0) return NULL; if (neverbleed_iobuf_size(buf) < *l) return NULL; ret = buf->start; buf->start += *l; return ret; } static int iobuf_write(neverbleed_iobuf_t *buf, int fd) { struct iovec vecs[2] = {{NULL}}; size_t bufsz = neverbleed_iobuf_size(buf); int vecindex; ssize_t r; vecs[0].iov_base = &bufsz; vecs[0].iov_len = sizeof(bufsz); vecs[1].iov_base = buf->start; vecs[1].iov_len = bufsz; for (vecindex = 0; vecindex != sizeof(vecs) / sizeof(vecs[0]);) { while ((r = writev(fd, vecs + vecindex, sizeof(vecs) / sizeof(vecs[0]) - vecindex)) == -1 && errno == EINTR) ; if (r == -1) return -1; assert(r != 0); while (r != 0 && r >= vecs[vecindex].iov_len) { r -= vecs[vecindex].iov_len; ++vecindex; } if (r != 0) { vecs[vecindex].iov_base = (char *)vecs[vecindex].iov_base + r; vecs[vecindex].iov_len -= r; } } return 0; } static int iobuf_read(neverbleed_iobuf_t *buf, int fd) { size_t sz; if (read_nbytes(fd, &sz, sizeof(sz)) != 0) return -1; iobuf_reserve(buf, sz); if (read_nbytes(fd, buf->end, sz) != 0) return -1; buf->end += sz; return 0; } void neverbleed_iobuf_dispose(neverbleed_iobuf_t *buf) { iobuf_dispose(buf); } static void iobuf_transaction_write(neverbleed_iobuf_t *buf, struct st_neverbleed_thread_data_t *thdata) { if (iobuf_write(buf, thdata->fd) == -1) { if (errno != 0) { dief("write error (%d) %s", errno, strerror(errno)); } else { dief("connection closed by daemon"); } } } static void iobuf_transaction_read(neverbleed_iobuf_t *buf, struct st_neverbleed_thread_data_t *thdata) { iobuf_dispose(buf); if (iobuf_read(buf, thdata->fd) == -1) { if (errno != 0) { dief("read error (%d) %s", errno, strerror(errno)); } else { dief("connection closed by daemon"); } } } /** * Only sends a request, does not read a response */ static void iobuf_transaction_no_response(neverbleed_iobuf_t *buf, struct st_neverbleed_thread_data_t *thdata) { if (neverbleed_transaction_cb != NULL) { neverbleed_transaction_cb(buf, 1); } else { iobuf_transaction_write(buf, thdata); iobuf_dispose(buf); } } /** * Sends a request and reads a response. */ static void iobuf_transaction(neverbleed_iobuf_t *buf, struct st_neverbleed_thread_data_t *thdata) { if (neverbleed_transaction_cb != NULL) { neverbleed_transaction_cb(buf, 0); } else { iobuf_transaction_write(buf, thdata); iobuf_transaction_read(buf, thdata); } } #if !defined(NAME_MAX) || defined(__linux__) /* readdir(3) is known to be thread-safe on Linux and should be thread-safe on a platform that does not have a predefined value for NAME_MAX */ #define FOREACH_DIRENT(dp, dent) \ struct dirent *dent; \ while ((dent = readdir(dp)) != NULL) #else #define FOREACH_DIRENT(dp, dent) \ struct { \ struct dirent d; \ char s[NAME_MAX + 1]; \ } dent_; \ struct dirent *dentp, *dent = &dent_.d; \ int ret; \ while ((ret = readdir_r(dp, dent, &dentp)) == 0 && dentp != NULL) #endif /* FOREACH_DIRENT */ static void unlink_dir(const char *path) { DIR *dp; char buf[PATH_MAX]; if ((dp = opendir(path)) != NULL) { FOREACH_DIRENT(dp, entp) { if (strcmp(entp->d_name, ".") == 0 || strcmp(entp->d_name, "..") == 0) continue; snprintf(buf, sizeof(buf), "%s/%s", path, entp->d_name); unlink_dir(buf); } closedir(dp); } unlink(path); rmdir(path); } static void dispose_thread_data(void *_thdata) { struct st_neverbleed_thread_data_t *thdata = _thdata; assert(thdata->fd >= 0); close(thdata->fd); thdata->fd = -1; free(thdata); } static struct st_neverbleed_thread_data_t *get_thread_data(neverbleed_t *nb) { struct st_neverbleed_thread_data_t *thdata; pid_t self_pid = getpid(); ssize_t r; if ((thdata = pthread_getspecific(nb->thread_key)) != NULL) { if (thdata->self_pid == self_pid) return thdata; /* we have been forked! */ close(thdata->fd); } else { if ((thdata = malloc(sizeof(*thdata))) == NULL) dief("malloc failed"); } thdata->self_pid = self_pid; #ifdef SOCK_CLOEXEC if ((thdata->fd = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)) == -1) dief("socket(2) failed"); #else if ((thdata->fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) dief("socket(2) failed"); set_cloexec(thdata->fd); #endif while (connect(thdata->fd, (void *)&nb->sun_, sizeof(nb->sun_)) != 0) if (errno != EINTR) dief("failed to connect to privsep daemon"); while ((r = write(thdata->fd, nb->auth_token, sizeof(nb->auth_token))) == -1 && errno == EINTR) ; if (r != sizeof(nb->auth_token)) dief("failed to send authentication token"); pthread_setspecific(nb->thread_key, thdata); return thdata; } int neverbleed_get_fd(neverbleed_t *nb) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); return thdata->fd; } void neverbleed_transaction_read(neverbleed_t *nb, neverbleed_iobuf_t *buf) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); iobuf_transaction_read(buf, thdata); } void neverbleed_transaction_write(neverbleed_t *nb, neverbleed_iobuf_t *buf) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); iobuf_transaction_write(buf, thdata); } static void do_exdata_free_callback(void *parent, void *ptr, CRYPTO_EX_DATA *ad, int idx, long argl, void *argp) { /* when other engines are used, this callback gets called without neverbleed data */ if (ptr == NULL) return; struct st_neverbleed_rsa_exdata_t *exdata = ptr; struct st_neverbleed_thread_data_t *thdata = get_thread_data(exdata->nb); neverbleed_iobuf_t buf = {NULL}; iobuf_push_str(&buf, "del_pkey"); iobuf_push_num(&buf, exdata->key_index); // "del_pkey" command is fire-and-forget, it cannot fail, so doesn't have a response iobuf_transaction_no_response(&buf, thdata); free(exdata); } static int get_rsa_exdata_idx(void); static void rsa_exdata_free_callback(void *parent, void *ptr, CRYPTO_EX_DATA *ad, int idx, long argl, void *argp) { assert(idx == get_rsa_exdata_idx()); do_exdata_free_callback(parent, ptr, ad, idx, argl, argp); } static int get_rsa_exdata_idx(void) { static volatile int index; NEVERBLEED_MULTITHREAD_ONCE({ index = RSA_get_ex_new_index(0, NULL, NULL, NULL, rsa_exdata_free_callback); }); return index; } static void get_privsep_data(const RSA *rsa, struct st_neverbleed_rsa_exdata_t **exdata, struct st_neverbleed_thread_data_t **thdata) { *exdata = RSA_get_ex_data(rsa, get_rsa_exdata_idx()); if (*exdata == NULL) { errno = 0; dief("invalid internal ref"); } *thdata = get_thread_data((*exdata)->nb); } static struct { struct { pthread_mutex_t lock; /** * if the slot is use contains a non-NULL key; if not in use, contains the index of the next empty slot or SIZE_MAX if there * are no more empty slots */ union { EVP_PKEY *pkey; size_t next_empty; } *slots; size_t num_slots; size_t first_empty; } keys; neverbleed_t *nb; } daemon_vars = {{.lock = PTHREAD_MUTEX_INITIALIZER, .first_empty = SIZE_MAX}}; static __thread struct { int sockfd; #ifdef __linux int epollfd; #endif struct { neverbleed_iobuf_t *first, **next; } responses; } conn_ctx; static int use_offload = 0; #if USE_OFFLOAD struct engine_request { neverbleed_iobuf_t *buf; int async_fd; #ifdef OPENSSL_IS_BORINGSSL struct { RSA *rsa; uint8_t output[512]; union { struct { uint8_t padded[512]; } digestsign; }; } data; async_ctx *async_ctx; #else int (*stub)(neverbleed_iobuf_t *); struct { ASYNC_WAIT_CTX *ctx; ASYNC_JOB *job; } async; #endif }; static void offload_free_request(struct engine_request *req) { #ifdef OPENSSL_IS_BORINGSSL bssl_qat_async_finish_job(req->async_ctx); RSA_free(req->data.rsa); #else ASYNC_WAIT_CTX_free(req->async.ctx); #endif OPENSSL_cleanse(req, sizeof(*req)); free(req); } static int do_epoll_ctl(int epollfd, int op, int fd, struct epoll_event *event) { int ret; while ((ret = epoll_ctl(epollfd, op, fd, event) != 0) && errno == EINTR) ; return ret; } static void register_wait_fd(struct engine_request *req) { #ifdef OPENSSL_IS_BORINGSSL ASYNC_WAIT_CTX *ctx = req->async_ctx->currjob->waitctx; #else ASYNC_WAIT_CTX *ctx = req->async.ctx; #endif size_t numfds; if (!ASYNC_WAIT_CTX_get_all_fds(ctx, NULL, &numfds) || numfds != 1) dief("unexpected number of fds (%zu) requested in async mode\n", numfds); if (!ASYNC_WAIT_CTX_get_all_fds(ctx, &req->async_fd, &numfds)) dief("ASYNC_WAIT_CTX_get_all_fds failed\n"); struct epoll_event ev = {.events = EPOLLIN, .data.ptr = req}; if (do_epoll_ctl(conn_ctx.epollfd, EPOLL_CTL_ADD, req->async_fd, &ev) != 0) dief("epoll_ctl failed:%d\n", errno); } #endif static int send_responses(int cleanup) { neverbleed_iobuf_t *buf; int result = 0; /* Send all buffers that have data being filled. The lock is held until everything is being done, as this function can be called * from multiple threads simultaneously. */ while ((buf = conn_ctx.responses.first) != NULL && !buf->processing) { if ((conn_ctx.responses.first = buf->next) == NULL) conn_ctx.responses.next = &conn_ctx.responses.first; if (!cleanup && iobuf_write(buf, conn_ctx.sockfd) != 0) { warnf(errno != 0 ? "write error" : "connection closed by client"); result = -1; } iobuf_dispose(buf); free(buf); if (result != 0) break; } return result; } static RSA *daemon_get_rsa(size_t key_index) { RSA *rsa = NULL; pthread_mutex_lock(&daemon_vars.keys.lock); if (key_index < daemon_vars.keys.num_slots) rsa = EVP_PKEY_get1_RSA(daemon_vars.keys.slots[key_index].pkey); pthread_mutex_unlock(&daemon_vars.keys.lock); return rsa; } size_t allocate_slot(void) { /* expand if all slots are in use */ if (daemon_vars.keys.first_empty == SIZE_MAX) { size_t new_capacity = (daemon_vars.keys.num_slots < 4 ? 4 : daemon_vars.keys.num_slots) * 2; if ((daemon_vars.keys.slots = realloc(daemon_vars.keys.slots, sizeof(daemon_vars.keys.slots[0]) * new_capacity)) == NULL) dief("no memory"); daemon_vars.keys.first_empty = daemon_vars.keys.num_slots; for (size_t i = daemon_vars.keys.num_slots; i < new_capacity - 1; ++i) daemon_vars.keys.slots[i].next_empty = i + 1; daemon_vars.keys.slots[new_capacity - 1].next_empty = SIZE_MAX; daemon_vars.keys.num_slots = new_capacity; } /* detach the first empty slot from the empty list */ size_t slot_index = daemon_vars.keys.first_empty; daemon_vars.keys.first_empty = daemon_vars.keys.slots[slot_index].next_empty; /* set bogus value in the allocated slot to help figure out what happened upon crash */ daemon_vars.keys.slots[slot_index].next_empty = SIZE_MAX - 1; return slot_index; } static size_t daemon_set_pkey(EVP_PKEY *pkey) { assert(pkey != NULL); pthread_mutex_lock(&daemon_vars.keys.lock); size_t index = allocate_slot(); daemon_vars.keys.slots[index].pkey = pkey; EVP_PKEY_up_ref(pkey); pthread_mutex_unlock(&daemon_vars.keys.lock); return index; } static int priv_encdec_proxy(const char *cmd, int flen, const unsigned char *from, unsigned char *_to, RSA *rsa, int padding) { struct st_neverbleed_rsa_exdata_t *exdata; struct st_neverbleed_thread_data_t *thdata; neverbleed_iobuf_t buf = {NULL}; size_t ret; unsigned char *to; size_t tolen; get_privsep_data(rsa, &exdata, &thdata); iobuf_push_str(&buf, cmd); iobuf_push_bytes(&buf, from, flen); iobuf_push_num(&buf, exdata->key_index); iobuf_push_num(&buf, padding); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &ret) != 0 || (to = iobuf_shift_bytes(&buf, &tolen)) == NULL) { errno = 0; dief("failed to parse response"); } memcpy(_to, to, tolen); iobuf_dispose(&buf); return (int)ret; } static int priv_encdec_stub(const char *name, int (*func)(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding), neverbleed_iobuf_t *buf) { unsigned char *from, to[4096]; size_t flen; size_t key_index, padding; RSA *rsa; int ret; if ((from = iobuf_shift_bytes(buf, &flen)) == NULL || iobuf_shift_num(buf, &key_index) != 0 || iobuf_shift_num(buf, &padding) != 0) { errno = 0; warnf("%s: failed to parse request", name); return -1; } if ((rsa = daemon_get_rsa(key_index)) == NULL) { errno = 0; warnf("%s: invalid key index:%zu\n", name, key_index); return -1; } ret = func((int)flen, from, to, rsa, (int)padding); iobuf_dispose(buf); RSA_free(rsa); iobuf_push_num(buf, ret); iobuf_push_bytes(buf, to, ret > 0 ? ret : 0); return 0; } #if !defined(OPENSSL_IS_BORINGSSL) static int priv_enc_proxy(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { return priv_encdec_proxy("priv_enc", flen, from, to, rsa, padding); } static int priv_enc_stub(neverbleed_iobuf_t *buf) { return priv_encdec_stub(__FUNCTION__, RSA_private_encrypt, buf); } static int priv_dec_proxy(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { return priv_encdec_proxy("priv_dec", flen, from, to, rsa, padding); } static int priv_dec_stub(neverbleed_iobuf_t *buf) { return priv_encdec_stub(__FUNCTION__, RSA_private_decrypt, buf); } static int sign_proxy(int type, const unsigned char *m, unsigned int m_len, unsigned char *_sigret, unsigned *_siglen, const RSA *rsa) { struct st_neverbleed_rsa_exdata_t *exdata; struct st_neverbleed_thread_data_t *thdata; neverbleed_iobuf_t buf = {NULL}; size_t ret, siglen; unsigned char *sigret; get_privsep_data(rsa, &exdata, &thdata); iobuf_push_str(&buf, "sign"); iobuf_push_num(&buf, type); iobuf_push_bytes(&buf, m, m_len); iobuf_push_num(&buf, exdata->key_index); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &ret) != 0 || (sigret = iobuf_shift_bytes(&buf, &siglen)) == NULL) { errno = 0; dief("failed to parse response"); } memcpy(_sigret, sigret, siglen); *_siglen = (unsigned)siglen; iobuf_dispose(&buf); return (int)ret; } static int sign_stub(neverbleed_iobuf_t *buf) { unsigned char *m, sigret[4096]; size_t type, m_len, key_index; RSA *rsa; unsigned siglen = 0; int ret; if (iobuf_shift_num(buf, &type) != 0 || (m = iobuf_shift_bytes(buf, &m_len)) == NULL || iobuf_shift_num(buf, &key_index) != 0) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } if ((rsa = daemon_get_rsa(key_index)) == NULL) { errno = 0; warnf("%s: invalid key index:%zu", __FUNCTION__, key_index); return -1; } ret = RSA_sign((int)type, m, (unsigned)m_len, sigret, &siglen, rsa); iobuf_dispose(buf); RSA_free(rsa); iobuf_push_num(buf, ret); iobuf_push_bytes(buf, sigret, ret == 1 ? siglen : 0); return 0; } #endif static EVP_PKEY *create_pkey(neverbleed_t *nb, size_t key_index, const char *ebuf, const char *nbuf) { struct st_neverbleed_rsa_exdata_t *exdata; RSA *rsa; EVP_PKEY *pkey; BIGNUM *e = NULL, *n = NULL; if ((exdata = malloc(sizeof(*exdata))) == NULL) { fprintf(stderr, "no memory\n"); abort(); } exdata->nb = nb; exdata->key_index = key_index; rsa = RSA_new_method(nb->engine); RSA_set_ex_data(rsa, get_rsa_exdata_idx(), exdata); if (BN_hex2bn(&e, ebuf) == 0) { fprintf(stderr, "failed to parse e:%s\n", ebuf); abort(); } if (BN_hex2bn(&n, nbuf) == 0) { fprintf(stderr, "failed to parse n:%s\n", nbuf); abort(); } RSA_set0_key(rsa, n, e, NULL); #if !defined(OPENSSL_IS_BORINGSSL) RSA_set_flags(rsa, RSA_FLAG_EXT_PKEY); #endif pkey = EVP_PKEY_new(); EVP_PKEY_set1_RSA(pkey, rsa); RSA_free(rsa); return pkey; } #ifdef NEVERBLEED_ECDSA static EC_KEY *daemon_get_ecdsa(size_t key_index) { EC_KEY *ec_key = NULL; pthread_mutex_lock(&daemon_vars.keys.lock); if (key_index < daemon_vars.keys.num_slots) ec_key = EVP_PKEY_get1_EC_KEY(daemon_vars.keys.slots[key_index].pkey); pthread_mutex_unlock(&daemon_vars.keys.lock); return ec_key; } static int ecdsa_sign_stub(neverbleed_iobuf_t *buf) { unsigned char *m, sigret[4096]; size_t type, m_len, key_index; EC_KEY *ec_key; unsigned siglen = 0; int ret; if (iobuf_shift_num(buf, &type) != 0 || (m = iobuf_shift_bytes(buf, &m_len)) == NULL || iobuf_shift_num(buf, &key_index) != 0) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } if ((ec_key = daemon_get_ecdsa(key_index)) == NULL) { errno = 0; warnf("%s: invalid key index:%zu", __FUNCTION__, key_index); return -1; } ret = ECDSA_sign((int)type, m, (unsigned)m_len, sigret, &siglen, ec_key); iobuf_dispose(buf); EC_KEY_free(ec_key); iobuf_push_num(buf, ret); iobuf_push_bytes(buf, sigret, ret == 1 ? siglen : 0); return 0; } static int get_ecdsa_exdata_idx(void); static void ecdsa_exdata_free_callback(void *parent, void *ptr, CRYPTO_EX_DATA *ad, int idx, long argl, void *argp) { assert(idx == get_ecdsa_exdata_idx()); do_exdata_free_callback(parent, ptr, ad, idx, argl, argp); } static int get_ecdsa_exdata_idx(void) { static volatile int index; NEVERBLEED_MULTITHREAD_ONCE({ index = EC_KEY_get_ex_new_index(0, NULL, NULL, NULL, ecdsa_exdata_free_callback); }); return index; } static void ecdsa_get_privsep_data(const EC_KEY *ec_key, struct st_neverbleed_rsa_exdata_t **exdata, struct st_neverbleed_thread_data_t **thdata) { *exdata = EC_KEY_get_ex_data(ec_key, get_ecdsa_exdata_idx()); if (*exdata == NULL) { errno = 0; dief("invalid internal ref"); } *thdata = get_thread_data((*exdata)->nb); } static int ecdsa_sign_proxy(int type, const unsigned char *m, int m_len, unsigned char *_sigret, unsigned int *_siglen, const BIGNUM *kinv, const BIGNUM *rp, EC_KEY *ec_key) { struct st_neverbleed_rsa_exdata_t *exdata; struct st_neverbleed_thread_data_t *thdata; neverbleed_iobuf_t buf = {NULL}; size_t ret, siglen; unsigned char *sigret; ecdsa_get_privsep_data(ec_key, &exdata, &thdata); /* as far as I've tested so far, kinv and rp are always NULL. Looks like setup_sign will precompute this, but it is only called sign_sig, and it seems to be not used in TLS ECDSA */ if (kinv != NULL || rp != NULL) { errno = 0; dief("unexpected non-NULL kinv and rp"); } iobuf_push_str(&buf, "ecdsa_sign"); iobuf_push_num(&buf, type); iobuf_push_bytes(&buf, m, m_len); iobuf_push_num(&buf, exdata->key_index); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &ret) != 0 || (sigret = iobuf_shift_bytes(&buf, &siglen)) == NULL) { errno = 0; dief("failed to parse response"); } memcpy(_sigret, sigret, siglen); *_siglen = (unsigned)siglen; iobuf_dispose(&buf); return (int)ret; } static EVP_PKEY *ecdsa_create_pkey(neverbleed_t *nb, size_t key_index, int curve_name, const void *pubkey, size_t pubkey_len) { struct st_neverbleed_rsa_exdata_t *exdata; EC_KEY *ec_key; EC_GROUP *ec_group; EC_POINT *ec_pubkey; EVP_PKEY *pkey; if ((exdata = malloc(sizeof(*exdata))) == NULL) { fprintf(stderr, "no memory\n"); abort(); } exdata->nb = nb; exdata->key_index = key_index; ec_key = EC_KEY_new_method(nb->engine); EC_KEY_set_ex_data(ec_key, get_ecdsa_exdata_idx(), exdata); ec_group = EC_GROUP_new_by_curve_name(curve_name); if (!ec_group) { fprintf(stderr, "could not create EC_GROUP\n"); abort(); } EC_KEY_set_group(ec_key, ec_group); ec_pubkey = EC_POINT_new(ec_group); assert(ec_pubkey != NULL); if (!EC_POINT_oct2point(ec_group, ec_pubkey, pubkey, pubkey_len, NULL)) { fprintf(stderr, "failed to get ECDSA ephemeral public key from BIGNUM\n"); abort(); } EC_KEY_set_public_key(ec_key, ec_pubkey); pkey = EVP_PKEY_new(); EVP_PKEY_set1_EC_KEY(pkey, ec_key); EC_POINT_free(ec_pubkey); EC_GROUP_free(ec_group); EC_KEY_free(ec_key); return pkey; } #endif static EVP_PKEY *daemon_get_pkey(size_t key_index) { EVP_PKEY *pkey = NULL; pthread_mutex_lock(&daemon_vars.keys.lock); if (key_index < daemon_vars.keys.num_slots) { pkey = daemon_vars.keys.slots[key_index].pkey; EVP_PKEY_up_ref(pkey); } pthread_mutex_unlock(&daemon_vars.keys.lock); return pkey; } #if USE_OFFLOAD && defined(OPENSSL_IS_BORINGSSL) static struct engine_request *bssl_offload_create_request(neverbleed_iobuf_t *buf, EVP_PKEY *pkey) { RSA *_rsa = EVP_PKEY_get1_RSA(pkey); struct engine_request *req = malloc(sizeof(*req)); if (req == NULL) dief("no memory\n"); *req = (struct engine_request){.buf = buf, .async_fd = -1, .async_ctx = bssl_qat_async_start_job(), .data.rsa = _rsa}; if (req->async_ctx == NULL) dief("failed to initialize async job\n"); if (RSA_size(req->data.rsa) > sizeof(req->data.output)) dief("RSA key too large\n"); return req; } static void bssl_offload_digestsign(neverbleed_iobuf_t *buf, EVP_PKEY *pkey, const EVP_MD *md, const void *signdata, size_t signlen, int rsa_pss) { uint8_t digest[EVP_MAX_MD_SIZE]; unsigned digestlen; { /* generate digest of signdata */ EVP_MD_CTX *mdctx = EVP_MD_CTX_new(); if (mdctx == NULL) dief("no memory\n"); if (!EVP_DigestInit_ex(mdctx, md, NULL) || !EVP_DigestUpdate(mdctx, signdata, signlen) || !EVP_DigestFinal_ex(mdctx, digest, &digestlen)) dief("digest calculation failed\n"); EVP_MD_CTX_free(mdctx); } struct engine_request *req = bssl_offload_create_request(buf, pkey); size_t rsa_size = RSA_size(req->data.rsa), padded_len; int padding; /* generate padded octets to be signed */ if (rsa_pss) { if (!RSA_padding_add_PKCS1_PSS_mgf1(req->data.rsa, req->data.digestsign.padded, digest, md, md, -1)) dief("RSA_paddding_add_PKCS1_PSS_mgf1 failed\n"); padded_len = rsa_size; padding = RSA_NO_PADDING; } else { /* PKCS1 padding */ int hash_nid = EVP_MD_type(md), is_alloced; uint8_t *tbs; if (!RSA_add_pkcs1_prefix(&tbs, &padded_len, &is_alloced, hash_nid, digest, digestlen)) dief("RSA_add_pkcs1_prefix failed\n"); if (padded_len > rsa_size) dief("output of RSA_add_pkcs1_prefix is unexpectedly large\n"); memcpy(req->data.digestsign.padded, tbs, padded_len); if (is_alloced) OPENSSL_free(tbs); padding = RSA_PKCS1_PADDING; } OPENSSL_cleanse(digest, sizeof(digest)); /* dispatch RSA calculation */ RSA_METHOD *meth = bssl_engine_get_rsa_method(); if (meth == NULL) dief("failed to obtain QAT RSA method table\n"); size_t siglen; if (!meth->sign_raw(req->data.rsa, &siglen, req->data.output, rsa_size, req->data.digestsign.padded, padded_len, padding)) dief("sign_raw failure\n"); if (siglen != 0) dief("sign_raw completed synchronously unexpectedly\n"); buf->processing = 1; register_wait_fd(req); } static int bssl_offload_decrypt(neverbleed_iobuf_t *buf, EVP_PKEY *pkey, const void *src, size_t len) { struct engine_request *req = bssl_offload_create_request(buf, pkey); /* dispatch RSA calculation */ RSA_METHOD *meth = bssl_engine_get_rsa_method(); if (meth == NULL) dief("failed to obtain QAT RSA method table\n"); size_t outlen; if (!meth->decrypt(req->data.rsa, &outlen, req->data.output, sizeof(req->data.output), src, len, RSA_NO_PADDING)) { warnf("RSA decrypt failure\n"); goto Exit; } if (outlen != 0) dief("RSA decrypt completed synchronously unexpectedly\n"); buf->processing = 1; register_wait_fd(req); return 1; Exit: offload_free_request(req); return 0; } #endif static int digestsign_stub(neverbleed_iobuf_t *buf) { size_t key_index, md_nid, signlen; void *signdata; size_t rsa_pss; EVP_PKEY *pkey; const EVP_MD *md; /* parse input */ if (iobuf_shift_num(buf, &key_index) != 0 || iobuf_shift_num(buf, &md_nid) != 0 || (signdata = iobuf_shift_bytes(buf, &signlen)) == NULL || iobuf_shift_num(buf, &rsa_pss) != 0) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } if ((pkey = daemon_get_pkey(key_index)) == NULL) { errno = 0; warnf("%s: invalid key index:%zu", __FUNCTION__, key_index); return -1; } if (md_nid != SIZE_MAX) { if ((md = EVP_get_digestbynid((int)md_nid)) == NULL) { errno = 0; warnf("%s: invalid EVP_MD nid", __FUNCTION__); return -1; } } else { md = NULL; } #if USE_OFFLOAD && defined(OPENSSL_IS_BORINGSSL) if (use_offload && EVP_PKEY_id(pkey) == EVP_PKEY_RSA) { bssl_offload_digestsign(buf, pkey, md, signdata, signlen, rsa_pss); goto Exit; } #endif /* generate signature */ EVP_MD_CTX *mdctx = NULL; EVP_PKEY_CTX *pkey_ctx = NULL; unsigned char digestbuf[4096]; size_t digestlen; if ((mdctx = EVP_MD_CTX_create()) == NULL) goto Softfail; if (EVP_DigestSignInit(mdctx, &pkey_ctx, md, NULL, pkey) != 1) goto Softfail; if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA && rsa_pss) { if (EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING) != 1 || EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, -1) != 1) goto Softfail; if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, md) != 1) goto Softfail; } /* ED25519 keys can never be loaded, so use the Update -> Final call chain without worrying about backward compatibility */ if (EVP_DigestSignUpdate(mdctx, signdata, signlen) != 1) goto Softfail; if (EVP_DigestSignFinal(mdctx, NULL, &digestlen) != 1) goto Softfail; if (sizeof(digestbuf) < digestlen) { warnf("%s: digest unexpectedly long as %zu bytes", __FUNCTION__, digestlen); goto Softfail; } if (EVP_DigestSignFinal(mdctx, digestbuf, &digestlen) != 1) goto Softfail; Respond: /* build response */ iobuf_dispose(buf); iobuf_push_bytes(buf, digestbuf, digestlen); if (mdctx != NULL) EVP_MD_CTX_destroy(mdctx); Exit: if (pkey != NULL) EVP_PKEY_free(pkey); return 0; Softfail: digestlen = 0; goto Respond; } void neverbleed_start_digestsign(neverbleed_iobuf_t *buf, EVP_PKEY *pkey, const EVP_MD *md, const void *input, size_t len, int rsa_pss) { struct st_neverbleed_rsa_exdata_t *exdata; struct st_neverbleed_thread_data_t *thdata; const char *cmd = "digestsign"; /* obtain reference */ switch (EVP_PKEY_base_id(pkey)) { case EVP_PKEY_RSA: { RSA *rsa = EVP_PKEY_get1_RSA(pkey); /* get0 is available not available in OpenSSL 1.0.2 */ get_privsep_data(rsa, &exdata, &thdata); RSA_free(rsa); cmd = "digestsign-rsa"; } break; #ifdef NEVERBLEED_ECDSA case EVP_PKEY_EC: ecdsa_get_privsep_data(EVP_PKEY_get0_EC_KEY(pkey), &exdata, &thdata); break; #endif default: dief("unexpected private key"); break; } *buf = (neverbleed_iobuf_t){NULL}; iobuf_push_str(buf, cmd); iobuf_push_num(buf, exdata->key_index); iobuf_push_num(buf, md != NULL ? (size_t)EVP_MD_nid(md) : SIZE_MAX); iobuf_push_bytes(buf, input, len); iobuf_push_num(buf, rsa_pss); } void neverbleed_finish_digestsign(neverbleed_iobuf_t *buf, void **digest, size_t *digest_len) { const void *src; if ((src = iobuf_shift_bytes(buf, digest_len)) == NULL) { errno = 0; dief("failed to parse response"); } if ((*digest = malloc(*digest_len)) == NULL) dief("no memory"); memcpy(*digest, src, *digest_len); iobuf_dispose(buf); } static int decrypt_stub(neverbleed_iobuf_t *buf) { size_t key_index, srclen; void *src; EVP_PKEY *pkey; RSA *rsa; uint8_t decryptbuf[1024]; int decryptlen; /* parse input */ if (iobuf_shift_num(buf, &key_index) != 0 || (src = iobuf_shift_bytes(buf, &srclen)) == NULL) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } if ((pkey = daemon_get_pkey(key_index)) == NULL) { errno = 0; warnf("%s: invalid key index:%zu", __FUNCTION__, key_index); return -1; } rsa = EVP_PKEY_get1_RSA(pkey); /* get0 is available not available in OpenSSL 1.0.2 */ assert(rsa != NULL); assert(sizeof(decryptbuf) >= RSA_size(rsa)); #if USE_OFFLOAD && defined(OPENSSL_IS_BORINGSSL) if (use_offload) { if (!bssl_offload_decrypt(buf, pkey, src, srclen)) goto Softfail; goto Exit; } #endif if ((decryptlen = RSA_private_decrypt(srclen, src, decryptbuf, rsa, RSA_NO_PADDING)) == -1) { errno = 0; warnf("RSA decryption error"); goto Softfail; } Respond: iobuf_dispose(buf); iobuf_push_bytes(buf, decryptbuf, decryptlen); Exit: RSA_free(rsa); EVP_PKEY_free(pkey); return 0; Softfail: decryptlen = 0; goto Respond; } void neverbleed_start_decrypt(neverbleed_iobuf_t *buf, EVP_PKEY *pkey, const void *input, size_t len) { struct st_neverbleed_rsa_exdata_t *exdata; struct st_neverbleed_thread_data_t *thdata; { RSA *rsa = EVP_PKEY_get1_RSA(pkey); /* get0 is available not available in OpenSSL 1.0.2 */ assert(rsa != NULL); get_privsep_data(rsa, &exdata, &thdata); RSA_free(rsa); } *buf = (neverbleed_iobuf_t){NULL}; iobuf_push_str(buf, "decrypt"); iobuf_push_num(buf, exdata->key_index); iobuf_push_bytes(buf, input, len); } void neverbleed_finish_decrypt(neverbleed_iobuf_t *buf, void **digest, size_t *digest_len) { neverbleed_finish_digestsign(buf, digest, digest_len); } int neverbleed_load_private_key_file(neverbleed_t *nb, SSL_CTX *ctx, const char *fn, char *errbuf) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); neverbleed_iobuf_t buf = {NULL}; int ret = 1; size_t index, type; EVP_PKEY *pkey; iobuf_push_str(&buf, "load_key"); iobuf_push_str(&buf, fn); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &type) != 0 || iobuf_shift_num(&buf, &index) != 0) { errno = 0; dief("failed to parse response"); } switch (type) { case NEVERBLEED_TYPE_RSA: { char *estr, *nstr; if ((estr = iobuf_shift_str(&buf)) == NULL || (nstr = iobuf_shift_str(&buf)) == NULL) { errno = 0; dief("failed to parse response"); } pkey = create_pkey(nb, index, estr, nstr); break; } #ifdef NEVERBLEED_ECDSA case NEVERBLEED_TYPE_ECDSA: { size_t curve_name, pubkey_len; void *pubkey_bytes; if (iobuf_shift_num(&buf, &curve_name) != 0 || (pubkey_bytes = iobuf_shift_bytes(&buf, &pubkey_len)) == NULL) { errno = 0; dief("failed to parse response"); } pkey = ecdsa_create_pkey(nb, index, (int)curve_name, pubkey_bytes, pubkey_len); break; } #endif default: { char *errstr; if ((errstr = iobuf_shift_str(&buf)) == NULL) { errno = 0; dief("failed to parse response"); } snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "%s", errstr); return -1; } } iobuf_dispose(&buf); /* success */ if (SSL_CTX_use_PrivateKey(ctx, pkey) != 1) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "SSL_CTX_use_PrivateKey failed"); ret = 0; } EVP_PKEY_free(pkey); return ret; } static int load_key_stub(neverbleed_iobuf_t *buf) { char *fn; FILE *fp = NULL; RSA *rsa = NULL; size_t key_index = SIZE_MAX; char *estr = NULL, *nstr = NULL, errbuf[NEVERBLEED_ERRBUF_SIZE] = ""; size_t type = NEVERBLEED_TYPE_ERROR; EVP_PKEY *pkey = NULL; #ifdef NEVERBLEED_ECDSA const EC_GROUP *ec_group; void *ec_pubkeybytes = NULL; size_t ec_pubkeylen; #endif if ((fn = iobuf_shift_str(buf)) == NULL) { warnf("%s: failed to parse request", __FUNCTION__); return -1; } if ((fp = fopen(fn, "rt")) == NULL) { strerror_r(errno, errbuf, sizeof(errbuf)); goto Respond; } if ((pkey = PEM_read_PrivateKey(fp, NULL, NULL, NULL)) == NULL) { snprintf(errbuf, sizeof(errbuf), "failed to parse the private key"); goto Respond; } switch (EVP_PKEY_base_id(pkey)) { case EVP_PKEY_RSA: { const BIGNUM *e, *n; rsa = EVP_PKEY_get1_RSA(pkey); type = NEVERBLEED_TYPE_RSA; RSA_get0_key(rsa, &n, &e, NULL); estr = BN_bn2hex(e); nstr = BN_bn2hex(n); break; } case EVP_PKEY_EC: { #ifdef NEVERBLEED_ECDSA const EC_POINT *ec_pubkey; EC_KEY *ec_key; ec_key = (EC_KEY *)EVP_PKEY_get0_EC_KEY(pkey); type = NEVERBLEED_TYPE_ECDSA; ec_group = EC_KEY_get0_group(ec_key); ec_pubkey = EC_KEY_get0_public_key(ec_key); ec_pubkeylen = EC_POINT_point2oct(ec_group, ec_pubkey, POINT_CONVERSION_UNCOMPRESSED, NULL, 0, NULL); if (!(ec_pubkeylen > 0 && (ec_pubkeybytes = malloc(ec_pubkeylen)) != NULL && EC_POINT_point2oct(ec_group, ec_pubkey, POINT_CONVERSION_UNCOMPRESSED, ec_pubkeybytes, ec_pubkeylen, NULL) == ec_pubkeylen)) dief("failed to serialize EC public key"); break; #else snprintf(errbuf, sizeof(errbuf), "ECDSA support requires OpenSSL >= 1.1.0, LibreSSL >= 2.9.1, or BoringSSL"); goto Respond; #endif } default: snprintf(errbuf, sizeof(errbuf), "unsupported private key: %d", EVP_PKEY_base_id(pkey)); goto Respond; } /* store the key */ key_index = daemon_set_pkey(pkey); Respond: iobuf_dispose(buf); iobuf_push_num(buf, type); iobuf_push_num(buf, key_index); switch (type) { case NEVERBLEED_TYPE_RSA: iobuf_push_str(buf, estr != NULL ? estr : ""); iobuf_push_str(buf, nstr != NULL ? nstr : ""); break; #ifdef NEVERBLEED_ECDSA case NEVERBLEED_TYPE_ECDSA: iobuf_push_num(buf, EC_GROUP_get_curve_name(ec_group)); iobuf_push_bytes(buf, ec_pubkeybytes, ec_pubkeylen); break; #endif default: iobuf_push_str(buf, errbuf); } if (rsa != NULL) RSA_free(rsa); if (pkey != NULL) EVP_PKEY_free(pkey); if (estr != NULL) OPENSSL_free(estr); if (nstr != NULL) OPENSSL_free(nstr); #ifdef NEVERBLEED_ECDSA if (ec_pubkeybytes != NULL) free(ec_pubkeybytes); #endif if (fp != NULL) fclose(fp); return 0; } int neverbleed_setuidgid(neverbleed_t *nb, const char *user, int change_socket_ownership) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); neverbleed_iobuf_t buf = {NULL}; size_t ret; iobuf_push_str(&buf, "setuidgid"); iobuf_push_str(&buf, user); iobuf_push_num(&buf, change_socket_ownership); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &ret) != 0) { errno = 0; dief("failed to parse response"); } iobuf_dispose(&buf); return (int)ret; } static int setuidgid_stub(neverbleed_iobuf_t *buf) { const char *user; size_t change_socket_ownership; struct passwd pwbuf, *pw; char pwstrbuf[65536]; /* should be large enough */ int ret = -1; if ((user = iobuf_shift_str(buf)) == NULL || iobuf_shift_num(buf, &change_socket_ownership) != 0) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } errno = 0; if (getpwnam_r(user, &pwbuf, pwstrbuf, sizeof(pwstrbuf), &pw) != 0) { warnf("%s: getpwnam_r failed", __FUNCTION__); goto Respond; } if (pw == NULL) { warnf("%s: failed to obtain information of user:%s", __FUNCTION__, user); goto Respond; } if (change_socket_ownership) { char *dir; if (chown(daemon_vars.nb->sun_.sun_path, pw->pw_uid, pw->pw_gid) != 0) dief("chown failed for:%s", daemon_vars.nb->sun_.sun_path); dir = dirname(daemon_vars.nb->sun_.sun_path); if (chown(dir, pw->pw_uid, pw->pw_gid) != 0) dief("chown failed for:%s", dir); free(dir); } /* setuidgid */ if (setgid(pw->pw_gid) != 0) { warnf("%s: setgid(%d) failed", __FUNCTION__, (int)pw->pw_gid); goto Respond; } if (initgroups(pw->pw_name, pw->pw_gid) != 0) { warnf("%s: initgroups(%s, %d) failed", __FUNCTION__, pw->pw_name, (int)pw->pw_gid); goto Respond; } if (setuid(pw->pw_uid) != 0) { warnf("%s: setuid(%d) failed\n", __FUNCTION__, (int)pw->pw_uid); goto Respond; } ret = 0; Respond: iobuf_dispose(buf); iobuf_push_num(buf, ret); return 0; } #if NEVERBLEED_HAS_PTHREAD_SETAFFINITY_NP int neverbleed_setaffinity(neverbleed_t *nb, NEVERBLEED_CPU_SET_T *cpuset) { struct st_neverbleed_thread_data_t *thdata = get_thread_data(nb); neverbleed_iobuf_t buf = {NULL}; size_t ret; iobuf_push_str(&buf, "setaffinity"); iobuf_push_bytes(&buf, cpuset, sizeof(*cpuset)); iobuf_transaction(&buf, thdata); if (iobuf_shift_num(&buf, &ret) != 0) { errno = 0; dief("failed to parse response"); } iobuf_dispose(&buf); return (int)ret; } static int setaffinity_stub(neverbleed_iobuf_t *buf) { char *cpuset_bytes; size_t cpuset_len; NEVERBLEED_CPU_SET_T cpuset; int ret = 1; if ((cpuset_bytes = iobuf_shift_bytes(buf, &cpuset_len)) == NULL) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } assert(cpuset_len == sizeof(NEVERBLEED_CPU_SET_T)); memcpy(&cpuset, cpuset_bytes, cpuset_len); #ifdef __NetBSD__ ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset); #else ret = pthread_setaffinity_np(pthread_self(), sizeof(NEVERBLEED_CPU_SET_T), &cpuset); #endif if (ret != 0) { ret = 1; goto Respond; } ret = 0; Respond: iobuf_dispose(buf); iobuf_push_num(buf, ret); return 0; } #endif __attribute__((noreturn)) static void *daemon_close_notify_thread(void *_close_notify_fd) { int close_notify_fd = (int)((char *)_close_notify_fd - (char *)NULL); char b; ssize_t r; Redo: r = read(close_notify_fd, &b, 1); if (r == -1 && errno == EINTR) goto Redo; if (r > 0) goto Redo; /* close or error */ /* unlink the temporary directory and socket file */ unlink_dir(dirname(daemon_vars.nb->sun_.sun_path)); _exit(0); } static int del_pkey_stub(neverbleed_iobuf_t *buf) { size_t key_index; if (iobuf_shift_num(buf, &key_index) != 0) { errno = 0; warnf("%s: failed to parse request", __FUNCTION__); return -1; } pthread_mutex_lock(&daemon_vars.keys.lock); /* set slot as available */ if (key_index < daemon_vars.keys.num_slots) { EVP_PKEY_free(daemon_vars.keys.slots[key_index].pkey); daemon_vars.keys.slots[key_index].next_empty = daemon_vars.keys.first_empty; daemon_vars.keys.first_empty = key_index; } else { warnf("%s: invalid key index %zu", __FUNCTION__, key_index); } pthread_mutex_unlock(&daemon_vars.keys.lock); return 0; } #define offload_start(stub, buf) ((stub)(buf)) #if USE_OFFLOAD #ifdef OPENSSL_IS_BORINGSSL static int offload_resume(struct engine_request *req) { size_t outlen; if (do_epoll_ctl(conn_ctx.epollfd, EPOLL_CTL_DEL, req->async_fd, NULL) != 0) dief("epoll_ctl failed:%d\n", errno); /* get result */ if (bssl_qat_async_ctx_copy_result(req->async_ctx, req->data.output, &outlen, sizeof(req->data.output)) != 0) dief("failed to obtain offload result\n"); if (outlen > sizeof(req->data.output)) dief("RSA output is unexpectedly large\n"); /* save the result */ iobuf_dispose(req->buf); iobuf_push_bytes(req->buf, req->data.output, outlen); req->buf->processing = 0; offload_free_request(req); return 0; } #else static int offload_jobfunc(void *_req) { struct engine_request *req = *(void **)_req; return req->stub(req->buf); } #undef offload_start static int offload_start(int (*stub)(neverbleed_iobuf_t *), neverbleed_iobuf_t *buf) { /* if engine is not used, run the stub synchronously */ if (!use_offload) return stub(buf); buf->processing = 1; struct engine_request *req = malloc(sizeof(*req)); if (req == NULL) dief("no memory"); *req = (struct engine_request){.buf = buf, .async_fd = -1, .stub = stub}; if ((req->async.ctx = ASYNC_WAIT_CTX_new()) == NULL) dief("failed to create ASYNC_WAIT_CTX\n"); int ret; switch (ASYNC_start_job(&req->async.job, req->async.ctx, &ret, offload_jobfunc, &req, sizeof(req))) { case ASYNC_PAUSE: /* operation running async; register fd and bail out */ register_wait_fd(req); return 0; case ASYNC_FINISH: /* completed synchronously */ buf->processing = 0; break; default: dief("ASYNC_start_job errored\n"); break; } offload_free_request(req); return ret; } static int offload_resume(struct engine_request *req) { int ret; switch (ASYNC_start_job(&req->async.job, req->async.ctx, &ret, offload_jobfunc, &req, sizeof(req))) { case ASYNC_PAUSE: /* assume that wait fd is unchanged */ return 0; case ASYNC_FINISH: if (do_epoll_ctl(conn_ctx.epollfd, EPOLL_CTL_DEL, req->async_fd, NULL) != 0) dief("epoll_ctl failed:%d\n", errno); break; default: dief("ASYNC_start_job failed\n"); break; } /* job done */ req->buf->processing = 0; offload_free_request(req); return ret; } #endif #endif /** * This function waits for the provided socket to become readable, then calls `nanosleep(1)` before returning. * The intention behind sleep is to provide the application to complete its event loop before the neverbleed process starts * spending CPU cycles on the time-consuming RSA operation. * In addition, when QAT is used, this function processes completion notifications from QAT and sends the responses. */ static int wait_for_data(int cleanup) { #if USE_OFFLOAD struct epoll_event events[20]; int has_read = 0, num_events; do { while ((num_events = epoll_wait(conn_ctx.epollfd, events, sizeof(events) / sizeof(events[0]), -1)) == -1 && (errno == EAGAIN || errno == EINTR)) ; if (num_events == -1) dief("epoll_wait(2):%d\n", errno); for (int i = 0; i < num_events; ++i) { if (events[i].data.ptr == NULL) { has_read = 1; } else { struct engine_request *req = events[i].data.ptr; int ret; if ((ret = offload_resume(req)) != 0) return ret; if ((ret = send_responses(0)) != 0) return ret; } } } while (!has_read); #else fd_set rfds; int ret; FD_ZERO(&rfds); if (!cleanup) FD_SET(conn_ctx.sockfd, &rfds); while ((ret = select(conn_ctx.sockfd + 1, &rfds, NULL, NULL, NULL)) == -1 && (errno == EAGAIN || errno == EINTR)) ; if (ret == -1) dief("select(2):%d\n", errno); #endif // yield when data is available struct timespec tv = {.tv_nsec = 1}; (void)nanosleep(&tv, NULL); return 0; } static void *daemon_conn_thread(void *_sock_fd) { conn_ctx.sockfd = (int)((char *)_sock_fd - (char *)NULL); conn_ctx.responses.next = &conn_ctx.responses.first; neverbleed_iobuf_t *buf = NULL; #if USE_OFFLOAD if ((conn_ctx.epollfd = epoll_create1(EPOLL_CLOEXEC)) == -1) dief("epoll_create1 failed:%d\n", errno); { struct epoll_event ev = {.events = EPOLLIN}; if (do_epoll_ctl(conn_ctx.epollfd, EPOLL_CTL_ADD, conn_ctx.sockfd, &ev) != 0) dief("epoll_ctl failed:%d\n", errno); } #endif { /* authenticate */ unsigned char auth_token[NEVERBLEED_AUTH_TOKEN_SIZE]; if (read_nbytes(conn_ctx.sockfd, &auth_token, sizeof(auth_token)) != 0) { warnf("failed to receive authencication token from client"); goto Exit; } if (memcmp(auth_token, daemon_vars.nb->auth_token, NEVERBLEED_AUTH_TOKEN_SIZE) != 0) { warnf("client authentication failed"); goto Exit; } } while (1) { if (wait_for_data(0) != 0) break; free(buf); buf = malloc(sizeof(*buf)); if (buf == NULL) dief("no memory"); *buf = (neverbleed_iobuf_t){}; char *cmd; if (iobuf_read(buf, conn_ctx.sockfd) != 0) { if (errno != 0) warnf("read error"); break; } if ((cmd = iobuf_shift_str(buf)) == NULL) { errno = 0; warnf("failed to parse request"); break; } #if !defined(OPENSSL_IS_BORINGSSL) if (strcmp(cmd, "priv_enc") == 0) { if (offload_start(priv_enc_stub, buf) != 0) break; } else if (strcmp(cmd, "priv_dec") == 0) { if (offload_start(priv_dec_stub, buf) != 0) break; } else if (strcmp(cmd, "sign") == 0) { if (offload_start(sign_stub, buf) != 0) break; #ifdef NEVERBLEED_ECDSA } else if (strcmp(cmd, "ecdsa_sign") == 0) { if (ecdsa_sign_stub(buf) != 0) break; #endif } else #endif if (strcmp(cmd, "digestsign") == 0) { if (digestsign_stub(buf) != 0) break; } else if (strcmp(cmd, "digestsign-rsa") == 0) { if (offload_start(digestsign_stub, buf) != 0) break; } else if (strcmp(cmd, "decrypt") == 0) { if (offload_start(decrypt_stub, buf) != 0) break; } else if (strcmp(cmd, "load_key") == 0) { if (load_key_stub(buf) != 0) break; } else if (strcmp(cmd, "del_pkey") == 0) { if (del_pkey_stub(buf) != 0) break; iobuf_dispose(buf); // "del_pkey" command is fire-and-forget, it cannot fail, so doesn't have a response continue; } else if (strcmp(cmd, "setuidgid") == 0) { if (setuidgid_stub(buf) != 0) break; #if NEVERBLEED_HAS_PTHREAD_SETAFFINITY_NP } else if (strcmp(cmd, "setaffinity") == 0) { if (setaffinity_stub(buf) != 0) break; #endif } else { warnf("unknown command:%s", cmd); break; } /* add response to chain */ *conn_ctx.responses.next = buf; conn_ctx.responses.next = &buf->next; buf = NULL; /* do not free */ /* send responses if possible */ if (send_responses(0) != 0) break; } Exit: free(buf); /* run the loop while async ops are running */ while (conn_ctx.responses.first != NULL) wait_for_data(1); close(conn_ctx.sockfd); #ifdef __linux close(conn_ctx.epollfd); #endif return NULL; } #if !(defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)) #define closefrom my_closefrom static void my_closefrom(int lowfd) { /* On linux, try close_range (2), then fall back to the slow loop if it fails. */ #if defined(__linux__) && defined(__NR_close_range) if (syscall(__NR_close_range, lowfd, ~0, 0) == 0) return; #endif for (int fd = (int)sysconf(_SC_OPEN_MAX) - 1; fd >= lowfd; --fd) (void)close(fd); } #endif static void cleanup_fds(int listen_fd, int close_notify_fd) { int maxfd, k; maxfd = 0; if (listen_fd > maxfd) { maxfd = listen_fd; } if (close_notify_fd > maxfd) { maxfd = close_notify_fd; } for (k = 0; k < maxfd; k++) { if (k == listen_fd || k == close_notify_fd) continue; switch (k) { case STDOUT_FILENO: case STDERR_FILENO: case STDIN_FILENO: break; default: (void)close(k); } } closefrom(maxfd + 1); } __attribute__((noreturn)) static void daemon_main(int listen_fd, int close_notify_fd, const char *tempdir) { pthread_t tid; pthread_attr_t thattr; int sock_fd; cleanup_fds(listen_fd, close_notify_fd); pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, 1); switch (neverbleed_offload) { case NEVERBLEED_OFFLOAD_QAT_ON: case NEVERBLEED_OFFLOAD_QAT_AUTO: { #if USE_OFFLOAD && defined(OPENSSL_IS_BORINGSSL) ENGINE_load_qat(); bssl_qat_set_default_string("RSA"); use_offload = ENGINE_QAT_PTR_GET() != NULL; #elif USE_OFFLOAD && !defined(OPENSSL_IS_BORINGSSL) ENGINE *qat = ENGINE_by_id("qatengine"); if (qat != NULL && ENGINE_init(qat)) { if (!ENGINE_set_default_RSA(qat)) dief("failed to assign RSA operations to QAT\n"); use_offload = 1; } #endif if (!use_offload && neverbleed_offload == NEVERBLEED_OFFLOAD_QAT_ON) dief("use of QAT is forced but unavailable\n"); } break; default: break; } if (pthread_create(&tid, &thattr, daemon_close_notify_thread, (char *)NULL + close_notify_fd) != 0) dief("pthread_create failed"); while (1) { while ((sock_fd = accept(listen_fd, NULL, NULL)) == -1) ; if (pthread_create(&tid, &thattr, daemon_conn_thread, (char *)NULL + sock_fd) != 0) dief("pthread_create failed"); } } #ifndef NEVERBLEED_OPAQUE_RSA_METHOD static RSA_METHOD static_rsa_method = { "privsep RSA method", /* name */ NULL, /* rsa_pub_enc */ NULL, /* rsa_pub_dec */ priv_enc_proxy, /* rsa_priv_enc */ priv_dec_proxy, /* rsa_priv_dec */ NULL, /* rsa_mod_exp */ NULL, /* bn_mod_exp */ NULL, /* init */ NULL, /* finish */ RSA_FLAG_SIGN_VER, /* flags */ NULL, /* app data */ sign_proxy, /* rsa_sign */ NULL, /* rsa_verify */ NULL /* rsa_keygen */ }; #endif int neverbleed_init(neverbleed_t *nb, char *errbuf) { int pipe_fds[2] = {-1, -1}, listen_fd = -1; char *tempdir = NULL; /* setup the daemon */ if (pipe(pipe_fds) != 0) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "pipe(2) failed:%s", strerror(errno)); goto Fail; } set_cloexec(pipe_fds[1]); if ((tempdir = strdup("/tmp/openssl-privsep.XXXXXX")) == NULL) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "no memory"); goto Fail; } if (mkdtemp(tempdir) == NULL) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "failed to create temporary directory under /tmp:%s", strerror(errno)); goto Fail; } memset(&nb->sun_, 0, sizeof(nb->sun_)); nb->sun_.sun_family = AF_UNIX; snprintf(nb->sun_.sun_path, sizeof(nb->sun_.sun_path), "%s/_", tempdir); RAND_bytes(nb->auth_token, sizeof(nb->auth_token)); if ((listen_fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "socket(2) failed:%s", strerror(errno)); goto Fail; } if (bind(listen_fd, (void *)&nb->sun_, sizeof(nb->sun_)) != 0) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "failed to bind to %s:%s", nb->sun_.sun_path, strerror(errno)); goto Fail; } if (listen(listen_fd, SOMAXCONN) != 0) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "listen(2) failed:%s", strerror(errno)); goto Fail; } nb->daemon_pid = fork(); switch (nb->daemon_pid) { case -1: snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "fork(2) failed:%s", strerror(errno)); goto Fail; case 0: close(pipe_fds[1]); #if defined(__linux__) prctl(PR_SET_DUMPABLE, 0, 0, 0, 0); #elif defined(__FreeBSD__) int dumpable = PROC_TRACE_CTL_DISABLE; procctl(P_PID, 0, PROC_TRACE_CTL, &dumpable); #elif defined(__sun) setpflags(__PROC_PROTECT, 1); #elif defined(__APPLE__) ptrace(PT_DENY_ATTACH, 0, 0, 0); #endif if (neverbleed_post_fork_cb != NULL) neverbleed_post_fork_cb(); daemon_vars.nb = nb; daemon_main(listen_fd, pipe_fds[0], tempdir); break; default: break; } close(listen_fd); listen_fd = -1; close(pipe_fds[0]); pipe_fds[0] = -1; #if defined(OPENSSL_IS_BORINGSSL) nb->engine = NULL; #else { /* setup engine */ const RSA_METHOD *rsa_default_method; RSA_METHOD *rsa_method; #ifdef NEVERBLEED_ECDSA const EC_KEY_METHOD *ecdsa_default_method; EC_KEY_METHOD *ecdsa_method; #endif #ifdef NEVERBLEED_OPAQUE_RSA_METHOD rsa_default_method = RSA_PKCS1_OpenSSL(); rsa_method = RSA_meth_dup(rsa_default_method); RSA_meth_set1_name(rsa_method, "privsep RSA method"); RSA_meth_set_priv_enc(rsa_method, priv_enc_proxy); RSA_meth_set_priv_dec(rsa_method, priv_dec_proxy); RSA_meth_set_sign(rsa_method, sign_proxy); #else rsa_default_method = RSA_PKCS1_SSLeay(); rsa_method = &static_rsa_method; rsa_method->rsa_pub_enc = rsa_default_method->rsa_pub_enc; rsa_method->rsa_pub_dec = rsa_default_method->rsa_pub_dec; rsa_method->rsa_verify = rsa_default_method->rsa_verify; rsa_method->bn_mod_exp = rsa_default_method->bn_mod_exp; #endif #ifdef NEVERBLEED_ECDSA ecdsa_default_method = EC_KEY_get_default_method(); ecdsa_method = EC_KEY_METHOD_new(ecdsa_default_method); /* it seems sign_sig and sign_setup is not used in TLS ECDSA. */ EC_KEY_METHOD_set_sign(ecdsa_method, ecdsa_sign_proxy, NULL, NULL); #endif if ((nb->engine = ENGINE_new()) == NULL || !ENGINE_set_id(nb->engine, "neverbleed") || !ENGINE_set_name(nb->engine, "privilege separation software engine") || !ENGINE_set_RSA(nb->engine, rsa_method) #ifdef NEVERBLEED_ECDSA || !ENGINE_set_EC(nb->engine, ecdsa_method) #endif ) { snprintf(errbuf, NEVERBLEED_ERRBUF_SIZE, "failed to initialize the OpenSSL engine"); goto Fail; } ENGINE_add(nb->engine); } #endif /* setup thread key */ pthread_key_create(&nb->thread_key, dispose_thread_data); free(tempdir); return 0; Fail: if (pipe_fds[0] != -1) close(pipe_fds[0]); if (pipe_fds[1] != -1) close(pipe_fds[1]); if (tempdir != NULL) { unlink_dir(tempdir); free(tempdir); } if (listen_fd != -1) close(listen_fd); if (nb->engine != NULL) { ENGINE_free(nb->engine); nb->engine = NULL; } return -1; } void (*neverbleed_post_fork_cb)(void) = NULL; void (*neverbleed_transaction_cb)(neverbleed_iobuf_t *, int) = NULL; enum neverbleed_offload_type neverbleed_offload = NEVERBLEED_OFFLOAD_OFF;