From 9da14566c7095e681e0459f5f354a9771ed6e5f5 Mon Sep 17 00:00:00 2001 From: Emeric Date: Thu, 12 Jul 2012 15:14:09 +0200 Subject: [PATCH 1/4] Shared Context Update: - Disable openssl internal cache: less memory usage and no perf impact. - Update shared context initialization API. --- shctx.c | 104 ++++++++++++++++++++++++++++++++------------------------ shctx.h | 22 +++++++++--- stud.c | 4 ++- 3 files changed, 80 insertions(+), 50 deletions(-) diff --git a/shctx.c b/shctx.c index 0bafb54..1245ad4 100644 --- a/shctx.c +++ b/shctx.c @@ -164,11 +164,11 @@ static inline void shared_context_unlock(void) /* SSL callback used on new session creation */ int shctx_new_cb(SSL *ssl, SSL_SESSION *sess) { - (void)ssl; struct shared_session *shsess; unsigned char *data,*p; unsigned int data_len; unsigned char encsess[SHSESS_MAX_ENCODED_LEN]; + (void)ssl; /* check if session reserved size in aligned buffer is large enougth for the ASN1 encode session */ data_len=i2d_SSL_SESSION(sess, NULL); @@ -216,13 +216,13 @@ int shctx_new_cb(SSL *ssl, SSL_SESSION *sess) { /* SSL callback used on lookup an existing session cause none found in internal cache */ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_copy) { - (void)ssl; struct shared_session *shsess; unsigned char data[SHSESS_MAX_DATA_LEN], *p; unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH]; unsigned int data_len; long cdate; SSL_SESSION *sess; + (void)ssl; /* allow the session to be freed automatically by openssl */ *do_copy = 0; @@ -269,10 +269,10 @@ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_cop /* SSL callback used to signal session is no more used in internal cache */ void shctx_remove_cb(SSL_CTX *ctx, SSL_SESSION *sess) { - (void)ctx; struct shared_session *shsess; unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH]; unsigned char *key = sess->session_id; + (void)ctx; /* tree key is zeros padded sessionid */ if ( sess->session_id_length < SSL_MAX_SSL_SESSION_ID_LENGTH ) { @@ -333,64 +333,80 @@ void shsess_set_new_cbk(void (*func)(unsigned char *, unsigned int, long)) { shared_session_new_cbk = func; } -/* Init shared memory context if not allocated and set SSL context callbacks - * size is the max number of stored session - * Returns: -1 on alloc failure, size if performs context alloc, and 0 if just perform - * callbacks registration */ -int shared_context_init(SSL_CTX *ctx, int size) -{ - int ret = 0; - - if (!shctx) { - int i; - +/* Allocate shared memory context. + * size is maximum cached sessions. + * if set less or equal to 0, SHCTX_DEFAULT_SIZE is used. + * Returns: -1 on alloc failure, size if it performs context alloc, + * and 0 if cache is already allocated */ +int shared_context_init(int size) { + int i; #ifndef USE_SYSCALL_FUTEX - pthread_mutexattr_t attr; + pthread_mutexattr_t attr; #endif /* USE_SYSCALL_FUTEX */ - struct shared_session *prev,*cur; + struct shared_session *prev,*cur; + + if (shctx) + return 0; + + if (size<=0) + size = SHCTX_DEFAULT_SIZE; - shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_session)), - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (!shctx || shctx == MAP_FAILED) - return -1; + shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_session)), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (!shctx || shctx == MAP_FAILED) { + shctx = NULL; + return -1; + } #ifdef USE_SYSCALL_FUTEX - shctx->waiters = 0; + shctx->waiters = 0; #else - pthread_mutexattr_init(&attr); - pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); - pthread_mutex_init(&shctx->mutex, &attr); + pthread_mutexattr_init(&attr); + pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); + pthread_mutex_init(&shctx->mutex, &attr); #endif - memset(&shctx->active.key, 0, sizeof(struct ebmb_node)); - memset(&shctx->free.key, 0, sizeof(struct ebmb_node)); + memset(&shctx->active.key, 0, sizeof(struct ebmb_node)); + memset(&shctx->free.key, 0, sizeof(struct ebmb_node)); - /* No duplicate authorized in tree: */ - shctx->active.key.node.branches.b[1] = (void *)1; + /* No duplicate authorized in tree: */ + //shctx->active.key.node.branches.b[1] = (void *)1; + shctx->active.key.node.branches = EB_ROOT_UNIQUE; - cur = &shctx->active; - cur->n = cur->p = cur; + cur = &shctx->active; + cur->n = cur->p = cur; - cur = &shctx->free; - for ( i = 0 ; i < size ; i++) { - prev = cur; - cur = (struct shared_session *)((char *)prev + sizeof(struct shared_session)); - prev->n = cur; - cur->p = prev; - } - cur->n = &shctx->free; - shctx->free.p = cur; - - ret = size; + cur = &shctx->free; + for ( i = 0 ; i < size ; i++) { + prev = cur; + cur = (struct shared_session *)((char *)prev + sizeof(struct shared_session)); + prev->n = cur; + cur->p = prev; } + cur->n = &shctx->free; + shctx->free.p = cur; + + return size; +} + - /* set SSL internal cache size to external cache / 8 + 123 */ - SSL_CTX_sess_set_cache_size(ctx, size >> 3 | 0x3ff); +/* Set session cache mode to server and disable openssl internal cache. + * Set shared cache callbacks on an ssl context. + * Shared context MUST be firstly initialized */ +void shared_context_set_cache(SSL_CTX *ctx) { + + SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER + |SSL_SESS_CACHE_NO_INTERNAL + |SSL_SESS_CACHE_NO_AUTO_CLEAR); + if (!shctx) + return; /* Set callbacks */ SSL_CTX_sess_set_new_cb(ctx, shctx_new_cb); SSL_CTX_sess_set_get_cb(ctx, shctx_get_cb); SSL_CTX_sess_set_remove_cb(ctx, shctx_remove_cb); - return ret; } + + + diff --git a/shctx.h b/shctx.h index 42a600c..37cdf7b 100644 --- a/shctx.h +++ b/shctx.h @@ -21,11 +21,16 @@ #define SHSESS_MAX_DATA_LEN 512 #endif +#ifndef SHCTX_DEFAULT_SIZE +#define SHCTX_DEFAULT_SIZE 20000 +#endif + #define SHSESS_MAX_ENCODED_LEN SSL_MAX_SSL_SESSION_ID_LENGTH \ + SHSESS_MAX_DATA_LEN \ + SHSESS_MAX_FOOTER_LEN + /* Callback called on a new session event: * session contains the sessionid zeros padded to SSL_MAX_SSL_SESSION_ID_LENGTH * followed by ASN1 session encoding. @@ -44,10 +49,17 @@ void shsess_set_new_cbk(void (*func)(unsigned char *session, unsigned int len, l * if cdate not 0, on get events session creation date will be reset to cdate */ void shctx_sess_add(const unsigned char *session, unsigned int session_len, long cdate); -/* Init shared memory context if not allocated and set SSL context callbacks - * size is the max number of stored session - * Returns: -1 on alloc failure, size if performs context alloc, and 0 if just - * perform callbacks registration */ -int shared_context_init(SSL_CTX *ctx, int size); +/* Allocate shared memory context. + * size is maximum cached sessions. + * if set less or equal to 0, SHCTX_DEFAULT_SIZE is used. + * Returns: -1 on alloc failure, size if it performs context alloc, + * and 0 if cache is already allocated */ +int shared_context_init(int size); + +/* Set shared cache callbacks on an ssl context. + * Set session cache mode to server and disable openssl internal cache. + * Shared context MUST be firstly initialized */ +void shared_context_set_cache(SSL_CTX *ctx); #endif /* SHCTX_H */ + diff --git a/stud.c b/stud.c index 482fcc0..a126b51 100644 --- a/stud.c +++ b/stud.c @@ -615,10 +615,12 @@ SSL_CTX * init_openssl() { } #ifdef USE_SHARED_CACHE else if (CONFIG->SHARED_CACHE) { - if (shared_context_init(ctx, CONFIG->SHARED_CACHE) < 0) { + if (shared_context_init(CONFIG->SHARED_CACHE) < 0) { ERR("Unable to alloc memory for shared cache.\n"); exit(1); } + shared_context_set_cache(ctx); + if (CONFIG->SHCUPD_PORT) { if (compute_secret(rsa, shared_secret) < 0) { ERR("Unable to compute shared secret.\n"); From 7559d6e32fb9bc8f73fb02925fca4c39afdb38af Mon Sep 17 00:00:00 2001 From: Emeric Date: Fri, 24 Aug 2012 17:53:50 +0200 Subject: [PATCH 2/4] Remove unused atomic inc function from share context code --- shctx.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/shctx.c b/shctx.c index 1245ad4..5b62b64 100644 --- a/shctx.c +++ b/shctx.c @@ -68,17 +68,6 @@ static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned return ret; } -static inline unsigned char atomic_inc(unsigned int *ptr) -{ - unsigned char ret; - __asm volatile("lock incl %0\n" - "setne %1\n" - : "+m" (*ptr), "=qm" (ret) - : - : "memory"); - return ret; -} - static inline unsigned char atomic_dec(unsigned int *ptr) { unsigned char ret; From 87854d34f8783fcc5072c559b711be265788552a Mon Sep 17 00:00:00 2001 From: Emeric Date: Fri, 24 Aug 2012 18:02:20 +0200 Subject: [PATCH 3/4] If Futex used but asm atomics ops not supported (arch not i586 nor x86_64) try to use less optimized gcc built-ins. Seems useful to use futex on linux/arm. --- shctx.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/shctx.c b/shctx.c index 5b62b64..f690ced 100644 --- a/shctx.c +++ b/shctx.c @@ -48,6 +48,7 @@ static void (*shared_session_new_cbk)(unsigned char *session, unsigned int sessi /* Lock functions */ #ifdef USE_SYSCALL_FUTEX +#if defined (__i586__) || defined (__x86_64__) static inline unsigned int xchg(unsigned int *ptr, unsigned int x) { __asm volatile("lock xchgl %0,%1" @@ -79,6 +80,30 @@ static inline unsigned char atomic_dec(unsigned int *ptr) return ret; } +#else /* if no x86_64 or i586 arch: use less optimized gcc >= 4.1 built-ins */ +static inline unsigned int xchg(unsigned int *ptr, unsigned int x) +{ + unsigned int old; + + do { + old = *ptr; + } while (__sync_val_compare_and_swap(ptr, old, x) != old); + + return old; +} + +static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new) +{ + return __sync_val_compare_and_swap(ptr, old, new); +} + +static inline unsigned char atomic_dec(unsigned int *ptr) +{ + return __sync_sub_and_fetch(ptr, 1) ? 1 : 0; +} + +#endif + static inline void shared_context_lock(void) { unsigned int x; From b751b759742f6e0b3fc27d9a9a15505383ebff57 Mon Sep 17 00:00:00 2001 From: stop Date: Sat, 25 Aug 2012 14:43:20 +0200 Subject: [PATCH 4/4] Shared cache: xchg gcc built-in equivalent found. --- shctx.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/shctx.c b/shctx.c index f690ced..ca0bb3c 100644 --- a/shctx.c +++ b/shctx.c @@ -83,13 +83,7 @@ static inline unsigned char atomic_dec(unsigned int *ptr) #else /* if no x86_64 or i586 arch: use less optimized gcc >= 4.1 built-ins */ static inline unsigned int xchg(unsigned int *ptr, unsigned int x) { - unsigned int old; - - do { - old = *ptr; - } while (__sync_val_compare_and_swap(ptr, old, x) != old); - - return old; + return __sync_lock_test_and_set(ptr, x); } static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new)