From e3575828e07be265fd6628a2c58103bf2fdf7535 Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 21 Sep 2012 15:13:28 -0500 Subject: [PATCH] (PR14571 partial fix) For dyninst, use TLS for map and stat data. * runtime/dyninst/tls_data.c: New file. * runtime/stat.c (struct _Stat): Add a tls_data_container_t structure. (_stp_stat_tls_object_init): New function. (_stp_stat_tls_object_free): Ditto. (_stp_stat_init): Instead of directly allocating percpu data, for dyninst set up tls data to be created when accessed by calling _stp_tls_data_container_init(). (_stp_stat_del): For dyninst, call _stp_tls_data_container_cleanup() to remove all the tls data. (_stp_stat_add): For dyninst, get the proper tls stat object. (_stp_stat_get_cpu): Deleted unused function. (_stp_stat_get): For dyninst, get the proper tls stat objects. (_stp_stat_clear): For dyninst, clear the stat in each thread's tls data. * runtime/stat.h (struct stat_data): Add a tls_data_object_t structure. * runtime/map.c (_stp_map_tls_object_init): New function. (_stp_map_tls_object_free): Ditto. (_stp_pmap_new): Instead of directly allocating percpu data, for dyninst set up tls data to be created when accessed by calling _stp_tls_data_container_init(). (_stp_pmap_clear): For dyninst, clear the map in each thread's tls data. (_stp_pmap_del): For dyninst, call _stp_tls_data_container_cleanup() to remove all the tls data. (_stp_pmap_agg): Add dyninst support. * runtime/map.h (struct map_root): Add a tls_data_object_t structure. (struct pmap): Add a tls_data_container_t structure. * runtime/map-stat.c (_stp_hstat_tls_object_init): New function. (_stp_pmap_new_hstat_linear): For dyninst, override the standard tls data object init function with _stp_hstat_tls_object_init(), which knows how to handle hstats. (_stp_pmap_new_hstat_log): Ditto. * runtime/pmap-gen.c (_stp_pmap_tls_object_init): New function. (_stp_pmap_new): For dyninst, override the standard tls data object init function with _stp_pmap_tls_object_init(), which knows how to handle pmaps. (_stp_pmap_set): For dyninst, get the proper tls pmap object. (_stp_pmap_add): Ditto. (_stp_pmap_get_cpu): Ditto. (_stp_pmap_get): Ditto. (_stp_pmap_del): Ditto. * runtime/dyninst/linux_defs.h: Added container_of(), list_entry(), list_for_each_entry(), and list_for_each_entry_safe(). --- runtime/dyninst/linux_defs.h | 19 ++++ runtime/dyninst/tls_data.c | 178 +++++++++++++++++++++++++++++++++++ runtime/map-stat.c | 42 ++++++++- runtime/map.c | 145 ++++++++++++++++++++++++---- runtime/map.h | 18 +++- runtime/pmap-gen.c | 82 +++++++++++++++- runtime/stat.c | 124 +++++++++++++++++++----- runtime/stat.h | 15 ++- 8 files changed, 580 insertions(+), 43 deletions(-) create mode 100644 runtime/dyninst/tls_data.c diff --git a/runtime/dyninst/linux_defs.h b/runtime/dyninst/linux_defs.h index 7ccdf56d9..0d0d41ff5 100644 --- a/runtime/dyninst/linux_defs.h +++ b/runtime/dyninst/linux_defs.h @@ -3,6 +3,7 @@ #ifndef _STAPDYN_LINUX_DEFS_H_ #define _STAPDYN_LINUX_DEFS_H_ +#include #include #include "linux_hash.h" @@ -41,6 +42,10 @@ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) + #define __must_be_array(arr) 0 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr)) @@ -294,10 +299,24 @@ static inline int list_empty(const struct list_head *head) return head->next == head; } +#define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + #define list_for_each_safe(pos, n, head) \ for (pos = (head)->next, n = pos->next; pos != (head); \ pos = n, n = pos->next) +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + static inline void INIT_HLIST_NODE(struct hlist_node *h) { h->next = NULL; diff --git a/runtime/dyninst/tls_data.c b/runtime/dyninst/tls_data.c new file mode 100644 index 000000000..7f3be9dd2 --- /dev/null +++ b/runtime/dyninst/tls_data.c @@ -0,0 +1,178 @@ +/* -*- linux-c -*- + * TLS Data Functions + * Copyright (C) 2012 Red Hat Inc. + * + * This file is part of systemtap, and is free software. You can + * redistribute it and/or modify it under the terms of the GNU General + * Public License (GPL); either version 2, or (at your option) any + * later version. + */ + +#ifndef _TLS_DATA_C_ +#define _TLS_DATA_C_ + +#include +#include + +struct tls_data_object_t; + +struct tls_data_container_t { + pthread_key_t key; /* key indexing TLS objects */ + size_t size; /* allocated size of a new TLS object */ + struct list_head head; /* list of tls_data_object_t structs */ + pthread_mutex_t lock; /* lock protecting list */ + int (*init_function)(struct tls_data_object_t *); + void (*free_function)(struct tls_data_object_t *); +}; + +struct tls_data_object_t { + struct list_head list; + struct tls_data_container_t *container; +}; + +#define TLS_DATA_CONTAINER_LOCK(con) pthread_mutex_lock(&(con)->lock) +#define TLS_DATA_CONTAINER_UNLOCK(con) pthread_mutex_unlock(&(con)->lock) + +#define for_each_tls_data(obj, container) \ + list_for_each_entry((obj), &(container)->head, list) + +#define for_each_tls_data_safe(obj, n, container) \ + list_for_each_entry_safe((obj), (n), &(container)->head, list) + +static void _stp_tls_free_per_thread_ptr(void *addr) +{ + struct tls_data_object_t *obj = addr; + + if (obj != NULL) { + struct tls_data_container_t *container = obj->container; + + /* Remove this object from the container's list of objects */ + if (container) { + pthread_mutex_lock(&container->lock); + list_del(&obj->list); + + /* Give the code above us a chance to cleanup. */ + if (container->free_function) + container->free_function(obj); + } + + /* Note that this free() call only works correctly if + * the struct tls_data_object is the first thing in + * its containing structure. */ + free(obj); + + if (container) + pthread_mutex_unlock(&container->lock); + } +} + +static struct tls_data_object_t * +_stp_tls_get_per_thread_ptr(struct tls_data_container_t *container) +{ + /* See if we've already got an object for this thread. */ + struct tls_data_object_t *obj = pthread_getspecific(container->key); + + /* If we haven't set up an tls object instance for the key for + * this thread yet, allocate one. */ + if (obj == NULL) { + int rc; + + /* The real alloc_percpu() allocates zero-filled + * memory, so we need to so the same. */ + obj = calloc(container->size, 1); + if (obj == NULL) { + _stp_error("Couldn't allocate tls object memory: %d\n", + errno); + goto exit; + } + + /* Give the code above us a chance to initialize the + * newly created object. */ + obj->container = container; + if (container->init_function) { + if (container->init_function(obj) != 0) { + free(obj); + obj = NULL; + goto exit; + } + } + + /* Inform pthreads about this instance. */ + pthread_mutex_lock(&container->lock); + if ((rc = pthread_setspecific(container->key, obj)) == 0) { + /* Add obj to container's list of objs (for + * use in looping over all threads). */ + list_add(&obj->list, &container->head); + } + else { + _stp_error("Couldn't setspecific on tls key: %d\n", + rc); + + /* Give the code above us a chance to cleanup. */ + if (container->free_function) + container->free_function(obj); + + free(obj); + obj = NULL; + } + pthread_mutex_unlock(&container->lock); + } +exit: + return obj; +} + +static void +_stp_tls_data_container_update(struct tls_data_container_t *container, + int (*init_function)(struct tls_data_object_t *), + void (*free_function)(struct tls_data_object_t *)) +{ + container->init_function = init_function; + container->free_function = free_function; +} + +static int +_stp_tls_data_container_init(struct tls_data_container_t *container, + size_t size, + int (*init_function)(struct tls_data_object_t *), + void (*free_function)(struct tls_data_object_t *)) +{ + int rc; + + INIT_LIST_HEAD(&container->head); + if ((rc = pthread_mutex_init(&container->lock, NULL)) != 0) { + _stp_error("Couldn't init tls mutex: %d\n", rc); + return 1; + } + if ((rc = pthread_key_create(&container->key, + &_stp_tls_free_per_thread_ptr)) != 0) { + _stp_error("Couldn't create tls key: %d\n", rc); + (void)pthread_mutex_destroy(&container->lock); + return 1; + } + + container->size = size; + container->init_function = init_function; + container->free_function = free_function; + return 0; +} + +static void +_stp_tls_data_container_cleanup(struct tls_data_container_t *container) +{ + struct tls_data_object_t *obj, *n; + + TLS_DATA_CONTAINER_LOCK(container); + (void) pthread_key_delete(container->key); + for_each_tls_data_safe(obj, n, container) { + list_del(&obj->list); + + /* Give the code above us a chance to cleanup. */ + if (container->free_function) + container->free_function(obj); + + free(obj); + } + TLS_DATA_CONTAINER_UNLOCK(container); + (void)pthread_mutex_destroy(&container->lock); +} +#endif /* _TLS_DATA_C_ */ diff --git a/runtime/map-stat.c b/runtime/map-stat.c index 2548a0015..4c48ed9af 100644 --- a/runtime/map-stat.c +++ b/runtime/map-stat.c @@ -1,6 +1,6 @@ /* -*- linux-c -*- * map functions to handle statistics - * Copyright (C) 2005 Red Hat Inc. + * Copyright (C) 2005, 2012 Red Hat Inc. * * This file is part of systemtap, and is free software. You can * redistribute it and/or modify it under the terms of the GNU General @@ -53,6 +53,28 @@ static MAP _stp_map_new_hstat_linear (unsigned max_entries, int ksize, int start return m; } +#ifndef __KERNEL__ +static int _stp_map_tls_object_init(struct tls_data_object_t *obj); +static void _stp_map_tls_object_free(struct tls_data_object_t *obj); + +static int _stp_hstat_tls_object_init(struct tls_data_object_t *obj) +{ + MAP m = container_of(obj, struct map_root, object); + PMAP p = container_of(obj->container, struct pmap, container); + + if (_stp_map_tls_object_init(obj) != 0) + return -1; + + /* Copy the hist params from the agg. */ + m->hist.type = p->agg.hist.type; + m->hist.start = p->agg.hist.start; + m->hist.stop = p->agg.hist.stop; + m->hist.interval = p->agg.hist.interval; + m->hist.buckets = p->agg.hist.buckets; + return 0; +} +#endif + static PMAP _stp_pmap_new_hstat_linear (unsigned max_entries, int ksize, int start, int stop, int interval) { PMAP pmap; @@ -68,6 +90,7 @@ static PMAP _stp_pmap_new_hstat_linear (unsigned max_entries, int ksize, int sta if (pmap) { int i; MAP m; +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = (MAP)per_cpu_ptr (pmap->map, i); m->hist.type = HIST_LINEAR; @@ -76,7 +99,14 @@ static PMAP _stp_pmap_new_hstat_linear (unsigned max_entries, int ksize, int sta m->hist.interval = interval; m->hist.buckets = buckets; } - /* now set agg map params */ +#else + /* Override the tls data object init function with one + * that knows how to handle hstats. */ + _stp_tls_data_container_update(&pmap->container, + &_stp_hstat_tls_object_init, + &_stp_map_tls_object_free); +#endif + /* now set agg map params */ m = &pmap->agg; m->hist.type = HIST_LINEAR; m->hist.start = start; @@ -95,11 +125,19 @@ static PMAP _stp_pmap_new_hstat_log (unsigned max_entries, int key_size) if (pmap) { int i; MAP m; +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = (MAP)per_cpu_ptr (pmap->map, i); m->hist.type = HIST_LOG; m->hist.buckets = HIST_LOG_BUCKETS; } +#else + /* Override the tls data object init function with one + * that knows how to handle hstats. */ + _stp_tls_data_container_update(&pmap->container, + &_stp_hstat_tls_object_init, + &_stp_map_tls_object_free); +#endif /* now set agg map params */ m = &pmap->agg; m->hist.type = HIST_LOG; diff --git a/runtime/map.c b/runtime/map.c index 9bf625ebd..3a12a35b2 100644 --- a/runtime/map.c +++ b/runtime/map.c @@ -1,6 +1,6 @@ /* -*- linux-c -*- * Map Functions - * Copyright (C) 2005-2009 Red Hat Inc. + * Copyright (C) 2005-2009, 2012 Red Hat Inc. * * This file is part of systemtap, and is free software. You can * redistribute it and/or modify it under the terms of the GNU General @@ -18,6 +18,19 @@ #include "stat-common.c" #include "map-stat.c" +#if NEED_MAP_LOCKS +#ifdef __KERNEL__ +#define MAP_LOCK(m) spin_lock(&(m)->lock) +#define MAP_UNLOCK(m) spin_unlock(&(m)->lock) +#else +#define MAP_LOCK(m) pthread_mutex_lock(&(m)->lock) +#define MAP_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) +#endif +#else +#define MAP_LOCK(m) do {} while (0) +#define MAP_UNLOCK(m) do {} while (0) +#endif + static int map_sizes[] = { sizeof(int64_t), MAP_STRING_LENGTH, @@ -209,8 +222,10 @@ static int _stp_map_init(MAP m, unsigned max_entries, int type, int key_size, in else tmp = _stp_kmalloc_node_gfp(size, cpu_to_node(cpu), STP_ALLOC_SLEEP_FLAGS); - if (!tmp) + if (!tmp) { + _stp_error("error allocating map entry\n"); return -1; + } // dbug ("allocated %lx\n", (long)tmp); list_add((struct list_head *)tmp, &m->pool); @@ -239,6 +254,50 @@ static MAP _stp_map_new(unsigned max_entries, int type, int key_size, int data_s return m; } +#ifndef __KERNEL__ +static int _stp_map_tls_object_init(struct tls_data_object_t *obj) +{ + MAP m = container_of(obj, struct map_root, object); + PMAP p = container_of(obj->container, struct pmap, container); + + INIT_LIST_HEAD(&m->pool); + INIT_LIST_HEAD(&m->head); + m->hashes = NULL; + +#if NEED_MAP_LOCKS + { + int rc; + if ((rc = pthread_mutex_init(&m->lock, NULL)) != 0) { + _stp_error("Couldn't initialize map mutex: %d\n", rc); + return -1; + } + } +#endif + + /* To get the correct parameters for _stp_map_init(), get them + * from the cached values in PMAP. */ + if (_stp_map_init(m, p->max_entries, p->type, p->key_size, + p->data_size, -1) != 0) { + __stp_map_del(m); +#if NEED_MAP_LOCKS + (void)pthread_mutex_destroy(&m->lock); +#endif + return -1; + } + + return 0; +} + +static void _stp_map_tls_object_free(struct tls_data_object_t *obj) +{ + MAP m = container_of(obj, struct map_root, object); + __stp_map_del(m); +#if NEED_MAP_LOCKS + (void)pthread_mutex_destroy(&m->lock); +#endif +} +#endif + static PMAP _stp_pmap_new(unsigned max_entries, int type, int key_size, int data_size) { int i; @@ -249,10 +308,19 @@ static PMAP _stp_pmap_new(unsigned max_entries, int type, int key_size, int data if (pmap == NULL) return NULL; +#ifdef __KERNEL__ pmap->map = map = (MAP) _stp_alloc_percpu (sizeof(struct map_root)); if (map == NULL) goto err; +#else + if (_stp_tls_data_container_init(&pmap->container, + sizeof(struct map_root), + &_stp_map_tls_object_init, + &_stp_map_tls_object_free) != 0) + goto err; +#endif +#ifdef __KERNEL__ /* initialize the memory lists first so if allocations fail */ /* at some point, it is easy to clean up. */ for_each_possible_cpu(i) { @@ -260,15 +328,24 @@ static PMAP _stp_pmap_new(unsigned max_entries, int type, int key_size, int data INIT_LIST_HEAD(&m->pool); INIT_LIST_HEAD(&m->head); } +#endif INIT_LIST_HEAD(&pmap->agg.pool); INIT_LIST_HEAD(&pmap->agg.head); +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = per_cpu_ptr (map, i); if (_stp_map_init(m, max_entries, type, key_size, data_size, i)) { goto err1; } } +#else + /* Cache values for use by _stp_map_tls_object_init(). */ + pmap->max_entries = max_entries; + pmap->type = type; + pmap->key_size = key_size; + pmap->data_size = data_size; +#endif if (_stp_map_init(&pmap->agg, max_entries, type, key_size, data_size, -1)) goto err1; @@ -276,11 +353,15 @@ static PMAP _stp_pmap_new(unsigned max_entries, int type, int key_size, int data return pmap; err1: +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = per_cpu_ptr (map, i); __stp_map_del(m); } _stp_free_percpu(map); +#else + _stp_tls_data_container_cleanup(&pmap->container); +#endif err: _stp_kfree(pmap); return NULL; @@ -363,16 +444,28 @@ static void _stp_pmap_clear(PMAP pmap) if (pmap == NULL) return; +#ifdef __KERNEL__ for_each_possible_cpu(i) { MAP m = per_cpu_ptr (pmap->map, i); -#if NEED_MAP_LOCKS - spin_lock(&m->lock); -#endif + + MAP_LOCK(m); _stp_map_clear(m); -#if NEED_MAP_LOCKS - spin_unlock(&m->lock); -#endif + MAP_UNLOCK(m); + } +#else + { + struct tls_data_object_t *obj; + TLS_DATA_CONTAINER_LOCK(&pmap->container); + for_each_tls_data(obj, &pmap->container) { + MAP m = container_of(obj, struct map_root, object); + + MAP_LOCK(m); + _stp_map_clear(m); + MAP_UNLOCK(m); + } + TLS_DATA_CONTAINER_UNLOCK(&pmap->container); } +#endif _stp_map_clear(&pmap->agg); } @@ -417,11 +510,15 @@ static void _stp_pmap_del(PMAP pmap) if (pmap == NULL) return; +#ifdef __KERNEL__ for_each_possible_cpu(i) { MAP m = per_cpu_ptr (pmap->map, i); __stp_map_del(m); } _stp_free_percpu(pmap->map); +#else + _stp_tls_data_container_cleanup(&pmap->container); +#endif /* free agg map elements */ __stp_map_del(&pmap->agg); @@ -740,6 +837,9 @@ static MAP _stp_pmap_agg (PMAP pmap) struct map_node *ptr, *aptr = NULL; struct hlist_head *head, *ahead; struct hlist_node *e, *f; +#ifndef __KERNEL__ + struct tls_data_object_t *obj; +#endif agg = &pmap->agg; @@ -747,11 +847,15 @@ static MAP _stp_pmap_agg (PMAP pmap) /* every time we aggregate. which would be best? */ _stp_map_clear (agg); +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = per_cpu_ptr (pmap->map, i); -#if NEED_MAP_LOCKS - spin_lock(&m->lock); +#else + TLS_DATA_CONTAINER_LOCK(&pmap->container); + for_each_tls_data(obj, &pmap->container) { + m = container_of(obj, struct map_root, object); #endif + MAP_LOCK(m); /* walk the hash chains. */ for (hash = 0; hash < HASH_TABLE_SIZE; hash++) { head = &m->hashes[hash]; @@ -770,18 +874,17 @@ static MAP _stp_pmap_agg (PMAP pmap) _stp_add_agg(aptr, ptr); else { if (!_stp_new_agg(agg, ahead, ptr)) { -#if NEED_MAP_LOCKS - spin_unlock(&m->lock); -#endif + MAP_UNLOCK(m); return NULL; } } } } -#if NEED_MAP_LOCKS - spin_unlock(&m->lock); -#endif + MAP_UNLOCK(m); } +#ifndef __KERNEL__ + TLS_DATA_CONTAINER_UNLOCK(&pmap->container); +#endif return agg; } @@ -923,10 +1026,20 @@ static int _stp_pmap_size (PMAP pmap) { int i, num = 0; +#ifdef __KERNEL__ for_each_possible_cpu(i) { MAP m = per_cpu_ptr (pmap->map, i); num += m->num; } +#else + struct tls_data_object_t *obj; + TLS_DATA_CONTAINER_LOCK(&pmap->container); + for_each_tls_data(obj, &pmap->container) { + MAP m = container_of(obj, struct map_root, object); + num += m->num; + } + TLS_DATA_CONTAINER_UNLOCK(&pmap->container); +#endif return num; } #endif /* _MAP_C_ */ diff --git a/runtime/map.h b/runtime/map.h index 51d657f8c..e8c4450f7 100644 --- a/runtime/map.h +++ b/runtime/map.h @@ -1,6 +1,6 @@ /* -*- linux-c -*- * Map Header File - * Copyright (C) 2005 Red Hat Inc. + * Copyright (C) 2005, 2012 Red Hat Inc. * * This file is part of systemtap, and is free software. You can * redistribute it and/or modify it under the terms of the GNU General @@ -15,6 +15,7 @@ #include #elif defined(__DYNINST__) #include "dyninst/ilog2.h" +#include "dyninst/tls_data.c" #endif /** @file map.h @@ -94,6 +95,11 @@ struct map_node { * It is allocated once when _stp_map_new() is called. */ struct map_root { +#ifndef __KERNEL__ + /* Note that the tls_data_object_t must be first in struct + * map_root. */ + struct tls_data_object_t object; +#endif /* type of the value stored in the array */ int type; @@ -143,7 +149,17 @@ struct map_root { typedef struct map_root *MAP; struct pmap { +#ifdef __KERNEL__ MAP map; /* per-cpu maps */ +#else + struct tls_data_container_t container; + + /* Cached _stp_map_init() values. */ + unsigned max_entries; + int type; + int key_size; + int data_size; +#endif struct map_root agg; /* aggregation map */ }; typedef struct pmap *PMAP; diff --git a/runtime/pmap-gen.c b/runtime/pmap-gen.c index c44b6339c..9f81b1543 100644 --- a/runtime/pmap-gen.c +++ b/runtime/pmap-gen.c @@ -1,6 +1,6 @@ /* -*- linux-c -*- * pmap API generator - * Copyright (C) 2005-2008 Red Hat Inc. + * Copyright (C) 2005-2008, 2012 Red Hat Inc. * * This file is part of systemtap, and is free software. You can * redistribute it and/or modify it under the terms of the GNU General @@ -653,6 +653,24 @@ static unsigned int KEYSYM(phash) (ALLKEYSD(key)) return (unsigned int) (hash % HASH_TABLE_SIZE); } +#ifndef __KERNEL__ +static int _stp_map_tls_object_init(struct tls_data_object_t *obj); +static void _stp_map_tls_object_free(struct tls_data_object_t *obj); + +static int KEYSYM(_stp_pmap_tls_object_init)(struct tls_data_object_t *obj) +{ + MAP m = container_of(obj, struct map_root, object); + PMAP p = container_of(obj->container, struct pmap, container); + + if (_stp_map_tls_object_init(obj) != 0) + return -1; + + m->get_key = KEYSYM(pmap_get_key); + m->copy = KEYSYM(pmap_copy_keys); + m->cmp = KEYSYM(pmap_key_cmp); + return 0; +} +#endif #if VALUE_TYPE == INT64 || VALUE_TYPE == STRING static PMAP KEYSYM(_stp_pmap_new) (unsigned max_entries) @@ -714,6 +732,7 @@ static PMAP KEYSYM(_stp_pmap_new) (unsigned max_entries, int htype, ...) if (pmap) { int i; MAP m; +#ifdef __KERNEL__ for_each_possible_cpu(i) { m = per_cpu_ptr (pmap->map, i); m->get_key = KEYSYM(pmap_get_key); @@ -723,6 +742,13 @@ static PMAP KEYSYM(_stp_pmap_new) (unsigned max_entries, int htype, ...) spin_lock_init(m->lock); #endif } +#else + /* Override the tls data object init function with one + * that knows how to handle pmaps. */ + _stp_tls_data_container_update(&pmap->container, + &KEYSYM(_stp_pmap_tls_object_init), + &_stp_map_tls_object_free); +#endif m = &pmap->agg; m->get_key = KEYSYM(pmap_get_key); m->copy = KEYSYM(pmap_copy_keys); @@ -792,7 +818,17 @@ static int KEYSYM(__stp_pmap_set) (MAP map, ALLKEYSD(key), VSTYPE val, int add) static int KEYSYM(_stp_pmap_set) (PMAP pmap, ALLKEYSD(key), VSTYPE val) { int res; +#ifdef __KERNEL__ MAP m = per_cpu_ptr (pmap->map, MAP_GET_CPU ()); +#else + struct tls_data_object_t *obj; + MAP m; + + obj = _stp_tls_get_per_thread_ptr(&pmap->container); + if (!obj) + return -ENOMEM; + m = container_of(obj, struct map_root, object); +#endif #if NEED_MAP_LOCKS if (!spin_trylock(&m->lock)) return -3; @@ -808,7 +844,17 @@ static int KEYSYM(_stp_pmap_set) (PMAP pmap, ALLKEYSD(key), VSTYPE val) static int KEYSYM(_stp_pmap_add) (PMAP pmap, ALLKEYSD(key), VSTYPE val) { int res; - MAP m = per_cpu_ptr (pmap->map, MAP_GET_CPU()); +#ifdef __KERNEL__ + MAP m = per_cpu_ptr (pmap->map, MAP_GET_CPU ()); +#else + struct tls_data_object_t *obj; + MAP m; + + obj = _stp_tls_get_per_thread_ptr(&pmap->container); + if (!obj) + return -ENOMEM; + m = container_of(obj, struct map_root, object); +#endif #if NEED_MAP_LOCKS if (!spin_trylock(&m->lock)) return -3; @@ -830,11 +876,21 @@ static VALTYPE KEYSYM(_stp_pmap_get_cpu) (PMAP pmap, ALLKEYSD(key)) struct KEYSYM(pmap_node) *n; VALTYPE res; MAP map; +#ifndef __KERNEL__ + struct tls_data_object_t *obj; +#endif if (pmap == NULL) return NULLRET; +#ifdef __KERNEL__ map = per_cpu_ptr (pmap->map, MAP_GET_CPU ()); +#else + obj = _stp_tls_get_per_thread_ptr(&pmap->container); + if (!obj) + return NULLRET; + map = container_of(obj, struct map_root, object); +#endif hv = KEYSYM(phash) (ALLKEYS(key)); head = &map->hashes[hv]; @@ -896,6 +952,9 @@ static VALTYPE KEYSYM(_stp_pmap_get) (PMAP pmap, ALLKEYSD(key)) struct KEYSYM(pmap_node) *n; struct map_node *anode = NULL; MAP map, agg; +#ifndef __KERNEL__ + struct tls_data_object_t *obj; +#endif if (pmap == NULL) return NULLRET; @@ -940,8 +999,14 @@ static VALTYPE KEYSYM(_stp_pmap_get) (PMAP pmap, ALLKEYSD(key)) } /* now total each cpu */ +#ifdef __KERNEL__ for_each_possible_cpu(cpu) { map = per_cpu_ptr (pmap->map, cpu); +#else + TLS_DATA_CONTAINER_LOCK(&pmap->container); + for_each_tls_data(obj, &pmap->container) { + map = container_of(obj, struct map_root, object); +#endif head = &map->hashes[hv]; #if NEED_MAP_LOCKS @@ -992,6 +1057,9 @@ static VALTYPE KEYSYM(_stp_pmap_get) (PMAP pmap, ALLKEYSD(key)) spin_unlock(&map->lock); #endif } +#ifndef __KERNEL__ + TLS_DATA_CONTAINER_UNLOCK(&pmap->container); +#endif if (anode && !clear_agg) return MAP_GET_VAL(anode); @@ -1055,7 +1123,17 @@ static int KEYSYM(__stp_pmap_del) (MAP map, ALLKEYSD(key)) static int KEYSYM(_stp_pmap_del) (PMAP pmap, ALLKEYSD(key)) { int res; +#ifdef __KERNEL__ MAP m = per_cpu_ptr (pmap->map, MAP_GET_CPU ()); +#else + struct tls_data_object_t *obj; + MAP m; + + obj = _stp_tls_get_per_thread_ptr(&pmap->container); + if (!obj) + return -ENOMEM; + m = container_of(obj, struct map_root, object); +#endif #if NEED_MAP_LOCKS if (!spin_trylock(&m->lock)) return -1; diff --git a/runtime/stat.c b/runtime/stat.c index 58c71eb0c..d8a9fe4b0 100644 --- a/runtime/stat.c +++ b/runtime/stat.c @@ -1,6 +1,6 @@ /* -*- linux-c -*- * Statistics Aggregation - * Copyright (C) 2005-2008 Red Hat Inc. + * Copyright (C) 2005-2008, 2012 Red Hat Inc. * Copyright (C) 2006 Intel Corporation * * This file is part of systemtap, and is free software. You can @@ -38,13 +38,22 @@ * @{ */ +#ifndef __KERNEL__ +#include +#endif + #include "stat-common.c" /* for the paranoid. */ #if NEED_STAT_LOCKS == 1 +#ifdef __KERNEL__ #define STAT_LOCK(sd) spin_lock(&sd->lock) #define STAT_UNLOCK(sd) spin_unlock(&sd->lock) #else +#define STAT_LOCK(sd) pthread_mutex_lock(&sd->lock) +#define STAT_UNLOCK(sd) pthread_mutex_unlock(&sd->lock) +#endif +#else #define STAT_LOCK(sd) ; #define STAT_UNLOCK(sd) ; #endif @@ -52,14 +61,44 @@ /** Stat struct for stat.c. Maps do not need this */ struct _Stat { struct _Hist hist; - /* per-cpu data. allocated with _stp_alloc_percpu() */ + + /* + * In kernel-mode, the stat data is per-cpu data (allocated + * with _stp_alloc_percpu()) stored in 'sd'. In dyninst-mode, + * the stat data is thread local storage. + */ +#ifdef __KERNEL__ stat_data *sd; +#else + struct tls_data_container_t container; +#endif /* aggregated data */ stat_data *agg; }; typedef struct _Stat *Stat; +#ifndef __KERNEL__ +#if NEED_STAT_LOCKS == 1 +static int _stp_stat_tls_object_init(struct tls_data_object_t *obj) +{ + stat_data *sd = container_of(obj, stat_data, object); + + int rc; + if ((rc = pthread_mutex_init(&sd->lock, NULL)) != 0) { + _stp_error("Couldn't initialize stat mutex: %d\n", rc); + return -1; + } + return 0; +} + +static void _stp_stat_tls_object_free(struct tls_data_object_t *obj) +{ + stat_data *sd = container_of(obj, stat_data, object); + (void)pthread_mutex_destroy(&sd->lock); +} +#endif /* NEED_STAT_LOCKS == 1 */ +#endif /* !__KERNEL__ */ /** Initialize a Stat. * Call this during probe initialization to create a Stat. @@ -77,7 +116,11 @@ typedef struct _Stat *Stat; static Stat _stp_stat_init (int type, ...) { int size, buckets=0, start=0, stop=0, interval=0; +#ifdef __KERNEL__ stat_data *sd, *agg; +#else + stat_data *agg; +#endif Stat st; if (type != HIST_NONE) { @@ -103,9 +146,11 @@ static Stat _stp_stat_init (int type, ...) return NULL; size = buckets * sizeof(int64_t) + sizeof(stat_data); +#ifdef __KERNEL__ sd = (stat_data *) _stp_alloc_percpu (size); if (sd == NULL) goto exit1; + st->sd = sd; #if NEED_STAT_LOCKS == 1 { @@ -115,7 +160,20 @@ static Stat _stp_stat_init (int type, ...) spin_lock_init(sdp->lock); } } -#endif +#endif /* NEED_STAT_LOCKS == 1 */ + +#else /* !__KERNEL__ */ + +#if NEED_STAT_LOCKS == 1 + if (_stp_tls_data_container_init(&st->container, size, + &_stp_stat_tls_object_init, + &_stp_stat_tls_object_free) != 0) +#else /* NEED_STAT_LOCKS !=1 */ + if (_stp_tls_data_container_init(&st->container, size, + NULL, NULL) != 0) +#endif /* NEED_STAT_LOCKS != 1 */ + goto exit1; +#endif /* !__KERNEL__ */ agg = (stat_data *)_stp_kmalloc_gfp(size, STP_ALLOC_SLEEP_FLAGS); if (agg == NULL) @@ -126,12 +184,15 @@ static Stat _stp_stat_init (int type, ...) st->hist.stop = stop; st->hist.interval = interval; st->hist.buckets = buckets; - st->sd = sd; st->agg = agg; return st; exit2: +#ifdef __KERNEL__ _stp_kfree (sd); +#else + _stp_tls_data_container_cleanup(&st->container); +#endif exit1: _stp_kfree (st); return NULL; @@ -145,7 +206,11 @@ exit1: static void _stp_stat_del (Stat st) { if (st) { +#ifdef __KERNEL__ _stp_free_percpu (st->sd); +#else /* !__KERNEL__ */ + _stp_tls_data_container_cleanup(&st->container); +#endif /* !__KERNEL__ */ _stp_kfree (st->agg); _stp_kfree (st); } @@ -159,29 +224,23 @@ static void _stp_stat_del (Stat st) */ static void _stp_stat_add (Stat st, int64_t val) { +#ifdef __KERNEL__ stat_data *sd = per_cpu_ptr (st->sd, get_cpu()); +#else + struct tls_data_object_t *obj; + stat_data *sd; + + obj = _stp_tls_get_per_thread_ptr(&st->container); + if (!obj) + return; + sd = container_of(obj, stat_data, object); +#endif STAT_LOCK(sd); __stp_stat_add (&st->hist, sd, val); STAT_UNLOCK(sd); put_cpu(); } -/** Get per-cpu Stats. - * Gets the Stats for a specific CPU. - * - * If NEED_STAT_LOCKS is set, you MUST call STAT_UNLOCK() - * when you are finished with the returned pointer. - * - * @param st Stat - * @param cpu CPU number - * @returns A pointer to a stat. - */ -static stat_data *_stp_stat_get_cpu (Stat st, int cpu) -{ - stat_data *sd = per_cpu_ptr (st->sd, cpu); - STAT_LOCK(sd); - return sd; -} static void _stp_stat_clear_data (Stat st, stat_data *sd) { @@ -208,11 +267,21 @@ static stat_data *_stp_stat_get (Stat st, int clear) { int i, j; stat_data *agg = st->agg; + stat_data *sd; +#ifndef __KERNEL__ + struct tls_data_object_t *obj; +#endif STAT_LOCK(agg); _stp_stat_clear_data (st, agg); +#ifdef __KERNEL__ for_each_possible_cpu(i) { - stat_data *sd = per_cpu_ptr (st->sd, i); + sd = per_cpu_ptr (st->sd, i); +#else + TLS_DATA_CONTAINER_LOCK(&st->container); + for_each_tls_data(obj, &st->container) { + sd = container_of(obj, stat_data, object); +#endif STAT_LOCK(sd); if (sd->count) { if (agg->count == 0) { @@ -234,6 +303,9 @@ static stat_data *_stp_stat_get (Stat st, int clear) } STAT_UNLOCK(sd); } +#ifndef __KERNEL__ + TLS_DATA_CONTAINER_UNLOCK(&st->container); +#endif return agg; } @@ -245,13 +317,23 @@ static stat_data *_stp_stat_get (Stat st, int clear) */ static void _stp_stat_clear (Stat st) { +#ifdef __KERNEL__ int i; for_each_possible_cpu(i) { stat_data *sd = per_cpu_ptr (st->sd, i); +#else + struct tls_data_object_t *obj; + TLS_DATA_CONTAINER_LOCK(&st->container); + for_each_tls_data(obj, &st->container) { + stat_data *sd = container_of(obj, stat_data, object); +#endif STAT_LOCK(sd); _stp_stat_clear_data (st, sd); STAT_UNLOCK(sd); } +#ifndef __KERNEL__ + TLS_DATA_CONTAINER_UNLOCK(&st->container); +#endif } /** @} */ #endif /* _STAT_C_ */ diff --git a/runtime/stat.h b/runtime/stat.h index d61bfcfd5..96371e9d1 100644 --- a/runtime/stat.h +++ b/runtime/stat.h @@ -1,6 +1,6 @@ /* -*- linux-c -*- * Statistics Header - * Copyright (C) 2005 Red Hat Inc. + * Copyright (C) 2005, 2012 Red Hat Inc. * * This file is part of systemtap, and is free software. You can * redistribute it and/or modify it under the terms of the GNU General @@ -11,6 +11,10 @@ #ifndef _STAT_H_ #define _STAT_H_ +#ifndef __KERNEL__ +#include "dyninst/tls_data.c" +#endif + #ifndef NEED_STAT_LOCKS #define NEED_STAT_LOCKS 0 #endif @@ -30,11 +34,20 @@ enum histtype { HIST_NONE, HIST_LOG, HIST_LINEAR }; /** Statistics are stored in this struct. This is per-cpu or per-node data and is variable length due to the unknown size of the histogram. */ struct stat_data { +#ifndef __KERNEL__ + /* Note that the tls_data_object_t must be first in struct + * stat_data. */ + struct tls_data_object_t object; +#endif int64_t count; int64_t sum; int64_t min, max; #if NEED_STAT_LOCKS == 1 +#ifdef __KERNEL__ spinlock_t lock; +#else /* !__KERNEL__ */ + pthread_mutex_t lock; +#endif /* !__KERNEL__ */ #endif int64_t histogram[]; }; -- 2.43.5