memory tracing

.gitignore

@@ -1,3 +1,5 @@
 build
 *.user
 version.h
+*.todo
+*.pyc

CMakeLists.txt

@@ -147,6 +147,9 @@ set( SOURCES src/config.c
              src/util/cache.c
              src/util/color.c
              src/util/gradient.c
+             src/util/addr2line.c
+             src/util/print.c
+             src/util/mem.c
              src/util/uevent.c
              src/util/window.c )
 
@@ -265,9 +268,14 @@ if( RT_LIBRARY )
 endif( RT_LIBRARY )
 
 target_link_libraries( tint2 m )
+if(ENABLE_BACKTRACE)
+  target_link_libraries( tint2 dl )
+  target_link_libraries( tint2 z )
+  target_link_libraries( tint2 bfd )
+endif(ENABLE_BACKTRACE)
 
 add_dependencies( tint2 version )
-set_target_properties( tint2 PROPERTIES COMPILE_FLAGS "-Wall -Wpointer-arith -fno-strict-aliasing -pthread -std=c99 ${ASAN_C_FLAGS} ${TRACING_C_FLAGS}" )
+set_target_properties( tint2 PROPERTIES COMPILE_FLAGS "-Wall -Wpointer-arith -fno-strict-aliasing -pthread -std=c11 ${ASAN_C_FLAGS} ${TRACING_C_FLAGS}" )
 set_target_properties( tint2 PROPERTIES LINK_FLAGS "-pthread -fno-strict-aliasing ${ASAN_L_FLAGS} ${BACKTRACE_L_FLAGS}  ${TRACING_L_FLAGS}" )
 
 install( TARGETS tint2 DESTINATION bin )

src/util/addr2line.c

@@ -0,0 +1,238 @@
+/* addr2line.c -- convert addresses to line number and function name
+   Copyright 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2009  Free Software Foundation, Inc.
+   Contributed by Ulrich Lauther <Ulrich.Lauther@mchp.siemens.de>
+
+   This file is part of GNU Binutils.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 51 Franklin Street - Fifth Floor, Boston,
+   MA 02110-1301, USA.  */
+
+/* Derived from objdump.c and nm.c by Ulrich.Lauther@mchp.siemens.de */
+
+#ifdef ENABLE_EXECINFO
+
+#include <bfd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "addr2line.h"
+#include "print.h"
+
+static bfd_boolean unwind_inlines = 1; /* -i, unwind inlined functions. */
+static bfd_boolean with_addresses = 0; /* -a, show addresses.  */
+static bfd_boolean with_functions = 1; /* -f, show function names.  */
+static bfd_boolean do_demangle = 1;    /* -C, demangle names.  */
+static bfd_boolean base_names = 1;     /* -s, strip directory names.  */
+
+typedef struct Lookup {
+    char *exe_file_name;
+    void *address;
+    char *result;
+    bfd *abfd;
+    asymbol **syms;
+    bfd_vma pc;
+    const char *filename;
+    const char *functionname;
+    unsigned int line;
+    unsigned int discriminator;
+    bfd_boolean found;
+} Lookup;
+
+static asymbol **slurp_symtab(bfd *);
+static void find_address_in_section(bfd *, asection *, void *unused);
+static void translate_address(Lookup *lookup);
+
+static asymbol **slurp_symtab(bfd *abfd)
+{
+    if ((bfd_get_file_flags(abfd) & HAS_SYMS) == 0)
+        return NULL;
+
+    long storage = bfd_get_symtab_upper_bound(abfd);
+    bfd_boolean dynamic = FALSE;
+    if (storage == 0) {
+        storage = bfd_get_dynamic_symtab_upper_bound(abfd);
+        dynamic = TRUE;
+    }
+    if (storage < 0)
+        return NULL;
+
+    asymbol **syms = (asymbol **)malloc(storage);
+    long symcount;
+    if (dynamic)
+        symcount = bfd_canonicalize_dynamic_symtab(abfd, syms);
+    else
+        symcount = bfd_canonicalize_symtab(abfd, syms);
+    if (symcount < 0)
+        return syms;
+
+    // If there are no symbols left after canonicalization and
+    // we have not tried the dynamic symbols then give them a go.
+    if (symcount == 0 && !dynamic && (storage = bfd_get_dynamic_symtab_upper_bound(abfd)) > 0) {
+        free(syms);
+        syms = (asymbol **)malloc(storage);
+        symcount = bfd_canonicalize_dynamic_symtab(abfd, syms);
+    }
+    return syms;
+}
+
+static void find_address_in_section(bfd *abfd, asection *section, void *unused)
+{
+    Lookup *lookup = (Lookup *)abfd->usrdata;
+
+    bfd_vma vma;
+    bfd_size_type size;
+
+    if (lookup->found)
+        return;
+
+    if ((bfd_get_section_flags(abfd, section) & SEC_ALLOC) == 0)
+        return;
+
+    vma = bfd_get_section_vma(abfd, section);
+    if (lookup->pc < vma)
+        return;
+
+    size = bfd_get_section_size(section);
+    if (lookup->pc >= vma + size)
+        return;
+
+    lookup->found = bfd_find_nearest_line_discriminator(abfd,
+                                                        section,
+                                                        lookup->syms,
+                                                        lookup->pc - vma,
+                                                        &lookup->filename,
+                                                        &lookup->functionname,
+                                                        &lookup->line,
+                                                        &lookup->discriminator);
+}
+
+static void translate_address(Lookup *lookup)
+{
+    Buffer *buffer = NULL;
+    lookup->pc = (bfd_vma)lookup->address;
+
+    if (with_addresses) {
+        char tmp[256];
+        bfd_sprintf_vma(lookup->abfd, tmp, lookup->pc);
+        buffer = buffer_printf(buffer, "0x%s: ", tmp);
+    }
+
+    lookup->found = FALSE;
+    bfd_map_over_sections(lookup->abfd, find_address_in_section, NULL);
+
+    if (!lookup->found) {
+        if (with_functions)
+            buffer = buffer_printf(buffer, "?? ");
+        buffer = buffer_printf(buffer, "??:0");
+    } else {
+        while (1) {
+            if (with_functions) {
+                const char *name;
+                char *alloc = NULL;
+
+                name = lookup->functionname;
+                if (name == NULL || *name == '\0')
+                    name = "??";
+                else if (do_demangle) {
+                    alloc = bfd_demangle(lookup->abfd, name, 3);
+                    if (alloc != NULL)
+                        name = alloc;
+                }
+
+                buffer = buffer_printf(buffer, "%s", name);
+                buffer = buffer_printf(buffer, " at ");
+
+                if (alloc != NULL)
+                    free(alloc);
+            }
+
+            if (base_names && lookup->filename != NULL) {
+                const char *h = strrchr(lookup->filename, '/');
+                if (h != NULL)
+                    lookup->filename = h + 1;
+            }
+
+            buffer = buffer_printf(buffer, "%s:", lookup->filename ? lookup->filename : "??");
+            if (lookup->line != 0) {
+                if (lookup->discriminator != 0)
+                    buffer = buffer_printf(buffer, "%u (discriminator %u)", lookup->line, lookup->discriminator);
+                else
+                    buffer = buffer_printf(buffer, "%u", lookup->line);
+            } else {
+                buffer = buffer_printf(buffer, "??");
+            }
+            if (!unwind_inlines)
+                lookup->found = FALSE;
+            else
+                lookup->found =
+                    bfd_find_inliner_info(lookup->abfd, &lookup->filename, &lookup->functionname, &lookup->line);
+            if (!lookup->found)
+                break;
+            buffer = buffer_printf(buffer, " (inlined by) ");
+        }
+    }
+    if (buffer)
+        lookup->result = buffer->data;
+}
+
+Lookup *addr2line_init(const char *file_name)
+{
+    bfd *abfd = bfd_openr(file_name, NULL);
+    if (abfd == NULL)
+        return NULL;
+
+    // Decompress sections.
+    abfd->flags |= BFD_DECOMPRESS;
+
+    if (bfd_check_format(abfd, bfd_archive))
+        goto err;
+
+    char **matching = NULL;
+    if (!bfd_check_format_matches(abfd, bfd_object, &matching)) {
+        free(matching);
+        goto err;
+    }
+
+    Lookup *lookup = (Lookup *)calloc(1, sizeof(Lookup));
+    lookup->abfd = abfd;
+    lookup->abfd->usrdata = lookup;
+    lookup->syms = slurp_symtab(abfd);
+    lookup->exe_file_name = strdup(file_name);
+    return lookup;
+
+err:
+    bfd_close(abfd);
+    return NULL;
+}
+
+void addr2line_destroy(Lookup *lookup)
+{
+    free(lookup->syms);
+    bfd_close(lookup->abfd);
+    free(lookup->exe_file_name);
+    free(lookup);
+}
+
+char *addr2line_lookup(Lookup *lookup, void *address)
+{
+    lookup->address = address;
+    lookup->result = NULL;
+    translate_address(lookup);
+    return lookup->result;
+}
+
+#endif

src/util/addr2line.h

@@ -0,0 +1,14 @@
+#ifndef ADDR2LINE
+#define ADDR2LINE
+
+#ifdef ENABLE_EXECINFO
+
+typedef struct Lookup Lookup;
+
+Lookup *addr2line_init(const char *file_name);
+void addr2line_destroy(Lookup *lookup);
+char *addr2line_lookup(Lookup *lookup, void *address);
+
+#endif
+
+#endif

src/util/mem.c

@@ -0,0 +1,471 @@
+#ifdef ENABLE_EXECINFO
+
+#include <dlfcn.h>
+#include <execinfo.h>
+#include <pthread.h>
+#include <stdc-predef.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <zlib.h>
+
+#include "addr2line.h"
+#include "mem.h"
+#include "print.h"
+
+#ifndef RTLD_NEXT
+# define RTLD_NEXT	((void *) -1l)
+#endif
+
+#ifndef thread_local
+# if __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__
+#  define thread_local _Thread_local
+# elif defined _WIN32 && ( \
+       defined _MSC_VER || \
+       defined __ICL || \
+       defined __DMC__ || \
+       defined __BORLANDC__ )
+#  define thread_local __declspec(thread)
+/* note that ICC (linux) and Clang are covered by __GNUC__ */
+# elif defined __GNUC__ || \
+       defined __SUNPRO_C || \
+       defined __xlC__
+#  define thread_local __thread
+# else
+#  error "Cannot define thread_local"
+# endif
+#endif
+
+#define get_caller() __builtin_extract_return_addr(__builtin_return_address(0))
+
+#define size_t_mul_overflow(a, b) (a > 0 && b > SIZE_MAX / a)
+
+typedef struct ListItem {
+    struct ListItem *next;
+    struct ListItem *prev;
+    void *data;
+} ListItem;
+
+typedef struct List {
+    ListItem *head;
+    ListItem *tail;
+} List;
+
+void list_append(List *list, void *data)
+{
+    ListItem *n = (ListItem*)calloc(1, sizeof(ListItem));
+    n->data = data;
+    if (!list->tail) {
+        list->head = list->tail = n;
+    } else {
+        list->tail->next = n;
+        n->prev = list->tail;
+        list->tail = list->tail->next;
+    }
+}
+
+typedef size_t (*HashFunc)(void *data);
+typedef int (*EqualFunc)(void *data1, void *data2);
+typedef void *(*CopyFunc)(void *data);
+
+typedef struct HashTable {
+    List *buckets;
+    size_t num_buckets;
+    size_t (*hash_func)(void *data);
+    int (*equal_func)(void *data1, void *data2);
+    void *(*copy_func)(void *data);
+} HashTable;
+
+size_t hash_void(void *data)
+{
+    return (size_t)data;
+}
+
+int equal_void(void *data1, void *data2)
+{
+    return data1 == data2;
+}
+
+void *copy_void(void *data)
+{
+    return data;
+}
+
+void hash_table_init(HashTable *table, size_t num_buckets, HashFunc hash_func, EqualFunc equal_func, CopyFunc copy_func)
+{
+    if (table->buckets)
+        return;
+    if (num_buckets < 1)
+        num_buckets = 4096;
+    table->num_buckets = num_buckets;
+    table->buckets = (List*)calloc(table->num_buckets, sizeof(List));
+    table->hash_func = hash_func ? hash_func : hash_void;
+    table->equal_func = equal_func ? equal_func : equal_void;
+    table->copy_func = copy_func ? copy_func : copy_void;
+}
+
+void *hash_table_add(HashTable *table, void *data)
+{
+    size_t hash = table->hash_func(data);
+    size_t bucket = hash % table->num_buckets;
+    for (ListItem *existing = table->buckets[bucket].head; existing; existing = existing->next) {
+        if (table->equal_func(existing->data, data))
+            return existing->data;
+    }
+    list_append(&table->buckets[bucket], table->copy_func(data));
+    return data;
+}
+
+#define MAX_TRACE_SIZE 120
+
+typedef struct Trace {
+    void *entries[MAX_TRACE_SIZE+8];
+    size_t id;
+} Trace;
+
+#define LARGE_THRESH (10 * 1024 * 1024)
+
+static void *(*calloc_original)(size_t count, size_t size) = NULL;
+static void *(*realloc_original)(void *p, size_t size) = NULL;
+static void *(*malloc_original)(size_t size) = NULL;
+static void *(*free_original)(void *p) = NULL;
+static gzFile mem_log = NULL;
+static pid_t mem_log_owner = 0;
+pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static HashTable traces;
+static size_t next_trace_id = 0;
+pthread_mutex_t trace_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static HashTable addr2lines;
+pthread_mutex_t addr2lines_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+thread_local int paused = 0;
+
+static double get_unix_time()
+{
+    struct timeval tv;
+    if (gettimeofday(&tv, NULL) != 0)
+        return 0;
+    return tv.tv_sec + 1.0e-6 * tv.tv_usec;
+}
+
+__attribute__ ((noinline))
+static void get_backtrace(Trace *trace) {
+    int size = backtrace(trace->entries, MAX_TRACE_SIZE);
+    trace->entries[size] = NULL;
+    for (void **p = trace->entries; *p; p++) {
+        *p = *(p+2);
+    }
+    trace->id = 0;
+}
+
+int trace_size(Trace *trace)
+{
+    int size = 0;
+    for (void **p = trace->entries; *p; p++) {
+        size++;
+    }
+    return size;
+}
+
+typedef struct FilenameLookup {
+    char *file_name;
+    Lookup *lookup;
+} FilenameLookup;
+
+size_t hash_lookup(void *data)
+{
+    FilenameLookup *lookup = (FilenameLookup *)data;
+    size_t seed = 14695981039346656037ULL;
+    size_t hash = seed;
+    for (char *p = lookup->file_name; *p; p++) {
+        hash = (hash ^ (size_t)(*p)) * 1099511628211ULL;
+    }
+    return hash;
+}
+
+int equal_lookup(void *data1, void *data2)
+{
+    FilenameLookup *lookup1 = (FilenameLookup *)data1;
+    FilenameLookup *lookup2 = (FilenameLookup *)data2;
+    return strcmp(lookup1->file_name, lookup2->file_name) == 0;
+}
+
+char *resolve_symbol(char *file_name, void *address)
+{
+    FilenameLookup *lookup = (FilenameLookup *)calloc(1, sizeof(FilenameLookup));
+    lookup->file_name = strdup(file_name);
+
+    pthread_mutex_lock(&addr2lines_mutex);
+    hash_table_init(&addr2lines, 0, hash_lookup, equal_lookup, NULL);
+    FilenameLookup *existing = (FilenameLookup *)hash_table_add(&addr2lines, lookup);
+    if (existing == lookup) {
+        lookup->lookup = addr2line_init(lookup->file_name);
+    } else {
+        free(lookup->file_name);
+        free(lookup);
+    }
+    char *result = addr2line_lookup(existing->lookup, address);
+    pthread_mutex_unlock(&addr2lines_mutex);
+    return result;
+}
+
+char *backtrace_to_string(Trace *trace)
+{
+    int size = trace_size(trace);
+    char **strings = backtrace_symbols(trace->entries, size);
+    if (!strings)
+        return NULL;
+    Buffer *buffer = NULL;
+    buffer = buffer_printf(buffer, "t %lu ", trace->id);
+    for (int i = 0; i < size; i++) {
+        buffer = buffer_printf(buffer, "%s", strings[i]);
+        // /lib/x86_64-linux-gnu/libglib-2.0.so.0(g_realloc+0xf) [0x7efda85c96af]
+        char *func_start = strstr(strings[i], "(");
+        if (func_start) {
+            *func_start = 0;
+            char *file_name = strings[i];
+            char *resolved = resolve_symbol(file_name, trace->entries[i]);
+            if (resolved) {
+                buffer = buffer_printf(buffer, " %s", resolved);
+                free(resolved);
+            }
+        }
+        buffer = buffer_printf(buffer, "|");
+    }
+    free(strings);
+    buffer = buffer_printf(buffer, "\n");
+    return buffer->data;
+}
+
+static size_t hash_trace(Trace *trace)
+{
+    size_t seed = 14695981039346656037ULL;
+    size_t hash = seed;
+    for (void **p = trace->entries; *p; p++) {
+        hash = (hash ^ (size_t)(*p)) * 1099511628211ULL;
+    }
+    return hash;
+}
+
+static int equal_traces(Trace *a, Trace *b)
+{
+    void **pa, **pb;
+    for (pa = a->entries, pb = b->entries; *pa && *pb; pa++, pb++) {
+        if (*pa != *pb)
+            return 0;
+    }
+    if (*pa || *pb)
+        return 0;
+    return 1;
+}
+
+static Trace *copy_trace(Trace *trace)
+{
+    Trace *result = (Trace*)calloc(1, sizeof(Trace));
+    *result = *trace;
+    return result;
+}
+
+static int trace_assign_id(Trace *trace)
+{
+    int created = 0;
+    pthread_mutex_lock(&trace_mutex);
+    hash_table_init(&traces, 1024 * 1024, (HashFunc)hash_trace, (EqualFunc)equal_traces, (CopyFunc)copy_trace);
+    trace->id = 0;
+    Trace *existing = (Trace*)hash_table_add(&traces, trace);
+    if (!existing->id) {
+        next_trace_id++;
+        existing->id = trace->id = next_trace_id;
+        created = 1;
+    } else {
+        trace->id = existing->id;
+    }
+    pthread_mutex_unlock(&trace_mutex);
+    return created;
+}
+
+static void log_flush();
+
+static void log_init()
+{
+    char path[256];
+    sprintf(path, "/tmp/tint2.%lu.memtrace", (size_t)getpid());
+    pthread_mutex_lock(&log_mutex);
+    if (!mem_log) {
+        mem_log = gzopen(path, "w");
+        mem_log_owner = getpid();
+        atexit(log_flush);
+    }
+    pthread_mutex_unlock(&log_mutex);
+    if (mem_log)
+        fprintf(stderr, "Writing memory log to %s\n", path);
+}
+
+static void log_write(char *buffer)
+{
+    if (mem_log) {
+        pthread_mutex_lock(&log_mutex);
+        if (getpid() == mem_log_owner)
+            gzwrite(mem_log, buffer, (unsigned)strlen(buffer));
+        pthread_mutex_unlock(&log_mutex);
+    }
+}
+
+static void log_flush()
+{
+    if (mem_log) {
+        pthread_mutex_lock(&log_mutex);
+        if (getpid() == mem_log_owner)
+            gzflush(mem_log, Z_SYNC_FLUSH);
+        pthread_mutex_unlock(&log_mutex);
+    }
+}
+
+static void malloc_profile_load_pointers()
+{
+    realloc_original = dlsym(RTLD_NEXT, "realloc");
+    malloc_original = dlsym(RTLD_NEXT, "malloc");
+    calloc_original = dlsym(RTLD_NEXT, "calloc");
+    free_original = dlsym(RTLD_NEXT, "free");
+    int trace_mem = getenv("TRACE_MEM") != NULL;
+    if (trace_mem)
+        log_init();
+    else
+        paused = 1;
+}
+
+void *calloc(size_t count, size_t size)
+{
+    if (paused)
+        return calloc_original ? calloc_original(count, size) : 0;
+    paused = 1;
+    if (!calloc_original) {
+        malloc_profile_load_pointers();
+        if (!calloc_original)
+            return NULL;
+    }
+    void *result = calloc_original(count, size);
+    if (mem_log) {
+        Trace trace;
+        get_backtrace(&trace);
+        int created = trace_assign_id(&trace);
+        if (created) {
+            char *str = backtrace_to_string(&trace);
+            if (str) {
+                log_write(str);
+                free(str);
+            }
+        }
+        char buffer[1024];
+        sprintf(buffer, "[%f] t %lu : %p = c %lu %lu\n", get_unix_time(), trace.id, result, count, size);
+        log_write(buffer);
+        if (size_t_mul_overflow(count, size) || (count * size) > LARGE_THRESH)
+            log_flush();
+    }
+    paused = 0;
+    return result;
+}
+
+void *realloc(void *p, size_t size)
+{
+    if (paused)
+        return realloc_original ? realloc_original(p, size) : 0;
+    paused = 1;
+    if (!realloc_original) {
+        malloc_profile_load_pointers();
+        if (!realloc_original)
+            return NULL;
+    }
+    void *result = realloc_original(p, size);
+    if (mem_log) {
+        Trace trace;
+        get_backtrace(&trace);
+        int created = trace_assign_id(&trace);
+        if (created) {
+            char *str = backtrace_to_string(&trace);
+            if (str) {
+                log_write(str);
+                free(str);
+            }
+        }
+        char buffer[1024];
+        sprintf(buffer, "[%f] t %lu : %p = r %p %lu\n", get_unix_time(), trace.id, result, p, size);
+        log_write(buffer);
+        if (size > LARGE_THRESH)
+            log_flush();
+    }
+    paused = 0;
+    return result;
+}
+
+void *malloc(size_t size)
+{
+    if (paused)
+        return malloc_original ? malloc_original(size) : 0;
+    paused = 1;
+    if (!malloc_original) {
+        malloc_profile_load_pointers();
+        if (!malloc_original)
+            return NULL;
+    }
+    void *result = malloc_original(size);
+    if (mem_log) {
+        Trace trace;
+        get_backtrace(&trace);
+        int created = trace_assign_id(&trace);
+        if (created) {
+            char *str = backtrace_to_string(&trace);
+            if (str) {
+                log_write(str);
+                free(str);
+            }
+        }
+        char buffer[1024];
+        sprintf(buffer, "[%f] t %lu : %p = m %lu\n", get_unix_time(), trace.id, result, size);
+        log_write(buffer);
+        if (size > LARGE_THRESH)
+            log_flush();
+    }
+    paused = 0;
+    return result;
+}
+
+void free(void *p)
+{
+    if (paused) {
+        if (free_original)
+            free_original(p);
+        return;
+    }
+    paused = 1;
+    if (!free_original) {
+        malloc_profile_load_pointers();
+        if (!free_original)
+            return;
+    }
+    free_original(p);
+    if (mem_log) {
+        Trace trace;
+        get_backtrace(&trace);
+        int created = trace_assign_id(&trace);
+        if (created) {
+            char *str = backtrace_to_string(&trace);
+            if (str) {
+                log_write(str);
+                free(str);
+            }
+        }
+        char buffer[1024];
+        sprintf(buffer, "[%f] t %lu : 0 = f %p\n", get_unix_time(), trace.id, p);
+        log_write(buffer);
+    }
+    paused = 0;
+}
+
+#endif

src/util/mem.h

@@ -0,0 +1,4 @@
+#ifndef MEM_H
+#define MEM_H
+
+#endif

src/util/print.c

@@ -0,0 +1,40 @@
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "print.h"
+
+Buffer *vbuffer_printf(Buffer *buffer, const char *fmt, va_list ap)
+{
+    va_list ap1;
+    va_copy(ap1, ap);
+    int ret = vsnprintf(NULL, 0, fmt, ap1) + 1;
+    va_end(ap1);
+    if (ret < 1)
+        return buffer;
+
+    size_t size = (size_t)ret;
+    if (!buffer)
+        buffer = (Buffer*)calloc(1, sizeof(Buffer));
+    if (!buffer->data) {
+        buffer->size = 2 * size + 128;
+        buffer->data = (char*) calloc(buffer->size, 1);
+    } else if (strlen(buffer->data) + size >= buffer->size + 1) {
+        buffer->size = 2 * (size + buffer->size) + 128;
+        buffer->data = (char*) realloc(buffer->data, buffer->size);
+    }
+
+    vsnprintf(buffer->data + strlen(buffer->data), size, fmt, ap);
+    return buffer;
+}
+
+Buffer *buffer_printf(Buffer *buffer, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    buffer = vbuffer_printf(buffer, fmt, ap);
+    va_end(ap);
+
+    return buffer;
+}

src/util/print.h

@@ -0,0 +1,16 @@
+#ifndef PRINT_H
+#define PRINT_H
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <sys/types.h>
+
+typedef struct Buffer {
+    char *data;
+    size_t size;
+} Buffer;
+
+Buffer *vbuffer_printf(Buffer *buffer, const char *fmt, va_list ap);
+Buffer *buffer_printf(Buffer *buffer, const char *fmt, ...);
+
+#endif

test/memtrace-analyze.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python2
+
+import gzip
+import sys
+
+
+def commas(v):
+  return "{:12,}".format(v)
+
+
+class Event:
+  def __init__(self):
+    self.alloc = False
+    self.free = False
+    self.ts = None
+    self.size = None
+    self.addr = None
+    self.trace_name = None
+
+
+class Analyzer:
+  def __init__(self):
+    self.traces = {}
+    self.events = []
+    self.all_allocations = {}
+    self.new_allocations = {}
+    self.all_mem = 0
+    self.old_mem = 0
+
+  def read(self, path):
+    with gzip.open(path, "rb") as f:
+      try:
+        for line in f:
+          line = line.strip()
+          if line.startswith("t"):
+            self.add_trace(line)
+          elif line.startswith("["):
+            self.add_call(line)
+      except:
+        pass
+
+  def add_trace(self, line):
+    t, name, content = line.split(" ", 2)
+    name = int(name)
+    assert(name > 0)
+    assert(name not in self.traces)
+    self.traces[name] = content.split("|")
+
+  def add_call(self, line):
+    first, second = line.split(":", 1)
+    first = first.strip()
+    ts, t, tname = first.split(" ")
+    ts = float(ts.replace("[", "").replace("]", ""))
+    assert(t == "t")
+    tname = int(tname)
+    assert(tname in self.traces)
+    second = second.strip()
+    result, eq, func, params = second.split(" ", 3)
+    assert(eq == "=")
+    params = params.split(" ")
+    e = Event()
+    e.ts = ts
+    e.trace_name = tname
+    if func == "m" and result != "(nil)":
+      e.alloc = True
+      e.addr = int(result, 16)
+      e.size = int(params[0])
+    elif func == "c" and result != "(nil)":
+      e.alloc = True
+      e.addr = int(result, 16)
+      e.size = int(params[0]) * int(params[1])
+    elif func == "r":
+      old_addr = params[0]
+      size = int(params[1])
+      if old_addr == "(nil)":
+        if result != "(nil)":
+          e.alloc = True
+          e.addr = int(result, 16)
+          e.size = size
+      elif old_addr != "(nil)":
+        if result != "(nil)":
+          e.free = True
+          e.addr = int(old_addr, 16)
+          self.events.append(e)
+          e = Event()
+          e.ts = ts
+          e.trace_name = tname
+          e.alloc = True
+          e.addr = int(result, 16)
+          e.size = size
+    elif func == "f":
+      addr = params[0]
+      if addr != "(nil)":
+        e.free = True
+        e.addr = int(addr, 16)
+    if e.alloc or e.free:
+      self.events.append(e)
+
+
+  def analyze(self):
+    self.all_allocations = {}
+    self.new_allocations = {}
+    self.all_mem = 0
+    self.old_mem = 0
+    self.peak_mem = 0
+    period = 1
+    next_ts_print = self.events[0].ts
+    for e in self.events:
+      if e.ts >= next_ts_print:
+        self.print_allocations()
+        next_ts_print += period
+      self.process_event(e)
+    self.print_allocations()
+    self.print_leaks()
+
+
+  def print_allocations(self):
+    print "Memory usage: current:", commas(self.all_mem), " | delta:", commas(self.all_mem - self.old_mem), " | peak:", commas(self.peak_mem)
+    self.old_mem = self.all_mem
+    self.new_alllocations = {}
+
+
+  def print_leaks(self):
+    if not self.all_allocations:
+      return
+    print "Memory leaked:", commas(self.all_mem)
+    leaks = [e for addr, e in self.all_allocations.iteritems()]
+    leaks.sort(key=lambda e: e.size)
+    for e in leaks:
+      print "Leak:", commas(e.size)
+      print "Allocated in:\n  ", "\n  ".join(self.traces[e.trace_name])
+
+  def process_event(self, e):
+    if e.alloc:
+      assert(e.addr not in self.all_allocations)
+      self.all_allocations[e.addr] = e
+      assert(e.addr not in self.new_allocations)
+      self.new_allocations[e.addr] = e
+      self.all_mem += e.size
+      self.peak_mem = max(self.peak_mem, self.all_mem)
+    else:
+      assert(e.addr in self.all_allocations)
+      e_alloc = self.all_allocations[e.addr]
+      if e.ts - e_alloc.ts > 1. and e_alloc.size > 1e3 and False:
+          print "Long-lived alloc:", commas(e_alloc.size), "freed after", commas(e.ts - e_alloc.ts) + "s"
+          print "Allocated in:\n  ", "\n  ".join(self.traces[e_alloc.trace_name])
+          print "Freed in:\n  ", "\n  ".join(self.traces[e.trace_name])
+      if e_alloc.addr in self.new_allocations:
+        if e_alloc.size > 1e7 and False:
+          print "Large alloc:", commas(e_alloc.size), "freed after", commas(e.ts - e_alloc.ts) + "s"
+          print "Allocated in:\n  ", "\n  ".join(self.traces[e_alloc.trace_name])
+          print "Freed in:\n  ", "\n  ".join(self.traces[e.trace_name])
+        del self.new_allocations[e_alloc.addr]
+      del self.all_allocations[e_alloc.addr]
+      self.all_mem -= e_alloc.size
+
+
+def main():
+  a = Analyzer()
+  a.read(sys.argv[1])
+  a.analyze()
+
+
+if __name__ == "__main__":
+  main()

tint2.files

@@ -237,3 +237,9 @@ src/signals.c
 src/signals.h
 src/tracing.c
 src/tracing.h
+src/util/mem.c
+src/util/mem.h
+src/util/addr2line.c
+src/util/addr2line.h
+src/util/print.h
+src/util/print.c