Intercepting syscalls in glibc example

[Tom Kraljevic <tom dot kraljevic at gmail dot com>]

Goal:  On an ubuntu system (release "hardy"), modify and build a glibc
which intercepts syscalls.

Getting this to work how I wanted took a little while, and there wasn't
much related info that I could find, so I figured I would share it with
the email group for posterity.

This example hijacks the write syscall and adds a proxy layer which
allows the user to compute statistics, etc.  In my particular example,
I am statically linking the program.

I reduced the set of items built to a minimum (libc.a, libm.a, and
some crt object files) because I experienced build problems in the elf
directory when the dynamic linker was being built.  I'm happy to use
the dynamic linker already on the host system when necessary, so I
just skipped that part.

I'd say the most difficult aspect of this was figuring out how to work
within the established make framework of glibc.


[ Apologies for using a somewhat out-of-date glibc for the example, but
this is the version my system is shipping with, so that's the version
I was interested in...

Note:  My local work area is /var/tmp/tomk/glibcexample ]


1.  GET ACCESS TO UBUNTU PATCHED BUILDABLE SOURCE CODE

cd /usr/src
sudo apt-get source libc6
sudo chown -R tomk glibc-2.7
cd glibc-2.7/
./debian/rules binary
rm -f build-tree/glibc-2.7/debian


2.  COPY REQUIRED SOURCE CODE TO A LOCAL BUILD AREA

cd <workarea>
./scripts/freshglibcsource.sh


3.  MODIFY GLIBC

cd <workarea>
cd src

Modify source with this patch:
glibc-2.7/sysdeps/unix/syscalls.list
64c64
< write         -       write           Ci:ibn  __libc_write    __write write
---
> write         -       write           Ci:ibn  __native_libc_write     __native_write native_write


4.  BUILD GLIBC

cd <workarea>
./scripts/buildglibc.sh


5.  BUILD TEST PROGRAM AND RUN

cd <workarea>
cd test
make clean
make
./hello

Now, set a breakpoint on the intercept routine and watch the breakpoint get hit.

gdb ./hello
b proxy_write
r
[ hits breakpoint ]
c
q


6.  DIRECTORY AND FILE LISTINGS FROM MY WORK AREA
---------------------------------------------------------------------------
% ls
total 24
4 build/  4 doc/  4 release/  4 scripts/  4 src/  4 test/
---------------------------------------------------------------------------
% ls scripts/
total 8
4 buildglibc.sh*  4 freshglibcsource.sh*
---------------------------------------------------------------------------
% cat scripts/freshglibcsource.sh
#!/bin/sh

set -x
set -e

tmproot=`dirname $0`/..
cd $tmproot
root=`pwd`
srcroot=$root/src

# Clean up
cd $srcroot
rm -fr glibc-2.7
rm -fr debian

# Copy new source
rsync -rt --copy-links --copy-dirlinks /usr/src/glibc-2.7/build-tree/glibc-2.7 .

mkdir debian
rsync -rt --copy-links --copy-dirlinks /usr/src/glibc-2.7/debian/include debian
---------------------------------------------------------------------------
% cat scripts/buildglibc.sh
#!/bin/sh

# Print all commands
set -x

# Stop on error
set -e

# Variables
tmproot=`dirname $0`/..
cd $tmproot
root=`pwd`
srcroot=$root/src
releaseroot=$root/release
buildroot=$root/build
glibcroot=glibc-2.7
srcdir=$srcroot/$glibcroot
builddir=$buildroot/$glibcroot
releasedir=$releaseroot/$glibcroot

# Clean up
rm -fr $builddir
rm -fr $releasedir
mkdir $builddir
cd $builddir

# Configure
CC="gcc-4.2 -fno-stack-protector" CXX="g++-4.2 -fno-stack-protector"
LDFLAGS="" AUTOCONF=false $srcdir/configure --host=x86_64-linux-gnu
--build=x86_64-linux-gnu --prefix=$releaseroot/$glibcroot
--without-cvs --enable-add-ons=libidn,"nptl " --enable-profile
--without-selinux --with-headers=$srcroot/debian/include
--enable-kernel=2.6.8 2>&1 | tee configure.out

# Enable parallel builds
cores=`cat /proc/cpuinfo | grep processor | wc -l`
parallelism=`expr $cores + 1`
echo Using make parallelism $parallelism
mv Makefile MakefileT1
perl -p -e "s/# PARALLELMFLAGS = -j 4/PARALLELMFLAGS = -j
$parallelism/" MakefileT1 > MakefileT2
mv MakefileT2 Makefile
rm -f MakefileT1

#
# For make and make install, just do a specific needed subset.
#

# make
make lib 2>&1 | tee make_lib.out
make objdir=`pwd` subdir=math -C $srcdir/math 2>&1 | tee make_math.out

# make install
make install-headers 2>&1 | tee make_install-headers.out
touch $releasedir/include/gnu/stubs.h
cp -p $builddir/bits/stdio_lim.h $releasedir/include/bits
mkdir -p $releasedir/lib
cp -p $builddir/libc.a $releasedir/lib
cp -p $builddir/math/libm.a $releasedir/lib
cp -p $builddir/csu/crt1.o $releasedir/lib
cp -p $builddir/csu/crti.o $releasedir/lib
cp -p $builddir/csu/crtn.o $releasedir/lib
---------------------------------------------------------------------------
% ls test/
total 16
4 hello.c  4 Makefile  4 proxy.c  4 shim.c
---------------------------------------------------------------------------
% cat test/Makefile
GCC = gcc
GXX = g++

GLIBCDIR = /var/tmp/tomk/glibcexample/release/glibc-2.7
GLIBCLIBDIR = $(GLIBCDIR)/lib

GCCLIBDIR = /usr/lib/gcc/x86_64-linux-gnu/4.2

CFLAGS = \
       -nostdinc \
       -isystem $(GLIBCDIR)/include \
       -isystem $(GCCLIBDIR)/include

LDFLAGS = \
       -static \
       -nostdlib

CRTLIBS = \
       $(GLIBCLIBDIR)/crti.o \
       $(GLIBCLIBDIR)/crt1.o \
       $(GCCLIBDIR)/crtbegin.o \
       $(GCCLIBDIR)/crtend.o \
       $(GLIBCLIBDIR)/crtn.o

GLIBCLIBS = \
       $(GLIBCLIBDIR)/libc.a \
       $(GLIBCLIBDIR)/libm.a

GCCLIBS = \
       -lgcc -lgcc_eh -lstdc++ -lsupc++

TARGET_LDFLAGS_START_GROUP = -Xlinker --start-group
TARGET_LDFLAGS_END_GROUP = -Xlinker --end-group

all:    hello

hello.o:        hello.c
       $(GCC) $(CFLAGS) -c -o $@ $<

proxy.o:        proxy.c
       $(GCC) $(CFLAGS) -c -o $@ $<

shim.o: shim.c
       $(GCC) $(CFLAGS) -c -o $@ $<

hello:  hello.o proxy.o shim.o
       $(GXX) $(LDFLAGS) -o $@ $(TARGET_LDFLAGS_START_GROUP)
$(CRTLIBS) $^ $(GLIBCLIBS) $(GCCLIBS) $(TARGET_LDFLAGS_END_GROUP)

clean:
       rm -f hello.o proxy.o shim.o
       rm -f hello
---------------------------------------------------------------------------
% cat test/hello.c
#include <stdio.h>

int
main()
{
   printf ("hello\n");
   return 0;
}
---------------------------------------------------------------------------
% cat test/proxy.c
#include <stdlib.h>

extern ssize_t __native_libc_write (int __fd, __const void *__buf, size_t __n);

ssize_t proxy_write (int __fd, __const void *__buf, size_t __n)
{
   return __native_libc_write (__fd, __buf, __n);
}
---------------------------------------------------------------------------
% cat test/shim.c
#include <stdlib.h>

extern ssize_t proxy_write (int __fd, __const void *__buf, size_t __n);

ssize_t __libc_write (int __fd, __const void *__buf, size_t __n)
{
   return proxy_write (__fd, __buf, __n);
}

ssize_t __write (int __fd, __const void *__buf, size_t __n)
{
   return proxy_write (__fd, __buf, __n);
}

ssize_t write (int __fd, __const void *__buf, size_t __n)
{
   return proxy_write (__fd, __buf, __n);
}
---------------------------------------------------------------------------