RFC: GCC plugin to find encrypted function pointer calls in glibc

[Aldy Hernandez]

Hi folks!

I'm working on a GCC plugin to pinpoint places in glibc where indirect 
function calls are made through a function pointer that has not been 
demangled.

As a reference, I am attaching a test case with various things that my 
plugin currently warns on that may serve as examples to what I am trying 
to accomplish.

The general idea is that I have defined a decryption operation as the 
output of a binary operator (xor, ior, and), so while we would warn for 
something like this:

	typedef void (*callback_t) (void);
	void foo (callback_t cb)
	{
	  cb ();
	}

...the following would be OK:

	#define PTR_DEMANGLE(var) \
	  (var) = (__typeof(var)) ((unsigned long) (var) ^ MAGIC)

	typedef void (*callback_t) (void);
	void foo (callback_t cb)
	{
	  callback_t tmp = cb;
	  PTR_DEMANGLE (tmp);
	  tmp ();
	}

However, I see glibc uses the following idiom:

#  define PTR_DEMANGLE(reg)	asm ("ror $2*" LP_SIZE "+1....)

Since I would prefer not to assume the output of all inline asm's are a 
demangling operation, I would like to get feedback from the community on 
what would be preferred.

My preferred approach is to add an attribute to an inline function that 
would wrap the asm:

	__attribute__((decrypt)) static inline funcp demangler (funcp f)
	{
		asm("blah");
	}

This is straightforward, clean, and follows language semantics (not to 
mention that I already have it implemented into my plugin :)), but 
Florian made funny faces when I showed it to him, so here I am :).

It would be neat if GCC had a way of tagging individual gimple 
statements with an attribute, so we could tag them with 
__attribute__((decrypt)), but alas we don't have such mechanism, and I'd 
prefer not to perform major surgery to GCC to make it so.

Another alternative would be to tag inline asms with commented out magic 
at the end, such that the plugin would notice and take appropriate action:

#ifdef FUNCTION_POINTER_CHECKS
#define DEMANGLE_TAG " ##_DEMANGLE_##"
#else
#define DEMANGLE_TAG ""
#endif
#  define PTR_DEMANGLE(var)  asm("ror $2*..."##DEMANGLE_TAG \
				: "=r" (var)
				: "0" (var)
				etc
				etc

This is straightforward, but I still prefer the inline function plus 
attribute idea.

If anyone is interested, I can post the code to my plugin.

What do y'all think?

Aldy
-------------- next part --------------
typedef void (*funcp) (void);

/* Binary operations on a function pointer is a recognized form of
   pointer encryption.

   The decryption must be done in place (to a local/temporary or a
   register), and never written to memory, otherwise it is considered
   unsafe.  */
#define PTR_DEMANGLE(var) \
  (var) = (__typeof(var)) ((unsigned long) (var) ^ 0xc00ffee)

/* A function with __attribute__((encrypt)) is considered an
   acceptable way to encrypt a function pointer.  Similar to PTR_DEMANGLE
   above, the decryption must be done in place.

   Note: It would be neat if we could tag the attribute directly onto the
   asm statement, but GCC currently doesn't do this, and it would
   probably be over kill.  */
__attribute__((encrypt))
static funcp asm_demangler (funcp f)
{
  asm ("#decrypt_operation $2, $0" : "=r" (f) : "0" (f));
  return f;
}

void (*encrypted_funcp) (void);
void (*plain_funcp) (void);
void direct_function (void);

extern int bar();

void foo()
{
  PTR_DEMANGLE (encrypted_funcp);
  encrypted_funcp ();		// WARN, written to memory.

  encrypted_funcp = asm_demangler (encrypted_funcp);
  encrypted_funcp ();		// WARN, written to memory.

  plain_funcp ();		// WARN, no decryption at all.

  direct_function();		// OK.  Don't care about plain func calls.

  funcp f = encrypted_funcp;
  f = asm_demangler (f);
  f();				// OK.  Decryption to a temporary, not memory.

  f = encrypted_funcp;
  f();				// WARN, no decryption at all.

  PTR_DEMANGLE (f);
  f();				// OK. Decryption in place.

  f = encrypted_funcp;
  if (bar())
    PTR_DEMANGLE (f);
  f ();				// WARN, path not demangled.

  // Test that phi convergence code works.
  f = encrypted_funcp;
  if (bar())
    PTR_DEMANGLE (f);
  else
    {
      funcp g = f;
      PTR_DEMANGLE (g);
      f = g;
    }
  f ();				// OK, all paths demangled.

  f = encrypted_funcp;
  if (bar())
    PTR_DEMANGLE (f);
  f();				// WARN, only one path mangled.

  extern void qsort(funcp callback);
  qsort (encrypted_funcp); // WARN, passing non decrypted funcp in argument

  qsort(0);		    // OK to pass a constant even though it's
			    // not decrypted.

  if (encrypted_funcp == 0) // WARN, comparing non-decrypted with constant.
    direct_function();

  f = encrypted_funcp;
  PTR_DEMANGLE (f);
  if (f != 0)		    // OK to compare with constant, we've been
			    // demangled.
    direct_function();
}

void foo2()
{
  funcp f = encrypted_funcp;
  funcp g = plain_funcp;
  PTR_DEMANGLE (f);
  if (f == g)		   // WARN, incompatible encryptness comparison
    bar();

  f = encrypted_funcp;     // OK to compare same encryption type.
  if (f == g)
    bar();
}

// Test that the propagator doesn't go into an infinite loop.

void
bad_inifinite_loop (const signed char *inptr)
{
  while (1)
    {
      signed char ch = *inptr;

      if (!ch)
        continue;

      ch &= 0x1f;

      for (unsigned i = 1; i < 10000; ++i)
        ch |= inptr[0] & 0x3f;
    }
}