[PATCH] libdwfl: Don't crash on out of memory when frame unwinding.

[Mark Wielaard <mjw at redhat dot com>]

new_unwound didn't check whether malloc was successful. Check that we
could allocate memory before use and return DWFL_E_NOMEM otherwise.

Signed-off-by: Mark Wielaard <mjw@redhat.com>
---
 libdwfl/ChangeLog      |  6 ++++++
 libdwfl/frame_unwind.c | 23 ++++++++++++++++++-----
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/libdwfl/ChangeLog b/libdwfl/ChangeLog
index cc8eec7..903c0cc 100644
--- a/libdwfl/ChangeLog
+++ b/libdwfl/ChangeLog
@@ -1,3 +1,9 @@
+2016-02-13  Mark Wielaard  <mjw@redhat.com>
+
+	* frame_unwind.c (new_unwound): Check and return unwound.
+	(handle_cfi): Check new_unwound was able to allocate new memory
+	before use. Return DWFL_E_NOMEM otherwise.
+
 2016-02-11  Mark Wielaard  <mjw@redhat.com>
 
 	* relocate.c (relocate_section): Check result of all gelf_get* calls.
diff --git a/libdwfl/frame_unwind.c b/libdwfl/frame_unwind.c
index 0e470b9..a885446 100644
--- a/libdwfl/frame_unwind.c
+++ b/libdwfl/frame_unwind.c
@@ -1,5 +1,5 @@
 /* Get previous frame state for an existing frame state.
-   Copyright (C) 2013, 2014 Red Hat, Inc.
+   Copyright (C) 2013, 2014, 2016 Red Hat, Inc.
    This file is part of elfutils.
 
    This file is free software; you can redistribute it and/or modify
@@ -511,7 +511,7 @@ expr_eval (Dwfl_Frame *state, Dwarf_Frame *frame, const Dwarf_Op *ops,
 #undef pop
 }
 
-static void
+static Dwfl_Frame *
 new_unwound (Dwfl_Frame *state)
 {
   assert (state->unwound == NULL);
@@ -522,6 +522,8 @@ new_unwound (Dwfl_Frame *state)
   assert (nregs > 0);
   Dwfl_Frame *unwound;
   unwound = malloc (sizeof (*unwound) + sizeof (*unwound->regs) * nregs);
+  if (unlikely (unwound == NULL))
+    return NULL;
   state->unwound = unwound;
   unwound->thread = thread;
   unwound->unwound = NULL;
@@ -529,6 +531,7 @@ new_unwound (Dwfl_Frame *state)
   unwound->initial_frame = false;
   unwound->pc_state = DWFL_FRAME_STATE_ERROR;
   memset (unwound->regs_set, 0, sizeof (unwound->regs_set));
+  return unwound;
 }
 
 /* The logic is to call __libdwfl_seterrno for any CFI bytecode interpretation
@@ -545,8 +548,14 @@ handle_cfi (Dwfl_Frame *state, Dwarf_Addr pc, Dwarf_CFI *cfi, Dwarf_Addr bias)
       __libdwfl_seterrno (DWFL_E_LIBDW);
       return;
     }
-  new_unwound (state);
-  Dwfl_Frame *unwound = state->unwound;
+
+  Dwfl_Frame *unwound = new_unwound (state);
+  if (unwound == NULL)
+    {
+      __libdwfl_seterrno (DWFL_E_NOMEM);
+      return;
+    }
+
   unwound->signal_frame = frame->fde->cie->signal_frame;
   Dwfl_Thread *thread = state->thread;
   Dwfl_Process *process = thread->process;
@@ -730,7 +739,11 @@ __libdwfl_frame_unwind (Dwfl_Frame *state)
   Dwfl_Thread *thread = state->thread;
   Dwfl_Process *process = thread->process;
   Ebl *ebl = process->ebl;
-  new_unwound (state);
+  if (new_unwound (state) == NULL)
+    {
+      __libdwfl_seterrno (DWFL_E_NOMEM);
+      return;
+    }
   state->unwound->pc_state = DWFL_FRAME_STATE_PC_UNDEFINED;
   // &Dwfl_Frame.signal_frame cannot be passed as it is a bitfield.
   bool signal_frame = false;
-- 
2.5.0