[binutils-gdb] libctf: fix ctf_open endianness problems with raw CTF files

[Jose E.Marchesi]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=cf02c44dfd2876867ebd6baa3ce19cf41ff978e1

commit cf02c44dfd2876867ebd6baa3ce19cf41ff978e1
Author: Nick Alcock <nick.alcock@oracle.com>
Date:   Wed Jun 19 15:56:52 2019 +0100

    libctf: fix ctf_open endianness problems with raw CTF files
    
    ctf_open (or, rather, ctf_fdopen, which underlies it) has several
    endianness problems, even though it was written after the
    endian-swapping code was implemented, so should have been endian-aware.
    
    Even though the comment right above the relevant check says that it wil
    check for CTF magic in any endianness, it only checks in the native
    endianness, so opening raw LE CTF files on BE, or vice-versa, will fail.
    It also checks the CTF version by hand, without ever endianness-swapping
    the header, so that too will fail, and is entirely redundant because
    ctf_simple_open does the job properly in any case.  We have a similar
    problem in the next if block, which checks for raw CTF archives: we are
    checking in the native endianness while we should be doing a le64toh()
    on it to check in little-endian form only: so opening CTF archives
    created on the local machine will fail if the local machine is
    big-endian.
    
    Adding insult to injury, if ctf_simple_open then fails, we go on and try
    to turn it into a single-element CTF archive regardless, throwing the
    error away.  Since this involves dereferencing null pointers it is not
    likely to work very well.
    
    libctf/
    	* ctf-open-bfd.c: Add swap.h and ctf-endian.h.
    	(ctf_fdopen): Check for endian-swapped raw CTF magic, and
    	little-endian CTF archive magic.  Do not check the CTF version:
    	ctf_simple_open does that in endian-safe ways.  Do not dereference
    	null pointers on open failure.

Diff:
---
 libctf/ChangeLog      |  8 ++++++++
 libctf/ctf-open-bfd.c | 23 ++++++++++++++---------
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/libctf/ChangeLog b/libctf/ChangeLog
index 0d55ea0..2324aeb 100644
--- a/libctf/ChangeLog
+++ b/libctf/ChangeLog
@@ -1,5 +1,13 @@
 2019-06-19  Nick Alcock <nick.alcock@oracle.com>
 
+	* ctf-open-bfd.c: Add swap.h and ctf-endian.h.
+	(ctf_fdopen): Check for endian-swapped raw CTF magic, and
+	little-endian CTF archive magic.  Do not check the CTF version:
+	ctf_simple_open does that in endian-safe ways.  Do not dereference
+	null pointers on open failure.
+
+2019-06-19  Nick Alcock <nick.alcock@oracle.com>
+
 	* ctf-open.c (get_vbytes_common): Return the new slice size.
 	(ctf_bufopen): Flip the endianness of the CTF-section header copy.
 	Remember to copy in the CTF data when opening an uncompressed
diff --git a/libctf/ctf-open-bfd.c b/libctf/ctf-open-bfd.c
index 76b7f9d..bb2d7e6 100644
--- a/libctf/ctf-open-bfd.c
+++ b/libctf/ctf-open-bfd.c
@@ -26,6 +26,8 @@
 #include <fcntl.h>
 #include <elf.h>
 #include <bfd.h>
+#include "swap.h"
+#include "ctf-endian.h"
 
 #include "elf-bfd.h"
 
@@ -243,24 +245,27 @@ ctf_fdopen (int fd, const char *filename, const char *target, int *errp)
   if ((nbytes = ctf_pread (fd, &ctfhdr, sizeof (ctfhdr), 0)) <= 0)
     return (ctf_set_open_errno (errp, nbytes < 0 ? errno : ECTF_FMT));
 
-  /* If we have read enough bytes to form a CTF header and the magic
-     string matches, attempt to interpret the file as raw CTF.  */
+  /* If we have read enough bytes to form a CTF header and the magic string
+     matches, in either endianness, attempt to interpret the file as raw
+     CTF.  */
 
-  if ((size_t) nbytes >= sizeof (ctf_preamble_t) &&
-      ctfhdr.ctp_magic == CTF_MAGIC)
+  if ((size_t) nbytes >= sizeof (ctf_preamble_t)
+      && (ctfhdr.ctp_magic == CTF_MAGIC
+	  || ctfhdr.ctp_magic == bswap_16 (CTF_MAGIC)))
     {
       ctf_file_t *fp = NULL;
       void *data;
 
-      if (ctfhdr.ctp_version > CTF_VERSION)
-	return (ctf_set_open_errno (errp, ECTF_CTFVERS));
-
       if ((data = ctf_mmap (st.st_size, 0, fd)) == NULL)
 	return (ctf_set_open_errno (errp, errno));
 
       if ((fp = ctf_simple_open (data, (size_t) st.st_size, NULL, 0, 0,
 				 NULL, 0, errp)) == NULL)
-	ctf_munmap (data, (size_t) st.st_size);
+	{
+	  ctf_munmap (data, (size_t) st.st_size);
+	  return NULL;			/* errno is set for us.  */
+	}
+
       fp->ctf_data_mmapped = data;
       fp->ctf_data_mmapped_len = (size_t) st.st_size;
 
@@ -270,7 +275,7 @@ ctf_fdopen (int fd, const char *filename, const char *target, int *errp)
   if ((nbytes = ctf_pread (fd, &arc_magic, sizeof (arc_magic), 0)) <= 0)
     return (ctf_set_open_errno (errp, nbytes < 0 ? errno : ECTF_FMT));
 
-  if ((size_t) nbytes >= sizeof (uint64_t) && arc_magic == CTFA_MAGIC)
+  if ((size_t) nbytes >= sizeof (uint64_t) && le64toh (arc_magic) == CTFA_MAGIC)
     {
       struct ctf_archive *arc;