[binutils-gdb] VMS/BFD: Fix a sign extension issue with archive symbol lookup

[Maciej W. Rozycki]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=66eb41a42b3931e37c7b5660c84c37bf24469e59

commit 66eb41a42b3931e37c7b5660c84c37bf24469e59
Author: Maciej W. Rozycki <macro@redhat.com>
Date:   Mon Dec 15 19:04:30 2025 +0000

    VMS/BFD: Fix a sign extension issue with archive symbol lookup
    
    Symbol binary search code for VMS archive files uses plain `char' data
    type to cast a difference between characters to data of the `int' type.
    Consequently the difference is consider unsigned in the range between 0
    and 255 on hosts where plain `char' data type is unsigned, resulting in
    symbol lookup failures, such as with the test expansion included with
    this change causing regressions as follows:
    
    FAIL: Regular archive link
    FAIL: Thin archive link
    FAIL: Regular archive plus regular link
    FAIL: Regular archive plus thin link
    FAIL: Thin archive plus regular link
    FAIL: Thin archive plus thin link
    
    owing to link failures such as:
    
    .../ld/ld-new: tmpdir/abc.o:($DATA$+0x0): undefined reference to `aa'
    .../ld/ld-new: tmpdir/ab.a(b.obj):($DATA$+0x10): undefined reference to `aa'
    .../ld/ld-new: tmpdir/abc.o:($DATA$+0x0): undefined reference to `aa'
    .../ld/ld-new: tmpdir/ab.a(b.obj):($DATA$+0x10): undefined reference to `aa'
    
    with the `alpha-dec-vms' target on the `powerpc64le-linux-gnu' host.
    
    Use explicit `signed char' data type for the cast then, removing the
    failures.

Diff:
---
 bfd/vms-lib.c                       |  2 +-
 ld/testsuite/ld-archive/archive.exp | 12 ++++++------
 ld/testsuite/ld-archive/x.s         |  4 ++++
 ld/testsuite/ld-archive/y.s         |  4 ++++
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/bfd/vms-lib.c b/bfd/vms-lib.c
index 2ffb4e4acf9..3c1f8ef007a 100644
--- a/bfd/vms-lib.c
+++ b/bfd/vms-lib.c
@@ -800,7 +800,7 @@ _bfd_vms_lib_find_symbol (bfd *abfd, const char *name)
       int mid = lo + (hi - lo) / 2;
       int diff;
 
-      diff = (char)(name[0] - syms[mid].name[0]);
+      diff = (signed char) (name[0] - syms[mid].name[0]);
       if (diff == 0)
 	diff = strcmp (name, syms[mid].name);
       if (diff == 0)
diff --git a/ld/testsuite/ld-archive/archive.exp b/ld/testsuite/ld-archive/archive.exp
index b8685e94f42..035b271f775 100644
--- a/ld/testsuite/ld-archive/archive.exp
+++ b/ld/testsuite/ld-archive/archive.exp
@@ -23,12 +23,12 @@ remote_file host delete \
     "tmpdir/abn.a" "tmpdir/abnt.a"
 
 run_ld_link_tests {
-    {"First regular archive create"     ""   "" "" {a.s b.s} {} "ab.a"  }
-    {"Second regular archive create"    ""   "" "" {c.s d.s} {} "cd.a"  }
-    {"First thin archive create"        "T"  "" "" {a.s b.s} {} "abt.a" }
-    {"Second thin archive create"       "T"  "" "" {c.s d.s} {} "cdt.a" }
-    {"Regular archive w/o index create" "S"  "" "" {a.s b.s} {} "abn.a" }
-    {"Thin archive w/o index create"    "ST" "" "" {a.s b.s} {} "abnt.a"}
+    {"First regular archive create"     ""   "" "" {a.s b.s x.s} {} "ab.a"   }
+    {"Second regular archive create"    ""   "" "" {c.s d.s y.s} {} "cd.a"   }
+    {"First thin archive create"        "T"  "" "" {a.s b.s x.s} {} "abt.a"  }
+    {"Second thin archive create"       "T"  "" "" {c.s d.s y.s} {} "cdt.a"  }
+    {"Regular archive w/o index create" "S"  "" "" {a.s b.s x.s} {} "abn.a"  }
+    {"Thin archive w/o index create"    "ST" "" "" {a.s b.s x.s} {} "abnt.a" }
 }
 
 set old_ldflags $LDFLAGS
diff --git a/ld/testsuite/ld-archive/x.s b/ld/testsuite/ld-archive/x.s
new file mode 100644
index 00000000000..49ace626872
--- /dev/null
+++ b/ld/testsuite/ld-archive/x.s
@@ -0,0 +1,4 @@
+	.data
+	.globl	xx
+xx:
+	.dc.a	0
diff --git a/ld/testsuite/ld-archive/y.s b/ld/testsuite/ld-archive/y.s
new file mode 100644
index 00000000000..45d7ec7eef9
--- /dev/null
+++ b/ld/testsuite/ld-archive/y.s
@@ -0,0 +1,4 @@
+	.data
+	.globl	yy
+yy:
+	.dc.a	0