[PATCH] iconv: Add --bom CLI option

jtmoon79 jtm.moon@gmail.com
Tue Nov 17 02:32:46 GMT 2026


Allow iconv to prepend a byte order mark for supported output encodings.
Warn when the requested encoding has no known BOM, unless warnings are
suppressed. Add tests for BOM output across encodings and edge cases.

For charmap input converted to generic UTF-16 or UTF-32, emit one leading
BOM instead of the previous odd behavior of a BOM before every character.
This changes output even without `--bom`, including empty input, which now
produces a BOM instead of an empty file. Explicit-endian targets remain
BOM-free unless `--bom` is requested.
---
 iconv/Makefile                         |   6 +
 iconv/iconv_charmap.c                  |   6 +-
 iconv/iconv_prog.c                     | 161 +++++++++++++-
 iconv/iconv_prog.h                     |   4 +-
 iconv/test_data/abc.ASCII.txt          |   1 +
 iconv/test_data/abc.GB18030_BOM.txt    |   1 +
 iconv/test_data/abc.UTF-16BE.txt       | Bin 0 -> 8 bytes
 iconv/test_data/abc.UTF-16BE_BOM.txt   | Bin 0 -> 10 bytes
 iconv/test_data/abc.UTF-16LE.txt       | Bin 0 -> 8 bytes
 iconv/test_data/abc.UTF-16LE_BOM.txt   | Bin 0 -> 10 bytes
 iconv/test_data/abc.UTF-32BE.txt       | Bin 0 -> 16 bytes
 iconv/test_data/abc.UTF-32BE_BOM.txt   | Bin 0 -> 20 bytes
 iconv/test_data/abc.UTF-32LE.txt       | Bin 0 -> 16 bytes
 iconv/test_data/abc.UTF-32LE_BOM.txt   | Bin 0 -> 20 bytes
 iconv/test_data/abc.UTF-8.txt          |   1 +
 iconv/test_data/abc.UTF-8_BOM.txt      |   1 +
 iconv/test_data/empty.GB18030_BOM.txt  |   1 +
 iconv/test_data/empty.UTF-16BE_BOM.txt |   1 +
 iconv/test_data/empty.UTF-16LE_BOM.txt |   1 +
 iconv/test_data/empty.UTF-32BE_BOM.txt | Bin 0 -> 4 bytes
 iconv/test_data/empty.UTF-32LE_BOM.txt | Bin 0 -> 4 bytes
 iconv/test_data/empty.UTF-8_BOM.txt    |   1 +
 iconv/test_data/empty.txt              |   0
 iconv/tst-iconv_prog-bom.sh            | 278 +++++++++++++++++++++++++
 24 files changed, 455 insertions(+), 8 deletions(-)
 create mode 100644 iconv/test_data/abc.ASCII.txt
 create mode 100644 iconv/test_data/abc.GB18030_BOM.txt
 create mode 100644 iconv/test_data/abc.UTF-16BE.txt
 create mode 100644 iconv/test_data/abc.UTF-16BE_BOM.txt
 create mode 100644 iconv/test_data/abc.UTF-16LE.txt
 create mode 100644 iconv/test_data/abc.UTF-16LE_BOM.txt
 create mode 100644 iconv/test_data/abc.UTF-32BE.txt
 create mode 100644 iconv/test_data/abc.UTF-32BE_BOM.txt
 create mode 100644 iconv/test_data/abc.UTF-32LE.txt
 create mode 100644 iconv/test_data/abc.UTF-32LE_BOM.txt
 create mode 100644 iconv/test_data/abc.UTF-8.txt
 create mode 100644 iconv/test_data/abc.UTF-8_BOM.txt
 create mode 100644 iconv/test_data/empty.GB18030_BOM.txt
 create mode 100644 iconv/test_data/empty.UTF-16BE_BOM.txt
 create mode 100644 iconv/test_data/empty.UTF-16LE_BOM.txt
 create mode 100644 iconv/test_data/empty.UTF-32BE_BOM.txt
 create mode 100644 iconv/test_data/empty.UTF-32LE_BOM.txt
 create mode 100644 iconv/test_data/empty.UTF-8_BOM.txt
 create mode 100644 iconv/test_data/empty.txt
 create mode 100755 iconv/tst-iconv_prog-bom.sh

diff --git a/iconv/Makefile b/iconv/Makefile
index 69555cbe57..03749e1668 100644
--- a/iconv/Makefile
+++ b/iconv/Makefile
@@ -81,6 +81,7 @@ include $(patsubst %,$(..)libof-iterator.mk,$(cpp-srcs-left))
 ifeq ($(run-built-tests),yes)
 xtests-special += $(objpfx)test-iconvconfig.out
 tests-special += \
+	$(objpfx)tst-iconv_prog-bom.out \
 	$(objpfx)tst-iconv_prog-buffer-large.out \
 	$(objpfx)tst-iconv_prog-buffer-tiny.out \
 	$(objpfx)tst-iconv_prog-buffer.out \
@@ -167,6 +168,11 @@ $(objpfx)tst-iconv_prog-buffer-large.out: \
 	$(BASH) $< $(common-objdir) '$(run-program-prefix)' '' '22' > $@; \
 	$(evaluate-test)
 
+$(objpfx)tst-iconv_prog-bom.out: \
+  tst-iconv_prog-bom.sh $(objpfx)iconv_prog
+	$(BASH) $< $(common-objdir) '$(run-program-prefix)' > $@; \
+	$(evaluate-test)
+
 $(objpfx)tst-iconvconfig.out: tst-iconvconfig.sh $(objpfx)iconvconfig
 	$(BASH) $< $(common-objdir) '$(test-wrapper-env)' \
 		'$(run-program-env)' '$(rpath-link)' > $@; \
diff --git a/iconv/iconv_charmap.c b/iconv/iconv_charmap.c
index cc16d08db1..a987810b17 100644
--- a/iconv/iconv_charmap.c
+++ b/iconv/iconv_charmap.c
@@ -99,7 +99,8 @@ int
 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
 		    const char *to_code, struct charmap_t *to_charmap,
 		    int argc, int remaining, char *argv[],
-		    const char *output_file)
+		    const char *output_file,
+		    const char *bom, size_t bom_length)
 {
   struct convtable *cvtbl;
   int status = EXIT_SUCCESS;
@@ -148,6 +149,9 @@ charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
   else
     output = stdout;
 
+  if (bom_length > 0 && fwrite (bom, 1, bom_length, output) != bom_length)
+    error (EXIT_FAILURE, errno, _("cannot open output file"));
+
   /* We can now start the conversion.  */
   if (remaining == argc)
     {
diff --git a/iconv/iconv_prog.c b/iconv/iconv_prog.c
index 0aaca468e4..a0ebc932a9 100644
--- a/iconv/iconv_prog.c
+++ b/iconv/iconv_prog.c
@@ -18,6 +18,7 @@
 #include <argp.h>
 #include <assert.h>
 #include <ctype.h>
+#include <endian.h>
 #include <errno.h>
 #include <error.h>
 #include <fcntl.h>
@@ -51,6 +52,7 @@ enum
   {
     OPT_VERBOSE = 1000,
     OPT_BUFFER_SIZE,
+    OPT_BOM,
   };
 #define OPT_LIST	'l'
 
@@ -64,6 +66,7 @@ static const struct argp_option options[] =
   { "list", 'l', NULL, 0, N_("list all known coded character sets") },
   { NULL, 0, NULL, 0, N_("Output control:") },
   { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
+  { "bom", OPT_BOM, NULL, 0, N_("prepend byte order mark to output") },
   { "output", 'o', N_("FILE"), 0, N_("output file") },
   { "silent", 's', NULL, 0, N_("suppress warnings") },
   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
@@ -108,6 +111,12 @@ static int list;
 /* If nonzero omit invalid character from output.  */
 int omit_invalid;
 
+/* If true, prepend a byte order mark to the output.  */
+static bool emit_bom;
+
+/* If true, suppress warnings.  */
+static bool silent;
+
 /* Current index in argv (after command line processing) with the
    input file name.  */
 static int current_input_file_index;
@@ -119,6 +128,7 @@ static size_t output_buffer_size = 1024 * 1024;
 /* Prototypes for the functions doing the actual work.  */
 static void prepare_output_file (char **argv);
 static void close_output_file (__gconv_t cd, int status);
+static void output_bytes (const char *bytes, size_t length);
 static int process_block (iconv_t cd, char **addr, size_t *len,
 			  off64_t file_offset, bool *incomplete);
 static int process_fd (iconv_t cd, int fd);
@@ -126,6 +136,102 @@ static int process_file (iconv_t cd, FILE *input);
 static void print_known_names (void);
 
 
+struct bom_spec
+{
+  const char *bytes;
+  size_t length;
+  const char *rewrite_tocode;
+};
+
+static const char bom_utf8[] = { '\xef', '\xbb', '\xbf' };
+static const char bom_utf16be[] = { '\xfe', '\xff' };
+static const char bom_utf16le[] = { '\xff', '\xfe' };
+static const char bom_utf32be[] = { '\x00', '\x00', '\xfe', '\xff' };
+static const char bom_utf32le[] = { '\xff', '\xfe', '\x00', '\x00' };
+static const char bom_gb18030[] = { '\x84', '\x31', '\x95', '\x33' };
+
+static bool
+code_is (const char *code, const char *name)
+{
+  return strcmp (code, name) == 0;
+}
+
+static bool
+bom_for_code (const char *code, struct bom_spec *bom)
+{
+  *bom = (struct bom_spec) { NULL, 0, NULL };
+
+  if (code_is (code, "UTF-8//")
+      || code_is (code, "UTF8//")
+      || code_is (code, "ISO-10646/UTF8/")
+      || code_is (code, "ISO-10646/UTF-8/"))
+    *bom = (struct bom_spec) { bom_utf8, sizeof (bom_utf8), NULL };
+  else if (code_is (code, "UTF-16BE//") || code_is (code, "UTF16BE//"))
+    *bom = (struct bom_spec) { bom_utf16be, sizeof (bom_utf16be), NULL };
+  else if (code_is (code, "UTF-16LE//") || code_is (code, "UTF16LE//"))
+    *bom = (struct bom_spec) { bom_utf16le, sizeof (bom_utf16le), NULL };
+  else if (code_is (code, "UTF-32BE//") || code_is (code, "UTF32BE//"))
+    *bom = (struct bom_spec) { bom_utf32be, sizeof (bom_utf32be), NULL };
+  else if (code_is (code, "UTF-32LE//") || code_is (code, "UTF32LE//"))
+    *bom = (struct bom_spec) { bom_utf32le, sizeof (bom_utf32le), NULL };
+  else if (code_is (code, "UTF-16//") || code_is (code, "UTF16//"))
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    *bom = (struct bom_spec) { bom_utf16le, sizeof (bom_utf16le),
+				       "UTF-16LE//" };
+#else
+    *bom = (struct bom_spec) { bom_utf16be, sizeof (bom_utf16be),
+				       "UTF-16BE//" };
+#endif
+  else if (code_is (code, "UTF-32//") || code_is (code, "UTF32//"))
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    *bom = (struct bom_spec) { bom_utf32le, sizeof (bom_utf32le),
+				       "UTF-32LE//" };
+#else
+    *bom = (struct bom_spec) { bom_utf32be, sizeof (bom_utf32be),
+				       "UTF-32BE//" };
+#endif
+  else if (code_is (code, "GB18030//"))
+    *bom = (struct bom_spec) { bom_gb18030, sizeof (bom_gb18030), NULL };
+
+  return bom->bytes != NULL;
+}
+
+static bool
+bom_for_user_code (const char *code, struct bom_spec *bom)
+{
+  struct gconv_spec conv_spec;
+  bool result;
+
+  if (__gconv_create_spec (&conv_spec, "", code) == NULL)
+    error (EXIT_FAILURE, errno, _("failed to start conversion processing"));
+
+  result = bom_for_code (conv_spec.tocode, bom);
+  __gconv_destroy_spec (&conv_spec);
+  return result;
+}
+
+static void
+replace_tocode (struct gconv_spec *conv_spec, const char *tocode)
+{
+  char *copy = strdup (tocode);
+  if (copy == NULL)
+    error (EXIT_FAILURE, errno, _("failed to start conversion processing"));
+  free (conv_spec->tocode);
+  conv_spec->tocode = copy;
+}
+
+static void
+warn_missing_bom (void)
+{
+  if (emit_bom && !silent)
+    {
+      const char *to_pretty = to_code[0] ? to_code : nl_langinfo (CODESET);
+      error (0, 0, _("warning: no byte order mark for encoding `%s'"),
+	     to_pretty);
+    }
+}
+
+
 int
 main (int argc, char *argv[])
 {
@@ -174,13 +280,27 @@ main (int argc, char *argv[])
      doing things.  The other case allows converting using the iconv()
      function.  */
   if (from_charmap != NULL || to_charmap != NULL)
-    /* Construct the conversion table and do the conversion.  */
-    status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
-				 argc, current_input_file_index, argv,
-				 output_file);
+    {
+      struct bom_spec bom;
+
+      if (!bom_for_user_code (to_code, &bom))
+	warn_missing_bom ();
+
+      if (!emit_bom && bom.rewrite_tocode == NULL)
+	bom.length = 0;
+
+      const char *charmap_tocode = (bom.rewrite_tocode != NULL
+				    ? bom.rewrite_tocode : to_code);
+
+      /* Construct the conversion table and do the conversion.  */
+      status = charmap_conversion (from_code, from_charmap, charmap_tocode,
+				   to_charmap, argc, current_input_file_index,
+				   argv, output_file, bom.bytes, bom.length);
+    }
   else
     {
       struct gconv_spec conv_spec;
+      struct bom_spec bom = { NULL, 0, NULL };
       int res;
 
       if (__gconv_create_spec (&conv_spec, from_code, to_code) == NULL)
@@ -193,6 +313,10 @@ main (int argc, char *argv[])
       if (omit_invalid)
         conv_spec.ignore = true;
 
+      if (emit_bom && bom_for_code (conv_spec.tocode, &bom)
+	  && bom.rewrite_tocode != NULL)
+	replace_tocode (&conv_spec, bom.rewrite_tocode);
+
       /* Let's see whether we have these coded character sets.  */
       res = __gconv_open (&conv_spec, &cd, 0);
 
@@ -251,7 +375,11 @@ conversions from `%s' and to `%s' are not supported"),
 		   _("failed to start conversion processing"));
 	}
 
+      if (emit_bom && bom.bytes == NULL)
+    	warn_missing_bom ();
+
       prepare_output_file (argv);
+      output_bytes (bom.bytes, bom.length);
 
       /* Now process the remaining files.  Write them to stdout or the file
 	 specified with the `-o' parameter.  If we have no file given as
@@ -335,8 +463,7 @@ parse_opt (int key, char *arg, struct argp_state *state)
       output_file = arg;
       break;
     case 's':
-      /* Nothing, for now at least.  We are not giving out any information
-	 about missing character or so.  */
+      silent = true;
       break;
     case 'c':
       /* Omit invalid characters from output.  */
@@ -353,6 +480,9 @@ parse_opt (int key, char *arg, struct argp_state *state)
     case OPT_VERBOSE:
       verbose = 1;
       break;
+    case OPT_BOM:
+      emit_bom = true;
+      break;
     case OPT_LIST:
       list = 1;
       break;
@@ -599,6 +729,25 @@ flush_output (void)
   output_buffer_remaining = output_buffer_size;
 }
 
+static void
+output_bytes (const char *bytes, size_t length)
+{
+  while (length > 0)
+    {
+      if (output_buffer_remaining == 0)
+	flush_output ();
+
+      size_t count = output_buffer_remaining;
+      if (count > length)
+	count = length;
+      memcpy (output_buffer_current, bytes, count);
+      output_buffer_current += count;
+      output_buffer_remaining -= count;
+      bytes += count;
+      length -= count;
+    }
+}
+
 static void
 close_output_file (__gconv_t cd, int status)
 {
diff --git a/iconv/iconv_prog.h b/iconv/iconv_prog.h
index ea1c3ad321..72c6eea911 100644
--- a/iconv/iconv_prog.h
+++ b/iconv/iconv_prog.h
@@ -19,6 +19,7 @@
 #define _ICONV_PROG_H	1
 
 #include <stdio.h>
+#include <stddef.h>
 
 #include <charmap.h>
 #include <record-status.h>
@@ -32,7 +33,8 @@ extern int charmap_conversion (const char *from_code,
 			       const char *to_code,
 			       struct charmap_t *to_charmap,
 			       int argc, int remaining, char *argv[],
-			       const char *output_file);
+			       const char *output_file,
+			       const char *bom, size_t bom_length);
 
 
 #endif	/* iconv_prog.h */
diff --git a/iconv/test_data/abc.ASCII.txt b/iconv/test_data/abc.ASCII.txt
new file mode 100644
index 0000000000..8baef1b4ab
--- /dev/null
+++ b/iconv/test_data/abc.ASCII.txt
@@ -0,0 +1 @@
+abc
diff --git a/iconv/test_data/abc.GB18030_BOM.txt b/iconv/test_data/abc.GB18030_BOM.txt
new file mode 100644
index 0000000000..df002a1b95
--- /dev/null
+++ b/iconv/test_data/abc.GB18030_BOM.txt
@@ -0,0 +1 @@
+„1•3abc
diff --git a/iconv/test_data/abc.UTF-16BE.txt b/iconv/test_data/abc.UTF-16BE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f712558484aa542108b6cc79292dc3cddccffe86
GIT binary patch
literal 8
PcmZQbWJqF2X5az<1<V04

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-16BE_BOM.txt b/iconv/test_data/abc.UTF-16BE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7a254b33f44589b0f37997457ae2d4ac2e727091
GIT binary patch
literal 10
RcmezOpCOSUi6NPR3ji3g11<mn

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-16LE.txt b/iconv/test_data/abc.UTF-16LE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2912a891841f1538b35df13d217604b35e43752b
GIT binary patch
literal 8
PcmYdfNMcB4;9>v(2K)gr

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-16LE_BOM.txt b/iconv/test_data/abc.UTF-16LE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..22223aaa75966d97946bd825a9b544d2158ed5de
GIT binary patch
literal 10
RcmezWFOeaMA(?@T0RS1|11<mn

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-32BE.txt b/iconv/test_data/abc.UTF-32BE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e05bbfe9c1fb7c734eccabb4c8aa1e3c50ae0a14
GIT binary patch
literal 16
UcmZQzU`S+OU`PVuWFY1O019XUF#rGn

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-32BE_BOM.txt b/iconv/test_data/abc.UTF-32BE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..58f98f02b15a1812e54092aeaed7e50017e145f6
GIT binary patch
literal 20
WcmZQz`1hZIfgurylYlrGh`9hQHv=vJ

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-32LE.txt b/iconv/test_data/abc.UTF-32LE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..77b9cf14b82ef94505a1daeed01fc44436d738e2
GIT binary patch
literal 16
ScmYdfU|>iB;$$G^0%8CS_yI8h

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-32LE_BOM.txt b/iconv/test_data/abc.UTF-32LE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1c2aa3f59cfdea31eed76e964a2cfcea8f05954b
GIT binary patch
literal 20
WcmezWkAWePfq@|jh?9Ys3y1+U!UHY<

literal 0
HcmV?d00001

diff --git a/iconv/test_data/abc.UTF-8.txt b/iconv/test_data/abc.UTF-8.txt
new file mode 100644
index 0000000000..8baef1b4ab
--- /dev/null
+++ b/iconv/test_data/abc.UTF-8.txt
@@ -0,0 +1 @@
+abc
diff --git a/iconv/test_data/abc.UTF-8_BOM.txt b/iconv/test_data/abc.UTF-8_BOM.txt
new file mode 100644
index 0000000000..dad473fcef
--- /dev/null
+++ b/iconv/test_data/abc.UTF-8_BOM.txt
@@ -0,0 +1 @@
+abc
diff --git a/iconv/test_data/empty.GB18030_BOM.txt b/iconv/test_data/empty.GB18030_BOM.txt
new file mode 100644
index 0000000000..f98355e62f
--- /dev/null
+++ b/iconv/test_data/empty.GB18030_BOM.txt
@@ -0,0 +1 @@
+„1•3
\ No newline at end of file
diff --git a/iconv/test_data/empty.UTF-16BE_BOM.txt b/iconv/test_data/empty.UTF-16BE_BOM.txt
new file mode 100644
index 0000000000..e2df5e2ee7
--- /dev/null
+++ b/iconv/test_data/empty.UTF-16BE_BOM.txt
@@ -0,0 +1 @@
+þÿ
\ No newline at end of file
diff --git a/iconv/test_data/empty.UTF-16LE_BOM.txt b/iconv/test_data/empty.UTF-16LE_BOM.txt
new file mode 100644
index 0000000000..46b134b197
--- /dev/null
+++ b/iconv/test_data/empty.UTF-16LE_BOM.txt
@@ -0,0 +1 @@
+ÿþ
\ No newline at end of file
diff --git a/iconv/test_data/empty.UTF-32BE_BOM.txt b/iconv/test_data/empty.UTF-32BE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2fa7e8adff3fd2feb70d43073809e01639e59c72
GIT binary patch
literal 4
LcmZQz`1c<G0{;R2

literal 0
HcmV?d00001

diff --git a/iconv/test_data/empty.UTF-32LE_BOM.txt b/iconv/test_data/empty.UTF-32LE_BOM.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e37e73e3b5686eee9f67334e1f682a790d31f52b
GIT binary patch
literal 4
LcmezWkAVRI2KoX1

literal 0
HcmV?d00001

diff --git a/iconv/test_data/empty.UTF-8_BOM.txt b/iconv/test_data/empty.UTF-8_BOM.txt
new file mode 100644
index 0000000000..5f282702bb
--- /dev/null
+++ b/iconv/test_data/empty.UTF-8_BOM.txt
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/iconv/test_data/empty.txt b/iconv/test_data/empty.txt
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/iconv/tst-iconv_prog-bom.sh b/iconv/tst-iconv_prog-bom.sh
new file mode 100755
index 0000000000..282f73d4f2
--- /dev/null
+++ b/iconv/tst-iconv_prog-bom.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+# Test for iconv (the program) byte order mark output.
+# Copyright (C) 2026 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <https://www.gnu.org/licenses/>.
+
+# Arguments:
+#   root of the build tree ($(objpfx-common))
+#   test command wrapper (for running on the board/with new ld.so)
+
+exec 2>&1
+set -e
+
+exec {logfd}>&1
+
+codir=$1
+test_program_prefix="$2"
+logs="$(dirname "$0")/test_data"
+charmap="$(dirname "$0")/../localedata/charmaps/ANSI_X3.4-1968"
+
+required_files=(
+  abc.ASCII.txt
+  abc.UTF-8.txt
+  abc.UTF-8_BOM.txt
+  abc.UTF-16LE.txt
+  abc.UTF-16LE_BOM.txt
+  abc.UTF-16BE.txt
+  abc.UTF-16BE_BOM.txt
+  abc.UTF-32LE.txt
+  abc.UTF-32LE_BOM.txt
+  abc.UTF-32BE.txt
+  abc.UTF-32BE_BOM.txt
+  abc.GB18030_BOM.txt
+  empty.txt
+  empty.UTF-8_BOM.txt
+  empty.UTF-16LE_BOM.txt
+  empty.UTF-16BE_BOM.txt
+  empty.UTF-32LE_BOM.txt
+  empty.UTF-32BE_BOM.txt
+  empty.GB18030_BOM.txt
+)
+
+for file in "${required_files[@]}" ; do
+  if ! test -r "$logs/$file" ; then
+    echo "missing BOM fixture: $logs/$file" >&$logfd
+    exit 77
+  fi
+done
+
+failure=false
+tmp=`mktemp -d`
+trap 'rm -rf "$tmp"' 0
+
+run_iconv () {
+  echo "${BASH_SOURCE[1]}:${BASH_LINENO[0]}: iconv $@" >&$logfd
+  $test_program_prefix "$codir/iconv/iconv_prog" "$@"
+}
+
+compare_files () {
+  local actual=$1
+  local expected=$2
+  local message=$3
+  if ! cmp -s "$actual" "$expected" ; then
+    echo "error: $message" >&$logfd
+    echo "*** expected: $expected" >&$logfd
+    echo "*** actual:   $actual" >&$logfd
+    cmp -l "$expected" "$actual" | sed -n '1,20p' >&$logfd || true
+    failure=true
+  fi
+}
+
+make_source () {
+  local input=$1
+  local skip=$2
+  local output=$3
+  if test "$skip" -eq 0 ; then
+    cp "$input" "$output"
+  else
+    dd if="$input" of="$output" bs=1 skip="$skip" 2>/dev/null
+  fi
+}
+
+# Focused BOM byte tests, including tiny output buffers.
+: > "$tmp/empty"
+run_iconv --buffer-size=1 -f UTF-8 -t UTF-32LE --bom \
+  -o "$tmp/out" "$tmp/empty"
+printf '\xff\xfe\x00\x00' > "$tmp/expected"
+compare_files "$tmp/out" "$tmp/expected" "UTF-32LE empty BOM output"
+
+printf AB > "$tmp/part1"
+printf CD > "$tmp/part2"
+run_iconv -f UTF-8 -t UTF-8 --bom -o "$tmp/out" \
+  "$tmp/part1" "$tmp/part2"
+printf '\xef\xbb\xbfABCD' > "$tmp/expected"
+compare_files "$tmp/out" "$tmp/expected" "BOM written once for multiple files"
+
+cp "$logs/abc.UTF-8.txt" "$tmp/in-place"
+run_iconv -f UTF-8 -t UTF-16LE --bom -o "$tmp/in-place" "$tmp/in-place"
+compare_files "$tmp/in-place" "$logs/abc.UTF-16LE_BOM.txt" \
+  "in-place UTF-16LE BOM conversion"
+
+set +e
+run_iconv -f UTF-8 -t ASCII --bom -o "$tmp/ascii" \
+  "$logs/abc.ASCII.txt" > "$tmp/ascii-warning" 2>&1
+status=$?
+set -e
+if test "$status" -ne 0 ; then
+  echo "error: ASCII warning conversion failed with status $status" >&$logfd
+  failure=true
+fi
+if ! grep -q "no byte order mark" "$tmp/ascii-warning" ; then
+  echo "error: missing ASCII no-BOM warning" >&$logfd
+  failure=true
+fi
+
+run_iconv -s -f UTF-8 -t ASCII --bom -o "$tmp/ascii" \
+  "$logs/abc.ASCII.txt" > "$tmp/ascii-silent" 2>&1
+if grep -q "no byte order mark" "$tmp/ascii-silent" ; then
+  echo "error: -s did not suppress no-BOM warning" >&$logfd
+  failure=true
+fi
+
+# Generic charmap output keeps the default BOM; --bom adds one for explicit
+# endian encodings.
+printf '\xff' > "$tmp/invalid"
+cat "$logs/abc.ASCII.txt" "$logs/abc.ASCII.txt" > "$tmp/joined"
+cat > "$tmp/literal.charmap" <<'EOF'
+<code_set_name> BOM-TEST
+<escape_char> /
+<mb_cur_min> 1
+<mb_cur_max> 1
+CHARMAP
+<U0041> /x41
+<UFEFF> /x58
+END CHARMAP
+EOF
+printf XAX > "$tmp/literal"
+printf '\xef\xbb\xbfA\xef\xbb\xbf' > "$tmp/literal-utf8"
+
+for target in UTF-16 UTF16 UTF-32 UTF32 ; do
+  for bom_flag in '' --bom ; do
+    run_iconv -f ASCII -t "$target" -o "$tmp/with-bom" "$logs/abc.ASCII.txt"
+    cp "$tmp/with-bom" "$tmp/expected"
+    run_iconv -f "$charmap" -t "$target" $bom_flag -o "$tmp/out" \
+      "$logs/abc.ASCII.txt"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag file conversion"
+    run_iconv -f "$charmap" -t "$target" $bom_flag \
+      < "$logs/abc.ASCII.txt" > "$tmp/out"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag stdin conversion"
+    run_iconv -f "$target" -t UTF-8 -o "$tmp/decoded" "$tmp/out"
+    compare_files "$tmp/decoded" "$logs/abc.ASCII.txt" \
+      "charmap $target $bom_flag decoded text"
+
+    run_iconv -f ASCII -t "$target" -o "$tmp/with-bom" "$tmp/joined"
+    cp "$tmp/with-bom" "$tmp/expected"
+    run_iconv -f "$charmap" -t "$target" $bom_flag -o "$tmp/out" \
+      "$tmp/empty" "$logs/abc.ASCII.txt" "$logs/abc.ASCII.txt"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag multiple files"
+
+    run_iconv -f ASCII -t "$target" --bom -o "$tmp/expected" \
+      "$tmp/empty"
+    run_iconv -f "$charmap" -t "$target" $bom_flag -o "$tmp/out" \
+      "$tmp/empty"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag empty output"
+    run_iconv -c -f "$charmap" -t "$target" $bom_flag -o "$tmp/out" \
+      "$tmp/invalid"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag discarded output"
+
+    run_iconv -f UTF-8 -t "$target" -o "$tmp/with-bom" "$tmp/literal-utf8"
+    cp "$tmp/with-bom" "$tmp/expected"
+    run_iconv -f "$tmp/literal.charmap" -t "$target" $bom_flag \
+      -o "$tmp/out" "$tmp/literal"
+    compare_files "$tmp/out" "$tmp/expected" \
+      "charmap $target $bom_flag literal U+FEFF"
+  done
+done
+
+sources=(
+  "ASCII|$logs/abc.ASCII.txt|UTF-8|0"
+  "charmap|$logs/abc.ASCII.txt|$charmap|0"
+  "UTF-8|$logs/abc.UTF-8.txt|UTF-8|0"
+  "UTF-8_BOM|$logs/abc.UTF-8_BOM.txt|UTF-8|3"
+  "UTF-16LE|$logs/abc.UTF-16LE.txt|UTF-16LE|0"
+  "UTF-16LE_BOM|$logs/abc.UTF-16LE_BOM.txt|UTF-16|0"
+  "UTF-16BE|$logs/abc.UTF-16BE.txt|UTF-16BE|0"
+  "UTF-16BE_BOM|$logs/abc.UTF-16BE_BOM.txt|UTF-16|0"
+  "UTF-32LE|$logs/abc.UTF-32LE.txt|UTF-32LE|0"
+  "UTF-32LE_BOM|$logs/abc.UTF-32LE_BOM.txt|UTF-32|0"
+  "UTF-32BE|$logs/abc.UTF-32BE.txt|UTF-32BE|0"
+  "UTF-32BE_BOM|$logs/abc.UTF-32BE_BOM.txt|UTF-32|0"
+  "GB18030_BOM|$logs/abc.GB18030_BOM.txt|GB18030|4"
+)
+
+targets=(
+  "UTF-8|UTF-8||$logs/abc.UTF-8.txt"
+  "UTF-8_BOM|UTF-8|--bom|$logs/abc.UTF-8_BOM.txt"
+  "UTF-16LE|UTF-16LE||$logs/abc.UTF-16LE.txt"
+  "UTF-16LE_BOM|UTF-16LE|--bom|$logs/abc.UTF-16LE_BOM.txt"
+  "UTF-16BE|UTF-16BE||$logs/abc.UTF-16BE.txt"
+  "UTF-16BE_BOM|UTF-16BE|--bom|$logs/abc.UTF-16BE_BOM.txt"
+  "UTF-32LE|UTF-32LE||$logs/abc.UTF-32LE.txt"
+  "UTF-32LE_BOM|UTF-32LE|--bom|$logs/abc.UTF-32LE_BOM.txt"
+  "UTF-32BE|UTF-32BE||$logs/abc.UTF-32BE.txt"
+  "UTF-32BE_BOM|UTF-32BE|--bom|$logs/abc.UTF-32BE_BOM.txt"
+  "GB18030_BOM|GB18030|--bom|$logs/abc.GB18030_BOM.txt"
+)
+
+for source_entry in "${sources[@]}" ; do
+  IFS='|' read -r source_name source_file from_code skip_bytes \
+    <<< "$source_entry"
+  source_path="$tmp/source-$source_name"
+  make_source "$source_file" "$skip_bytes" "$source_path"
+
+  for target_entry in "${targets[@]}" ; do
+    IFS='|' read -r target_name to_code bom_flag expected \
+      <<< "$target_entry"
+    run_iconv -f "$from_code" -t "$to_code" $bom_flag \
+      -o "$tmp/out" "$source_path"
+    compare_files "$tmp/out" "$expected" \
+      "matrix $source_name to $target_name"
+  done
+done
+
+empty_sources=(
+  "ASCII|$logs/empty.txt|UTF-8|0"
+  "charmap|$logs/empty.txt|$charmap|0"
+  "UTF-8_BOM|$logs/empty.UTF-8_BOM.txt|UTF-8|3"
+  "UTF-16LE_BOM|$logs/empty.UTF-16LE_BOM.txt|UTF-16|0"
+  "UTF-16BE_BOM|$logs/empty.UTF-16BE_BOM.txt|UTF-16|0"
+  "UTF-32LE_BOM|$logs/empty.UTF-32LE_BOM.txt|UTF-32|0"
+  "UTF-32BE_BOM|$logs/empty.UTF-32BE_BOM.txt|UTF-32|0"
+  "GB18030_BOM|$logs/empty.GB18030_BOM.txt|GB18030|4"
+)
+
+for source_entry in "${empty_sources[@]}" ; do
+  IFS='|' read -r source_name source_file from_code skip_bytes \
+    <<< "$source_entry"
+  source_path="$tmp/empty-source-$source_name"
+  make_source "$source_file" "$skip_bytes" "$source_path"
+
+  for target_entry in "${targets[@]}" ; do
+    IFS='|' read -r target_name to_code bom_flag _ \
+      <<< "$target_entry"
+    if [[ "$target_name" == *_BOM ]] ; then
+      expected="$logs/empty.$target_name.txt"
+    else
+      expected="$logs/empty.txt"
+    fi
+    run_iconv -f "$from_code" -t "$to_code" $bom_flag \
+      -o "$tmp/out" "$source_path"
+    compare_files "$tmp/out" "$expected" \
+      "empty matrix $source_name to $target_name"
+  done
+done
+
+if $failure ; then
+  exit 1
+fi
+
+exit 0
\ No newline at end of file
-- 
2.43.0



More information about the Libc-alpha mailing list