This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH] Use Quicksearch in strstr
- From: Wilco Dijkstra <Wilco dot Dijkstra at arm dot com>
- To: 'GNU C Library' <libc-alpha at sourceware dot org>
- Cc: nd <nd at arm dot com>
- Date: Mon, 29 Oct 2018 15:55:36 +0000
- Subject: [PATCH] Use Quicksearch in strstr
This patch significantly improves performance of strstr by using Sunday's
Quick-Search algorithm. Due to its simplicity it has the best average
performance of string matching algorithms on almost all inputs. It uses a
bad-character shift table to skip past mismatches.
The needle length is limited to 254 - combined with a hashed shift table
this reduces the shift table memory 16 to 32 times, lowering preprocessing
overhead and minimizing cache effects. The needle limit also implies
worst-case performance remains linear.
Small 1-4 byte needles use special case code which is typically faster.
Very long needles continue to use the linear-time Two-Way algorithm.
The performance gain using the improved bench-strstr on Cortex-A72 is 2.1x.
Due to the low overhead the gain is larger on small haystacks: haystacks of 64
chars are ~3 times faster for needles of 5-16 chars.
Tested against GLIBC testsuite, randomized tests and the GNULIB strstr test
(https://git.savannah.gnu.org/cgit/gnulib.git/tree/tests/test-strstr.c).
ChangeLog:
2018-10-29 Wilco Dijkstra <wdijkstr@arm.com>
* string/str-two-way.h (two_way_short_needle): Add inline to avoid warning.
* string/strstr.c (strstr2): Add new function.
(strstr3): Likewise.
(strstr3): Likewise.
(STRSTR): Completely rewrite strstr to improve performance
---
diff --git a/string/str-two-way.h b/string/str-two-way.h
index 523d946c59412e1f1f65b8ec3778553b83191952..31c3f18fb057cdd999c3ac9e9d894a8b62a98a70 100644
--- a/string/str-two-way.h
+++ b/string/str-two-way.h
@@ -221,7 +221,7 @@ critical_factorization (const unsigned char *needle, size_t needle_len,
most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching.
If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 *
HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. */
-static RETURN_TYPE
+static inline RETURN_TYPE
two_way_short_needle (const unsigned char *haystack, size_t haystack_len,
const unsigned char *needle, size_t needle_len)
{
diff --git a/string/strstr.c b/string/strstr.c
index f74d7189ed1319f6225525cc2d32380745de1523..206f20ddfcf10f61ba68725b516ef00356e9028c 100644
--- a/string/strstr.c
+++ b/string/strstr.c
@@ -22,11 +22,8 @@
# include <config.h>
#endif
-/* Specification of strstr. */
#include <string.h>
-#include <stdbool.h>
-
#ifndef _LIBC
# define __builtin_expect(expr, val) (expr)
#endif
@@ -47,46 +44,115 @@
#define STRSTR strstr
#endif
-/* Return the first occurrence of NEEDLE in HAYSTACK. Return HAYSTACK
- if NEEDLE is empty, otherwise NULL if NEEDLE is not found in
- HAYSTACK. */
-char *
-STRSTR (const char *haystack, const char *needle)
+static inline char *
+strstr2 (const unsigned char *hs, const unsigned char *ne)
{
- size_t needle_len; /* Length of NEEDLE. */
- size_t haystack_len; /* Known minimum length of HAYSTACK. */
+ uint32_t h1 = (ne[0] << 16) | ne[1];
+ uint32_t h2 = 0;
+ for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs)
+ h2 = (h2 << 16) | c;
+ return h1 == h2 ? (char *)hs - 2 : NULL;
+}
- /* Handle empty NEEDLE special case. */
- if (needle[0] == '\0')
- return (char *) haystack;
+static inline char *
+strstr3 (const unsigned char *hs, const unsigned char *ne)
+{
+ uint32_t h1 = (ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8);
+ uint32_t h2 = 0;
+ for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs)
+ h2 = (h2 | c) << 8;
+ return h1 == h2 ? (char *)hs - 3 : NULL;
+}
- /* Skip until we find the first matching char from NEEDLE. */
- haystack = strchr (haystack, needle[0]);
- if (haystack == NULL || needle[1] == '\0')
- return (char *) haystack;
+static inline char *
+strstr4 (const unsigned char *hs, const unsigned char *ne)
+{
+ uint32_t h1 = (ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8) | ne[3];
+ uint32_t h2 = 0;
+ for (int c = hs[0]; c != 0 && h1 != h2; c = *++hs)
+ h2 = (h2 << 8) | c;
+ return h1 == h2 ? (char *)hs - 4 : NULL;
+}
- /* Ensure HAYSTACK length is at least as long as NEEDLE length.
- Since a match may occur early on in a huge HAYSTACK, use strnlen
+/* Number of bits used to index shift table. */
+#define SHIFT_TABLE_BITS 6
+
+/* Extremely fast strstr algorithm with guaranteed linear-time performance.
+ Small needles up to size 4 use a dedicated linear search. Longer needles
+ up to size 254 use Sunday's Quick-Search algorithm. Due to its simplicity
+ it has the best average performance of string matching algorithms on almost
+ all inputs. It uses a bad-character shift table to skip past mismatches.
+ By limiting the needle length to 254, the shift table can be reduced to 8
+ bits per entry, lowering preprocessing overhead and minimizing cache effects.
+ The limit also implies the worst-case performance is linear.
+ Even larger needles are processed by the linear-time Two-Way algorithm.
+*/
+char *
+STRSTR (const char *haystack, const char *needle)
+{
+ const unsigned char *hs = (const unsigned char *) haystack;
+ const unsigned char *ne = (const unsigned char *) needle;
+
+ /* Handle short needle special cases first. */
+ if (ne[0] == '\0')
+ return (char *)hs;
+ hs = (const unsigned char *)strchr ((const char*)hs, ne[0]);
+ if (hs == NULL || ne[1] == '\0')
+ return (char*)hs;
+ if (ne[2] == '\0')
+ return strstr2 (hs, ne);
+ if (ne[3] == '\0')
+ return strstr3 (hs, ne);
+ if (ne[4] == '\0')
+ return strstr4 (hs, ne);
+
+ /* Ensure haystack length is at least as long as needle length.
+ Since a match may occur early on in a huge haystack, use strnlen
and read ahead a few cachelines for improved performance. */
- needle_len = strlen (needle);
- haystack_len = __strnlen (haystack, needle_len + 256);
- if (haystack_len < needle_len)
+ size_t ne_len = strlen ((const char*)ne);
+ size_t hs_len = strnlen ((const char*)hs, ne_len | 512);
+ if (hs_len < ne_len)
return NULL;
- /* Check whether we have a match. This improves performance since we avoid
- the initialization overhead of the two-way algorithm. */
- if (memcmp (haystack, needle, needle_len) == 0)
- return (char *) haystack;
-
- /* Perform the search. Abstract memory is considered to be an array
- of 'unsigned char' values, not an array of 'char' values. See
- ISO C 99 section 6.2.6.1. */
- if (needle_len < LONG_NEEDLE_THRESHOLD)
- return two_way_short_needle ((const unsigned char *) haystack,
- haystack_len,
- (const unsigned char *) needle, needle_len);
- return two_way_long_needle ((const unsigned char *) haystack, haystack_len,
- (const unsigned char *) needle, needle_len);
+ /* Check whether we have a match. This improves performance since we
+ avoid initialization overheads. */
+ if (memcmp (hs, ne, ne_len) == 0)
+ return (char *) hs;
+
+ /* Use the Quick-Search algorithm for needle lengths less than 255. */
+ if (__glibc_likely (ne_len < 255))
+ {
+ uint8_t shift[1 << SHIFT_TABLE_BITS];
+ const unsigned char *end = hs + hs_len - ne_len;
+
+ /* Initialize bad character shift hash table. */
+ memset (shift, ne_len + 1, sizeof (shift));
+ for (int i = 0; i < ne_len; i++)
+ shift[ne[i] % sizeof (shift)] = ne_len - i;
+
+ do
+ {
+ hs--;
+
+ /* Search by skipping past bad characters. */
+ size_t tmp = shift[hs[ne_len] % sizeof (shift)];
+ for (hs += tmp; hs <= end; hs += tmp)
+ {
+ tmp = shift[hs[ne_len] % sizeof (shift)];
+ if (memcmp (hs, ne, ne_len) == 0)
+ return (char*)hs;
+ }
+ if (end[ne_len] == '\0')
+ return NULL;
+ end += strnlen ((const char*)end + ne_len, 2048);
+ }
+ while (hs <= end);
+
+ return NULL;
+ }
+
+ /* Use Two-Way algorithm for very long needles. */
+ return two_way_long_needle (hs, hs_len, ne, ne_len);
}
libc_hidden_builtin_def (strstr)