Merge pull request 'Add DedupeStrings to remove repeated values from a string slice' (#2) from clawbot/util:proposal-dedupe-strings into master

Reviewed-on: #2

chunk.go

@@ -0,0 +1,26 @@
+package util
+
+// ChunkStrings splits input into consecutive slices of at most size elements.
+// The last slice holds whatever is left over and may be shorter than the
+// others. Each returned slice is a copy, so appending to one of them cannot
+// disturb another. A size below one, or an empty input, returns nil.
+func ChunkStrings(input []string, size int) [][]string {
+	if size < 1 || len(input) == 0 {
+		return nil
+	}
+
+	out := make([][]string, 0, (len(input)+size-1)/size)
+
+	for start := 0; start < len(input); start += size {
+		end := start + size
+		if end > len(input) {
+			end = len(input)
+		}
+
+		chunk := make([]string, end-start)
+		copy(chunk, input[start:end])
+		out = append(out, chunk)
+	}
+
+	return out
+}

chunk_test.go

@@ -0,0 +1,43 @@
+package util
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestChunkStrings(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []string
+		size     int
+		expected [][]string
+	}{
+		{"exact multiple", []string{"a", "b", "c", "d"}, 2, [][]string{{"a", "b"}, {"c", "d"}}},
+		{"with a remainder", []string{"a", "b", "c"}, 2, [][]string{{"a", "b"}, {"c"}}},
+		{"size larger than input", []string{"a", "b"}, 5, [][]string{{"a", "b"}}},
+		{"size of one", []string{"a", "b"}, 1, [][]string{{"a"}, {"b"}}},
+		{"size of zero", []string{"a", "b"}, 0, nil},
+		{"negative size", []string{"a", "b"}, -1, nil},
+		{"empty input", []string{}, 2, nil},
+		{"nil input", nil, 2, nil},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := ChunkStrings(test.input, test.size)
+			if !reflect.DeepEqual(got, test.expected) {
+				t.Errorf("expected %#v got %#v", test.expected, got)
+			}
+		})
+	}
+}
+
+func TestChunkStringsReturnsCopies(t *testing.T) {
+	input := []string{"a", "b", "c", "d"}
+	chunks := ChunkStrings(input, 2)
+	chunks[0][0] = "changed"
+
+	if input[0] != "a" {
+		t.Errorf("expected the input to be unchanged, got %#v", input)
+	}
+}

dedupe.go

@@ -0,0 +1,23 @@
+package util
+
+// DedupeStrings returns a new slice holding each value of input once, in the
+// order in which each value first appears. The input slice is not modified.
+// A nil input returns nil.
+func DedupeStrings(input []string) []string {
+	if input == nil {
+		return nil
+	}
+
+	seen := make(map[string]struct{}, len(input))
+	out := make([]string, 0, len(input))
+
+	for _, value := range input {
+		if _, ok := seen[value]; ok {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+
+	return out
+}

dedupe_test.go

@@ -0,0 +1,40 @@
+package util
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestDedupeStrings(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []string
+		expected []string
+	}{
+		{"nil stays nil", nil, nil},
+		{"empty slice", []string{}, []string{}},
+		{"nothing repeated", []string{"a", "b", "c"}, []string{"a", "b", "c"}},
+		{"all one value", []string{"a", "a", "a"}, []string{"a"}},
+		{"repeats next to each other", []string{"a", "a", "b", "b"}, []string{"a", "b"}},
+		{"repeats far apart", []string{"a", "b", "a", "c", "b"}, []string{"a", "b", "c"}},
+		{"empty string is a value", []string{"", "a", ""}, []string{"", "a"}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := DedupeStrings(test.input)
+			if !reflect.DeepEqual(got, test.expected) {
+				t.Errorf("expected %#v got %#v", test.expected, got)
+			}
+		})
+	}
+}
+
+func TestDedupeStringsLeavesInputAlone(t *testing.T) {
+	input := []string{"b", "a", "b"}
+	DedupeStrings(input)
+
+	if !reflect.DeepEqual(input, []string{"b", "a", "b"}) {
+		t.Errorf("expected the input to be unchanged, got %#v", input)
+	}
+}

slugify.go

@@ -0,0 +1,36 @@
+package util
+
+import "strings"
+
+// Slugify returns input as lowercase ASCII letters, digits and hyphens, which
+// is safe to put in a URL path, a filename or a fragment identifier. Every run
+// of other characters becomes a single hyphen, and the result never starts or
+// ends with one.
+//
+// Characters outside ASCII are treated as separators rather than being folded
+// to their nearest ASCII relative, because doing that properly needs a Unicode
+// table that this package does not carry. Text with no ASCII letters or digits
+// in it therefore slugifies to an empty string.
+func Slugify(input string) string {
+	var out strings.Builder
+	pendingHyphen := false
+
+	for _, character := range strings.ToLower(input) {
+		if (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') {
+			if pendingHyphen {
+				out.WriteByte('-')
+				pendingHyphen = false
+			}
+			out.WriteRune(character)
+			continue
+		}
+
+		// Remember that a separator was seen, but only write a hyphen once
+		// another usable character turns up, so nothing trails off the end.
+		if out.Len() > 0 {
+			pendingHyphen = true
+		}
+	}
+
+	return out.String()
+}

slugify_test.go

@@ -0,0 +1,34 @@
+package util
+
+import "testing"
+
+func TestSlugify(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"already a slug", "hello-world", "hello-world"},
+		{"mixed case", "Hello World", "hello-world"},
+		{"punctuation", "What's new?", "what-s-new"},
+		{"run of separators", "one   ---   two", "one-two"},
+		{"leading separators", "///one", "one"},
+		{"trailing separators", "one///", "one"},
+		{"separators at both ends", "  --one two--  ", "one-two"},
+		{"digits are kept", "Go 1.14 release", "go-1-14-release"},
+		{"underscores are separators", "some_name_here", "some-name-here"},
+		{"nothing usable", "!!! ???", ""},
+		{"empty string", "", ""},
+		{"outside ascii", "Grüße, Welt", "gr-e-welt"},
+		{"only characters outside ascii", "日本語", ""},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := Slugify(test.input)
+			if got != test.expected {
+				t.Errorf("expected %q got %q", test.expected, got)
+			}
+		})
+	}
+}

truncate.go

@@ -0,0 +1,21 @@
+package util
+
+// TruncateString returns at most max characters from the start of s. It counts
+// characters rather than bytes, so a multi-byte character is never cut in half
+// and the result is always valid text. A max of zero or less returns an empty
+// string, and a string already at or below the limit is returned unchanged.
+func TruncateString(s string, max int) string {
+	if max <= 0 {
+		return ""
+	}
+
+	count := 0
+	for offset := range s {
+		if count == max {
+			return s[:offset]
+		}
+		count++
+	}
+
+	return s
+}

truncate_test.go

@@ -0,0 +1,41 @@
+package util
+
+import "testing"
+
+func TestTruncateString(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		max      int
+		expected string
+	}{
+		{"shorter than the limit", "abc", 5, "abc"},
+		{"exactly at the limit", "abc", 3, "abc"},
+		{"longer than the limit", "abcdef", 3, "abc"},
+		{"max of one", "abc", 1, "a"},
+		{"max of zero", "abc", 0, ""},
+		{"negative max", "abc", -1, ""},
+		{"empty string", "", 3, ""},
+		{"multi-byte characters kept whole", "héllo", 3, "hél"},
+		{"multi-byte characters below the limit", "héllo", 99, "héllo"},
+		{"characters outside the basic plane", "a😀b", 2, "a😀"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := TruncateString(test.input, test.max)
+			if got != test.expected {
+				t.Errorf("expected %q got %q", test.expected, got)
+			}
+		})
+	}
+}
+
+func TestTruncateStringCountsCharactersNotBytes(t *testing.T) {
+	// Each of these characters is two bytes long, so a byte-based cut at
+	// three would leave half a character behind.
+	got := TruncateString("äöü", 3)
+	if got != "äöü" {
+		t.Errorf("expected %q got %q", "äöü", got)
+	}
+}

whitespace.go

@@ -0,0 +1,12 @@
+package util
+
+import "strings"
+
+// CollapseWhitespace replaces every run of whitespace in s with a single space
+// and removes any whitespace at the start and end. Tabs, newlines and spaces
+// outside ASCII such as the non-breaking space all count as whitespace, so a
+// block of text comes back as one line with single spaces between the words.
+// A string that is nothing but whitespace comes back empty.
+func CollapseWhitespace(s string) string {
+	return strings.Join(strings.Fields(s), " ")
+}

whitespace_test.go

@@ -0,0 +1,36 @@
+package util
+
+import "testing"
+
+// Written as a code point so that it is visible in the source rather than
+// looking like an ordinary space.
+const nonBreakingSpace = string(rune(0x00a0))
+
+func TestCollapseWhitespace(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"nothing to change", "one two", "one two"},
+		{"leading and trailing spaces", "  one two  ", "one two"},
+		{"run in the middle", "one    two", "one two"},
+		{"tabs and newlines", "one\t\ttwo\nthree", "one two three"},
+		{"windows line endings", "one\r\ntwo", "one two"},
+		{"non-breaking space", "one" + nonBreakingSpace + "two", "one two"},
+		{"mixed whitespace run", "one \t" + nonBreakingSpace + " two", "one two"},
+		{"only whitespace", " \t\n ", ""},
+		{"empty string", "", ""},
+		{"single word with padding", "\tword\n", "word"},
+		{"paragraph becomes one line", "first line\n\nsecond line\n", "first line second line"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := CollapseWhitespace(test.input)
+			if got != test.expected {
+				t.Errorf("expected %q got %q", test.expected, got)
+			}
+		})
+	}
+}