fix(runner): decode input with a byte order mark

runner/runner.go

@@ -10,6 +10,7 @@ import (
 	"html/template"
 	"image"
 	"io"
+	"iter"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -26,6 +27,8 @@ import (
 	"time"
 
 	"golang.org/x/exp/maps"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corona10/goimagehash"
@@ -779,7 +782,7 @@ func (r *Runner) streamInput() (chan string, error) {
 					return
 				}
 			} else {
-				for item, err := range fileutil.Lines(r.options.InputFile) {
+				for item, err := range inputFileLines(r.options.InputFile) {
 					if err != nil || !sendLine(item) {
 						return
 					}
@@ -791,15 +794,15 @@ func (r *Runner) streamInput() (chan string, error) {
 				gologger.Fatal().Msgf("No input provided: %s", err)
 			}
 			for _, file := range files {
-				for item, err := range fileutil.Lines(file) {
+				for item, err := range inputFileLines(file) {
 					if err != nil || !sendLine(item) {
 						return
 					}
 				}
 			}
 		}
 		if fileutil.HasStdin() {
-			for item, err := range fileutil.LinesReader(os.Stdin) {
+			for item, err := range fileutil.LinesReader(decodeInput(os.Stdin)) {
 				if err != nil || !sendLine(item) {
 					return
 				}
@@ -810,7 +813,7 @@ func (r *Runner) streamInput() (chan string, error) {
 }
 
 func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
-	scanner := bufio.NewScanner(finput)
+	scanner := bufio.NewScanner(decodeInput(finput))
 	for scanner.Scan() {
 		target, ok := r.readInputLine(scanner.Text())
 		if !ok {
@@ -833,6 +836,30 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	return numTargets, err
 }
 
+// decodeInput converts input starting with a UTF-16 byte order mark to UTF-8
+// (Windows PowerShell writes UTF-16) and strips a UTF-8 one. Input without a
+// byte order mark is passed through.
+func decodeInput(r io.Reader) io.Reader {
+	return transform.NewReader(r, unicode.BOMOverride(transform.Nop))
+}
+
+// inputFileLines streams the lines of an input file, decoded as decodeInput does.
+func inputFileLines(name string) iter.Seq2[string, error] {
+	return func(yield func(string, error) bool) {
+		f, err := os.Open(name)
+		if err != nil {
+			yield("", err)
+			return
+		}
+		defer f.Close() //nolint:errcheck
+		for line, err := range fileutil.LinesReader(decodeInput(f)) {
+			if !yield(line, err) {
+				return
+			}
+		}
+	}
+}
+
 func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) {
 	finput, err := os.Open(filePath)
 	if err != nil {

runner/targetline_test.go

@@ -219,6 +219,50 @@ func TestRunner_skipsInvalidLines(t *testing.T) {
 	})
 }
 
+// A file saved with a byte order mark, such as UTF-16 written by Windows
+// PowerShell, is decoded before its lines are read.
+func TestRunner_readsInputWithBOM(t *testing.T) {
+	want := []string{"a.example", "b.example"}
+	encodings := map[string][]byte{
+		"utf-16le": {0xff, 0xfe, 'a', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\r', 0, '\n', 0, 'b', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\r', 0, '\n', 0},
+		"utf-16be": {0xfe, 0xff, 0, 'a', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\n', 0, 'b', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\n'},
+		"utf-8":    append([]byte{0xef, 0xbb, 0xbf}, "a.example\nb.example\n"...),
+	}
+	for name, content := range encodings {
+		file := filepath.Join(t.TempDir(), "targets.txt")
+		require.NoError(t, os.WriteFile(file, content, 0o600))
+
+		t.Run(name+" -l", func(t *testing.T) {
+			r, err := New(&Options{})
+			require.NoError(t, err)
+			defer r.Close()
+			input, err := os.Open(file)
+			require.NoError(t, err)
+
+			count, err := r.loadAndCloseFile(input)
+			require.NoError(t, err)
+			require.Equal(t, len(want), count)
+			for _, key := range want {
+				_, ok := r.hm.Get(key)
+				require.True(t, ok, key)
+			}
+		})
+		t.Run(name+" -stream", func(t *testing.T) {
+			r, err := New(&Options{Stream: true, InputFile: file})
+			require.NoError(t, err)
+			defer r.Close()
+			out, err := r.streamInput()
+			require.NoError(t, err)
+
+			var got []string
+			for item := range out {
+				got = append(got, item)
+			}
+			require.Equal(t, want, got)
+		})
+	}
+}
+
 func TestRunner_skip_pinned(t *testing.T) {
 	r, err := New(&Options{Exclude: []string{"cdn"}})
 	require.NoError(t, err)