🐛 fix(intake): skip style, script and comment text when extracting HTML

pkg/intake/intake.go

@@ -256,29 +256,86 @@ func StripTags(s string) string {
 	var b strings.Builder
 	inTag := false
 	spacePending := false
-	for _, r := range s {
-		switch r {
-		case '<':
-			inTag = true
+	for i := 0; i < len(s); {
+		c := s[i]
+		if !inTag && c == '<' {
 			spacePending = true
-		case '>':
-			inTag = false
-		default:
-			if inTag {
+			if strings.HasPrefix(s[i:], "<!--") {
+				i = skipHTMLComment(s, i)
 				continue
 			}
-			if spacePending {
-				if b.Len() > 0 {
-					b.WriteByte(' ')
-				}
-				spacePending = false
+			if name := htmlTagName(s, i+1); htmlSkippedElement(name) {
+				i = skipHTMLElement(s, i, name)
+				continue
 			}
-			b.WriteRune(r)
+			inTag = true
+			i++
+			continue
+		}
+		if inTag {
+			if c == '>' {
+				inTag = false
+			}
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if c == '>' {
+			i++
+			continue
+		}
+		if spacePending {
+			if b.Len() > 0 {
+				b.WriteByte(' ')
+			}
+			spacePending = false
 		}
+		b.WriteRune(r)
+		i += size
 	}
 	return html.UnescapeString(b.String())
 }
 
+func skipHTMLComment(s string, i int) int {
+	end := strings.Index(s[i+4:], "-->")
+	if end < 0 {
+		return len(s)
+	}
+	return i + 4 + end + 3
+}
+
+func htmlTagName(s string, i int) string {
+	start := i
+	for i < len(s) && (s[i] >= 'a' && s[i] <= 'z' || s[i] >= 'A' && s[i] <= 'Z' || s[i] >= '0' && s[i] <= '9') {
+		i++
+	}
+	return strings.ToLower(s[start:i])
+}
+
+func htmlSkippedElement(name string) bool {
+	switch name {
+	case "script", "style", "noscript", "template":
+		return true
+	}
+	return false
+}
+
+// skipHTMLElement returns the index just past the closing tag matching the
+// element opened at s[i], or len(s) when it is never closed.
+func skipHTMLElement(s string, i int, name string) int {
+	for j := i + 1; j < len(s); j++ {
+		if s[j] != '<' || j+1 >= len(s) || s[j+1] != '/' || htmlTagName(s, j+2) != name {
+			continue
+		}
+		end := strings.IndexByte(s[j:], '>')
+		if end < 0 {
+			return len(s)
+		}
+		return j + end + 1
+	}
+	return len(s)
+}
+
 func StripRTF(s string) string {
 	var b strings.Builder
 	for i := 0; i < len(s); {

pkg/intake/intake_test.go

@@ -39,6 +39,31 @@ func TestExtractTextFormats(t *testing.T) {
 	}
 }
 
+func TestStripTagsSkipsNonTextRegions(t *testing.T) {
+	t.Parallel()
+	cases := []struct{ name, in, want string }{
+		{"entities", "<h1>Idea</h1><p>Help &amp; ship</p>", "Idea Help & ship"},
+		{"style", "<p>a</p><style>p > b { color: red }</style><p>b</p>", "a b"},
+		{"script", "<p>a</p><script>if (x > 1) { y() }</script><p>b</p>", "a b"},
+		{"noscript", "<p>a</p><noscript>enable js</noscript><p>b</p>", "a b"},
+		{"template", "<p>a</p><template><p>hidden</p></template><p>b</p>", "a b"},
+		{"uppercase", "<p>a</p><SCRIPT type=x>secret</SCRIPT ><p>b</p>", "a b"},
+		{"comment", "<p>a</p><!-- if x > 1 --><p>b</p>", "a b"},
+		{"unterminated comment", "<p>a</p><!-- hidden > text", "a"},
+		{"unterminated script", "<p>a</p><script>secret", "a"},
+		{"unclosed closing tag", "<p>a</p><script>secret</script", "a"},
+		{"similar tag name", "<p>a</p><style>x</styles>y</style><p>b</p>", "a b"},
+		{"stray lt", "a < b", "a "},
+		{"trailing lt", "a</", "a"},
+		{"unicode and stray gt", "caf\u00e9 > ok", "caf\u00e9  ok"},
+	}
+	for _, tc := range cases {
+		if got := StripTags(tc.in); got != tc.want {
+			t.Errorf("%s: StripTags(%q) = %q, want %q", tc.name, tc.in, got, tc.want)
+		}
+	}
+}
+
 func TestExtractDOCX(t *testing.T) {
 	t.Parallel()
 	var buf bytes.Buffer