fix(intake): drop style, script, template and comment text from HTML

pkg/intake/intake.go

@@ -252,11 +252,23 @@ func ExtractDOCX(data []byte) (string, error) {
 	return "", fmt.Errorf("word/document.xml not found")
 }
 
+// StripTags reduces HTML to its text. Comments and the contents of
+// htmlSkippedElements are dropped as a whole, since they are not rendered
+// text and a "Save as HTML" page is mostly inline CSS and JS.
 func StripTags(s string) string {
 	var b strings.Builder
 	inTag := false
 	spacePending := false
-	for _, r := range s {
+	for i := 0; i < len(s); {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r == '<' && !inTag {
+			if end := skipHTMLRegion(s, i); end > i {
+				i = end
+				spacePending = true
+				continue
+			}
+		}
+		i += size
 		switch r {
 		case '<':
 			inTag = true
@@ -279,6 +291,69 @@ func StripTags(s string) string {
 	return html.UnescapeString(b.String())
 }
 
+// htmlSkippedElements hold no rendered document text. Only template can
+// nest; the others are raw text and end at their first closing tag.
+var htmlSkippedElements = []string{"script", "style", "noscript", "template"}
+
+// skipHTMLRegion returns the index just past the comment or skipped element
+// that starts at s[i] (a '<'), the end of input if it is unterminated, or i
+// if s[i] starts neither.
+func skipHTMLRegion(s string, i int) int {
+	if strings.HasPrefix(s[i:], "<!--") {
+		// Searching from the first '-' also ends the empty comments
+		// "<!-->" and "<!--->" where a browser does.
+		if end := strings.Index(s[i+2:], "-->"); end >= 0 {
+			return i + 2 + end + len("-->")
+		}
+		return len(s)
+	}
+	for _, name := range htmlSkippedElements {
+		if !htmlTagAt(s, i+1, name) {
+			continue
+		}
+		depth := 0
+		for j := i; j < len(s); j++ {
+			if s[j] != '<' {
+				continue
+			}
+			switch {
+			case j+1 < len(s) && s[j+1] == '/' && htmlTagAt(s, j+2, name):
+				depth--
+			case j == i || (name == "template" && htmlTagAt(s, j+1, name)):
+				depth++
+			default:
+				continue
+			}
+			end := strings.IndexByte(s[j:], '>')
+			if end < 0 {
+				return len(s)
+			}
+			if depth == 0 {
+				return j + end + 1
+			}
+			j += end
+		}
+		return len(s)
+	}
+	return i
+}
+
+// htmlTagAt reports whether the tag name starting at s[i] is name, matched
+// case-insensitively and ending at whitespace, '/', '>' or end of input.
+func htmlTagAt(s string, i int, name string) bool {
+	if len(s)-i < len(name) || !strings.EqualFold(s[i:i+len(name)], name) {
+		return false
+	}
+	if i+len(name) == len(s) {
+		return true
+	}
+	switch s[i+len(name)] {
+	case ' ', '\t', '\n', '\r', '\f', '/', '>':
+		return true
+	}
+	return false
+}
+
 func StripRTF(s string) string {
 	var b strings.Builder
 	for i := 0; i < len(s); {

pkg/intake/intake_test.go

@@ -30,6 +30,16 @@ func TestExtractTextFormats(t *testing.T) {
 	if !strings.Contains(got, "Idea") || !strings.Contains(got, "Help & ship") || strings.Contains(got, "<p>") {
 		t.Fatalf("html stripped to %q", got)
 	}
+	got, err = ExtractDocument(".html", []byte(`<html><head><style>body{color:red}</style>
+   <script>window.dataLayer=[];gtag('js')</script>
+   <!-- generator: Word 16 --></head>
+   <body><p>Idea: add a dark mode</p></body></html>`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := cleanText(got); got != "Idea: add a dark mode" {
+		t.Fatalf("html with style/script/comment extracted %q", got)
+	}
 	got, err = ExtractDocument(".rtf", []byte(`{\rtf1\ansi This is \b bold\b0  idea\par done}`))
 	if err != nil {
 		t.Fatal(err)
@@ -39,6 +49,35 @@ func TestExtractTextFormats(t *testing.T) {
 	}
 }
 
+func TestStripTagsSkipsNonTextRegions(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name, in, want string
+	}{
+		{"style", `<p>a</p><style type="text/css">p > b{}</style><p>b</p>`, "a b"},
+		{"script", `<p>a</p><script>if (x < 1 && y > 2) { w("<p>no</p>") }</script><p>b</p>`, "a b"},
+		{"noscript", `<p>a</p><noscript><p>enable JS</p></noscript><p>b</p>`, "a b"},
+		{"template", `<p>a</p><template><p>row</p></template><p>b</p>`, "a b"},
+		{"nested template", `a<template><template>x</template>y</template>b`, "a b"},
+		{"comment with gt", `a<!-- if x > 1 -->b`, "a b"},
+		{"empty comments", `a<!-->b<!--->c`, "a b c"},
+		{"uppercase", `a<SCRIPT>x()</Script >b`, "a b"},
+		{"similar names kept", `<scripts>a</scripts><styled>b</styled>`, "a b"},
+		{"unterminated comment", `a<!-- b`, "a"},
+		{"unterminated element", `a<script>b`, "a"},
+		{"unterminated open tag", `a<style`, "a"},
+		{"unterminated close tag", `a<style>b</style`, "a"},
+		{"unterminated nested template", `a<template><template`, "a"},
+		{"comment marker inside tag", `<a title="<!--">a</a>b<p>c`, "a b c"},
+		{"entities", `<style>x</style>Help &amp; ship`, "Help & ship"},
+	}
+	for _, tc := range cases {
+		if got := cleanText(StripTags(tc.in)); got != tc.want {
+			t.Errorf("%s: StripTags(%q) = %q, want %q", tc.name, tc.in, got, tc.want)
+		}
+	}
+}
+
 func TestExtractDOCX(t *testing.T) {
 	t.Parallel()
 	var buf bytes.Buffer