From b5b7f69ffbdb903d95ed451e7a2951eefc33075e Mon Sep 17 00:00:00 2001 From: Bobby Bones <159389674+saariuslystoned@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:00:40 -0500 Subject: [PATCH 1/2] fix(gmail): read every history page before advancing the watch cursor watch serve and watch pull made one users.history.list call (100 records) per notification and then stored the response historyId, which is the mailbox's current history ID. Records after the first page were never fetched, and the stored cursor skipped them for good. Gmail's reference says the historyId can be stored once no nextPageToken is returned. Page through the history with the repeated-token guard before the processor advances the cursor. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + docs/watch.md | 3 + .../cmd/gmail_watch_history_pages_test.go | 62 +++++++++++++ .../gmail_watch_server_history_pages_test.go | 88 +++++++++++++++++++ internal/cmd/gmail_watch_source.go | 30 +++++-- 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 internal/cmd/gmail_watch_history_pages_test.go create mode 100644 internal/cmd/gmail_watch_server_history_pages_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 861521fad..984b13004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.48.1 - Unreleased +- Gmail: make `watch serve` and `watch pull` read every page of `users.history.list` before advancing the saved history cursor. They previously read only the first 100 history records of a notification and then stored the mailbox's current history ID, so changes past the first page (a burst of mail, bulk label edits, or a catch-up after downtime) were never delivered to the hook. — thanks @saariuslystoned. + ## 0.48.0 - 2026-11-05 **Highlights:** New backups stay local until you configure a remote; existing configurations keep working. Store service-account keys in the keyring, search Gmail IDs efficiently, and use environment-backed watch tokens, with Calendar fixes and security dependency updates. diff --git a/docs/watch.md b/docs/watch.md index be0d1b3af..92f5fbbe2 100644 --- a/docs/watch.md +++ b/docs/watch.md @@ -105,6 +105,9 @@ Notes: - `watch renew` reuses stored topic/labels. - Push and pull inspect notifications whose history ID equals the saved cursor; lower IDs remain stale. Duplicate Pub/Sub message IDs are still ignored. +- Push and pull read every page of Gmail history after the saved cursor and + advance the cursor only after the last page, so bursts of more than one page + of changes are delivered instead of skipped. - `watch stop` calls Gmail stop + clears state. - `watch serve` and `watch pull` use stored hook config if `--hook-url` is not provided. diff --git a/internal/cmd/gmail_watch_history_pages_test.go b/internal/cmd/gmail_watch_history_pages_test.go new file mode 100644 index 000000000..fd3a27ef0 --- /dev/null +++ b/internal/cmd/gmail_watch_history_pages_test.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +// A history listing that spans two pages: every record must be returned, and +// the stored cursor may only be the historyId of the page without a +// nextPageToken ("If you receive no nextPageToken in the response, there are no +// updates to retrieve and you can store the returned historyId"). +func TestGmailWatchSourceListHistoryReadsEveryPage(t *testing.T) { + var calls []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/gmail/v1/users/me/history" { + http.NotFound(w, r) + return + } + token := r.URL.Query().Get("pageToken") + calls = append(calls, token) + w.Header().Set("Content-Type", "application/json") + switch token { + case "": + _, _ = w.Write([]byte(`{"history":[{"id":"101","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`)) + case "p2": + _, _ = w.Write([]byte(`{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`)) + default: + http.Error(w, "unexpected page token", http.StatusBadRequest) + } + })) + t.Cleanup(srv.Close) + + source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)} + page, err := source.ListHistory(context.Background(), 100, 1, nil) + if err != nil { + t.Fatalf("ListHistory: %v", err) + } + if len(calls) != 2 || calls[1] != "p2" { + t.Fatalf("history.list calls = %q, want first page then p2", calls) + } + if len(page.Records) != 2 || page.Records[1].Added[0] != "m2" { + t.Fatalf("records = %#v, want both pages", page.Records) + } + if page.HistoryID != "300" { + t.Fatalf("historyId = %q, want 300", page.HistoryID) + } +} + +func TestGmailWatchSourceListHistoryRejectsRepeatedPageToken(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"history":[{"id":"101"}],"nextPageToken":"same","historyId":"300"}`)) + })) + t.Cleanup(srv.Close) + + source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)} + if _, err := source.ListHistory(context.Background(), 100, 1, nil); err == nil { + t.Fatal("ListHistory accepted a repeating page token; the cursor would advance past unread history") + } +} diff --git a/internal/cmd/gmail_watch_server_history_pages_test.go b/internal/cmd/gmail_watch_server_history_pages_test.go new file mode 100644 index 000000000..dd19abe5a --- /dev/null +++ b/internal/cmd/gmail_watch_server_history_pages_test.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "google.golang.org/api/gmail/v1" + + "github.com/openclaw/gogcli/internal/gmailwatch" +) + +// A push whose history spans two pages must deliver the messages from both +// pages to the hook and only then store the mailbox's historyId. +func TestGmailWatchServer_DeliversHistoryBeyondFirstPage(t *testing.T) { + setWatchTestConfigHome(t) + store := newGmailWatchTestStore(t, "a@b.com") + _ = store.Update(func(s *gmailWatchState) error { + s.Account = "a@b.com" + s.HistoryID = "100" + return nil + }) + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "": + _, _ = io.WriteString(w, `{"history":[{"id":"150","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`) + case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "p2": + _, _ = io.WriteString(w, `{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`) + case strings.HasPrefix(r.URL.Path, "/gmail/v1/users/me/messages/"): + id := strings.TrimPrefix(r.URL.Path, "/gmail/v1/users/me/messages/") + _, _ = io.WriteString(w, `{"id":"`+id+`","threadId":"t-`+id+`","payload":{"headers":[{"name":"Subject","value":"s"}]}}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(api.Close) + gsvc := newGmailServiceFromServer(t, api) + + var delivered gmailwatch.Payload + hook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&delivered) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(hook.Close) + + server := &gmailWatchServer{ + cfg: gmailWatchServeConfig{ + Account: "a@b.com", + Path: "/hook", + SharedToken: "tok", + HookURL: hook.URL, + HistoryMax: 1, + }, + store: store, + newService: func(context.Context, string) (*gmail.Service, error) { return gsvc, nil }, + hookClient: hook.Client(), + logf: func(string, ...any) {}, + warnf: func(string, ...any) {}, + } + + push := pubsubPushEnvelope{} + push.Message.Data = base64.StdEncoding.EncodeToString([]byte(`{"emailAddress":"a@b.com","historyId":"300"}`)) + body, _ := json.Marshal(push) + rr := httptest.NewRecorder() + server.ServeHTTP(rr, httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/hook?token=tok", bytes.NewReader(body))) + if rr.Code != http.StatusOK { + t.Fatalf("status: %d %s", rr.Code, rr.Body.String()) + } + + ids := make([]string, 0, len(delivered.Messages)) + for _, m := range delivered.Messages { + ids = append(ids, m.ID) + } + if strings.Join(ids, ",") != "m1,m2" { + t.Fatalf("hook got messages %v, want m1,m2", ids) + } + if got := store.Get().HistoryID; got != "300" { + t.Fatalf("stored historyId = %q, want 300", got) + } +} diff --git a/internal/cmd/gmail_watch_source.go b/internal/cmd/gmail_watch_source.go index 1606c122f..9f979ba62 100644 --- a/internal/cmd/gmail_watch_source.go +++ b/internal/cmd/gmail_watch_source.go @@ -35,17 +35,35 @@ func newGmailWatchSource(service *gmail.Service, cfg gmailWatchServeConfig, excl } } +// ListHistory reads every page of history after startID. The returned +// historyId is the mailbox's current one, which is only a safe cursor once no +// nextPageToken remains; storing it after the first page would skip the rest. func (s *gmailWatchSource) ListHistory(ctx context.Context, startID uint64, maxResults int64, historyTypes []string) (gmailwatch.HistoryPage, error) { - call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults) - if len(historyTypes) > 0 { - call.HistoryTypes(historyTypes...) - } - response, err := call.Context(ctx).Do() + historyID := "" + records, err := collectAllPages("", func(pageToken string) ([]gmailwatch.HistoryRecord, string, error) { + call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults) + if len(historyTypes) > 0 { + call.HistoryTypes(historyTypes...) + } + if pageToken != "" { + call.PageToken(pageToken) + } + response, callErr := call.Context(ctx).Do() + if callErr != nil { + return nil, "", callErr + } + page := historyPageFromGmail(response) + historyID = page.HistoryID + if response == nil { + return page.Records, "", nil + } + return page.Records, response.NextPageToken, nil + }) if err != nil { return gmailwatch.HistoryPage{}, err } - return historyPageFromGmail(response), nil + return gmailwatch.HistoryPage{HistoryID: historyID, Records: records}, nil } func (s *gmailWatchSource) ListRecentMessageIDs(ctx context.Context, maxResults int64) ([]string, error) { From e41deea8963619f897cade18ba7f2f5b32d68cca Mon Sep 17 00:00:00 2001 From: Bobby Bones <159389674+saariuslystoned@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:01:04 -0500 Subject: [PATCH 2/2] docs(changelog): reference #1312 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 984b13004..1444032d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.48.1 - Unreleased -- Gmail: make `watch serve` and `watch pull` read every page of `users.history.list` before advancing the saved history cursor. They previously read only the first 100 history records of a notification and then stored the mailbox's current history ID, so changes past the first page (a burst of mail, bulk label edits, or a catch-up after downtime) were never delivered to the hook. — thanks @saariuslystoned. +- Gmail: make `watch serve` and `watch pull` read every page of `users.history.list` before advancing the saved history cursor. They previously read only the first 100 history records of a notification and then stored the mailbox's current history ID, so changes past the first page (a burst of mail, bulk label edits, or a catch-up after downtime) were never delivered to the hook. (#1312) — thanks @saariuslystoned. ## 0.48.0 - 2026-11-05