diff --git a/CHANGELOG.md b/CHANGELOG.md index 861521fad..1444032d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.48.1 - Unreleased +- Gmail: make `watch serve` and `watch pull` read every page of `users.history.list` before advancing the saved history cursor. They previously read only the first 100 history records of a notification and then stored the mailbox's current history ID, so changes past the first page (a burst of mail, bulk label edits, or a catch-up after downtime) were never delivered to the hook. (#1312) — thanks @saariuslystoned. + ## 0.48.0 - 2026-11-05 **Highlights:** New backups stay local until you configure a remote; existing configurations keep working. Store service-account keys in the keyring, search Gmail IDs efficiently, and use environment-backed watch tokens, with Calendar fixes and security dependency updates. diff --git a/docs/watch.md b/docs/watch.md index be0d1b3af..92f5fbbe2 100644 --- a/docs/watch.md +++ b/docs/watch.md @@ -105,6 +105,9 @@ Notes: - `watch renew` reuses stored topic/labels. - Push and pull inspect notifications whose history ID equals the saved cursor; lower IDs remain stale. Duplicate Pub/Sub message IDs are still ignored. +- Push and pull read every page of Gmail history after the saved cursor and + advance the cursor only after the last page, so bursts of more than one page + of changes are delivered instead of skipped. - `watch stop` calls Gmail stop + clears state. - `watch serve` and `watch pull` use stored hook config if `--hook-url` is not provided. diff --git a/internal/cmd/gmail_watch_history_pages_test.go b/internal/cmd/gmail_watch_history_pages_test.go new file mode 100644 index 000000000..fd3a27ef0 --- /dev/null +++ b/internal/cmd/gmail_watch_history_pages_test.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +// A history listing that spans two pages: every record must be returned, and +// the stored cursor may only be the historyId of the page without a +// nextPageToken ("If you receive no nextPageToken in the response, there are no +// updates to retrieve and you can store the returned historyId"). +func TestGmailWatchSourceListHistoryReadsEveryPage(t *testing.T) { + var calls []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/gmail/v1/users/me/history" { + http.NotFound(w, r) + return + } + token := r.URL.Query().Get("pageToken") + calls = append(calls, token) + w.Header().Set("Content-Type", "application/json") + switch token { + case "": + _, _ = w.Write([]byte(`{"history":[{"id":"101","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`)) + case "p2": + _, _ = w.Write([]byte(`{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`)) + default: + http.Error(w, "unexpected page token", http.StatusBadRequest) + } + })) + t.Cleanup(srv.Close) + + source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)} + page, err := source.ListHistory(context.Background(), 100, 1, nil) + if err != nil { + t.Fatalf("ListHistory: %v", err) + } + if len(calls) != 2 || calls[1] != "p2" { + t.Fatalf("history.list calls = %q, want first page then p2", calls) + } + if len(page.Records) != 2 || page.Records[1].Added[0] != "m2" { + t.Fatalf("records = %#v, want both pages", page.Records) + } + if page.HistoryID != "300" { + t.Fatalf("historyId = %q, want 300", page.HistoryID) + } +} + +func TestGmailWatchSourceListHistoryRejectsRepeatedPageToken(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"history":[{"id":"101"}],"nextPageToken":"same","historyId":"300"}`)) + })) + t.Cleanup(srv.Close) + + source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)} + if _, err := source.ListHistory(context.Background(), 100, 1, nil); err == nil { + t.Fatal("ListHistory accepted a repeating page token; the cursor would advance past unread history") + } +} diff --git a/internal/cmd/gmail_watch_server_history_pages_test.go b/internal/cmd/gmail_watch_server_history_pages_test.go new file mode 100644 index 000000000..dd19abe5a --- /dev/null +++ b/internal/cmd/gmail_watch_server_history_pages_test.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "google.golang.org/api/gmail/v1" + + "github.com/openclaw/gogcli/internal/gmailwatch" +) + +// A push whose history spans two pages must deliver the messages from both +// pages to the hook and only then store the mailbox's historyId. +func TestGmailWatchServer_DeliversHistoryBeyondFirstPage(t *testing.T) { + setWatchTestConfigHome(t) + store := newGmailWatchTestStore(t, "a@b.com") + _ = store.Update(func(s *gmailWatchState) error { + s.Account = "a@b.com" + s.HistoryID = "100" + return nil + }) + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "": + _, _ = io.WriteString(w, `{"history":[{"id":"150","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`) + case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "p2": + _, _ = io.WriteString(w, `{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`) + case strings.HasPrefix(r.URL.Path, "/gmail/v1/users/me/messages/"): + id := strings.TrimPrefix(r.URL.Path, "/gmail/v1/users/me/messages/") + _, _ = io.WriteString(w, `{"id":"`+id+`","threadId":"t-`+id+`","payload":{"headers":[{"name":"Subject","value":"s"}]}}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(api.Close) + gsvc := newGmailServiceFromServer(t, api) + + var delivered gmailwatch.Payload + hook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&delivered) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(hook.Close) + + server := &gmailWatchServer{ + cfg: gmailWatchServeConfig{ + Account: "a@b.com", + Path: "/hook", + SharedToken: "tok", + HookURL: hook.URL, + HistoryMax: 1, + }, + store: store, + newService: func(context.Context, string) (*gmail.Service, error) { return gsvc, nil }, + hookClient: hook.Client(), + logf: func(string, ...any) {}, + warnf: func(string, ...any) {}, + } + + push := pubsubPushEnvelope{} + push.Message.Data = base64.StdEncoding.EncodeToString([]byte(`{"emailAddress":"a@b.com","historyId":"300"}`)) + body, _ := json.Marshal(push) + rr := httptest.NewRecorder() + server.ServeHTTP(rr, httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/hook?token=tok", bytes.NewReader(body))) + if rr.Code != http.StatusOK { + t.Fatalf("status: %d %s", rr.Code, rr.Body.String()) + } + + ids := make([]string, 0, len(delivered.Messages)) + for _, m := range delivered.Messages { + ids = append(ids, m.ID) + } + if strings.Join(ids, ",") != "m1,m2" { + t.Fatalf("hook got messages %v, want m1,m2", ids) + } + if got := store.Get().HistoryID; got != "300" { + t.Fatalf("stored historyId = %q, want 300", got) + } +} diff --git a/internal/cmd/gmail_watch_source.go b/internal/cmd/gmail_watch_source.go index 1606c122f..9f979ba62 100644 --- a/internal/cmd/gmail_watch_source.go +++ b/internal/cmd/gmail_watch_source.go @@ -35,17 +35,35 @@ func newGmailWatchSource(service *gmail.Service, cfg gmailWatchServeConfig, excl } } +// ListHistory reads every page of history after startID. The returned +// historyId is the mailbox's current one, which is only a safe cursor once no +// nextPageToken remains; storing it after the first page would skip the rest. func (s *gmailWatchSource) ListHistory(ctx context.Context, startID uint64, maxResults int64, historyTypes []string) (gmailwatch.HistoryPage, error) { - call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults) - if len(historyTypes) > 0 { - call.HistoryTypes(historyTypes...) - } - response, err := call.Context(ctx).Do() + historyID := "" + records, err := collectAllPages("", func(pageToken string) ([]gmailwatch.HistoryRecord, string, error) { + call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults) + if len(historyTypes) > 0 { + call.HistoryTypes(historyTypes...) + } + if pageToken != "" { + call.PageToken(pageToken) + } + response, callErr := call.Context(ctx).Do() + if callErr != nil { + return nil, "", callErr + } + page := historyPageFromGmail(response) + historyID = page.HistoryID + if response == nil { + return page.Records, "", nil + } + return page.Records, response.NextPageToken, nil + }) if err != nil { return gmailwatch.HistoryPage{}, err } - return historyPageFromGmail(response), nil + return gmailwatch.HistoryPage{HistoryID: historyID, Records: records}, nil } func (s *gmailWatchSource) ListRecentMessageIDs(ctx context.Context, maxResults int64) ([]string, error) {