Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## 0.48.1 - Unreleased

- Gmail: make `watch serve` and `watch pull` read every page of `users.history.list` before advancing the saved history cursor. They previously read only the first 100 history records of a notification and then stored the mailbox's current history ID, so changes past the first page (a burst of mail, bulk label edits, or a catch-up after downtime) were never delivered to the hook. (#1312) — thanks @saariuslystoned.

## 0.48.0 - 2026-11-05

**Highlights:** New backups stay local until you configure a remote; existing configurations keep working. Store service-account keys in the keyring, search Gmail IDs efficiently, and use environment-backed watch tokens, with Calendar fixes and security dependency updates.
Expand Down
3 changes: 3 additions & 0 deletions docs/watch.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ Notes:
- `watch renew` reuses stored topic/labels.
- Push and pull inspect notifications whose history ID equals the saved cursor;
lower IDs remain stale. Duplicate Pub/Sub message IDs are still ignored.
- Push and pull read every page of Gmail history after the saved cursor and
advance the cursor only after the last page, so bursts of more than one page
of changes are delivered instead of skipped.
- `watch stop` calls Gmail stop + clears state.
- `watch serve` and `watch pull` use stored hook config if `--hook-url` is not
provided.
Expand Down
62 changes: 62 additions & 0 deletions internal/cmd/gmail_watch_history_pages_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package cmd

import (
"context"
"net/http"
"net/http/httptest"
"testing"
)

// A history listing that spans two pages: every record must be returned, and
// the stored cursor may only be the historyId of the page without a
// nextPageToken ("If you receive no nextPageToken in the response, there are no
// updates to retrieve and you can store the returned historyId").
func TestGmailWatchSourceListHistoryReadsEveryPage(t *testing.T) {
var calls []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/gmail/v1/users/me/history" {
http.NotFound(w, r)
return
}
token := r.URL.Query().Get("pageToken")
calls = append(calls, token)
w.Header().Set("Content-Type", "application/json")
switch token {
case "":
_, _ = w.Write([]byte(`{"history":[{"id":"101","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`))
case "p2":
_, _ = w.Write([]byte(`{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`))
default:
http.Error(w, "unexpected page token", http.StatusBadRequest)
}
}))
t.Cleanup(srv.Close)

source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)}
page, err := source.ListHistory(context.Background(), 100, 1, nil)
if err != nil {
t.Fatalf("ListHistory: %v", err)
}
if len(calls) != 2 || calls[1] != "p2" {
t.Fatalf("history.list calls = %q, want first page then p2", calls)
}
if len(page.Records) != 2 || page.Records[1].Added[0] != "m2" {
t.Fatalf("records = %#v, want both pages", page.Records)
}
if page.HistoryID != "300" {
t.Fatalf("historyId = %q, want 300", page.HistoryID)
}
}

func TestGmailWatchSourceListHistoryRejectsRepeatedPageToken(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"history":[{"id":"101"}],"nextPageToken":"same","historyId":"300"}`))
}))
t.Cleanup(srv.Close)

source := &gmailWatchSource{service: newGmailServiceFromServer(t, srv)}
if _, err := source.ListHistory(context.Background(), 100, 1, nil); err == nil {
t.Fatal("ListHistory accepted a repeating page token; the cursor would advance past unread history")
}
}
88 changes: 88 additions & 0 deletions internal/cmd/gmail_watch_server_history_pages_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package cmd

import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"

"google.golang.org/api/gmail/v1"

"github.com/openclaw/gogcli/internal/gmailwatch"
)

// A push whose history spans two pages must deliver the messages from both
// pages to the hook and only then store the mailbox's historyId.
func TestGmailWatchServer_DeliversHistoryBeyondFirstPage(t *testing.T) {
setWatchTestConfigHome(t)
store := newGmailWatchTestStore(t, "a@b.com")
_ = store.Update(func(s *gmailWatchState) error {
s.Account = "a@b.com"
s.HistoryID = "100"
return nil
})

api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "":
_, _ = io.WriteString(w, `{"history":[{"id":"150","messagesAdded":[{"message":{"id":"m1"}}]}],"nextPageToken":"p2","historyId":"300"}`)
case r.URL.Path == "/gmail/v1/users/me/history" && r.URL.Query().Get("pageToken") == "p2":
_, _ = io.WriteString(w, `{"history":[{"id":"250","messagesAdded":[{"message":{"id":"m2"}}]}],"historyId":"300"}`)
case strings.HasPrefix(r.URL.Path, "/gmail/v1/users/me/messages/"):
id := strings.TrimPrefix(r.URL.Path, "/gmail/v1/users/me/messages/")
_, _ = io.WriteString(w, `{"id":"`+id+`","threadId":"t-`+id+`","payload":{"headers":[{"name":"Subject","value":"s"}]}}`)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(api.Close)
gsvc := newGmailServiceFromServer(t, api)

var delivered gmailwatch.Payload
hook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewDecoder(r.Body).Decode(&delivered)
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(hook.Close)

server := &gmailWatchServer{
cfg: gmailWatchServeConfig{
Account: "a@b.com",
Path: "/hook",
SharedToken: "tok",
HookURL: hook.URL,
HistoryMax: 1,
},
store: store,
newService: func(context.Context, string) (*gmail.Service, error) { return gsvc, nil },
hookClient: hook.Client(),
logf: func(string, ...any) {},
warnf: func(string, ...any) {},
}

push := pubsubPushEnvelope{}
push.Message.Data = base64.StdEncoding.EncodeToString([]byte(`{"emailAddress":"a@b.com","historyId":"300"}`))
body, _ := json.Marshal(push)
rr := httptest.NewRecorder()
server.ServeHTTP(rr, httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/hook?token=tok", bytes.NewReader(body)))
if rr.Code != http.StatusOK {
t.Fatalf("status: %d %s", rr.Code, rr.Body.String())
}

ids := make([]string, 0, len(delivered.Messages))
for _, m := range delivered.Messages {
ids = append(ids, m.ID)
}
if strings.Join(ids, ",") != "m1,m2" {
t.Fatalf("hook got messages %v, want m1,m2", ids)
}
if got := store.Get().HistoryID; got != "300" {
t.Fatalf("stored historyId = %q, want 300", got)
}
}
30 changes: 24 additions & 6 deletions internal/cmd/gmail_watch_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,35 @@ func newGmailWatchSource(service *gmail.Service, cfg gmailWatchServeConfig, excl
}
}

// ListHistory reads every page of history after startID. The returned
// historyId is the mailbox's current one, which is only a safe cursor once no
// nextPageToken remains; storing it after the first page would skip the rest.
func (s *gmailWatchSource) ListHistory(ctx context.Context, startID uint64, maxResults int64, historyTypes []string) (gmailwatch.HistoryPage, error) {
call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults)
if len(historyTypes) > 0 {
call.HistoryTypes(historyTypes...)
}
response, err := call.Context(ctx).Do()
historyID := ""
records, err := collectAllPages("", func(pageToken string) ([]gmailwatch.HistoryRecord, string, error) {
call := s.service.Users.History.List("me").StartHistoryId(startID).MaxResults(maxResults)
if len(historyTypes) > 0 {
call.HistoryTypes(historyTypes...)
}
if pageToken != "" {
call.PageToken(pageToken)
}
response, callErr := call.Context(ctx).Do()
if callErr != nil {
return nil, "", callErr
}
page := historyPageFromGmail(response)
historyID = page.HistoryID
if response == nil {
return page.Records, "", nil
}
return page.Records, response.NextPageToken, nil
})
if err != nil {
return gmailwatch.HistoryPage{}, err
}

return historyPageFromGmail(response), nil
return gmailwatch.HistoryPage{HistoryID: historyID, Records: records}, nil
}

func (s *gmailWatchSource) ListRecentMessageIDs(ctx context.Context, maxResults int64) ([]string, error) {
Expand Down
Loading