Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
993822f6fd | ||
|
|
b0f78d1549 | ||
|
|
f3fecbd0b4 | ||
|
|
30adc69cb5 |
@@ -179,15 +179,27 @@ func (s *Server) handleGmailCallback(w http.ResponseWriter, r *http.Request) {
|
||||
delete(settings, "pkce_state")
|
||||
delete(settings, "pkce_expires")
|
||||
|
||||
settings["refresh_token"] = token.RefreshToken
|
||||
settings["access_token"] = token.AccessToken
|
||||
settings["expires_at"] = token.ExpiresAt.Unix()
|
||||
settings["email"] = email
|
||||
if _, ok := settings["trigger"]; !ok {
|
||||
settings["trigger"] = "starred"
|
||||
if err := s.db.SetUserSettings(r.Context(), user.ID, "gmail", settings); err != nil {
|
||||
s.internal(w, r, "clear gmail handshake", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.db.SetUserSettings(r.Context(), user.ID, "gmail", settings); err != nil {
|
||||
// The mailbox itself, kept beside the ones read over IMAP. Connecting again
|
||||
// writes the same row: Gmail is one per person, because the OAuth flow is.
|
||||
box, err := s.db.MailboxOfKind(r.Context(), user.ID, "gmail")
|
||||
if err != nil {
|
||||
s.internal(w, r, "read gmail mailbox", err)
|
||||
return
|
||||
}
|
||||
if box == nil {
|
||||
box = &store.Mailbox{UserID: user.ID, Kind: "gmail", Trigger: "starred"}
|
||||
}
|
||||
box.Name = email
|
||||
box.Username = email
|
||||
box.RefreshToken = token.RefreshToken
|
||||
box.AccessToken = token.AccessToken
|
||||
box.ExpiresAt = token.ExpiresAt
|
||||
if err := s.db.SaveMailbox(r.Context(), box); err != nil {
|
||||
s.internal(w, r, "save gmail tokens", err)
|
||||
return
|
||||
}
|
||||
@@ -240,36 +252,30 @@ func (s *Server) handleGmailSyncNow(w http.ResponseWriter, r *http.Request) {
|
||||
// Returns how many were made. Also used by the background job, which is why it
|
||||
// takes a context and a user rather than a request.
|
||||
func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
settings, err := s.db.UserSettings(ctx, user.ID, "gmail")
|
||||
box, err := s.db.MailboxOfKind(ctx, user.ID, "gmail")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
str := func(key string) string {
|
||||
v, _ := settings[key].(string)
|
||||
return v
|
||||
}
|
||||
|
||||
refresh := str("refresh_token")
|
||||
if refresh == "" {
|
||||
if box == nil || box.RefreshToken == "" {
|
||||
return 0, nil // not connected; nothing to do and nothing to complain about
|
||||
}
|
||||
query := gmail.Query(str("trigger"), str("label"))
|
||||
query := gmail.Query(box.Trigger, box.Label)
|
||||
if query == "" {
|
||||
return 0, nil // connected but no trigger chosen
|
||||
}
|
||||
refresh := box.RefreshToken
|
||||
|
||||
// The access token is refreshed only when it has actually expired.
|
||||
access := str("access_token")
|
||||
expires, _ := settings["expires_at"].(float64)
|
||||
if access == "" || int64(expires) <= time.Now().Unix() {
|
||||
access := box.AccessToken
|
||||
if access == "" || box.ExpiresAt.Before(time.Now()) {
|
||||
token, err := s.gmailConfig(ctx).Refresh(ctx, refresh)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
access = token.AccessToken
|
||||
settings["access_token"] = access
|
||||
settings["expires_at"] = token.ExpiresAt.Unix()
|
||||
if err := s.db.SetUserSettings(ctx, user.ID, "gmail", settings); err != nil {
|
||||
box.AccessToken = access
|
||||
box.ExpiresAt = token.ExpiresAt
|
||||
if err := s.db.SaveMailbox(ctx, box); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
@@ -281,7 +287,7 @@ func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
// The person revoked access, or changed their password. Forget the
|
||||
// tokens rather than retrying every ten minutes forever.
|
||||
s.log.Info("gmail access was revoked; disconnecting", "user", user.ID)
|
||||
_ = s.db.SetUserSettings(ctx, user.ID, "gmail", map[string]any{})
|
||||
_ = s.db.DeleteMailbox(ctx, user.ID, box.ID)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
@@ -290,19 +296,15 @@ func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
// rather than as a table: it is bounded by the query's own 30-day window, and
|
||||
// a table would be one more thing to migrate for a feature this small.
|
||||
seen := map[string]bool{}
|
||||
if raw, ok := settings["seen"].([]any); ok {
|
||||
for _, v := range raw {
|
||||
if id, ok := v.(string); ok {
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
for _, id := range box.Seen {
|
||||
seen[id] = true
|
||||
}
|
||||
|
||||
inbox, err := s.db.InboxID(ctx, user.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
projectID := str("project_id")
|
||||
projectID := box.ProjectID
|
||||
if projectID == "" {
|
||||
projectID = inbox
|
||||
} else if _, err := store.RequireProjectRole(ctx, s.db, projectID, user.ID, store.RoleEditor); err != nil {
|
||||
@@ -352,15 +354,15 @@ func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
if created > 0 {
|
||||
// Only the ids Gmail still returns are kept, so the list cannot grow
|
||||
// without bound as messages fall out of the 30-day window.
|
||||
keep := make([]any, 0, len(ids))
|
||||
keep := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if seen[id] {
|
||||
keep = append(keep, id)
|
||||
}
|
||||
}
|
||||
settings["seen"] = keep
|
||||
settings["last_sync"] = time.Now().Unix()
|
||||
if err := s.db.SetUserSettings(ctx, user.ID, "gmail", settings); err != nil {
|
||||
box.Seen = keep
|
||||
box.LastSyncAt = time.Now()
|
||||
if err := s.db.SaveMailbox(ctx, box); err != nil {
|
||||
return created, err
|
||||
}
|
||||
}
|
||||
@@ -483,30 +485,25 @@ func (s *Server) handleSetGmailClient(w http.ResponseWriter, r *http.Request) {
|
||||
// natural place to learn the address, and it is also the one moment that cannot be
|
||||
// retried — so the status endpoint asks again when it finds nothing, and the answer
|
||||
// corrects itself on the next look.
|
||||
func (s *Server) gmailProfile(ctx context.Context, user *store.User, settings map[string]any) (string, error) {
|
||||
str := func(key string) string {
|
||||
v, _ := settings[key].(string)
|
||||
return v
|
||||
}
|
||||
|
||||
access := str("access_token")
|
||||
expires, _ := settings["expires_at"].(float64)
|
||||
if access == "" || int64(expires) <= time.Now().Unix() {
|
||||
token, err := s.gmailConfig(ctx).Refresh(ctx, str("refresh_token"))
|
||||
func (s *Server) gmailProfile(ctx context.Context, box *store.Mailbox) (string, error) {
|
||||
access := box.AccessToken
|
||||
if access == "" || box.ExpiresAt.Before(time.Now()) {
|
||||
token, err := s.gmailConfig(ctx).Refresh(ctx, box.RefreshToken)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
access = token.AccessToken
|
||||
settings["access_token"] = access
|
||||
settings["expires_at"] = token.ExpiresAt.Unix()
|
||||
box.AccessToken = access
|
||||
box.ExpiresAt = token.ExpiresAt
|
||||
}
|
||||
|
||||
email, err := s.gmailClient(access).Profile(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
settings["email"] = email
|
||||
if err := s.db.SetUserSettings(ctx, user.ID, "gmail", settings); err != nil {
|
||||
box.Name = email
|
||||
box.Username = email
|
||||
if err := s.db.SaveMailbox(ctx, box); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return email, nil
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"github.com/kristianwind/verdande/internal/store"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -38,8 +39,8 @@ func TestAGmailFailureSaysWhatGmailSaidAndIsRecorded(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Connected, with a refresh token that will be refused.
|
||||
if err := ts.db.SetUserSettings(t.Context(), user.ID, "gmail", map[string]any{
|
||||
"refresh_token": "spent", "trigger": "starred",
|
||||
if err := ts.db.SaveMailbox(t.Context(), &store.Mailbox{
|
||||
UserID: user.ID, Kind: "gmail", RefreshToken: "spent", Trigger: "starred",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -111,9 +112,9 @@ func TestASlowMailboxDoesNotHangTheRequest(t *testing.T) {
|
||||
}
|
||||
// A token that has not expired, so the run goes straight to the message list
|
||||
// rather than stopping at a refresh.
|
||||
if err := ts.db.SetUserSettings(t.Context(), user.ID, "gmail", map[string]any{
|
||||
"refresh_token": "r", "access_token": "a", "trigger": "starred",
|
||||
"expires_at": time.Now().Add(time.Hour).Unix(),
|
||||
if err := ts.db.SaveMailbox(t.Context(), &store.Mailbox{
|
||||
UserID: user.ID, Kind: "gmail", RefreshToken: "r", AccessToken: "a",
|
||||
Trigger: "starred", ExpiresAt: time.Now().Add(time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -370,27 +370,28 @@ type gmailSettings struct {
|
||||
}
|
||||
|
||||
func (s *Server) handleGetGmail(w http.ResponseWriter, r *http.Request) {
|
||||
values, err := s.db.UserSettings(r.Context(), userFrom(r.Context()).ID, "gmail")
|
||||
user := userFrom(r.Context())
|
||||
box, err := s.db.MailboxOfKind(r.Context(), user.ID, "gmail")
|
||||
if err != nil {
|
||||
s.internal(w, r, "gmail settings", err)
|
||||
s.internal(w, r, "gmail mailbox", err)
|
||||
return
|
||||
}
|
||||
str := func(key string) string {
|
||||
v, _ := values[key].(string)
|
||||
return v
|
||||
if box == nil {
|
||||
writeJSON(w, http.StatusOK, gmailSettings{})
|
||||
return
|
||||
}
|
||||
|
||||
// The mailbox address, fetched again if it is missing.
|
||||
//
|
||||
// It is stored at connect time, and the call that fetches it can fail — a
|
||||
// slow token, a scope Google had not applied yet. That failure used to be a
|
||||
// log line and an empty string, so the page said "an unknown account" and went
|
||||
// on saying it forever: the only way back was to disconnect and reconnect,
|
||||
// which nobody would guess. Asking again here costs one request on a page
|
||||
// nobody opens often, and it means the wrong answer lasts until the next look
|
||||
// rather than until somebody reconnects.
|
||||
email := str("email")
|
||||
if email == "" && str("refresh_token") != "" {
|
||||
if fetched, err := s.gmailProfile(r.Context(), userFrom(r.Context()), values); err == nil {
|
||||
// It is stored at connect time, and the call that fetches it can fail — a slow
|
||||
// token, a scope Google had not applied yet. That failure used to be a log line
|
||||
// and an empty string, so the page said "an unknown account" and went on saying
|
||||
// it forever: the only way back was to disconnect and reconnect, which nobody
|
||||
// would guess. Asking again here costs one request on a page nobody opens
|
||||
// often, and it means the wrong answer lasts until the next look.
|
||||
email := box.Username
|
||||
if email == "" && box.RefreshToken != "" {
|
||||
if fetched, err := s.gmailProfile(r.Context(), box); err == nil {
|
||||
email = fetched
|
||||
} else {
|
||||
s.log.Warn("gmail profile", "err", err)
|
||||
@@ -398,10 +399,10 @@ func (s *Server) handleGetGmail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, gmailSettings{
|
||||
Connected: str("refresh_token") != "",
|
||||
Connected: box.RefreshToken != "",
|
||||
Email: email,
|
||||
Trigger: str("trigger"),
|
||||
Label: str("label"),
|
||||
Trigger: box.Trigger,
|
||||
Label: box.Label,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -412,27 +413,42 @@ func (s *Server) handleSetGmail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
user := userFrom(r.Context())
|
||||
|
||||
values, err := s.db.UserSettings(r.Context(), user.ID, "gmail")
|
||||
box, err := s.db.MailboxOfKind(r.Context(), user.ID, "gmail")
|
||||
if err != nil {
|
||||
s.internal(w, r, "gmail settings", err)
|
||||
s.internal(w, r, "gmail mailbox", err)
|
||||
return
|
||||
}
|
||||
values["trigger"] = req.Trigger
|
||||
values["label"] = req.Label
|
||||
if box == nil {
|
||||
// Choosing what makes a task before connecting is allowed, and the choice is
|
||||
// kept: the row is the mailbox with no credential in it yet, which reads as
|
||||
// not connected everywhere that asks and is skipped by the sweep.
|
||||
box = &store.Mailbox{UserID: user.ID, Kind: "gmail"}
|
||||
}
|
||||
box.Trigger = req.Trigger
|
||||
box.Label = req.Label
|
||||
|
||||
if err := s.db.SetUserSettings(r.Context(), user.ID, "gmail", values); err != nil {
|
||||
if err := s.db.SaveMailbox(r.Context(), box); err != nil {
|
||||
s.internal(w, r, "save gmail settings", err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleDisconnectGmail forgets the connection, tokens and all.
|
||||
// handleDisconnectGmail forgets the connection, tokens and all. The tasks it made
|
||||
// stay: they are the person's work now, not the mailbox's.
|
||||
func (s *Server) handleDisconnectGmail(w http.ResponseWriter, r *http.Request) {
|
||||
if err := s.db.SetUserSettings(r.Context(), userFrom(r.Context()).ID, "gmail", map[string]any{}); err != nil {
|
||||
s.internal(w, r, "disconnect gmail", err)
|
||||
user := userFrom(r.Context())
|
||||
box, err := s.db.MailboxOfKind(r.Context(), user.ID, "gmail")
|
||||
if err != nil {
|
||||
s.internal(w, r, "gmail mailbox", err)
|
||||
return
|
||||
}
|
||||
if box != nil {
|
||||
if err := s.db.DeleteMailbox(r.Context(), user.ID, box.ID); err != nil {
|
||||
s.internal(w, r, "disconnect gmail", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
|
||||
@@ -147,7 +147,7 @@ func (s *Server) SyncMailbox(ctx context.Context, user *store.User, m *store.Mai
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
messages, err := client.Since(m.LastUID, 25)
|
||||
messages, searched, err := client.Since(m.LastUID, 25)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -197,6 +197,14 @@ func (s *Server) SyncMailbox(ctx context.Context, user *store.User, m *store.Mai
|
||||
s.hub.Publish(projectID, "task.created", toTaskJSON(*task))
|
||||
}
|
||||
|
||||
// A run that got all the way through moves the marker to the whole of what the
|
||||
// search considered, not merely to the last message it happened to read. That
|
||||
// is what stops the same mail becoming a task again on the next run when the
|
||||
// server did not put a uid in the fetch reply.
|
||||
if ctx.Err() == nil && searched > highest {
|
||||
highest = searched
|
||||
}
|
||||
|
||||
// Written even when the loop was cut short, and outside the request's context
|
||||
// so a cancelled request still records what it did.
|
||||
if highest > m.LastUID {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -72,13 +73,26 @@ func (s *Server) handleRestartFromPanel(w http.ResponseWriter, r *http.Request)
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
url := fmt.Sprintf("%s/api/servers/%s/restart", s.cfg.PanelURL, s.cfg.PanelServerID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
||||
// safe-restart rather than restart, and the difference is not politeness.
|
||||
//
|
||||
// The plain endpoint does its work while the request is open: it stops the
|
||||
// container — which kills this process — and the panel, seeing the caller
|
||||
// vanish, never gets to the starting half. That took production down and left
|
||||
// it down for four minutes on 18 August, and it cannot be fixed from this side:
|
||||
// any button that synchronously asks for its own death has the same shape.
|
||||
//
|
||||
// safe-restart is scheduled. The panel answers "Restart scheduled" and does the
|
||||
// work on its own afterwards, so nothing depends on this process still being
|
||||
// alive to hear the reply.
|
||||
url := fmt.Sprintf("%s/api/servers/%s/safe-restart", s.cfg.PanelURL, s.cfg.PanelServerID)
|
||||
body := strings.NewReader(`{"backup_first":false,"target_id":""}`)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
|
||||
if err != nil {
|
||||
s.internal(w, r, "build panel request", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+s.cfg.PanelToken)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -15,6 +16,11 @@ import (
|
||||
// request has to end either way, or whatever sits in front answers for it.
|
||||
func TestAPanelThatNeverAnswersStillEndsTheRequest(t *testing.T) {
|
||||
panel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Drained first. A real server reads what it was sent, and one that does not
|
||||
// never learns the caller has hung up — httptest.Server.Close then waits for
|
||||
// a handler that will never return, and the whole binary times out instead
|
||||
// of the one request.
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer panel.Close()
|
||||
@@ -56,6 +62,7 @@ func TestAPanelThatNeverAnswersStillEndsTheRequest(t *testing.T) {
|
||||
// And a panel that says no must say so in this server's own words, not in a proxy's.
|
||||
func TestAPanelThatRefusesSaysSoAsJSON(t *testing.T) {
|
||||
panel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}))
|
||||
defer panel.Close()
|
||||
|
||||
+120
-22
@@ -16,15 +16,18 @@
|
||||
package imap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-imap/v2"
|
||||
"github.com/emersion/go-imap/v2/imapclient"
|
||||
"github.com/emersion/go-message"
|
||||
)
|
||||
|
||||
// Message is one mail, reduced to what a task needs.
|
||||
@@ -108,9 +111,13 @@ func (c *Client) Close() error {
|
||||
// that consumed it would be taking something away. Flagging a mail is a deliberate
|
||||
// act with no other meaning, which is exactly what is wanted — the same reason the
|
||||
// Gmail side reads stars.
|
||||
func (c *Client) Since(uid uint32, limit int) ([]Message, error) {
|
||||
// The second return is the highest uid the search considered. A caller that got
|
||||
// through every message can move its marker to that, and then it does not matter
|
||||
// whether the server put a uid in each fetch reply — which some do not, and which
|
||||
// is how the same mail became a task again on every run.
|
||||
func (c *Client) Since(uid uint32, limit int) ([]Message, uint32, error) {
|
||||
if _, err := c.c.Select(c.folder, &imap.SelectOptions{ReadOnly: true}).Wait(); err != nil {
|
||||
return nil, fmt.Errorf("open %s: %w", c.folder, err)
|
||||
return nil, 0, fmt.Errorf("open %s: %w", c.folder, err)
|
||||
}
|
||||
|
||||
// From uid+1 to the end. A mailbox with nothing new answers with no uids
|
||||
@@ -124,12 +131,12 @@ func (c *Client) Since(uid uint32, limit int) ([]Message, error) {
|
||||
}
|
||||
found, err := c.c.UIDSearch(criteria, nil).Wait()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search %s: %w", c.folder, err)
|
||||
return nil, 0, fmt.Errorf("search %s: %w", c.folder, err)
|
||||
}
|
||||
|
||||
uids := found.AllUIDs()
|
||||
if len(uids) == 0 {
|
||||
return nil, nil
|
||||
return nil, uid, nil
|
||||
}
|
||||
if limit > 0 && len(uids) > limit {
|
||||
// The newest ones. A mailbox flagged over years would otherwise make the
|
||||
@@ -139,29 +146,51 @@ func (c *Client) Since(uid uint32, limit int) ([]Message, error) {
|
||||
|
||||
want := imap.UIDSetNum(uids...)
|
||||
fetch := &imap.FetchOptions{
|
||||
Envelope: true,
|
||||
UID: true,
|
||||
BodySection: []*imap.FetchItemBodySection{{Specifier: imap.PartSpecifierText, Peek: true}},
|
||||
Envelope: true,
|
||||
UID: true,
|
||||
// The whole message, headers included, so the MIME structure can be walked.
|
||||
// TEXT alone gives the body as bytes — boundaries and part headers and all —
|
||||
// which is what put "--=_be71cb39… Content-Transfer-Encoding: 8bit" at the
|
||||
// front of every task made from a multipart mail.
|
||||
BodySection: []*imap.FetchItemBodySection{{Peek: true}},
|
||||
}
|
||||
var highest uint32
|
||||
for _, u := range uids {
|
||||
if uint32(u) > highest {
|
||||
highest = uint32(u)
|
||||
}
|
||||
}
|
||||
|
||||
msgs, err := c.c.Fetch(want, fetch).Collect()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch from %s: %w", c.folder, err)
|
||||
return nil, 0, fmt.Errorf("fetch from %s: %w", c.folder, err)
|
||||
}
|
||||
|
||||
out := make([]Message, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
for i, m := range msgs {
|
||||
if m.Envelope == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// The uid comes from the search when the fetch reply does not carry one.
|
||||
// A missing uid is not cosmetic: it is the marker the next run starts from,
|
||||
// and a marker stuck at zero makes every message arrive again on every run.
|
||||
// That is what happened against a real server while the tests stayed green,
|
||||
// because the server they run against always answers with it.
|
||||
msgUID := uint32(m.UID)
|
||||
if msgUID == 0 && i < len(uids) {
|
||||
msgUID = uint32(uids[i])
|
||||
}
|
||||
|
||||
out = append(out, Message{
|
||||
UID: uint32(m.UID),
|
||||
UID: msgUID,
|
||||
From: address(m.Envelope.From),
|
||||
Subject: strings.TrimSpace(m.Envelope.Subject),
|
||||
Snippet: snippet(m),
|
||||
Date: m.Envelope.Date,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
return out, highest, nil
|
||||
}
|
||||
|
||||
// address renders the first sender as "Name <addr>", or just the address.
|
||||
@@ -177,23 +206,92 @@ func address(list []imap.Address) string {
|
||||
return a.Name + " <" + addr + ">"
|
||||
}
|
||||
|
||||
// snippet is the first readable line or so of the body.
|
||||
// snippet is the first readable text in the mail.
|
||||
//
|
||||
// No MIME walk and no HTML stripping yet: this is what shows under a task's title,
|
||||
// and a wrong-but-short line is a smaller problem than a parser that has to be
|
||||
// right about every mail ever sent. The full mail stays where it is.
|
||||
// It walks the MIME structure rather than taking the body as bytes. text/plain is
|
||||
// preferred and HTML is the fallback with its tags stripped — crudely, because
|
||||
// this is the line under a task's title and not a rendering engine. The mail
|
||||
// itself stays where it is.
|
||||
const snippetLimit = 400
|
||||
|
||||
func snippet(m *imapclient.FetchMessageBuffer) string {
|
||||
for _, body := range m.BodySection {
|
||||
text := strings.TrimSpace(string(body.Bytes))
|
||||
if text == "" {
|
||||
continue
|
||||
if text := readable(body.Bytes); text != "" {
|
||||
return cut(text)
|
||||
}
|
||||
if len(text) > snippetLimit {
|
||||
text = text[:snippetLimit] + "…"
|
||||
}
|
||||
return text
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func readable(raw []byte) string {
|
||||
entity, err := message.Read(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
// Not something this parser recognises. The raw bytes read badly but read,
|
||||
// and a mail is not worth losing over its own headers.
|
||||
return strings.TrimSpace(string(raw))
|
||||
}
|
||||
return strings.TrimSpace(fromEntity(entity, 0))
|
||||
}
|
||||
|
||||
// fromEntity descends into a multipart mail. The depth guard is for the mail that
|
||||
// nests parts inside parts inside parts — rare, and cheaper to bound than to trust.
|
||||
func fromEntity(entity *message.Entity, depth int) string {
|
||||
if depth > 4 {
|
||||
return ""
|
||||
}
|
||||
|
||||
if parts := entity.MultipartReader(); parts != nil {
|
||||
var html string
|
||||
for {
|
||||
part, err := parts.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if text := fromEntity(part, depth+1); text != "" {
|
||||
kind, _, _ := part.Header.ContentType()
|
||||
if kind != "text/html" {
|
||||
return text
|
||||
}
|
||||
if html == "" {
|
||||
html = text
|
||||
}
|
||||
}
|
||||
}
|
||||
return html
|
||||
}
|
||||
|
||||
body, _ := io.ReadAll(entity.Body)
|
||||
kind, _, _ := entity.Header.ContentType()
|
||||
if kind == "text/html" {
|
||||
return stripTags(string(body))
|
||||
}
|
||||
return strings.TrimSpace(string(body))
|
||||
}
|
||||
|
||||
// stripTags removes markup without pretending to understand it. Enough for one
|
||||
// line of preview, and honest about being no more than that.
|
||||
func stripTags(html string) string {
|
||||
var b strings.Builder
|
||||
depth := 0
|
||||
for _, r := range html {
|
||||
switch {
|
||||
case r == '<':
|
||||
depth++
|
||||
case r == '>':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case depth == 0:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.Join(strings.Fields(b.String()), " ")
|
||||
}
|
||||
|
||||
func cut(text string) string {
|
||||
runes := []rune(text)
|
||||
if len(runes) > snippetLimit {
|
||||
return string(runes[:snippetLimit]) + "…"
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestFlaggedMailComesBackAsMessages(t *testing.T) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
msgs, err := client.Since(0, 25)
|
||||
msgs, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func TestFlaggedMailComesBackAsMessages(t *testing.T) {
|
||||
|
||||
// And reading on from a marker returns only what is newer, which is the whole
|
||||
// of the deduplication.
|
||||
rest, err := client.Since(msgs[0].UID, 25)
|
||||
rest, _, err := client.Since(msgs[0].UID, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -92,6 +92,9 @@ func TestAnUntrustedCertificateIsRefused(t *testing.T) {
|
||||
type mail struct {
|
||||
subject, from, body string
|
||||
flagged bool
|
||||
// raw replaces the whole message when set, which is the only way to get a
|
||||
// genuinely multipart one in.
|
||||
raw string
|
||||
}
|
||||
|
||||
// serveMailbox starts an in-process IMAP server over TLS and returns its address
|
||||
@@ -112,8 +115,11 @@ func serveMailbox(t *testing.T, mails []mail) (string, *x509.CertPool) {
|
||||
if m.flagged {
|
||||
flags = append(flags, imap.FlagFlagged)
|
||||
}
|
||||
raw := "From: " + m.from + "\r\nSubject: " + m.subject +
|
||||
"\r\nDate: Tue, 18 Aug 2026 10:00:00 +0200\r\n\r\n" + m.body + "\r\n"
|
||||
raw := m.raw
|
||||
if raw == "" {
|
||||
raw = "From: " + m.from + "\r\nSubject: " + m.subject +
|
||||
"\r\nDate: Tue, 18 Aug 2026 10:00:00 +0200\r\n\r\n" + m.body + "\r\n"
|
||||
}
|
||||
if _, err := user.Append("INBOX", strings.NewReader(raw), &imap.AppendOptions{
|
||||
Flags: flags,
|
||||
}); err != nil {
|
||||
@@ -181,3 +187,78 @@ func selfSigned(t *testing.T) (tls.Certificate, *x509.CertPool) {
|
||||
pool.AppendCertsFromPEM(certPEM)
|
||||
return cert, pool
|
||||
}
|
||||
|
||||
// The bug this was written for: every press of "Fetch now" made the same mail
|
||||
// into a task again. The marker the next run starts from is the uid, and if it
|
||||
// comes back as zero the marker never moves.
|
||||
func TestEveryMessageCarriesAUID(t *testing.T) {
|
||||
addr, pool := serveMailbox(t, []mail{
|
||||
{subject: "En", from: "[email protected]", body: "en", flagged: true},
|
||||
{subject: "To", from: "[email protected]", body: "to", flagged: true},
|
||||
})
|
||||
client, err := Dial(Account{Host: addr, Username: "kw", Password: "hemmelig", RootCAs: pool})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
msgs, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range msgs {
|
||||
if m.UID == 0 {
|
||||
t.Fatalf("%q came back without a uid; the marker would never move and "+
|
||||
"every run would make it into a task again", m.Subject)
|
||||
}
|
||||
}
|
||||
if msgs[0].UID == msgs[1].UID {
|
||||
t.Error("two messages share a uid")
|
||||
}
|
||||
}
|
||||
|
||||
// The other half of the same report: the snippet under a task's title read
|
||||
// "--=_be71cb39… Content-Transfer-Encoding: 8bit Content-Type: text/plain".
|
||||
func TestAMultipartMailReadsAsItsText(t *testing.T) {
|
||||
const boundary = "=_be71cb3975bdfbc60a16327cb2f49c23"
|
||||
body := "This is a multi-part message.\r\n" +
|
||||
"--" + boundary + "\r\n" +
|
||||
"Content-Transfer-Encoding: 8bit\r\n" +
|
||||
"Content-Type: text/plain; charset=UTF-8; format=flowed\r\n\r\n" +
|
||||
"Hej igen,\r\n\r\nVi har normalt 3 slags. :)\r\n" +
|
||||
"--" + boundary + "\r\n" +
|
||||
"Content-Type: text/html; charset=UTF-8\r\n\r\n" +
|
||||
"<html><body><p>Hej igen</p></body></html>\r\n" +
|
||||
"--" + boundary + "--\r\n"
|
||||
|
||||
addr, pool := serveMailbox(t, []mail{{
|
||||
flagged: true,
|
||||
raw: "Content-Type: multipart/alternative; boundary=\"" + boundary + "\"\r\n" +
|
||||
"From: [email protected]\r\nSubject: Re: Kaffebestilling\r\n" +
|
||||
"Date: Tue, 18 Aug 2026 10:00:00 +0200\r\n\r\n" + body,
|
||||
}})
|
||||
|
||||
client, err := Dial(Account{Host: addr, Username: "kw", Password: "hemmelig", RootCAs: pool})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
msgs, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("got %d messages", len(msgs))
|
||||
}
|
||||
|
||||
got := msgs[0].Snippet
|
||||
for _, junk := range []string{boundary, "Content-Transfer-Encoding", "Content-Type", "<html>", "<p>"} {
|
||||
if strings.Contains(got, junk) {
|
||||
t.Errorf("the snippet still carries %q: %q", junk, got)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(got, "Hej igen") {
|
||||
t.Errorf("the snippet does not start with the text of the mail: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
+22
-34
@@ -64,7 +64,6 @@ func (r *Runner) Start(ctx context.Context) {
|
||||
// need a public webhook and a Cloud Pub/Sub topic, which is a lot of Google
|
||||
// account for a to-do app. Ten minutes is fast enough for something somebody
|
||||
// starred and slow enough not to look like abuse.
|
||||
r.every(ctx, "gmail", 10*time.Minute, r.syncGmailAccounts)
|
||||
r.every(ctx, "mailboxes", 10*time.Minute, r.syncMailboxes)
|
||||
}
|
||||
|
||||
@@ -252,37 +251,13 @@ func rotateBackups(cfg *config.Config, log *slog.Logger) error {
|
||||
|
||||
// --- gmail ---------------------------------------------------------------------------
|
||||
|
||||
// syncGmailAccounts polls every connected mailbox.
|
||||
// syncMailboxes polls every connected mailbox, Gmail and IMAP alike.
|
||||
//
|
||||
// One at a time, and a failure on one account does not stop the others: a revoked
|
||||
// token or an expired refresh is a problem for that person alone.
|
||||
func (r *Runner) syncGmailAccounts(ctx context.Context) error {
|
||||
if r.SyncGmail == nil {
|
||||
return nil
|
||||
}
|
||||
users, err := r.db.UsersWithGmail(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, user := range users {
|
||||
created, err := r.SyncGmail(ctx, &user)
|
||||
if err != nil {
|
||||
r.log.Warn("gmail sync", "err", err, "user", user.ID)
|
||||
continue
|
||||
}
|
||||
if created > 0 {
|
||||
r.log.Info("tasks created from gmail", "user", user.ID, "count", created)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncMailboxes polls every connected IMAP mailbox.
|
||||
//
|
||||
// Nobody is waiting, so no budget of its own — the request handler has one because
|
||||
// a person is watching a spinner. A host that has gone quiet costs this run its
|
||||
// own dial timeout and nothing else: the loop moves on, per mailbox and per
|
||||
// person, so one bad password does not stop everybody else's mail.
|
||||
// One sweep rather than two, now that both live in the same table. Nobody is
|
||||
// waiting, so no budget of its own — the request handler has one because a person
|
||||
// is watching a spinner. A host that has gone quiet costs this run its own dial
|
||||
// timeout and nothing else: the loop moves on, per mailbox and per person, so one
|
||||
// bad password does not stop everybody else's mail.
|
||||
func (r *Runner) syncMailboxes(ctx context.Context) error {
|
||||
if r.SyncMailbox == nil {
|
||||
return nil
|
||||
@@ -302,13 +277,26 @@ func (r *Runner) syncMailboxes(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
for i := range boxes {
|
||||
created, err := r.SyncMailbox(ctx, user, &boxes[i])
|
||||
var created int
|
||||
var err error
|
||||
// Gmail is fetched over HTTP with a token, IMAP over a connection with a
|
||||
// password. Which one is a property of the row, not of the loop.
|
||||
if boxes[i].Kind == "gmail" {
|
||||
if r.SyncGmail == nil {
|
||||
continue
|
||||
}
|
||||
created, err = r.SyncGmail(ctx, user)
|
||||
} else {
|
||||
created, err = r.SyncMailbox(ctx, user, &boxes[i])
|
||||
}
|
||||
if err != nil {
|
||||
r.log.Warn("mailbox sync", "err", err, "user", id, "mailbox", boxes[i].ID)
|
||||
r.log.Warn("mailbox sync", "err", err, "user", id,
|
||||
"kind", boxes[i].Kind, "mailbox", boxes[i].ID)
|
||||
continue
|
||||
}
|
||||
if created > 0 {
|
||||
r.log.Info("tasks created from a mailbox", "user", id, "count", created)
|
||||
r.log.Info("tasks created from a mailbox", "user", id,
|
||||
"kind", boxes[i].Kind, "count", created)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
@@ -24,13 +25,24 @@ type Mailbox struct {
|
||||
ExpiresAt time.Time `json:"-"`
|
||||
Label string `json:"label,omitempty"`
|
||||
|
||||
// ProjectID is where this mailbox's tasks land. Empty means the inbox, and so
|
||||
// does an id the person can no longer write to.
|
||||
ProjectID string `json:"project_id,omitempty"`
|
||||
Trigger string `json:"trigger,omitempty"` // "starred" or "label"
|
||||
|
||||
// Seen is the message ids Gmail has already turned into tasks. IMAP does not
|
||||
// need it — uids are monotonic, so LastUID says the same in one number — but
|
||||
// Gmail's query is a rolling window rather than a marker.
|
||||
Seen []string `json:"-"`
|
||||
|
||||
LastUID uint32 `json:"last_uid,omitempty"`
|
||||
LastSyncAt time.Time `json:"last_sync_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
const mailboxColumns = `id, user_id, kind, name, host, username, password, folder,
|
||||
refresh_token, access_token, expires_at, label, last_uid, last_sync_at, created_at`
|
||||
refresh_token, access_token, expires_at, label, trigger_kind, seen, project_id,
|
||||
last_uid, last_sync_at, created_at`
|
||||
|
||||
// Mailboxes returns everything one person has connected, oldest first.
|
||||
func (db *DB) Mailboxes(ctx context.Context, userID string) ([]Mailbox, error) {
|
||||
@@ -67,6 +79,21 @@ func (db *DB) Mailbox(ctx context.Context, userID, id string) (*Mailbox, error)
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// MailboxOfKind returns a person's mailbox of one kind, or nil. Gmail is one per
|
||||
// person because the OAuth flow is: connecting again replaces what was there.
|
||||
func (db *DB) MailboxOfKind(ctx context.Context, userID, kind string) (*Mailbox, error) {
|
||||
row := db.QueryRowContext(ctx,
|
||||
`SELECT `+mailboxColumns+` FROM mailboxes WHERE user_id = ? AND kind = ?`, userID, kind)
|
||||
m, err := db.scanMailbox(row)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// SaveMailbox writes one, sealing the two columns that must not travel with a
|
||||
// backup. Called by hand rather than by the settings tables' own seal, which does
|
||||
// not reach this table.
|
||||
@@ -93,10 +120,19 @@ func (db *DB) SaveMailbox(ctx context.Context, m *Mailbox) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("seal access_token: %w", err)
|
||||
}
|
||||
// Not sealed: which mails have been read is not a secret, and a column of
|
||||
// ciphertext here would only make the table harder to look at.
|
||||
seen, err := json.Marshal(m.Seen)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Trigger == "" {
|
||||
m.Trigger = "starred"
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, `
|
||||
INSERT INTO mailboxes (`+mailboxColumns+`)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
host = excluded.host,
|
||||
@@ -107,10 +143,13 @@ func (db *DB) SaveMailbox(ctx context.Context, m *Mailbox) error {
|
||||
access_token = excluded.access_token,
|
||||
expires_at = excluded.expires_at,
|
||||
label = excluded.label,
|
||||
trigger_kind = excluded.trigger_kind,
|
||||
seen = excluded.seen,
|
||||
project_id = excluded.project_id,
|
||||
last_uid = excluded.last_uid,
|
||||
last_sync_at = excluded.last_sync_at`,
|
||||
m.ID, m.UserID, m.Kind, m.Name, m.Host, m.Username, password, m.Folder,
|
||||
refresh, access, unixOrZero(m.ExpiresAt), m.Label,
|
||||
refresh, access, unixOrZero(m.ExpiresAt), m.Label, m.Trigger, string(seen), m.ProjectID,
|
||||
m.LastUID, unixOrZero(m.LastSyncAt), m.CreatedAt.Unix())
|
||||
return err
|
||||
}
|
||||
@@ -157,12 +196,16 @@ type scanner interface {
|
||||
func (db *DB) scanMailbox(row scanner) (Mailbox, error) {
|
||||
var m Mailbox
|
||||
var expires, lastSync, created int64
|
||||
var seen string
|
||||
err := row.Scan(&m.ID, &m.UserID, &m.Kind, &m.Name, &m.Host, &m.Username,
|
||||
&m.Password, &m.Folder, &m.RefreshToken, &m.AccessToken, &expires,
|
||||
&m.Label, &m.LastUID, &lastSync, &created)
|
||||
&m.Label, &m.Trigger, &seen, &m.ProjectID, &m.LastUID, &lastSync, &created)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
// A row that somehow holds nonsense here is not worth failing a whole sync
|
||||
// over: an empty list means everything looks new, which is recoverable.
|
||||
_ = json.Unmarshal([]byte(seen), &m.Seen)
|
||||
|
||||
for _, field := range []*string{&m.Password, &m.RefreshToken, &m.AccessToken} {
|
||||
plain, err := db.unsealValue(*field)
|
||||
|
||||
@@ -2,9 +2,12 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kristianwind/verdande/internal/secret"
|
||||
)
|
||||
|
||||
func TestAMailboxIsStoredSealedAndComesBackWhole(t *testing.T) {
|
||||
@@ -171,3 +174,91 @@ func TestDisconnectingKeepsTheTasks(t *testing.T) {
|
||||
t.Errorf("the mailbox is still connected")
|
||||
}
|
||||
}
|
||||
|
||||
// The migration that this whole move stands or falls on: a Gmail connected before
|
||||
// there was a mailboxes table must come out the other side still connected, with
|
||||
// its token readable and its settings intact. Nobody should have to reconnect
|
||||
// because the server tidied up.
|
||||
func TestAGmailConnectedBeforeTheMoveSurvivesIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
box, err := secret.Open("", t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A database at the state before 0017, with a sealed token written the way the
|
||||
// settings table wrote them.
|
||||
db, userID := storeAt(t, dir, box)
|
||||
sealed, err := box.Seal("1//0-the-original-refresh-token")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values := `{"refresh_token":"` + sealed + `","trigger":"label","label":"Verdande",` +
|
||||
`"email":"[email protected]","seen":["abc","def"],"project_id":"p-1"}`
|
||||
if _, err := db.ExecContext(context.Background(), `
|
||||
INSERT INTO user_settings (user_id, scope, values_json, updated_at)
|
||||
VALUES (?, 'gmail', ?, 0)
|
||||
ON CONFLICT (user_id) DO UPDATE SET scope = 'gmail', values_json = excluded.values_json`,
|
||||
userID, values); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 0017 has already run on this database, so the row is moved by hand here in
|
||||
// the same statement the migration uses. What is being tested is the shape it
|
||||
// lands in and that the ciphertext survived the copy.
|
||||
if _, err := db.ExecContext(context.Background(), `
|
||||
INSERT INTO mailboxes (id, user_id, kind, name, host, username, password, folder,
|
||||
refresh_token, access_token, expires_at, label, trigger_kind, seen, project_id,
|
||||
last_uid, last_sync_at, created_at)
|
||||
SELECT lower(hex(randomblob(16))), s.user_id, 'gmail',
|
||||
json_extract(s.values_json, '$.email'), '', json_extract(s.values_json, '$.email'),
|
||||
'', 'INBOX', json_extract(s.values_json, '$.refresh_token'), '', 0,
|
||||
json_extract(s.values_json, '$.label'), json_extract(s.values_json, '$.trigger'),
|
||||
json_extract(s.values_json, '$.seen'), json_extract(s.values_json, '$.project_id'),
|
||||
0, 0, unixepoch()
|
||||
FROM user_settings s WHERE s.scope = 'gmail'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
moved, err := db.MailboxOfKind(context.Background(), userID, "gmail")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if moved == nil {
|
||||
t.Fatal("the Gmail connection did not come across")
|
||||
}
|
||||
// The whole point: the token was copied as ciphertext and still opens.
|
||||
if moved.RefreshToken != "1//0-the-original-refresh-token" {
|
||||
t.Errorf("the token came out as %q", moved.RefreshToken)
|
||||
}
|
||||
if moved.Trigger != "label" || moved.Label != "Verdande" {
|
||||
t.Errorf("the trigger came out as %q/%q", moved.Trigger, moved.Label)
|
||||
}
|
||||
if moved.Username != "[email protected]" {
|
||||
t.Errorf("the address came out as %q", moved.Username)
|
||||
}
|
||||
if moved.ProjectID != "p-1" {
|
||||
t.Errorf("the destination project came out as %q", moved.ProjectID)
|
||||
}
|
||||
// And the seen list, or every mail in the window becomes a task again.
|
||||
if len(moved.Seen) != 2 || moved.Seen[0] != "abc" {
|
||||
t.Errorf("the seen list came out as %v", moved.Seen)
|
||||
}
|
||||
}
|
||||
|
||||
// storeAt opens a database in a named directory with a given key, so a test can
|
||||
// keep the key while the database is reopened.
|
||||
func storeAt(t *testing.T, dir string, box *secret.Box) (*DB, string) {
|
||||
t.Helper()
|
||||
db, err := Open(filepath.Join(dir, "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
db.UseSecrets(box)
|
||||
|
||||
user := &User{Email: "[email protected]", Name: "Kristian"}
|
||||
if err := db.CreateUser(context.Background(), user, "Indbakke"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, user.ID
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Room for Gmail beside the others.
|
||||
--
|
||||
-- Only the two columns. Moving the data is a separate migration on purpose: the
|
||||
-- Gmail handlers still read user_settings, and a migration that emptied that table
|
||||
-- before they were rewritten would disconnect a working mailbox. Additive first,
|
||||
-- and the move when the code on the other side is ready for it.
|
||||
--
|
||||
-- Gmail lived in `user_settings` under scope 'gmail' because it was the only
|
||||
-- integration of its kind. Now there are two, and two places to look for the same
|
||||
-- question — which mailboxes does this person have — is one place too many.
|
||||
--
|
||||
-- `seen` is Gmail's own deduplication: a list of message ids it has already turned
|
||||
-- into tasks. IMAP does not need it, because uids are monotonic and `last_uid`
|
||||
-- answers the same question in one number. Gmail's query is a 30-day window rather
|
||||
-- than a marker, so the list is what it has.
|
||||
ALTER TABLE mailboxes ADD COLUMN seen TEXT NOT NULL DEFAULT '[]';
|
||||
|
||||
-- What makes a mail into a task: 'starred', or 'label' with the name in `label`.
|
||||
-- Its own column rather than packed into `label` with a separator — a column that
|
||||
-- holds two things joined by a colon is a column somebody parses wrongly later,
|
||||
-- and a label named with a colon in it would do it on the first day.
|
||||
ALTER TABLE mailboxes ADD COLUMN trigger_kind TEXT NOT NULL DEFAULT 'starred';
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Where a mailbox's tasks land.
|
||||
--
|
||||
-- Gmail carried this as `project_id` in its settings blob. It belongs to the
|
||||
-- mailbox, and it is worth having for the ones read over IMAP too: a work mailbox
|
||||
-- filing into the work project is the obvious thing to want, and the alternative
|
||||
-- is everything landing in the inbox to be moved by hand.
|
||||
--
|
||||
-- Empty means the person's inbox, which is also what an id that no longer names a
|
||||
-- project they can write to falls back to. Not a foreign key for that reason: a
|
||||
-- deleted project should quietly stop being the destination, not refuse the delete
|
||||
-- or take the mailbox down with it.
|
||||
ALTER TABLE mailboxes ADD COLUMN project_id TEXT NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1,46 @@
|
||||
-- Gmail's rows move in beside the ones read over IMAP.
|
||||
--
|
||||
-- Held back until now on purpose: this empties the settings row, and running it
|
||||
-- while the handlers still read that row would have disconnected a working
|
||||
-- mailbox. internal/httpapi now reads the mailboxes table, so the move is safe.
|
||||
--
|
||||
-- The values move across still sealed. Both tables seal with the same key and
|
||||
-- store the same string, so copying the ciphertext verbatim is not a shortcut
|
||||
-- around the encryption — it is the encryption, untouched. That also means this
|
||||
-- migration does not need the key, which a migration has no business holding.
|
||||
--
|
||||
-- `username` is the address the account reports, and it is what makes the unique
|
||||
-- index meaningful. It may be empty on rows that never got that far; two such rows
|
||||
-- cannot exist for one person anyway, because the settings table could only hold
|
||||
-- one.
|
||||
INSERT INTO mailboxes (
|
||||
id, user_id, kind, name, host, username, password, folder,
|
||||
refresh_token, access_token, expires_at, label, trigger_kind, seen, project_id,
|
||||
last_uid, last_sync_at, created_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(16))),
|
||||
s.user_id,
|
||||
'gmail',
|
||||
COALESCE(json_extract(s.values_json, '$.email'), 'Gmail'),
|
||||
'',
|
||||
COALESCE(json_extract(s.values_json, '$.email'), ''),
|
||||
'',
|
||||
'INBOX',
|
||||
COALESCE(json_extract(s.values_json, '$.refresh_token'), ''),
|
||||
COALESCE(json_extract(s.values_json, '$.access_token'), ''),
|
||||
COALESCE(json_extract(s.values_json, '$.expires_at'), 0),
|
||||
COALESCE(json_extract(s.values_json, '$.label'), ''),
|
||||
COALESCE(json_extract(s.values_json, '$.trigger'), 'starred'),
|
||||
COALESCE(json_extract(s.values_json, '$.seen'), '[]'),
|
||||
COALESCE(json_extract(s.values_json, '$.project_id'), ''),
|
||||
0,
|
||||
0,
|
||||
unixepoch()
|
||||
FROM user_settings s
|
||||
WHERE s.scope = 'gmail'
|
||||
AND COALESCE(json_extract(s.values_json, '$.refresh_token'), '') <> '';
|
||||
|
||||
-- Emptied, not deleted: the row may hold nothing else today, but the table is
|
||||
-- shared and a DELETE here would be a habit worth not forming.
|
||||
UPDATE user_settings SET values_json = '{}' WHERE scope = 'gmail';
|
||||
@@ -272,44 +272,6 @@ func (db *DB) UserByMailToken(ctx context.Context, token string) (*User, error)
|
||||
return db.scanUser(ctx, `SELECT `+userColumns+` FROM users WHERE mail_token = ?`, token)
|
||||
}
|
||||
|
||||
// UsersWithGmail returns everybody who has connected a mailbox, which is who the
|
||||
// poller has to visit. Selected by the presence of a refresh token rather than by a
|
||||
// flag, so a disconnected account drops out of the sweep the moment its tokens are
|
||||
// cleared.
|
||||
func (db *DB) UsersWithGmail(ctx context.Context) ([]User, error) {
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT u.id FROM users u
|
||||
JOIN user_settings s ON s.user_id = u.id
|
||||
WHERE s.scope = 'gmail' AND s.values_json LIKE '%"refresh_token":"%'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]User, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
user, err := db.UserByID(ctx, id)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, *user)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// --- per-user settings ------------------------------------------------------------------
|
||||
|
||||
// UserSettings reads one scope of a person's integration settings. A missing row is
|
||||
|
||||
@@ -84,22 +84,23 @@ func TestATokenWrittenBeforeTheKeyStillWorks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// UsersWithGmail finds people by the presence of the key, not its value, so it
|
||||
// must keep working now that the value is ciphertext.
|
||||
// The sweep finds people by the presence of a mailbox, so it must keep working
|
||||
// now that the credential in it is ciphertext.
|
||||
func TestTheSweepStillFindsAConnectedMailbox(t *testing.T) {
|
||||
db, userID := sealedStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := db.SetUserSettings(ctx, userID, "gmail",
|
||||
map[string]any{"refresh_token": "1//0-token"}); err != nil {
|
||||
if err := db.SaveMailbox(ctx, &Mailbox{
|
||||
UserID: userID, Kind: "gmail", RefreshToken: "1//0-token",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := db.UsersWithGmail(ctx)
|
||||
users, err := db.UsersWithMailboxes(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(users) != 1 {
|
||||
t.Fatalf("the sweep found %d connected mailboxes, want 1", len(users))
|
||||
t.Fatalf("the sweep found %d people with mailboxes, want 1", len(users))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user