Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
241c5ed37d | ||
|
|
78891bae6f | ||
|
|
4b5f3f0864 | ||
|
|
052f8a43fb | ||
|
|
dee2d5e30b | ||
|
|
2bd8e48aba | ||
|
|
9b4e7686f9 |
@@ -66,3 +66,6 @@ node_modules.nosync/
|
||||
# Kept on disk because it is how a session hands over to the next one.
|
||||
HANDOFF.md
|
||||
NOTES-PLAN.md
|
||||
|
||||
# Agenternes arbejdstræer. Et git-repo inde i et git-repo, som ikke hører til her.
|
||||
.claude/worktrees/
|
||||
|
||||
@@ -76,6 +76,7 @@ func run() error {
|
||||
runner.SyncGmail = api.SyncGmail
|
||||
runner.SyncMailbox = api.SyncMailbox
|
||||
runner.Push = api.PushToUser
|
||||
runner.SendBeacon = api.SendBeacon
|
||||
runner.Start(jobCtx)
|
||||
|
||||
srv := &http.Server{
|
||||
|
||||
@@ -1396,6 +1396,114 @@ paths:
|
||||
description: The environment variables that are not set.
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
|
||||
/api/v1/beacon:
|
||||
post:
|
||||
tags: [auth]
|
||||
summary: Report that an installation exists. Unauthenticated.
|
||||
description: |
|
||||
The receiving end of the beacon. Open on purpose: the caller is a
|
||||
stranger's installation, which is the whole feature.
|
||||
|
||||
Answers 404 unless this instance has been told it is the collector, so an
|
||||
ordinary installation is not a spam sink that answers 200 to anything.
|
||||
Rate limited like the login routes, and the table behind it has a ceiling:
|
||||
the endpoint inserts a row keyed on an id the caller invented, so without
|
||||
one anybody could grow it until the volume is full.
|
||||
|
||||
Nothing about the request is recorded but the two values in the body. No
|
||||
source address is read or written here.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [instance_id]
|
||||
properties:
|
||||
instance_id:
|
||||
type: string
|
||||
description: A random id the installation generated for itself.
|
||||
version:
|
||||
type: string
|
||||
responses:
|
||||
"204":
|
||||
description: Recorded.
|
||||
"400":
|
||||
description: The id or the version is not the expected shape.
|
||||
"404":
|
||||
description: This instance is not a collector.
|
||||
|
||||
/api/v1/beacon/count:
|
||||
get:
|
||||
tags: [auth]
|
||||
summary: How many installations reported in the last thirty days.
|
||||
description: |
|
||||
Unauthenticated and cached for a minute, and only answered when the
|
||||
collector has been told to publish it.
|
||||
|
||||
A window rather than a running total: a total only climbs, so it stops
|
||||
meaning "how many people use this" and starts meaning "how many ever
|
||||
tried it".
|
||||
responses:
|
||||
"200":
|
||||
description: The count.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
installs: { type: integer }
|
||||
window_days: { type: integer }
|
||||
"404":
|
||||
description: Not a collector, or not published.
|
||||
|
||||
/api/v1/beacon/settings:
|
||||
get:
|
||||
tags: [auth]
|
||||
summary: What this instance reports, and to whom. Administrators only.
|
||||
description: |
|
||||
Includes the exact payload that will be sent, so the settings page can
|
||||
print it in full rather than describe it.
|
||||
responses:
|
||||
"200":
|
||||
description: The current configuration, and the counts if this is the collector.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
enabled: { type: boolean }
|
||||
collector_url: { type: string }
|
||||
instance_id: { type: string }
|
||||
version: { type: string }
|
||||
is_collector: { type: boolean }
|
||||
publish_count: { type: boolean }
|
||||
last_ping_at: { type: string, format: date-time }
|
||||
put:
|
||||
tags: [auth]
|
||||
summary: Turn the beacon off, or point it somewhere else. Administrators only.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
enabled: { type: boolean }
|
||||
collector_url:
|
||||
type: string
|
||||
description: |
|
||||
Where reports go. Fetched through a dialer that refuses
|
||||
loopback and private address space, so this cannot be used to
|
||||
make the server reach inside its own network.
|
||||
is_collector: { type: boolean }
|
||||
publish_count: { type: boolean }
|
||||
responses:
|
||||
"200":
|
||||
description: The configuration as it now stands.
|
||||
"422":
|
||||
description: The collector address is not an http or https URL.
|
||||
|
||||
/api/v1/panel/restart:
|
||||
post:
|
||||
tags: [auth]
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Package beacon answers one question for the project, and refuses to answer any
|
||||
// other: how many installations are out there.
|
||||
//
|
||||
// It sends two values once a day — an anonymous id the installation made up for
|
||||
// itself, and the version it is running — and nothing else. No IP is stored by
|
||||
// the collector, no hostname, no domain, no account, no usage. The payload is
|
||||
// small enough to be printed in full on the settings page, and it is, because a
|
||||
// promise about telemetry is worth exactly as much as the reader's ability to
|
||||
// check it.
|
||||
//
|
||||
// # On by default, and off in one click
|
||||
//
|
||||
// The alternative that was asked for first was mandatory, and the argument
|
||||
// against it is practical rather than principled: it cannot be enforced. The
|
||||
// domain can be blocked, the binary can be patched, and a fork removes it in its
|
||||
// first commit — so mandatory buys nothing over on-by-default, while costing the
|
||||
// trust the rest of this program works for. Secrets are sealed here, the key is
|
||||
// kept out of the database, and the backup page warns you what a copy contains.
|
||||
// Telemetry you are not allowed to refuse is the one sentence in that story that
|
||||
// does not fit.
|
||||
//
|
||||
// So: on by default, said plainly, one switch to turn it off. That collects
|
||||
// essentially the same number and leaves nobody feeling tricked.
|
||||
//
|
||||
// # Where this disagrees with the rest of the program
|
||||
//
|
||||
// `config.UpdateCheck` is off unless asked for, and its comment gives the reason:
|
||||
// "a self-hosted app that reaches out without being told to has broken the deal
|
||||
// its operator made by self-hosting." That is the same situation as this one, and
|
||||
// the two defaults do not agree.
|
||||
//
|
||||
// The disagreement is written down here rather than smoothed over, because it is
|
||||
// a real one and the reader deserves to see both halves. The case for treating
|
||||
// them differently is that the update check reveals an instance to a third party
|
||||
// (GitHub) and returns something the operator can act on, while this returns
|
||||
// nothing and goes to the project itself — and that a count nobody sends is a
|
||||
// count that does not exist, whereas an update nobody checks for is one you can
|
||||
// still go and look up. Whether that is a distinction or an excuse is a judgement
|
||||
// call, and it was made deliberately by the person whose instance this is.
|
||||
//
|
||||
// If it is ever decided the other way, the change is one word: the default in
|
||||
// `beaconSettings`. Nothing else in this package assumes which way it points.
|
||||
//
|
||||
// # What the collector can still see
|
||||
//
|
||||
// A request has a source address whether or not anybody writes it down. The
|
||||
// collector does not store or log it, and the code is arranged so that it never
|
||||
// reaches a function that could — but it passes through a reverse proxy and a
|
||||
// tunnel on the way, and those keep their own logs. Saying "we send no IP" would
|
||||
// be a half-truth; what is true is that nothing here records one.
|
||||
package beacon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultCollector is where a fresh installation reports, unless it is told
|
||||
// otherwise. Somebody running their own fleet points it at their own instance;
|
||||
// the field is on the settings page for exactly that.
|
||||
const DefaultCollector = "https://verdande.yggdrasilpanel.com/api/v1/beacon"
|
||||
|
||||
// Payload is the entire message. Two fields, and adding a third is a change to
|
||||
// what this program promises, not a change to a struct.
|
||||
type Payload struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// InstanceID and Version are checked on the way in as well as written on the way
|
||||
// out, because the collector's endpoint is unauthenticated: anything that reaches
|
||||
// it is a stranger's bytes, and it should be able to store only the two shapes it
|
||||
// expects.
|
||||
var (
|
||||
idShape = regexp.MustCompile(`^[0-9a-fA-F-]{16,64}$`)
|
||||
versionShape = regexp.MustCompile(`^v?[0-9]{1,4}(\.[0-9]{1,5}){0,3}(-[0-9A-Za-z.]{1,20})?$`)
|
||||
)
|
||||
|
||||
// ValidID reports whether an id is the shape this program writes.
|
||||
func ValidID(s string) bool { return idShape.MatchString(s) }
|
||||
|
||||
// ValidVersion allows a version number and the word this binary uses when it was
|
||||
// built outside a release.
|
||||
func ValidVersion(s string) bool { return s == "dev" || versionShape.MatchString(s) }
|
||||
|
||||
// Send delivers one ping. It is deliberately dull: a short timeout, no retry, no
|
||||
// queue, and a failure that is returned rather than made anybody's problem.
|
||||
//
|
||||
// No retry, because the next pass is tomorrow and a missed day changes a count of
|
||||
// installations by nothing. A telemetry ping that retries is a telemetry ping
|
||||
// that can hammer somebody's server on the day it is down.
|
||||
func Send(ctx context.Context, client *http.Client, collector string, p Payload) error {
|
||||
if !ValidID(p.InstanceID) {
|
||||
return fmt.Errorf("beacon: refusing to send a malformed id")
|
||||
}
|
||||
// Versionen kontrolleres også her, ikke kun hos modtageren. Kommentaren ved
|
||||
// mønstrene lovede begge veje, og gjorde det ikke — og en installation, der
|
||||
// sender noget mærkeligt, skal opdage det hos sig selv frem for at være noget,
|
||||
// en fremmed collector skal filtrere fra.
|
||||
if !ValidVersion(p.Version) {
|
||||
return fmt.Errorf("beacon: refusing to send a malformed version")
|
||||
}
|
||||
body, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, collector, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
// A plain user agent with the version already in the payload. Nothing is
|
||||
// learned from it that the body does not already say, and it makes the request
|
||||
// legible in a log rather than anonymous-looking.
|
||||
req.Header.Set("User-Agent", "verdande-beacon/1")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Read and discard, so the connection can be reused rather than dropped.
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("beacon: collector answered %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package beacon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Hvad der bliver sendt, tegn for tegn.
|
||||
//
|
||||
// Prøven er skrevet imod *hele* kroppen og ikke imod enkelte felter, fordi det er
|
||||
// den, der er løftet: siden siger "præcis det her og ikke mere", og et felt lagt
|
||||
// til senere skal vælte noget. En prøve, der kun så efter, at instance_id var med,
|
||||
// ville lade en ny nøgle glide igennem uden at nogen opdagede det.
|
||||
func TestSendsExactlyTwoFields(t *testing.T) {
|
||||
var got map[string]any
|
||||
var contentType string
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
contentType = r.Header.Get("Content-Type")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Errorf("kroppen er ikke JSON: %v", err)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
err := Send(context.Background(), srv.Client(), srv.URL, Payload{
|
||||
InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a",
|
||||
Version: "v0.26.3",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Send: %v", err)
|
||||
}
|
||||
|
||||
if contentType != "application/json" {
|
||||
t.Errorf("Content-Type = %q", contentType)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("der blev sendt %d felter, ikke to: %v", len(got), got)
|
||||
}
|
||||
if got["instance_id"] != "01a01b16-4e54-7000-9eaa-2f8eb0629c3a" {
|
||||
t.Errorf("instance_id = %v", got["instance_id"])
|
||||
}
|
||||
if got["version"] != "v0.26.3" {
|
||||
t.Errorf("version = %v", got["version"])
|
||||
}
|
||||
}
|
||||
|
||||
// En collector, der er nede, er ikke denne installations problem.
|
||||
func TestSendReportsAFailureRatherThanRetrying(t *testing.T) {
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
err := Send(context.Background(), srv.Client(), srv.URL, Payload{
|
||||
InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a", Version: "v1.0.0",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("en 500 skal give en fejl")
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("der blev prøvet %d gange; en telemetri-ping må ikke hamre på en server, der er nede", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Et id, der ikke er id-formet, sendes slet ikke — så en fejl her ikke bliver til
|
||||
// noget, en fremmed collector skal filtrere fra.
|
||||
func TestSendRefusesAMalformedID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("der blev sendt noget, som ikke skulle sendes")
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
if err := Send(context.Background(), srv.Client(), srv.URL, Payload{
|
||||
InstanceID: "; DROP TABLE beacon_installs", Version: "v1.0.0",
|
||||
}); err == nil {
|
||||
t.Fatal("et misformet id skal afvises før det forlader maskinen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShapes(t *testing.T) {
|
||||
ok := []string{"01a01b16-4e54-7000-9eaa-2f8eb0629c3a", "89df44d7-3742-4611-9a43-db0b07733c6d"}
|
||||
for _, s := range ok {
|
||||
if !ValidID(s) {
|
||||
t.Errorf("ValidID(%q) = false", s)
|
||||
}
|
||||
}
|
||||
bad := []string{"", "kort", "../../etc/passwd", "01a01b16-4e54-7000-9eaa-2f8eb0629c3a'; --",
|
||||
"<script>alert(1)</script>"}
|
||||
for _, s := range bad {
|
||||
if ValidID(s) {
|
||||
t.Errorf("ValidID(%q) = true", s)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range []string{"v0.26.3", "0.26.3", "v1.2.3-rc.1", "dev"} {
|
||||
if !ValidVersion(s) {
|
||||
t.Errorf("ValidVersion(%q) = false", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"", "v<script>", "v" + string(make([]byte, 200))} {
|
||||
if ValidVersion(s) {
|
||||
t.Errorf("ValidVersion(%q) = true", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kristianwind/verdande/internal/beacon"
|
||||
"github.com/kristianwind/verdande/internal/safedial"
|
||||
"github.com/kristianwind/verdande/internal/store"
|
||||
)
|
||||
|
||||
// The beacon, from the instance's side.
|
||||
//
|
||||
// Two halves that happen to live in the same binary: the part that reports in,
|
||||
// which every installation runs, and the part that receives, which only the one
|
||||
// acting as collector turns on. See internal/beacon for what is sent and why it
|
||||
// is on by default rather than compulsory.
|
||||
|
||||
const beaconScope = "beacon"
|
||||
|
||||
// beaconConfig is what this instance has been told to do. The zero value is what
|
||||
// a fresh installation gets, which is why Enabled defaults to true elsewhere and
|
||||
// not here — a zero struct must never mean "silently on".
|
||||
type beaconConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Collector string `json:"collector_url"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
IsCollector bool `json:"is_collector"`
|
||||
PublishCount bool `json:"publish_count"`
|
||||
LastPingAt int64 `json:"last_ping_at,omitempty"`
|
||||
}
|
||||
|
||||
// beaconSettings reads the configuration, filling in the defaults and minting the
|
||||
// anonymous id on first use.
|
||||
//
|
||||
// The id is generated here rather than derived from anything about the machine.
|
||||
// A hash of the hostname or the base URL would be stable and convenient and would
|
||||
// also be a value the collector could compare against a guess — which is the
|
||||
// difference between anonymous and pseudonymous, and it is the whole point.
|
||||
func (s *Server) beaconSettings(ctx context.Context) (beaconConfig, error) {
|
||||
cfg := beaconConfig{Enabled: true, Collector: beacon.DefaultCollector}
|
||||
|
||||
stored, err := s.db.InstanceSettings(ctx, beaconScope)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
if v, ok := stored["enabled"].(bool); ok {
|
||||
cfg.Enabled = v
|
||||
}
|
||||
if v, ok := stored["collector_url"].(string); ok && strings.TrimSpace(v) != "" {
|
||||
cfg.Collector = strings.TrimSpace(v)
|
||||
}
|
||||
if v, ok := stored["instance_id"].(string); ok && beacon.ValidID(v) {
|
||||
cfg.InstanceID = v
|
||||
}
|
||||
if v, ok := stored["is_collector"].(bool); ok {
|
||||
cfg.IsCollector = v
|
||||
}
|
||||
if v, ok := stored["publish_count"].(bool); ok {
|
||||
cfg.PublishCount = v
|
||||
}
|
||||
if v, ok := stored["last_ping_at"].(float64); ok {
|
||||
cfg.LastPingAt = int64(v)
|
||||
}
|
||||
|
||||
if cfg.InstanceID == "" {
|
||||
cfg.InstanceID = store.NewID()
|
||||
if err := s.saveBeaconSettings(ctx, cfg); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (s *Server) saveBeaconSettings(ctx context.Context, cfg beaconConfig) error {
|
||||
return s.db.SetInstanceSettings(ctx, beaconScope, map[string]any{
|
||||
"enabled": cfg.Enabled,
|
||||
"collector_url": cfg.Collector,
|
||||
"instance_id": cfg.InstanceID,
|
||||
"is_collector": cfg.IsCollector,
|
||||
"publish_count": cfg.PublishCount,
|
||||
"last_ping_at": cfg.LastPingAt,
|
||||
})
|
||||
}
|
||||
|
||||
// --- what the settings page shows ------------------------------------------------
|
||||
|
||||
type beaconStatusJSON struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Collector string `json:"collector_url"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
Version string `json:"version"`
|
||||
IsCollector bool `json:"is_collector"`
|
||||
PublishCount bool `json:"publish_count"`
|
||||
LastPingAt string `json:"last_ping_at,omitempty"`
|
||||
Stats *store.BeaconStats `json:"stats,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Server) handleBeaconStatus(w http.ResponseWriter, r *http.Request) {
|
||||
cfg, err := s.beaconSettings(r.Context())
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon settings", err)
|
||||
return
|
||||
}
|
||||
|
||||
out := beaconStatusJSON{
|
||||
Enabled: cfg.Enabled, Collector: cfg.Collector, InstanceID: cfg.InstanceID,
|
||||
Version: Version, IsCollector: cfg.IsCollector, PublishCount: cfg.PublishCount,
|
||||
}
|
||||
if cfg.LastPingAt > 0 {
|
||||
out.LastPingAt = time.Unix(cfg.LastPingAt, 0).Format(time.RFC3339)
|
||||
}
|
||||
if cfg.IsCollector {
|
||||
stats, err := s.db.BeaconSummary(r.Context())
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon summary", err)
|
||||
return
|
||||
}
|
||||
out.Stats = &stats
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
type beaconRequest struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
Collector *string `json:"collector_url"`
|
||||
IsCollector *bool `json:"is_collector"`
|
||||
PublishCount *bool `json:"publish_count"`
|
||||
}
|
||||
|
||||
func (s *Server) handleSetBeacon(w http.ResponseWriter, r *http.Request) {
|
||||
var req beaconRequest
|
||||
if err := decodeJSON(w, r, &req); err != nil {
|
||||
return
|
||||
}
|
||||
cfg, err := s.beaconSettings(r.Context())
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon settings", err)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Enabled != nil {
|
||||
cfg.Enabled = *req.Enabled
|
||||
}
|
||||
if req.IsCollector != nil {
|
||||
cfg.IsCollector = *req.IsCollector
|
||||
}
|
||||
if req.PublishCount != nil {
|
||||
cfg.PublishCount = *req.PublishCount
|
||||
}
|
||||
if req.Collector != nil {
|
||||
url := strings.TrimSpace(*req.Collector)
|
||||
if url == "" {
|
||||
url = beacon.DefaultCollector
|
||||
}
|
||||
// http and https only, and said as a field error rather than a 500: an
|
||||
// operator pointing this at their own collector will mistype it, and the
|
||||
// page should say which field is wrong.
|
||||
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
|
||||
writeFieldErrors(w, map[string]string{"collector_url": "must start with http:// or https://"})
|
||||
return
|
||||
}
|
||||
cfg.Collector = url
|
||||
}
|
||||
|
||||
if err := s.saveBeaconSettings(r.Context(), cfg); err != nil {
|
||||
s.internal(w, r, "save beacon settings", err)
|
||||
return
|
||||
}
|
||||
s.handleBeaconStatus(w, r)
|
||||
}
|
||||
|
||||
// --- the collector's side ---------------------------------------------------------
|
||||
|
||||
// handleBeaconPing receives one report. Unauthenticated, because the whole point
|
||||
// is that a stranger's installation can reach it.
|
||||
//
|
||||
// Nothing about the request is recorded except the two values in the body. The
|
||||
// source address is not read here, and there is no code path from this function
|
||||
// to anything that could write one down — which is the only version of that
|
||||
// promise worth making.
|
||||
func (s *Server) handleBeaconPing(w http.ResponseWriter, r *http.Request) {
|
||||
cfg, err := s.beaconSettings(r.Context())
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon settings", err)
|
||||
return
|
||||
}
|
||||
// An instance that is not a collector does not quietly accept and drop pings —
|
||||
// it says there is nothing here. Otherwise every installation in the world is a
|
||||
// spam sink that answers 200 to anything.
|
||||
if !cfg.IsCollector {
|
||||
writeError(w, http.StatusNotFound, CodeNotFound, "this instance is not a beacon collector")
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 4<<10)
|
||||
var p beacon.Payload
|
||||
if err := decodeJSON(w, r, &p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Checked, not trusted. This is the one endpoint anybody can post to, and the
|
||||
// two columns behind it should only ever hold the two shapes this program
|
||||
// writes — an id that is not id-shaped is somebody probing, not an install.
|
||||
if !beacon.ValidID(p.InstanceID) {
|
||||
writeFieldErrors(w, map[string]string{"instance_id": "malformed"})
|
||||
return
|
||||
}
|
||||
if p.Version != "" && !beacon.ValidVersion(p.Version) {
|
||||
writeFieldErrors(w, map[string]string{"version": "malformed"})
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.db.RecordBeacon(r.Context(), p.InstanceID, p.Version); err != nil {
|
||||
s.internal(w, r, "record beacon", err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleBeaconCount publishes the number, when the collector has been told to.
|
||||
//
|
||||
// Thirty days rather than a running total: a total only climbs, so it stops
|
||||
// meaning "how many people use this" and starts meaning "how many ever tried it".
|
||||
func (s *Server) handleBeaconCount(w http.ResponseWriter, r *http.Request) {
|
||||
cfg, err := s.beaconSettings(r.Context())
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon settings", err)
|
||||
return
|
||||
}
|
||||
if !cfg.IsCollector || !cfg.PublishCount {
|
||||
writeError(w, http.StatusNotFound, CodeNotFound, "not published")
|
||||
return
|
||||
}
|
||||
|
||||
n, err := s.db.BeaconCount(r.Context(), 30*24*time.Hour)
|
||||
if err != nil {
|
||||
s.internal(w, r, "beacon count", err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "public, max-age=60")
|
||||
writeJSON(w, http.StatusOK, map[string]any{"installs": n, "window_days": 30})
|
||||
}
|
||||
|
||||
// SendBeacon reports this installation, at most once a day.
|
||||
//
|
||||
// Exported and handed to the jobs runner the same way the mailbox syncs are: the
|
||||
// settings, the anonymous id and the version all live on this side, and the
|
||||
// runner should not have to know about any of them.
|
||||
//
|
||||
// The day is counted from the last successful send, not from a wall-clock hour.
|
||||
// A fleet of installations that all pinged at 03:00 would arrive in one spike;
|
||||
// counted this way they spread themselves out by when each was started.
|
||||
func (s *Server) SendBeacon(ctx context.Context) error {
|
||||
cfg, err := s.beaconSettings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
// The collector does not report to itself over the network. It would work, and
|
||||
// it would also mean the one instance whose count matters most depends on its
|
||||
// own tunnel being up to be counted.
|
||||
if cfg.IsCollector {
|
||||
if err := s.db.RecordBeacon(ctx, cfg.InstanceID, Version); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.LastPingAt = time.Now().Unix()
|
||||
return s.saveBeaconSettings(ctx, cfg)
|
||||
}
|
||||
|
||||
if time.Since(time.Unix(cfg.LastPingAt, 0)) < 24*time.Hour {
|
||||
return nil
|
||||
}
|
||||
|
||||
// safedial, ikke http.DefaultClient. Adressen kan skrives i fladen, og en
|
||||
// administrator, der peger den på 127.0.0.1 eller på 169.254.169.254, ville
|
||||
// ellers have en maskine, der henter indefra på kommando — og svaret på om den
|
||||
// svarede, er i sig selv det, sådan et forsøg leder efter.
|
||||
if err := beacon.Send(ctx, safedial.Client(15*time.Second), cfg.Collector, beacon.Payload{
|
||||
InstanceID: cfg.InstanceID,
|
||||
Version: Version,
|
||||
}); err != nil {
|
||||
// Logged at info, not error. The collector being unreachable is not this
|
||||
// installation's problem and must never look like one — it is somebody
|
||||
// else's server, and the only consequence is a number being one lower.
|
||||
s.log.Info("beacon not delivered", "err", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg.LastPingAt = time.Now().Unix()
|
||||
return s.saveBeaconSettings(ctx, cfg)
|
||||
}
|
||||
@@ -251,6 +251,17 @@ func (s *Server) handleGmailSyncNow(w http.ResponseWriter, r *http.Request) {
|
||||
//
|
||||
// Returns how many were made. Also used by the background job, which is why it
|
||||
// takes a context and a user rather than a request.
|
||||
// maxGmailSeen bounds the list of message ids already turned into tasks.
|
||||
//
|
||||
// Gmail is read-only here — the scope is `gmail.readonly`, deliberately, because
|
||||
// asking for less is what makes the consent screen something a person can agree
|
||||
// to — so verdande cannot unstar a message the way the IMAP side unflags one. The
|
||||
// record has to live on this side, and a record on this side has to be bounded.
|
||||
//
|
||||
// A count rather than the previous rule of "whatever Gmail returned this run",
|
||||
// which was not a bound but a way of forgetting.
|
||||
const maxGmailSeen = 2000
|
||||
|
||||
func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
box, err := s.db.MailboxOfKind(ctx, user.ID, "gmail")
|
||||
if err != nil {
|
||||
@@ -362,14 +373,35 @@ func (s *Server) SyncGmail(ctx context.Context, user *store.User) (int, error) {
|
||||
}
|
||||
|
||||
if created > 0 {
|
||||
// Only the ids Gmail still returns are kept, so the list cannot grow
|
||||
// without bound as messages fall out of the 30-day window.
|
||||
keep := make([]string, 0, len(ids))
|
||||
// The ids this run saw, on top of the ones already known.
|
||||
//
|
||||
// It used to keep only the ids Gmail returned *this run*, to stop the list
|
||||
// growing without bound. That threw away every id that was not on the page
|
||||
// it happened to fetch — and the query is capped, so on an account with more
|
||||
// starred mail than one page, each run forgot the messages the last one had
|
||||
// remembered. They came back unseen, and became tasks again, and again. That
|
||||
// is the duplicate.
|
||||
//
|
||||
// Bounded by a count instead. Newest first, and a ceiling far above any
|
||||
// plausible page, so an id leaves the list because it is old rather than
|
||||
// because it was not in the last answer.
|
||||
keep := make([]string, 0, len(ids)+len(box.Seen))
|
||||
added := map[string]bool{}
|
||||
for _, id := range ids {
|
||||
if seen[id] {
|
||||
if seen[id] && !added[id] {
|
||||
keep = append(keep, id)
|
||||
added[id] = true
|
||||
}
|
||||
}
|
||||
for _, id := range box.Seen {
|
||||
if !added[id] {
|
||||
keep = append(keep, id)
|
||||
added[id] = true
|
||||
}
|
||||
}
|
||||
if len(keep) > maxGmailSeen {
|
||||
keep = keep[:maxGmailSeen]
|
||||
}
|
||||
box.Seen = keep
|
||||
box.LastSyncAt = time.Now()
|
||||
if err := s.db.SaveMailbox(ctx, box); err != nil {
|
||||
|
||||
@@ -175,6 +175,10 @@ func (s *Server) SyncMailbox(ctx context.Context, user *store.User, m *store.Mai
|
||||
|
||||
created := 0
|
||||
highest := m.LastUID
|
||||
// The uids of the mails that actually became tasks. Only those get unflagged,
|
||||
// and only after the task is committed: a flag removed from a mail whose task
|
||||
// was never written is a mail nobody will ever see again.
|
||||
done := make([]uint32, 0, len(messages))
|
||||
for _, msg := range messages {
|
||||
if ctx.Err() != nil {
|
||||
// What was made so far is already committed and counted, and the marker
|
||||
@@ -204,12 +208,27 @@ func (s *Server) SyncMailbox(ctx context.Context, user *store.User, m *store.Mai
|
||||
continue
|
||||
}
|
||||
created++
|
||||
done = append(done, msg.UID)
|
||||
if msg.UID > highest {
|
||||
highest = msg.UID
|
||||
}
|
||||
s.hub.Publish(projectID, "task.created", toTaskJSON(*task))
|
||||
}
|
||||
|
||||
// The flag comes off what became a task. This is what actually stops a mail
|
||||
// arriving twice; the marker below is now a second line rather than the only
|
||||
// one. See imap.Unflag for why the record belongs in the mailbox and not here.
|
||||
//
|
||||
// Outside the request's context, like the marker: a cancelled request that
|
||||
// already created tasks must still take the flags off them, or the next run
|
||||
// makes them all again.
|
||||
if err := client.Unflag(done); err != nil {
|
||||
// Not fatal, and not the caller's problem. A folder somebody opened
|
||||
// read-only, or a server that refuses the STORE, still gets the uid marker
|
||||
// underneath — worse, but not broken.
|
||||
s.log.Warn("could not unflag imported mail", "err", err, "mailbox", m.ID, "count", len(done))
|
||||
}
|
||||
|
||||
// A run that got all the way through moves the marker to the whole of what the
|
||||
// search considered, not merely to the last message it happened to read. That
|
||||
// is what stops the same mail becoming a task again on the next run when the
|
||||
|
||||
@@ -353,7 +353,16 @@ func (s *Server) handleImportNotes(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
for _, f := range zr.File {
|
||||
name := f.Name
|
||||
if f.FileInfo().IsDir() || skipFile(name) || !strings.EqualFold(path.Ext(name), ".md") {
|
||||
if f.FileInfo().IsDir() || !strings.EqualFold(path.Ext(name), ".md") {
|
||||
continue
|
||||
}
|
||||
// Counted, not passed over in silence. A Markdown file the filter refuses is
|
||||
// a note that did not arrive, and the caller is told the same number either
|
||||
// way unless it is said here — which is how one note went missing without
|
||||
// anything looking wrong.
|
||||
if skipFile(name) {
|
||||
s.log.Warn("import skipped a note file", "file", name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -369,13 +378,24 @@ func (s *Server) handleImportNotes(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
|
||||
n := &store.Note{CreatedBy: user.ID, Body: string(body)}
|
||||
text, wrote, changed := readFrontMatter(string(body))
|
||||
|
||||
n := &store.Note{CreatedBy: user.ID, Body: text}
|
||||
if err := s.db.SaveNote(r.Context(), n); err != nil {
|
||||
s.log.Warn("import note", "err", err, "file", name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// After the save, not as part of it: SaveNote owns `updated_at` and is right
|
||||
// to, so the dates a note had before this program existed are written over
|
||||
// the top rather than passed through it.
|
||||
if !wrote.IsZero() || !changed.IsZero() {
|
||||
if err := s.db.SetNoteTimes(r.Context(), n.ID, wrote, changed); err != nil {
|
||||
s.log.Warn("import note dates", "err", err, "file", name)
|
||||
}
|
||||
}
|
||||
|
||||
// The pictures the note referred to, now that it has an id to hang them on.
|
||||
// The text is rewritten to the addresses they were given, so an image that
|
||||
// sat in the middle of a paragraph is still in the middle of it.
|
||||
@@ -398,10 +418,34 @@ func (s *Server) handleImportNotes(w http.ResponseWriter, r *http.Request) {
|
||||
// A zip made on a Mac carries a __MACOSX copy of every file, and importing those
|
||||
// would double a whole library with each copy full of binary rubbish.
|
||||
func skipFile(name string) bool {
|
||||
return strings.HasPrefix(name, "__MACOSX/") ||
|
||||
if strings.HasPrefix(name, "__MACOSX/") ||
|
||||
strings.HasPrefix(path.Base(name), "._") ||
|
||||
strings.HasPrefix(path.Base(name), ".") ||
|
||||
strings.Contains(name, "..")
|
||||
strings.HasPrefix(path.Base(name), ".") {
|
||||
return true
|
||||
}
|
||||
|
||||
// Traversal, read as path segments rather than as a substring.
|
||||
//
|
||||
// This was `strings.Contains(name, "..")`, which is the right idea aimed at the
|
||||
// wrong thing: it does not describe a path that climbs out of the archive, it
|
||||
// describes any name with two dots next to each other anywhere in it. A note
|
||||
// called "Så blev det endelig jul! Stay tuned... ☕️" was therefore dropped —
|
||||
// silently, and without even counting as skipped, so the number the import
|
||||
// reported still looked right. One note in twelve hundred, found only by
|
||||
// counting what came out.
|
||||
//
|
||||
// An ellipsis in a title is ordinary. A segment that *is* "..", or a path that
|
||||
// starts at the root, is not — and those are what a zip uses to write outside
|
||||
// the folder it was unpacked into.
|
||||
if strings.HasPrefix(name, "/") {
|
||||
return true
|
||||
}
|
||||
for _, part := range strings.Split(name, "/") {
|
||||
if part == ".." {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// attachToNote records the files a note's text points at and rewrites the text to
|
||||
@@ -466,6 +510,67 @@ func (s *Server) attachToNote(
|
||||
return body, attached
|
||||
}
|
||||
|
||||
// readFrontMatter takes the YAML block off the top of a Markdown file and reads
|
||||
// the two dates out of it, returning the rest of the text.
|
||||
//
|
||||
// Not a YAML parser, and deliberately not: the block this reads is the one this
|
||||
// program writes, plus whatever Obsidian and Bear write, and all of them use the
|
||||
// same handful of `key: value` lines. Pulling in a parser to read two dates would
|
||||
// be a dependency that can fail in ways nobody here would recognise.
|
||||
//
|
||||
// A file with no front matter comes back untouched — that is the normal case, and
|
||||
// it must never lose its first line to a rule about a block that is not there.
|
||||
func readFrontMatter(body string) (text string, created, modified time.Time) {
|
||||
if !strings.HasPrefix(body, "---\n") && !strings.HasPrefix(body, "---\r\n") {
|
||||
return body, time.Time{}, time.Time{}
|
||||
}
|
||||
rest := body[strings.Index(body, "\n")+1:]
|
||||
|
||||
end := strings.Index(rest, "\n---")
|
||||
if end < 0 {
|
||||
// An opening fence with no close is not front matter; it is a horizontal
|
||||
// rule somebody wrote at the top of their note.
|
||||
return body, time.Time{}, time.Time{}
|
||||
}
|
||||
block := rest[:end]
|
||||
after := rest[end+len("\n---"):]
|
||||
after = strings.TrimPrefix(after, "\r")
|
||||
after = strings.TrimPrefix(after, "\n")
|
||||
|
||||
for _, line := range strings.Split(block, "\n") {
|
||||
key, value, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
value = strings.TrimSpace(strings.Trim(strings.TrimSpace(value), `"'`))
|
||||
switch strings.ToLower(strings.TrimSpace(key)) {
|
||||
case "created", "created_at", "date":
|
||||
created = parseNoteTime(value)
|
||||
case "modified", "updated", "updated_at":
|
||||
modified = parseNoteTime(value)
|
||||
}
|
||||
}
|
||||
return strings.TrimLeft(after, "\n"), created, modified
|
||||
}
|
||||
|
||||
// parseNoteTime accepts the shapes these files actually carry. A date without a
|
||||
// time is that day at midnight, local — the note was written on that day, and
|
||||
// pretending to know the hour would be worse than not having one.
|
||||
func parseNoteTime(v string) time.Time {
|
||||
for _, layout := range []string{
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02 15:04",
|
||||
"2006-01-02",
|
||||
} {
|
||||
if t, err := time.ParseInLocation(layout, v, time.Local); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// mimeOf guesses from the extension. The archive does not carry a type, and
|
||||
// sniffing the bytes would mean reading every file twice.
|
||||
func mimeOf(name string) string {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package httpapi
|
||||
|
||||
import "testing"
|
||||
|
||||
// En titel med tre prikker i er ikke et forsøg på at skrive uden for mappen.
|
||||
//
|
||||
// `strings.Contains(name, "..")` var den rigtige idé rettet mod det forkerte:
|
||||
// den beskriver ikke en sti, der klatrer ud af arkivet, men ethvert navn med to
|
||||
// prikker ved siden af hinanden. Noten "Så blev det endelig jul! Stay tuned... ☕️"
|
||||
// blev derfor droppet lydløst — én ud af tolv hundrede, og kun fundet ved at
|
||||
// tælle efter, hvor mange der kom ud.
|
||||
func TestSkipFileTakesEllipsesButRefusesTraversal(t *testing.T) {
|
||||
keep := []string{
|
||||
"Så blev det endelig jul! Stay tuned... ☕️.md",
|
||||
"Møde 12.8..md",
|
||||
"vedhaeftninger/1-2-Pasted Graphic.png",
|
||||
"noter/Referat....md",
|
||||
}
|
||||
for _, name := range keep {
|
||||
if skipFile(name) {
|
||||
t.Errorf("skipFile(%q) = true, men det er et almindeligt filnavn", name)
|
||||
}
|
||||
}
|
||||
|
||||
drop := []string{
|
||||
"../uden for.md",
|
||||
"noter/../../etc/passwd",
|
||||
"/etc/passwd",
|
||||
"__MACOSX/._noget",
|
||||
".skjult.md",
|
||||
"noter/._AppleDouble.md",
|
||||
}
|
||||
for _, name := range drop {
|
||||
if !skipFile(name) {
|
||||
t.Errorf("skipFile(%q) = false, men den skal afvises", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -143,6 +143,19 @@ func New(cfg *config.Config, db *store.DB, log *slog.Logger, web fs.FS) *Server
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
})
|
||||
|
||||
// The beacon's receiving end, and the number it produces. Both without a
|
||||
// session, because the caller is a stranger's installation — that is the
|
||||
// whole feature. Rate limited like the login routes: it is the one endpoint
|
||||
// anybody on the internet can write to, and a row per request is a row per
|
||||
// request whether or not the request is honest.
|
||||
//
|
||||
// Refused with a 404 unless this instance has been told it is the collector,
|
||||
// so an ordinary installation is not a spam sink that answers 200 to
|
||||
// anything.
|
||||
r.Method(http.MethodPost, "/beacon",
|
||||
s.rateLimit(s.loginLimiter, http.HandlerFunc(s.handleBeaconPing)))
|
||||
r.Get("/beacon/count", s.handleBeaconCount)
|
||||
|
||||
// Reachable without being signed in — these are how you become signed in.
|
||||
r.Route("/auth", func(r chi.Router) {
|
||||
r.Get("/setup", s.handleSetupState)
|
||||
@@ -390,6 +403,11 @@ func New(cfg *config.Config, db *store.DB, log *slog.Logger, web fs.FS) *Server
|
||||
// `:latest` on the way, which is what makes a restart an update.
|
||||
r.Get("/panel", s.handlePanelStatus)
|
||||
r.Post("/panel/restart", s.handleRestartFromPanel)
|
||||
// Who this instance reports to, and whether it does. An
|
||||
// administrator's decision, not a person's: there is one
|
||||
// installation and it either phones home or it does not.
|
||||
r.Get("/beacon/settings", s.handleBeaconStatus)
|
||||
r.Put("/beacon/settings", s.handleSetBeacon)
|
||||
// A backup file is a complete copy of the database, so this whole
|
||||
// group is sessions-only as well as administrators-only: a leaked
|
||||
// token must not be able to download everybody's data.
|
||||
|
||||
@@ -310,3 +310,61 @@ func cut(text string) string {
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// Unflag takes the flag off the messages that have become tasks.
|
||||
//
|
||||
// This is what stops the same mail arriving twice, and it replaces a marker that
|
||||
// could not be made to hold. The marker was a uid: remember the highest one seen,
|
||||
// start above it next time. That is correct only if uids are what the server says
|
||||
// they are, and it broke in every way such a scheme can — a fetch reply without a
|
||||
// uid, a UIDVALIDITY that changed under a rebuilt mailbox, two mailbox rows
|
||||
// pointing at the same folder and each keeping its own marker. Every one of those
|
||||
// produced the same symptom: the same mail became a task again, and again.
|
||||
//
|
||||
// Unflagging moves the record out of our database and into the mailbox, where the
|
||||
// question is asked. "Flagged" means "turn this into a task"; once it is a task,
|
||||
// the flag has done its job and is not true any more. Nothing has to be
|
||||
// remembered, so there is nothing to get out of step — a second sync, a second
|
||||
// mailbox row, or a database restored from a backup all find the same answer,
|
||||
// because the answer is in the mailbox.
|
||||
//
|
||||
// It is destructive in the small way that matters: it changes something in
|
||||
// somebody's mail. That is the trade, and it is the right way round — a flag is a
|
||||
// note to self that this needs doing, and it has been done.
|
||||
//
|
||||
// The folder is re-selected writable. Since() opens it read-only on purpose, and
|
||||
// a search should not need write access just because a later step might.
|
||||
func (c *Client) Unflag(uids []uint32) error {
|
||||
if len(uids) == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.c.Select(c.folder, nil).Wait(); err != nil {
|
||||
return fmt.Errorf("open %s for writing: %w", c.folder, err)
|
||||
}
|
||||
|
||||
set := imap.UIDSetNum()
|
||||
for _, u := range uids {
|
||||
if u == 0 {
|
||||
// No uid means nothing to address. The message keeps its flag and will
|
||||
// be seen again — which is the failure this whole function exists to
|
||||
// prevent, so it is worth saying rather than skipping quietly.
|
||||
continue
|
||||
}
|
||||
set.AddNum(imap.UID(u))
|
||||
}
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Silent: the server has nothing to tell us that we would act on, and the
|
||||
// untagged replies would only have to be collected and thrown away.
|
||||
cmd := c.c.Store(set, &imap.StoreFlags{
|
||||
Op: imap.StoreFlagsDel,
|
||||
Silent: true,
|
||||
Flags: []imap.Flag{imap.FlagFlagged},
|
||||
}, nil)
|
||||
if err := cmd.Close(); err != nil {
|
||||
return fmt.Errorf("unflag in %s: %w", c.folder, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -316,3 +316,81 @@ func TestNothingAtOrBelowTheMarkerComesBack(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Den samme mail må ikke kunne blive en opgave to gange.
|
||||
//
|
||||
// Prøven kører hele vejen imod en rigtig server: den finder de flagede mails,
|
||||
// tager flaget af dem, og spørger så forfra som en ny synkronisering ville — med
|
||||
// markøren sat til nul, altså det værst tænkelige tilfælde, hvor alt, hvad vi
|
||||
// selv havde husket, er væk. Der skal ikke komme noget tilbage.
|
||||
//
|
||||
// Det er det, markøren ikke kunne love. Den holdt kun, hvis uids var, hvad
|
||||
// serveren sagde, og fejlede på et fetch-svar uden uid, på en UIDVALIDITY, der
|
||||
// skiftede, og på to postkasse-rækker, der pegede samme sted og hver især førte
|
||||
// deres eget regnskab. Alle tre gav det samme: mailen blev en opgave igen.
|
||||
func TestAnUnflaggedMailDoesNotComeBack(t *testing.T) {
|
||||
addr, pool := serveMailbox(t, []mail{
|
||||
{subject: "Faktura 4711", from: "[email protected]", body: "Vedhæftet.", flagged: true},
|
||||
{subject: "Nyhedsbrev", from: "[email protected]", body: "Læs mere", flagged: false},
|
||||
{subject: "Kontrakt", from: "[email protected]", body: "Til underskrift", flagged: true},
|
||||
})
|
||||
|
||||
client, err := Dial(Account{Host: addr, Username: "kw", Password: "hemmelig", RootCAs: pool})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
msgs, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("fandt %d flagede mails, ventede 2", len(msgs))
|
||||
}
|
||||
|
||||
uids := make([]uint32, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
uids = append(uids, m.UID)
|
||||
}
|
||||
if err := client.Unflag(uids); err != nil {
|
||||
t.Fatalf("Unflag: %v", err)
|
||||
}
|
||||
|
||||
// Forfra, og fra nul: selv en instans, der intet husker, må ikke finde dem igen.
|
||||
again, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(again) != 0 {
|
||||
t.Fatalf("%d mails kom igen efter at flaget var taget af — den første af dem er %q",
|
||||
len(again), again[0].Subject)
|
||||
}
|
||||
}
|
||||
|
||||
// Et tomt kald må ikke åbne mappen skrivbar for ingenting.
|
||||
func TestUnflagWithNothingToDoIsQuiet(t *testing.T) {
|
||||
addr, pool := serveMailbox(t, []mail{
|
||||
{subject: "Faktura", from: "[email protected]", body: "x", flagged: true},
|
||||
})
|
||||
client, err := Dial(Account{Host: addr, Username: "kw", Password: "hemmelig", RootCAs: pool})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
if err := client.Unflag(nil); err != nil {
|
||||
t.Errorf("Unflag(nil) = %v", err)
|
||||
}
|
||||
// Kun nuller: der er intet at adressere, og mailen skal beholde sit flag.
|
||||
if err := client.Unflag([]uint32{0, 0}); err != nil {
|
||||
t.Errorf("Unflag(nuller) = %v", err)
|
||||
}
|
||||
msgs, _, err := client.Since(0, 25)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(msgs) != 1 {
|
||||
t.Errorf("mailen mistede sit flag, uden at nogen havde adresseret den")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,11 @@ type Runner struct {
|
||||
// layer, which owns the VAPID keys and the subscription list.
|
||||
Push func(userID, title, body, projectID string)
|
||||
|
||||
// SendBeacon reports this installation to the collector, if it is switched on
|
||||
// and a day has passed. Supplied by the HTTP layer for the same reason as the
|
||||
// syncs: that is where the instance settings and the version live.
|
||||
SendBeacon func(ctx context.Context) error
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
@@ -69,6 +74,10 @@ func (r *Runner) Start(ctx context.Context) {
|
||||
// account for a to-do app. Ten minutes is fast enough for something somebody
|
||||
// starred and slow enough not to look like abuse.
|
||||
r.every(ctx, "mailboxes", 10*time.Minute, r.syncMailboxes)
|
||||
// Checked hourly, sent at most daily — the same shape as the backup, and for
|
||||
// the same reason: a container asleep at the scheduled minute would otherwise
|
||||
// skip the day. SendBeacon decides whether a day has passed.
|
||||
r.every(ctx, "beacon", time.Hour, r.sendBeacon)
|
||||
}
|
||||
|
||||
func (r *Runner) Wait() { r.wg.Wait() }
|
||||
@@ -103,6 +112,17 @@ func (r *Runner) every(ctx context.Context, name string, interval time.Duration,
|
||||
}()
|
||||
}
|
||||
|
||||
// sendBeacon reports that this installation exists, at most once a day.
|
||||
//
|
||||
// Every decision about what is sent, and why this is on by default rather than
|
||||
// compulsory, lives in internal/beacon. This is only the clock.
|
||||
func (r *Runner) sendBeacon(ctx context.Context) error {
|
||||
if r.SendBeacon == nil {
|
||||
return nil
|
||||
}
|
||||
return r.SendBeacon(ctx)
|
||||
}
|
||||
|
||||
// --- reminders ------------------------------------------------------------------
|
||||
|
||||
// sendDueReminders delivers everything that has come due since the last pass.
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Package safedial refuses to open a connection to this machine or to the
|
||||
// network it sits on.
|
||||
//
|
||||
// The problem it solves is the same one three times: somewhere a user supplies a
|
||||
// URL or a host, and the server fetches it. An AI provider's base address, a push
|
||||
// endpoint, an IMAP host, a beacon collector. Each of them is a feature, and each
|
||||
// of them is also a way to ask the server to reach something the caller cannot —
|
||||
// the panel next door, a metadata endpoint, a database on the same bridge — and
|
||||
// to report back whether it answered. On a homelab that is the whole point of the
|
||||
// box being where it is.
|
||||
//
|
||||
// # Why the check is on the address and not on the URL
|
||||
//
|
||||
// Parsing the URL and refusing "localhost" or "127.0.0.1" is the obvious version
|
||||
// and it does not work: a name resolves to whatever its owner says, so
|
||||
// `evil.example.com` can answer 127.0.0.1 — and can answer differently the second
|
||||
// time, after a parse-time check has passed. That is DNS rebinding, and the only
|
||||
// place it can be caught is after resolution and before the connection, which is
|
||||
// exactly where net.Dialer's Control hook runs.
|
||||
//
|
||||
// # What is refused
|
||||
//
|
||||
// Loopback, link-local (including 169.254.169.254, which is where cloud metadata
|
||||
// lives), the private ranges, unique-local IPv6, and the unspecified address.
|
||||
// Everything else is allowed: this is not an allowlist of the internet, it is a
|
||||
// wall around the inside.
|
||||
package safedial
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrPrivate is returned when a connection was refused for where it pointed. It
|
||||
// deliberately does not say which address it resolved to: the answer is itself
|
||||
// the thing an attacker is fishing for.
|
||||
type ErrPrivate struct{ Host string }
|
||||
|
||||
func (e *ErrPrivate) Error() string {
|
||||
return fmt.Sprintf("refusing to connect to %s: it resolves to an address on this machine or its private network", e.Host)
|
||||
}
|
||||
|
||||
// Blocked reports whether an address must not be dialled.
|
||||
func Blocked(ip net.IP) bool {
|
||||
return ip == nil ||
|
||||
ip.IsLoopback() ||
|
||||
ip.IsUnspecified() ||
|
||||
ip.IsLinkLocalUnicast() ||
|
||||
ip.IsLinkLocalMulticast() ||
|
||||
ip.IsInterfaceLocalMulticast() ||
|
||||
ip.IsMulticast() ||
|
||||
ip.IsPrivate()
|
||||
}
|
||||
|
||||
// Control is a net.Dialer Control function that refuses a private destination.
|
||||
//
|
||||
// It runs after the name has been resolved and before the socket is connected,
|
||||
// which is the only moment where the address that will actually be used is known.
|
||||
func Control(network, address string, _ syscall.RawConn) error {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if Blocked(ip) {
|
||||
return &ErrPrivate{Host: host}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dialer is a net.Dialer that will not reach inside.
|
||||
func Dialer(timeout time.Duration) *net.Dialer {
|
||||
return &net.Dialer{Timeout: timeout, Control: Control}
|
||||
}
|
||||
|
||||
// Client is an http.Client for fetching something a user named.
|
||||
//
|
||||
// Redirects are followed but capped, and every hop goes through the same dialer —
|
||||
// a redirect to 127.0.0.1 is the oldest way around a check that only looked at the
|
||||
// URL somebody typed.
|
||||
func Client(timeout time.Duration) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: &http.Transport{
|
||||
DialContext: Dialer(timeout).DialContext,
|
||||
TLSHandshakeTimeout: timeout,
|
||||
ResponseHeaderTimeout: timeout,
|
||||
},
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return fmt.Errorf("too many redirects")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// AllowPrivate is the same client without the wall, for a destination the
|
||||
// operator configured rather than a user — the panel, and nothing else. Named so
|
||||
// that using it is a decision somebody wrote down.
|
||||
func AllowPrivate(timeout time.Duration) *http.Client {
|
||||
return &http.Client{Timeout: timeout}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package safedial
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBlockedCoversTheInside(t *testing.T) {
|
||||
blocked := []string{
|
||||
"127.0.0.1", "::1", "0.0.0.0",
|
||||
// Skyens metadata-endepunkt. Den ene adresse, enhver SSRF sigter efter.
|
||||
"169.254.169.254",
|
||||
"10.0.0.5", "172.16.0.1", "192.168.1.10", "fd00::1", "fe80::1",
|
||||
}
|
||||
for _, s := range blocked {
|
||||
if !Blocked(net.ParseIP(s)) {
|
||||
t.Errorf("Blocked(%s) = false", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"93.184.216.34", "8.8.8.8", "2606:2800:220:1:248:1893:25c8:1946"} {
|
||||
if Blocked(net.ParseIP(s)) {
|
||||
t.Errorf("Blocked(%s) = true, men det er en almindelig adresse", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// En httptest-server lytter på loopback, hvilket gør den til den perfekte prøve:
|
||||
// den findes, den svarer, og den må ikke kunne nås.
|
||||
func TestClientRefusesLoopback(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("forbindelsen blev oprettet; den skulle være afvist")
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := Client(3 * time.Second).Get(srv.URL)
|
||||
if err == nil {
|
||||
t.Fatal("en forespørgsel til loopback skal afvises")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowPrivateStillReaches(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := AllowPrivate(3 * time.Second).Get(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("den skal kunne nå det, operatøren selv har peget på: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The collector's side of the beacon: what has reported in, and how many.
|
||||
//
|
||||
// Every method here is about counting installations, never about identifying one.
|
||||
// There is no lookup by anything but the id an installation made up for itself,
|
||||
// and there is nothing to join it against.
|
||||
|
||||
// BeaconStats is the whole of what the collector can say.
|
||||
type BeaconStats struct {
|
||||
Total int `json:"total"`
|
||||
Active7d int `json:"active_7d"`
|
||||
Active30d int `json:"active_30d"`
|
||||
ByVersion map[string]int `json:"by_version"`
|
||||
LastPingAt *time.Time `json:"last_ping_at,omitempty"`
|
||||
}
|
||||
|
||||
// maxBeaconInstalls bounds the table.
|
||||
//
|
||||
// The receiving endpoint is unauthenticated — it has to be, since the caller is a
|
||||
// stranger's installation — and it inserts a row keyed on an id the caller made
|
||||
// up. Anybody can therefore mint ids and grow this table until the volume the
|
||||
// database lives on is full. A ceiling turns that from a disk-fill into a number
|
||||
// that is merely wrong, which is the difference between an outage and an
|
||||
// annoyance. A hundred thousand is far above any plausible real count and far
|
||||
// below anything that matters on disk.
|
||||
const maxBeaconInstalls = 100_000
|
||||
|
||||
// RecordBeacon writes down that an installation is alive, and on which version.
|
||||
//
|
||||
// `first_seen` is kept from the original row on purpose: it is the only thing
|
||||
// that distinguishes an installation that has been running for a year from one
|
||||
// that appeared this morning, and it costs nothing to keep.
|
||||
func (db *DB) RecordBeacon(ctx context.Context, instanceID, version string) error {
|
||||
now := time.Now().Unix()
|
||||
|
||||
// An installation already known keeps reporting whatever the ceiling says: the
|
||||
// cap must never stop counting the real ones, only stop new rows from being
|
||||
// invented without limit.
|
||||
var known int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT count(*) FROM beacon_installs WHERE instance_id = ?`, instanceID).Scan(&known); err != nil {
|
||||
return err
|
||||
}
|
||||
if known == 0 {
|
||||
var total int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT count(*) FROM beacon_installs`).Scan(&total); err != nil {
|
||||
return err
|
||||
}
|
||||
if total >= maxBeaconInstalls {
|
||||
// Not an error to the caller. Their installation did nothing wrong, and
|
||||
// there is nothing for them to do about it.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
_, err := db.ExecContext(ctx, `
|
||||
INSERT INTO beacon_installs (instance_id, version, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (instance_id) DO UPDATE SET
|
||||
version = excluded.version,
|
||||
last_seen = excluded.last_seen`,
|
||||
instanceID, version, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
// BeaconCount is how many installations have reported within the window.
|
||||
//
|
||||
// A window rather than a running total, and that is not a detail. A total only
|
||||
// ever climbs: every instance anybody ever started for ten minutes stays in it
|
||||
// forever, so the number stops meaning "how many people use this" and starts
|
||||
// meaning "how many people have ever tried it". The question being asked is the
|
||||
// first one.
|
||||
func (db *DB) BeaconCount(ctx context.Context, within time.Duration) (int, error) {
|
||||
var n int
|
||||
err := db.QueryRowContext(ctx,
|
||||
`SELECT count(*) FROM beacon_installs WHERE last_seen >= ?`,
|
||||
time.Now().Add(-within).Unix()).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// BeaconSummary is everything the settings page shows.
|
||||
func (db *DB) BeaconSummary(ctx context.Context) (BeaconStats, error) {
|
||||
out := BeaconStats{ByVersion: map[string]int{}}
|
||||
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT count(*) FROM beacon_installs`).Scan(&out.Total); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
var err error
|
||||
if out.Active7d, err = db.BeaconCount(ctx, 7*24*time.Hour); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.Active30d, err = db.BeaconCount(ctx, 30*24*time.Hour); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Versions of the ones still alive, not of everything ever seen: an instance
|
||||
// that stopped reporting two years ago should not still be counted as running
|
||||
// the version it was on when it stopped.
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT version, count(*) FROM beacon_installs
|
||||
WHERE last_seen >= ? AND version <> ''
|
||||
GROUP BY version`, time.Now().Add(-30*24*time.Hour).Unix())
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v string
|
||||
var n int
|
||||
if err := rows.Scan(&v, &n); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.ByVersion[v] = n
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
var last *int64
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT max(last_seen) FROM beacon_installs`).Scan(&last); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if last != nil {
|
||||
t := time.Unix(*last, 0)
|
||||
out.LastPingAt = &t
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
-- One row per installation that has ever reported in.
|
||||
--
|
||||
-- This table only exists on the instance that is acting as the collector. Every
|
||||
-- other instance has it too, empty, because the schema is the same everywhere —
|
||||
-- and because "am I the collector" is a setting, not a build.
|
||||
--
|
||||
-- What is deliberately NOT here is the point of the whole feature: there is no
|
||||
-- column for an IP address, a hostname, a domain, an account, a count of tasks,
|
||||
-- or anything else. The two things stored are the two things sent, and the table
|
||||
-- is the promise written down where it cannot drift from the prose — a column
|
||||
-- added here later is a promise broken, and it should be as awkward to add as
|
||||
-- possible.
|
||||
--
|
||||
-- `instance_id` is the primary key rather than a rowid alias, because the same
|
||||
-- installation reporting daily must update its row rather than grow a new one.
|
||||
-- What is wanted is "how many installations", not "how many pings".
|
||||
CREATE TABLE beacon_installs (
|
||||
instance_id TEXT PRIMARY KEY,
|
||||
version TEXT NOT NULL DEFAULT '',
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- The count is always "seen in the last N days", so last_seen is what every query
|
||||
-- sorts and filters on.
|
||||
CREATE INDEX beacon_by_last_seen ON beacon_installs (last_seen DESC);
|
||||
@@ -247,6 +247,37 @@ func (db *DB) SearchNotes(ctx context.Context, query string, limit int) ([]Note,
|
||||
LIMIT ?`, expr, limit)
|
||||
}
|
||||
|
||||
// SetNoteTimes writes the dates a note had somewhere else.
|
||||
//
|
||||
// Only the import calls this, and it exists because SaveNote is right to refuse:
|
||||
// `updated_at` is when this program last wrote the note, and letting any caller
|
||||
// set it would make the column mean whatever the last writer felt like. A note
|
||||
// brought in from Apple Notes is the one case where the dates are facts from
|
||||
// before this database existed — and without them a move of twelve hundred notes
|
||||
// is a pile that was all written the same evening, which throws away the order
|
||||
// they were written in. That order is half of what an archive is.
|
||||
func (db *DB) SetNoteTimes(ctx context.Context, id string, created, updated time.Time) error {
|
||||
if created.IsZero() && updated.IsZero() {
|
||||
return nil
|
||||
}
|
||||
// COALESCE, so a note carrying only one of the two keeps what it already had
|
||||
// for the other rather than being stamped with the zero time — which would sort
|
||||
// it to the year one.
|
||||
var c, u any
|
||||
if !created.IsZero() {
|
||||
c = created.Unix()
|
||||
}
|
||||
if !updated.IsZero() {
|
||||
u = updated.Unix()
|
||||
}
|
||||
_, err := db.ExecContext(ctx, `
|
||||
UPDATE notes
|
||||
SET created_at = COALESCE(?, created_at),
|
||||
updated_at = COALESCE(?, updated_at)
|
||||
WHERE id = ?`, c, u, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteNote puts it in the trash. Nothing here removes a row.
|
||||
func (db *DB) DeleteNote(ctx context.Context, id string) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
|
||||
@@ -31,6 +31,9 @@ import sys
|
||||
from html.parser import HTMLParser
|
||||
|
||||
BLOCK_PREFIX = {"h1": "# ", "h2": "## ", "h3": "### ", "h4": "#### "}
|
||||
# Fra lukkemærke tilbage til åbnemærke, når et mærke skal deles over to linjer.
|
||||
OPENER = {"**": "**", "*": "*", "</u>": "<u>"}
|
||||
MARKS_ONLY = re.compile(r"(?:\*\*|\*|~~|<u>|</u>|\s)*")
|
||||
FORBIDDEN = set('/\\:*?"<>|')
|
||||
|
||||
|
||||
@@ -69,8 +72,26 @@ class NoteParser(HTMLParser):
|
||||
# --- linjer ---------------------------------------------------------------
|
||||
|
||||
def _flush_line(self):
|
||||
# Et mærke, der står åbent, når linjen slutter, lukkes her og åbnes igen på
|
||||
# den næste.
|
||||
#
|
||||
# Noter skriver <b>Møde forberedelse<br></b>: linjeskiftet ligger *inde i*
|
||||
# det fede. Skrev man linjen ud uden videre, blev de to stjerner, der lukker,
|
||||
# til den næste linjes første tegn — "**Møde forberedelse" og så en linje med
|
||||
# "**" på. Markdown er linjebaseret; et mærke, der skal over et linjeskift,
|
||||
# findes ikke, og skal derfor deles i to.
|
||||
reopen = list(self.marks)
|
||||
for closer in reversed(self.marks):
|
||||
self.line.append(closer)
|
||||
|
||||
text = "".join(self.line).rstrip()
|
||||
self.line = []
|
||||
|
||||
# En linje, der kun er mærker, er en tom linje. Det sker for den sidste
|
||||
# stump af et fedt afsnit, hvor der ikke står andet end linjeskiftet.
|
||||
if MARKS_ONLY.fullmatch(text):
|
||||
text = ""
|
||||
|
||||
self.line = [OPENER[c] for c in reopen]
|
||||
mono, self.mono = self.mono, False
|
||||
|
||||
if mono:
|
||||
@@ -121,11 +142,42 @@ class NoteParser(HTMLParser):
|
||||
self.lines.append("")
|
||||
self.mono_lines = []
|
||||
|
||||
def _open(self, opener: str, closer: str):
|
||||
"""Åbner et mærke — eller fortryder det, der lige lukkede.
|
||||
|
||||
Noter deler også fremhævet tekst op i kørsler: "Møde" står som
|
||||
<b>M</b><b>øde</b>, fordi der er en attributkørsel bag hver stump. Skrevet
|
||||
ligeud bliver det **M****øde**, som er ét ord i fed sagt på den grimmest
|
||||
mulige måde — og for kursiv bliver det *M**øde*, hvor de to stjerner i
|
||||
midten læses som fed af enhver Markdown-læser. Ordet kom ud forkert.
|
||||
|
||||
Lukkes et mærke og åbnes det samme igen med det samme, er der ikke sket
|
||||
noget imellem. Så tages lukningen tilbage, og de to kørsler er ét mærke.
|
||||
"""
|
||||
if self.line and self.line[-1] == closer:
|
||||
self.line.pop()
|
||||
else:
|
||||
self.line.append(opener)
|
||||
self.marks.append(closer)
|
||||
|
||||
# --- tags -----------------------------------------------------------------
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in BLOCK_PREFIX:
|
||||
self._flush_line()
|
||||
# En overskrift, der fortsætter den forrige, er den samme linje.
|
||||
#
|
||||
# Noter deler en linje op i så mange stumper, den har brug for — en
|
||||
# overskrift skrevet i ét hug står som
|
||||
# <h1>A</h1><h1>alb</h1><h1>org Kom</h1><h1>mune</h1>, fordi der er en
|
||||
# attributkørsel bag hver stump. Læste man et linjeskift ved hver <h1>,
|
||||
# blev "Aalborg Kommune" til fire overskrifter oven på hinanden, og
|
||||
# noten så ødelagt ud på den mest synlige måde, der findes: i sin egen
|
||||
# titel.
|
||||
#
|
||||
# Der brydes derfor kun, når blokken skifter til en anden slags. Selve
|
||||
# linjeskiftene kommer fra <div> og <br>, som er dem, der betyder det.
|
||||
if self.block != tag:
|
||||
self._flush_line()
|
||||
self.block = tag
|
||||
elif tag in ("ul", "ol"):
|
||||
self._flush_line()
|
||||
@@ -138,23 +190,24 @@ class NoteParser(HTMLParser):
|
||||
# Mærket sættes på linjen og aflæses, når den skrives ud.
|
||||
self.mono = True
|
||||
elif tag in ("b", "strong"):
|
||||
self.marks.append("**")
|
||||
self.line.append("**")
|
||||
self._open("**", "**")
|
||||
elif tag in ("i", "em"):
|
||||
self.marks.append("*")
|
||||
self.line.append("*")
|
||||
self._open("*", "*")
|
||||
elif tag == "u":
|
||||
self.marks.append("</u>")
|
||||
self.line.append("<u>")
|
||||
self._open("<u>", "</u>")
|
||||
elif tag == "br":
|
||||
self._flush_line()
|
||||
self.block = None
|
||||
elif tag == "div":
|
||||
self._flush_line()
|
||||
self.block = None
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in BLOCK_PREFIX:
|
||||
self._flush_line()
|
||||
self.block = None
|
||||
# Ikke her. Lukkes overskriften ved sit eget sluttag, kan den næste
|
||||
# stump af samme linje ikke se, at den hører til — det er <div> og <br>,
|
||||
# der afgør, hvor linjen holder op.
|
||||
pass
|
||||
elif tag in ("ul", "ol"):
|
||||
self._flush_line()
|
||||
if self.list_kind:
|
||||
@@ -172,6 +225,7 @@ class NoteParser(HTMLParser):
|
||||
self.line.append(self.marks.pop())
|
||||
elif tag == "div":
|
||||
self._flush_line()
|
||||
self.block = None
|
||||
|
||||
def handle_data(self, data):
|
||||
self.line.append(data)
|
||||
@@ -208,6 +262,95 @@ def safe_name(title: str, fallback: str) -> str:
|
||||
return name[:80] or fallback
|
||||
|
||||
|
||||
# Skilletegn, der ikke står i en note: enhedsadskillere fra ASCII, som ingen
|
||||
# skriver med et tastatur.
|
||||
FIELD, RECORD = "\u241f", "\u241e"
|
||||
|
||||
|
||||
def osascript_file(script: str) -> str:
|
||||
"""Som osascript(), men til et manuskript, der er for stort til -e."""
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".applescript", delete=False,
|
||||
encoding="utf-8") as f:
|
||||
f.write(script)
|
||||
path = f.name
|
||||
try:
|
||||
out = subprocess.run(["osascript", path], capture_output=True, text=True,
|
||||
timeout=600, check=False)
|
||||
return out.stdout
|
||||
except subprocess.SubprocessError:
|
||||
return ""
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def fetch(ids: list[str]) -> list[dict]:
|
||||
"""Navn, krop og datoer for en håndfuld noter i ét kald.
|
||||
|
||||
Ét kald pr. note var det oplagte, og det tog halvanden time for tolv hundrede
|
||||
— ikke fordi Noter er langsom, men fordi hvert kald er en proces, der skal
|
||||
startes. Det gjorde det umuligt at rette en fejl i oversættelsen og se
|
||||
resultatet: hver runde var en frokostpause. Nu hentes de i klumper, og hele
|
||||
eksporten tager minutter.
|
||||
|
||||
Klumper frem for alle på én gang, fordi en enkelt uleselig note ellers ville
|
||||
tage hele kørslen med sig — og fordi svaret skal kunne være i hukommelsen.
|
||||
|
||||
Datoerne læses som tal frem for som tekst. `creation date` giver en dato
|
||||
formateret efter maskinens sprog, og den ville skulle gættes tilbage; year,
|
||||
month og day er de samme tal på enhver maskine.
|
||||
"""
|
||||
quoted = ", ".join('"' + i.replace('"', '\\"') + '"' for i in ids)
|
||||
script = f'''
|
||||
tell application "Notes"
|
||||
set out to ""
|
||||
repeat with theId in {{{quoted}}}
|
||||
try
|
||||
set n to note id theId
|
||||
set c to creation date of n
|
||||
set m to modification date of n
|
||||
set out to out & (name of n) & "{FIELD}" & ¬
|
||||
((year of c) & "-" & (month of c as integer) & "-" & (day of c) & "T" & ¬
|
||||
(hours of c) & ":" & (minutes of c) & ":" & (seconds of c)) & "{FIELD}" & ¬
|
||||
((year of m) & "-" & (month of m as integer) & "-" & (day of m) & "T" & ¬
|
||||
(hours of m) & ":" & (minutes of m) & ":" & (seconds of m)) & "{FIELD}" & ¬
|
||||
(count of attachments of n) & "{FIELD}" & ¬
|
||||
(body of n) & "{RECORD}"
|
||||
on error
|
||||
set out to out & "{FIELD}{FIELD}{FIELD}{FIELD}" & "{RECORD}"
|
||||
end try
|
||||
end repeat
|
||||
return out
|
||||
end tell
|
||||
'''
|
||||
raw = osascript_file(script)
|
||||
rows = []
|
||||
for chunk in raw.split(RECORD):
|
||||
if not chunk.strip("\n "):
|
||||
continue
|
||||
parts = chunk.split(FIELD)
|
||||
if len(parts) < 5:
|
||||
rows.append({"name": "", "created": "", "modified": "", "files": 0, "body": ""})
|
||||
continue
|
||||
rows.append({
|
||||
"name": parts[0].lstrip("\n"), "created": parts[1], "modified": parts[2],
|
||||
"files": int(parts[3]) if parts[3].strip().isdigit() else 0,
|
||||
"body": parts[4],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def iso(stamp: str) -> str:
|
||||
""""2026-8-17T12:55:14" → "2026-08-17T12:55:14". Nul foran, så det sorterer."""
|
||||
try:
|
||||
date, _, time = stamp.partition("T")
|
||||
y, mo, d = (int(x) for x in date.split("-"))
|
||||
h, mi, se = (int(x) for x in time.split(":"))
|
||||
return f"{y:04d}-{mo:02d}-{d:02d}T{h:02d}:{mi:02d}:{se:02d}"
|
||||
except (ValueError, AttributeError):
|
||||
return ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
out_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/Desktop/apple-noter")
|
||||
limit = int(sys.argv[2]) if len(sys.argv) > 2 else 0
|
||||
@@ -238,16 +381,15 @@ def main() -> int:
|
||||
used: dict[str, int] = {}
|
||||
attachments_dir = os.path.join(out_dir, "vedhaeftninger")
|
||||
|
||||
for i, note_id in enumerate(ids, start=1):
|
||||
# Navn og krop i ét kald, adskilt af noget, der ikke står i en note.
|
||||
# Hentet hver for sig kunne de nå at komme fra hver sin note.
|
||||
both = osascript(
|
||||
f'tell application "Notes" to tell note id "{note_id}" '
|
||||
f'to return name & "\u241E" & body'
|
||||
)
|
||||
title, _, body = both.partition("\u241E")
|
||||
title = title.strip() or f"uden titel {i}"
|
||||
text = to_markdown(body)
|
||||
CHUNK = 40
|
||||
fetched: list[dict] = []
|
||||
for start in range(0, len(ids), CHUNK):
|
||||
fetched.extend(fetch(ids[start:start + CHUNK]))
|
||||
print(f" hentet {min(start + CHUNK, len(ids))} / {total}")
|
||||
|
||||
for i, (note_id, row) in enumerate(zip(ids, fetched), start=1):
|
||||
title = row["name"].strip() or f"uden titel {i}"
|
||||
text = to_markdown(row["body"])
|
||||
|
||||
# Kroppen begynder med notens egen titel. Skrives der en overskrift
|
||||
# ovenover, står titlen to gange i hver eneste note.
|
||||
@@ -293,18 +435,34 @@ def main() -> int:
|
||||
used[name] = used.get(name, 0) + 1
|
||||
path = os.path.join(out_dir, name if used[name] == 1 else f"{name} ({used[name]})") + ".md"
|
||||
|
||||
# Datoerne i et hoved, som enhver anden Markdown-læser kender.
|
||||
#
|
||||
# Uden dem er en flytning af tolv hundrede noter en bunke, der alle er
|
||||
# skrevet samme aften: rækkefølgen, som er halvdelen af hvad et arkiv er,
|
||||
# findes ikke længere. Obsidian, Bear og alt andet læser den her blok, og
|
||||
# verdandes egen import gør nu også.
|
||||
created, modified = iso(row["created"]), iso(row["modified"])
|
||||
head = ""
|
||||
if created or modified:
|
||||
head = "---\n"
|
||||
if created:
|
||||
head += f"created: {created}\n"
|
||||
if modified:
|
||||
head += f"modified: {modified}\n"
|
||||
head += "---\n\n"
|
||||
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(f"# {title}\n\n{text}\n")
|
||||
f.write(f"{head}# {title}\n\n{text}\n")
|
||||
|
||||
# Billeder og bilag. En note fra Noter er ofte fuld af fotografier, og en
|
||||
# note, der kommer over som ord alene, er ikke den note.
|
||||
count = osascript(
|
||||
f'tell application "Notes" to return count of attachments of note id "{note_id}"'
|
||||
)
|
||||
if count.isdigit() and int(count) > 0:
|
||||
# Antallet kom med i klumpen. Det var ét kald pr. note, og da kun hver
|
||||
# tiende note har et bilag, var de ni af dem en proces startet for at få
|
||||
# svaret nul.
|
||||
if row["files"] > 0:
|
||||
os.makedirs(attachments_dir, exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
for j in range(1, int(count) + 1):
|
||||
for j in range(1, row["files"] + 1):
|
||||
a_name = osascript(
|
||||
f'tell application "Notes" to return name of attachment {j} '
|
||||
f'of note id "{note_id}"'
|
||||
|
||||
+47
-1
@@ -655,7 +655,15 @@ test('et invitationslink opretter kontoen og giver adgang til projektet', async
|
||||
|
||||
await page.getByLabel('Ny opgave').fill('males inden fredag #Fælleshuset');
|
||||
await page.getByLabel('Ny opgave').press('Enter');
|
||||
await page.getByText('males inden fredag', { exact: true }).click();
|
||||
// Titlen alene, ikke rækken med datomærket i.
|
||||
//
|
||||
// Hurtig tilføjelse læser "fredag" som forfaldsdatoen og tager ordet ud af
|
||||
// titlen, så rækken er "males inden" plus et mærke, der siger hvornår. Mærkets
|
||||
// ord skifter med ugedagen — "fredag", når der er langt, og "I morgen", når man
|
||||
// kører prøven på en torsdag. Den her linje ledte efter hele rækken og fandt
|
||||
// den derfor ikke én dag om ugen. Prøven handler om invitationer, ikke om
|
||||
// datoer; den skal ikke vide hvilken dag det er.
|
||||
await page.getByText('males inden', { exact: true }).click();
|
||||
|
||||
const drawer = page.getByRole('complementary', { name: 'Opgave' });
|
||||
await drawer.getByLabel('Ansvarlig').selectOption({ label: 'Nabo' });
|
||||
@@ -2388,3 +2396,41 @@ test('lister overlever en gemning — også i et citat, og også med numre', asy
|
||||
|
||||
expect(trouble).toEqual([]);
|
||||
});
|
||||
|
||||
test('en note kan ikke smugle et script ind gennem et billedes alt-tekst', async ({ page }) => {
|
||||
const trouble = watchForTrouble(page);
|
||||
await page.goto('/noter');
|
||||
|
||||
// Undslap `alt` ikke sine anførselstegn, lukkede den første af dem attributten,
|
||||
// og resten stod som markup: et img med en onerror på. En note deles gennem et
|
||||
// projekt, så det ville køre på vores eget domæne, hos den der åbnede noten.
|
||||
const id = await page.evaluate(async () => {
|
||||
const r = await fetch('/api/v1/notes', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json', 'Sec-Fetch-Site': 'same-origin' },
|
||||
body: JSON.stringify({
|
||||
body: '# Ondsindet\n\n\n'
|
||||
})
|
||||
});
|
||||
return (await r.json()).id;
|
||||
});
|
||||
|
||||
await page.goto('/noter?note=' + id);
|
||||
const ed = page.getByRole('textbox', { name: 'Notens tekst' });
|
||||
await expect(ed.locator('img')).toHaveCount(1);
|
||||
// Billedet peger ingen steder, så det fejler at indlæse — hvilket er præcis
|
||||
// den vej, en onerror ville blive kaldt ad.
|
||||
await page.waitForTimeout(800);
|
||||
|
||||
expect(await page.evaluate(() => window.__pwned)).toBeUndefined();
|
||||
// Og hændelsen står som tekst i alt-attributten, hvor den hører hjemme.
|
||||
const alt = await ed.locator('img').getAttribute('alt');
|
||||
expect(alt).toContain('onerror');
|
||||
expect(await ed.locator('img').evaluate((el) => el.hasAttribute('onerror'))).toBe(false);
|
||||
|
||||
// Billedets egen 404 hører til prøven: adressen peger med vilje på et bilag,
|
||||
// der ikke findes, fordi det er den vej en onerror ville blive kaldt ad. Alt
|
||||
// andet skal stadig være stille.
|
||||
expect(trouble.filter((t) => !t.includes('/api/v1/attachments/'))).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -227,6 +227,9 @@ export const api = {
|
||||
gmailClient: () => get('/gmail/client'),
|
||||
setGmailClient: (data) => put('/gmail/client', data),
|
||||
|
||||
beacon: () => get('/beacon/settings'),
|
||||
setBeacon: (body) => put('/beacon/settings', body),
|
||||
|
||||
listBackups: () => get('/backups'),
|
||||
runBackup: () => post('/backups'),
|
||||
// A link rather than a fetch: the browser's own download handles a file of
|
||||
|
||||
@@ -805,6 +805,16 @@
|
||||
margin-bottom: var(--s1);
|
||||
}
|
||||
|
||||
/* Et billede fylder sin egen bredde og ikke mere. Et foto fra en telefon er
|
||||
fire tusind pixels bredt, og uden det her stikker det ud af arket. */
|
||||
.page :global(img) {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
border-radius: var(--radius-sm);
|
||||
display: block;
|
||||
margin: var(--s2) 0;
|
||||
}
|
||||
|
||||
.page :global(blockquote) {
|
||||
margin: 0 0 var(--s2);
|
||||
padding-left: var(--s3);
|
||||
|
||||
@@ -508,14 +508,22 @@
|
||||
sidebar is already a drop target for tasks — a task dragged onto I dag
|
||||
stopped landing, because the row had claimed the gesture. A grip claims
|
||||
nothing until it is grabbed. -->
|
||||
<!-- Håndtaget har ingen prik i sig længere.
|
||||
⠿ sad i den to pixels brede rende til venstre for teksten og lå oven i
|
||||
rækkens egen kant — idéen var rigtig, pladsen var det ikke. Men elementet
|
||||
bliver: det er dét, der ejer trækket, og gjorde man i stedet hele rækken
|
||||
draggable, ville alt indeni også blive det — og en opgave trukket hen på
|
||||
I dag holdt op med at lande, fordi rækken havde taget gesten. Nu er
|
||||
håndtaget usynligt og lidt bredere, og hånden, der kommer ved hover,
|
||||
er hele forklaringen. -->
|
||||
{#snippet grip(key)}
|
||||
<span
|
||||
class="grip"
|
||||
draggable="true"
|
||||
ondragstart={(e) => onNavDragStart(e, key)}
|
||||
ondragend={clearNavDrag}
|
||||
aria-hidden="true">⠿</span
|
||||
>
|
||||
aria-hidden="true"
|
||||
></span>
|
||||
{/snippet}
|
||||
|
||||
{#snippet projectRow(project, sortable)}
|
||||
@@ -1004,19 +1012,15 @@
|
||||
so nothing moves when it appears. */
|
||||
.grip {
|
||||
position: absolute;
|
||||
left: -2px;
|
||||
width: 12px;
|
||||
text-align: center;
|
||||
font-size: 10px;
|
||||
line-height: 1;
|
||||
color: var(--ink-faint);
|
||||
left: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 14px;
|
||||
cursor: grab;
|
||||
opacity: 0;
|
||||
transition: opacity var(--fast) var(--ease);
|
||||
}
|
||||
|
||||
.navrow:hover .grip {
|
||||
opacity: 1;
|
||||
.grip:active {
|
||||
cursor: grabbing;
|
||||
}
|
||||
|
||||
.navrow > :global(a) {
|
||||
|
||||
@@ -25,6 +25,25 @@ export const da = {
|
||||
'notes.bold': 'Fed',
|
||||
'notes.italic': 'Kursiv',
|
||||
'notes.underline': 'Understreget',
|
||||
'notes.sortBy': 'Rækkefølge',
|
||||
'notes.sortUpdated': 'Senest rørt',
|
||||
'notes.sortCreated': 'Oprettet',
|
||||
'notes.sortTitle': 'Navn',
|
||||
'beacon.title': 'Tælling af installationer',
|
||||
'beacon.hint': 'Én gang i døgnet melder denne installation, at den findes, så projektet kan se hvor mange der bruger det. Der sendes **kun** de to værdier herunder — et tilfældigt id og versionen — og intet andet: ingen IP bliver gemt, ingen navne, ingen adresser, intet om hvad du bruger det til. Slået til som udgangspunkt, og du kan slå den fra her.',
|
||||
'beacon.send': 'Send en anonym daglig melding',
|
||||
'beacon.exactly': 'Præcis det her — ikke mere:',
|
||||
'beacon.lastPing': 'Sidst sendt: {when}',
|
||||
'beacon.collector': 'Modtagerens adresse',
|
||||
'beacon.collectorHint': 'Hvor meldingen sendes hen. Lad den stå, medmindre du kører din egen tæller.',
|
||||
'beacon.beCollector': 'Denne instans er tælleren',
|
||||
'beacon.receiving': 'Modtager anonyme meldinger — id og version, aldrig en IP.',
|
||||
'beacon.total': 'I alt',
|
||||
'beacon.active7': 'Aktive 7 dage',
|
||||
'beacon.active30': 'Aktive 30 dage',
|
||||
'beacon.byVersion': 'Efter version (30 dage):',
|
||||
'beacon.publish': 'Offentliggør tallet på /api/v1/beacon/count',
|
||||
'beacon.publishHint': 'Uden login, mellemlagret et minut. Viser installationer set de seneste 30 dage — ikke et løbende total, som kun ville stige.',
|
||||
'notes.strike': 'Gennemstreget',
|
||||
'notes.showSource': 'Vis som kode',
|
||||
'notes.source': 'Notens kildetekst',
|
||||
|
||||
@@ -21,6 +21,25 @@ export const en = {
|
||||
'notes.bold': 'Bold',
|
||||
'notes.italic': 'Italic',
|
||||
'notes.underline': 'Underline',
|
||||
'notes.sortBy': 'Order',
|
||||
'notes.sortUpdated': 'Last touched',
|
||||
'notes.sortCreated': 'Created',
|
||||
'notes.sortTitle': 'Name',
|
||||
'beacon.title': 'Counting installations',
|
||||
'beacon.hint': 'Once a day this installation reports that it exists, so the project can see how many people run it. It sends **only** the two values below — a random id and the version — and nothing else: no IP is stored, no names, no addresses, nothing about what you use it for. On by default, and you can turn it off here.',
|
||||
'beacon.send': 'Send an anonymous daily report',
|
||||
'beacon.exactly': 'Exactly this — nothing more:',
|
||||
'beacon.lastPing': 'Last sent: {when}',
|
||||
'beacon.collector': 'Collector address',
|
||||
'beacon.collectorHint': 'Where the report goes. Leave it unless you run your own collector.',
|
||||
'beacon.beCollector': 'This instance is the collector',
|
||||
'beacon.receiving': 'Receiving anonymous reports — id and version, never an IP.',
|
||||
'beacon.total': 'Total',
|
||||
'beacon.active7': 'Active 7 days',
|
||||
'beacon.active30': 'Active 30 days',
|
||||
'beacon.byVersion': 'By version (30 days):',
|
||||
'beacon.publish': 'Publish the count at /api/v1/beacon/count',
|
||||
'beacon.publishHint': 'No login, cached for a minute. Reports installations seen in the last 30 days — not a running total, which would only ever climb.',
|
||||
'notes.strike': 'Strikethrough',
|
||||
'notes.showSource': 'Show as code',
|
||||
'notes.source': 'Note source',
|
||||
|
||||
+59
-2
@@ -37,14 +37,61 @@ const BLOCKS = [
|
||||
const escapeHtml = (s) =>
|
||||
s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
|
||||
/**
|
||||
* En værdi, der skal stå inde i en attribut.
|
||||
*
|
||||
* escapeHtml er skrevet til tekst mellem to tags, hvor et anførselstegn ikke
|
||||
* betyder noget, og lader dem derfor stå. Inde i en attribut lukker de den — og
|
||||
* så står resten af strengen som markup. Det er den ene forskel, og den er hele
|
||||
* forskellen mellem tekst og et onerror.
|
||||
*/
|
||||
const attr = (s) => escapeHtml(s).replace(/"/g, '"').replace(/'/g, ''');
|
||||
|
||||
/**
|
||||
* Billeder løftes ud af teksten, før noget andet rører den.
|
||||
*
|
||||
* De stod før først i kæden, og så kørte de fire resterende regler hen over en
|
||||
* streng, der nu indeholdt rigtig HTML — ind i den `alt`, der lige var lavet.
|
||||
* Det gik godt, så længe ingen af erstatningerne indeholdt et anførselstegn, og
|
||||
* det er en betingelse, der stod skrevet ingen steder: den dag nogen tilføjer en
|
||||
* regel for links, er den brudt, og det er præcis den vej, XSS-en kom ind ad.
|
||||
*
|
||||
* Værre endnu var undslupningen. `escapeHtml` kørte på hele teksten, og `attr`
|
||||
* kørte den igen på `alt` — så et og-tegn blev til en dobbelt undslupning, der
|
||||
* voksede ved hver eneste åbning og gemning. En note med "Kaffe & te" i et
|
||||
* billede blev længere, hver gang nogen så på den.
|
||||
*
|
||||
* En pladsholder løser begge dele. Billedet tages ud af den rå tekst, resten
|
||||
* undslippes og formateres som altid, og billedet lægges tilbage til sidst med en
|
||||
* `alt`, der er undsluppet nøjagtig én gang. Rækkefølgen af de øvrige regler kan
|
||||
* så ikke længere betyde noget.
|
||||
*
|
||||
* Nul-tegnet som mærke, fordi det er det ene tegn, en note ikke kan indeholde:
|
||||
* det overlever hverken en contenteditable eller en Markdown-fil.
|
||||
*/
|
||||
const IMAGE = /!\[([^\]]*)\]\((\/api\/v1\/attachments\/[0-9a-f-]+)\)/g;
|
||||
const MARK = "\u0000";
|
||||
|
||||
/** Inline marks, innermost first so a bold word inside italics survives both. */
|
||||
function inlineToHtml(text) {
|
||||
return escapeHtml(text)
|
||||
// Kun vores egen adresse bliver til et billede. Et `` fra en
|
||||
// indsat tekst ville ellers hente fra en fremmed vært, når noten åbnes, og
|
||||
// fortælle den, at den er blevet læst — og en note er det sidste sted, man vil
|
||||
// have en sporingspixel. Alt andet står som den tekst, det er.
|
||||
const images = [];
|
||||
const lifted = (text ?? '').replace(IMAGE, (_, alt, src) => {
|
||||
images.push(`<img src="${src}" alt="${attr(alt)}">`);
|
||||
return `${MARK}${images.length - 1}${MARK}`;
|
||||
});
|
||||
|
||||
const marked = escapeHtml(lifted)
|
||||
.replace(/<u>(.+?)<\/u>/g, '<u>$1</u>')
|
||||
.replace(/`([^`]+)`/g, '<code>$1</code>')
|
||||
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
|
||||
.replace(/~~(.+?)~~/g, '<s>$1</s>')
|
||||
.replace(/(^|[^*])\*([^*]+)\*/g, '$1<em>$2</em>');
|
||||
|
||||
return marked.replace(new RegExp(MARK + '(\\d+)' + MARK, 'g'), (_, i) => images[Number(i)] ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,7 +154,12 @@ export function markdownToHtml(markdown) {
|
||||
if (fenced) {
|
||||
closeList();
|
||||
fence = fenced[1] ?? '';
|
||||
out.push(`<pre data-lang="${escapeHtml(fence)}"><code>`);
|
||||
// attr, ikke escapeHtml: værdien står inde i en attribut. Den er sikker i
|
||||
// dag alene fordi hegn-regexen er \w*, som ikke rummer anførselstegn — og
|
||||
// den dag nogen udvider den for at tage imod "c++" eller "f#", er det en
|
||||
// XSS. Undslupningen skal passe til stedet, ikke til det, der
|
||||
// tilfældigvis kan stå der nu.
|
||||
out.push(`<pre data-lang="${attr(fence)}"><code>`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -184,6 +236,11 @@ function inlineToMarkdown(node) {
|
||||
|
||||
const inner = [...node.childNodes].map(inlineToMarkdown).join('');
|
||||
switch (node.tagName) {
|
||||
// Et billede har ingen tekst i sig, så uden det her læses det som ingenting
|
||||
// — og den første gemning ville tage hvert billede ud af noten uden at
|
||||
// nogen havde rørt det.
|
||||
case 'IMG':
|
||||
return ` ?? ''})`;
|
||||
case 'STRONG':
|
||||
case 'B':
|
||||
return inner ? `**${inner}**` : '';
|
||||
|
||||
@@ -35,10 +35,29 @@
|
||||
* A flat list of two hundred rows says when nothing. "I dag" and "I går" are
|
||||
* how anybody looking for something they just closed would describe it.
|
||||
*/
|
||||
/**
|
||||
* Dagen en opgave blev lukket, målt hvor personen står.
|
||||
*
|
||||
* Ikke `completed_at.slice(0, 10)`. Tidsstemplet er i UTC, og de ti første tegn
|
||||
* er derfor UTC's dato — mens overskriften nedenfor sammenligner med *lokal*
|
||||
* dato. I to timer efter midnat i dansk sommertid er de to ikke den samme dag,
|
||||
* så en opgave lukket 00.30 lå under "I går", indtil klokken blev to.
|
||||
*
|
||||
* Fundet af prøven, som kørte forbi midnat for første gang. Den havde været
|
||||
* grøn hver eneste gang før, fordi de to datoer er ens tyve timer i døgnet.
|
||||
*/
|
||||
const localDay = (iso) => {
|
||||
if (!iso) return '';
|
||||
const d = new Date(iso);
|
||||
if (Number.isNaN(d.getTime())) return '';
|
||||
const pad = (n) => String(n).padStart(2, '0');
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
||||
};
|
||||
|
||||
let days = $derived.by(() => {
|
||||
const groups = new Map();
|
||||
for (const task of done) {
|
||||
const key = (task.completed_at ?? '').slice(0, 10);
|
||||
const key = localDay(task.completed_at);
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(task);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,29 @@
|
||||
import { app } from '$lib/stores.svelte.js';
|
||||
import { goto } from '$app/navigation';
|
||||
import { ago } from '$lib/when.js';
|
||||
import { t } from '$lib/i18n.svelte.js';
|
||||
import { t, tag } from '$lib/i18n.svelte.js';
|
||||
|
||||
// --- beaconet -----------------------------------------------------------------
|
||||
|
||||
// Hvad denne installation melder, og til hvem. Se internal/beacon for hvorfor
|
||||
// den er slået til som udgangspunkt frem for at være obligatorisk.
|
||||
let beacon = $state(null);
|
||||
|
||||
$effect(() => {
|
||||
if (!app.user?.is_admin) return;
|
||||
api.beacon().then((r) => (beacon = r)).catch(() => {});
|
||||
});
|
||||
|
||||
async function saveBeacon(change) {
|
||||
try {
|
||||
beacon = await api.setBeacon(change);
|
||||
} catch (e) {
|
||||
app.toast(humanMessage(e));
|
||||
// Læst forfra, så en afvist ændring ikke bliver stående som et flueben,
|
||||
// der ikke svarer til, hvad serveren gør.
|
||||
api.beacon().then((r) => (beacon = r)).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// --- backups ---------------------------------------------------------------------
|
||||
|
||||
@@ -486,6 +508,95 @@
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<!-- Beaconet.
|
||||
Under Data, fordi det er her alt om "hvad forlader denne instans" står — en
|
||||
sikkerhedskopi, en eksport, en import. Telemetri hører til i samme rude som
|
||||
dem, ikke gemt væk under en fane, ingen åbner. -->
|
||||
<section class="panel">
|
||||
<header>
|
||||
<h2>{t('beacon.title')}</h2>
|
||||
<p class="hint">{t('beacon.hint')}</p>
|
||||
</header>
|
||||
|
||||
{#if beacon}
|
||||
<label class="check">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={beacon.enabled}
|
||||
onchange={(e) => saveBeacon({ enabled: e.currentTarget.checked })}
|
||||
/>
|
||||
<span>{t('beacon.send')}</span>
|
||||
</label>
|
||||
|
||||
<!-- Hele beskeden, ord for ord. Et løfte om telemetri er præcis så meget
|
||||
værd som læserens mulighed for at efterprøve det, så den står her frem
|
||||
for som en sætning om, hvad der cirka bliver sendt. -->
|
||||
<pre class="payload">{t('beacon.exactly')}
|
||||
{JSON.stringify({ instance_id: beacon.instance_id, version: beacon.version }, null, 2)}</pre>
|
||||
|
||||
{#if beacon.last_ping_at}
|
||||
<p class="hint">{t('beacon.lastPing', { when: new Date(beacon.last_ping_at).toLocaleString(tag()) })}</p>
|
||||
{/if}
|
||||
|
||||
<div class="field">
|
||||
<label for="collector">{t('beacon.collector')}</label>
|
||||
<input
|
||||
id="collector"
|
||||
value={beacon.collector_url}
|
||||
onchange={(e) => saveBeacon({ collector_url: e.currentTarget.value })}
|
||||
/>
|
||||
<p class="hint">{t('beacon.collectorHint')}</p>
|
||||
</div>
|
||||
|
||||
<label class="check">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={beacon.is_collector}
|
||||
onchange={(e) => saveBeacon({ is_collector: e.currentTarget.checked })}
|
||||
/>
|
||||
<span>{t('beacon.beCollector')}</span>
|
||||
</label>
|
||||
|
||||
{#if beacon.is_collector && beacon.stats}
|
||||
<p class="hint">{t('beacon.receiving')}</p>
|
||||
<div class="counts">
|
||||
<div class="count">
|
||||
<strong>{beacon.stats.total}</strong>
|
||||
<span>{t('beacon.total')}</span>
|
||||
</div>
|
||||
<div class="count">
|
||||
<strong>{beacon.stats.active_7d}</strong>
|
||||
<span>{t('beacon.active7')}</span>
|
||||
</div>
|
||||
<div class="count">
|
||||
<strong>{beacon.stats.active_30d}</strong>
|
||||
<span>{t('beacon.active30')}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{#if Object.keys(beacon.stats.by_version ?? {}).length}
|
||||
<p class="hint">
|
||||
{t('beacon.byVersion')}
|
||||
{Object.entries(beacon.stats.by_version)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(([v, n]) => `${v} ×${n}`)
|
||||
.join(', ')}
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<label class="check">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={beacon.publish_count}
|
||||
onchange={(e) => saveBeacon({ publish_count: e.currentTarget.checked })}
|
||||
/>
|
||||
<span>{t('beacon.publish')}</span>
|
||||
</label>
|
||||
<p class="hint">{t('beacon.publishHint')}</p>
|
||||
{/if}
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
/* The backup list, which is a list of rows with a right-hand action — the same
|
||||
shape the settings pages use elsewhere, said once here because this is the
|
||||
@@ -596,4 +707,48 @@
|
||||
.link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Beskeden som den sendes. Monospace, fordi den skal kunne sammenlignes tegn
|
||||
for tegn med det, der står i dokumentationen. */
|
||||
.payload {
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--text-xs);
|
||||
background: var(--surface-sunken);
|
||||
border: 1px solid var(--line);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: var(--s2) var(--s3);
|
||||
overflow-x: auto;
|
||||
margin: var(--s2) 0;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.counts {
|
||||
display: flex;
|
||||
gap: var(--s2);
|
||||
margin: var(--s2) 0;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.count {
|
||||
flex: 1 1 7rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
padding: var(--s3);
|
||||
border: 1px solid var(--line);
|
||||
border-radius: var(--radius);
|
||||
}
|
||||
|
||||
.count strong {
|
||||
font-size: var(--text-xl);
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.count span {
|
||||
font-size: var(--text-xs);
|
||||
color: var(--ink-faint);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -19,6 +19,50 @@
|
||||
import NoteEditor from '$lib/components/NoteEditor.svelte';
|
||||
|
||||
let notes = $state([]);
|
||||
|
||||
/**
|
||||
* Rækkefølgen, og hvorfor den kan vælges.
|
||||
*
|
||||
* Serveren svarer i én rækkefølge — senest rørt øverst — og det er den rigtige,
|
||||
* når man arbejder: den note, man var i gang med, er den, man skal tilbage til.
|
||||
* Den er også den eneste, der ikke kan svare på "hvad skrev jeg dengang".
|
||||
* Tolv hundrede noter flyttet ind fra et andet program er et arkiv, og et arkiv
|
||||
* læses efter, hvornår noget blev skrevet, og efter navn.
|
||||
*
|
||||
* Sorteret her frem for i en forespørgsel: hele listen er allerede hentet, den
|
||||
* fylder ingenting, og et valg, der ikke koster et kald, kan skiftes så hurtigt
|
||||
* som man kan ombestemme sig.
|
||||
*/
|
||||
const ORDERS = [
|
||||
{ key: 'updated', label: 'notes.sortUpdated' },
|
||||
{ key: 'created', label: 'notes.sortCreated' },
|
||||
{ key: 'title', label: 'notes.sortTitle' }
|
||||
];
|
||||
|
||||
let order = $state('updated');
|
||||
|
||||
$effect(() => {
|
||||
const saved = localStorage.getItem('notes.order');
|
||||
if (saved && ORDERS.some((o) => o.key === saved)) order = saved;
|
||||
});
|
||||
|
||||
function setOrder(next) {
|
||||
order = next;
|
||||
localStorage.setItem('notes.order', next);
|
||||
}
|
||||
|
||||
// Favoritter står øverst uanset rækkefølge. Det er hele meningen med at gøre en
|
||||
// note til favorit, og en sortering, der blandede dem ind igen, ville tage den.
|
||||
let ordered = $derived(
|
||||
[...notes].sort((a, b) => {
|
||||
if (a.pinned !== b.pinned) return a.pinned ? -1 : 1;
|
||||
if (order === 'title') {
|
||||
return (a.title || '').localeCompare(b.title || '', 'da', { sensitivity: 'base' });
|
||||
}
|
||||
const field = order === 'created' ? 'created_at' : 'updated_at';
|
||||
return new Date(b[field]).getTime() - new Date(a[field]).getTime();
|
||||
})
|
||||
);
|
||||
let selected = $state(null);
|
||||
let query = $state('');
|
||||
let status = $state('loading');
|
||||
@@ -69,6 +113,10 @@
|
||||
*/
|
||||
function plain(body) {
|
||||
return (body ?? '')
|
||||
// Et billede har ingen ord i sig. Uden det her stod der
|
||||
// "]*\)/g, '')
|
||||
.replace(/^#{1,6}\s+/gm, '')
|
||||
.replace(/^\s*[-*+]\s+/gm, '')
|
||||
.replace(/^\s*>\s?/gm, '')
|
||||
@@ -233,7 +281,22 @@
|
||||
<aside>
|
||||
<div class="head">
|
||||
<h1>{t('notes.title')}</h1>
|
||||
<button class="new" onclick={create} aria-label={t('notes.new')}>+</button>
|
||||
<div class="tools">
|
||||
<!-- En select frem for tre knapper: rækkefølgen vælges sjældent, og tre
|
||||
knapper ville tage plads fra listen hver eneste dag for at spare et
|
||||
klik en gang imellem. -->
|
||||
<select
|
||||
class="order"
|
||||
value={order}
|
||||
onchange={(e) => setOrder(e.currentTarget.value)}
|
||||
aria-label={t('notes.sortBy')}
|
||||
>
|
||||
{#each ORDERS as o}
|
||||
<option value={o.key}>{t(o.label)}</option>
|
||||
{/each}
|
||||
</select>
|
||||
<button class="new" onclick={create} aria-label={t('notes.new')}>+</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<input
|
||||
@@ -250,7 +313,7 @@
|
||||
<p class="hint">{query ? t('notes.noneFound') : t('notes.noneYet')}</p>
|
||||
{:else}
|
||||
<ul>
|
||||
{#each notes as note (note.id)}
|
||||
{#each ordered as note (note.id)}
|
||||
<li>
|
||||
<div class="rowline">
|
||||
<button
|
||||
@@ -372,6 +435,23 @@
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: var(--s2);
|
||||
}
|
||||
|
||||
.tools {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--s1);
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.order {
|
||||
font-size: var(--text-xs);
|
||||
color: var(--ink-muted);
|
||||
border: 0;
|
||||
background: none;
|
||||
padding: 2px var(--s1);
|
||||
max-width: 13ch;
|
||||
}
|
||||
|
||||
h1 {
|
||||
|
||||
Reference in New Issue
Block a user