Cloud sync of profiles and preferences through byte.vet

.gitignore

@@ -9,6 +9,7 @@ lerna-debug.log*
 
 node_modules
 dist
+dist-firefox
 dist.zip
 dist-ssr
 coverage

CLAUDE.md

@@ -11,6 +11,8 @@ A Manifest V3 browser extension (Chrome, Edge, Firefox) for proxy profile manage
 ```bash
 npm run build           # Type-check (vue-tsc) + build for Chrome/Edge
 npm run build:firefox   # Type-check + build for Firefox (transforms manifest)
+npm run build:preview   # Chrome build against the preview server (northernlight.byte.vet), fixed dev extension ID
+npm run build:preview:firefox  # The same for Firefox, into dist-firefox/ (so it doesn't replace the Chrome dist/)
 npm run dev             # Vite dev server
 npm test                # Run vitest in watch mode
 npm run coverage        # Single run with coverage report
@@ -20,6 +22,8 @@ npm run test:e2e            # Build into .e2e-build/, then run all e2e tests
 npm run test:e2e:chromium   # Chromium suite only (Playwright)
 npm run test:e2e:firefox    # Firefox smoke suite only (Selenium + geckodriver)
 npm run test:e2e:typecheck  # Type-check e2e/ (not covered by vue-tsc)
+npm run test:e2e:live       # Live sync smoke vs northernlight.byte.vet: headed, you sign in twice; never in CI
+npm run sync:api-types      # Regenerate src/services/sync/api.gen.ts from docs/sync-api.openapi.yaml
 E2E_SKIP_BUILD=1 npx playwright test -c e2e --project=chromium e2e/specs/pac.e2e.ts  # Reuse the last build, one spec
 ```
 
@@ -51,7 +55,7 @@ The core complexity lives here:
 - **`profile2config.ts`** -- `ProfileConverter` turns a `ProxyProfile` into a `ProxyConfig` (for `chrome.proxy.settings`). For non-PAC profiles and auto-switch profiles, it **generates PAC scripts via AST** using `escodegen`/`acorn` node builders in `scriptHelper.ts`. Auto-switch profiles compose multiple sub-profiles by generating `register()` calls that build a lookup table.
 - **Rule types** -- `domain` (`shExpMatch` on the host), `url` (`shExpMatch` on the URL), `regex` (`RegExp(...).test(url)`), `cidr` (`isInNet`), `disabled`. Browsers strip the path and query of https URLs before a PAC script sees them (Chrome 52+, Firefox 51+), so `url`/`regex` rules can't match https paths; the editor's help popover per type says so.
 - **`pacSimulator.ts`** -- JS reimplementations of PAC functions (`shExpMatch`, `isInNet`) used to simulate PAC evaluation in-extension (e.g., for tab badge resolution). `isInNet` returns `UNKNOWN` when given a hostname instead of an IP (can't do DNS in extension context).
-- **`auth.ts`** -- Resolves proxy auth credentials by walking the profile tree (auto-switch profiles delegate to sub-profiles).
+- **`auth.ts`** -- Resolves proxy auth credentials by walking the profile tree (auto-switch profiles delegate to sub-profiles). Hosts match case-insensitively (Chrome reports the challenger lower-cased). Chrome prompts on requests extensions can't see (e.g. the new tab page's), so `warmUpProxyAuth` (`index.ts`) sends one request of ours to `proxyverse-auth.invalid` through the proxy at browser start and on every proxy change, to get the credentials cached.
 - **`index.ts`** -- `setProxy`/`refreshProxy` apply a profile and save the `"active-profile"` snapshot (`refreshProxy` updates the snapshot too). `createProfileFinder(profile)` simulates which profile serves a URL (popup and badge). `refreshProxyOnProfileUpdate` re-applies the active profile when a storage change touches it or any profile it routes to (`profileTreeIDs`); the background calls it from `onProfileUpdate`, so writers of profiles don't need to.
 - **`ruleEditor.ts`** -- Writes domain rules to the active auto-switch profile from the popup: exact host, or a whole site (`example.com` + `*.example.com`, registrable domain via `tldts`). Rules are updated in place or prepended so they win.
 - **`pacFetcher.ts`** -- PAC URL fetching and `pac-refresh:<profileID>` alarms.
@@ -60,11 +64,11 @@ The core complexity lives here:
 ### Request stats and badge
 
 - **`src/services/stats.ts`** -- The background records every request per tab (`TabStats`), reset on each main-frame navigation, persisted to `storage.session` under `tabstats:<tabID>` (survives service worker restarts). The popup reads it via `subscribeTabStats` for the failed-resources alert and the Resources View. Benign errors (aborted, blocked by client) don't count as failures. Opening the popup sends `stats:ack`, marking the tab's failures as seen until it reloads.
-- **`src/services/indicator.ts`** -- `TabIndicator` owns the toolbar badge: profile initials (2 chars) globally and per tab, alternating every 2s with the count of unseen failures. It refreshes on proxy changes and on writes of the active-profile snapshot (e.g. a color edit leaves the proxy setting unchanged).
+- **`src/services/indicator.ts`** -- `TabIndicator` owns the toolbar badge: profile initials (2 chars) globally and per tab, alternating every 2s with the count of unseen failures. It refreshes on proxy changes, on writes of the active-profile snapshot (e.g. a color edit leaves the proxy setting unchanged) and on the `ui.failureBadge` setting. A tab's own badge wins over the global one, so a refresh repaints every tab with the profile predicted for its page (except at worker start: tabs keep their badges while it sleeps). Failures don't alternate when the setting is off, under System, or when another extension controls the proxy.
 
 ### Profile system (`src/services/profile.ts`)
 
-Profiles are stored in `chrome.storage.local` under key `"profiles"`. Types: `ProfileSimple` (proxy/pac), `ProfilePreset` (system/direct), `ProfileAutoSwitch` (rule-based routing). System profiles `DIRECT` and `SYSTEM` have fixed IDs `"direct"` and `"system"`. `onProfileUpdate(cb)` fires in every extension context with the new and old profiles.
+Profiles are stored in `chrome.storage.local` under key `"profiles"`. Write them only through `saveProfile`/`saveManyProfiles`/`deleteProfile`/`updateProfiles`: they hold a Web Lock (`withLock` in `utils.ts`) across extension contexts, so the config page, PAC refresh and sync don't overwrite each other. Deleting the active profile (here or by sync) switches the proxy to Direct (`refreshProxy`). Types: `ProfileSimple` (proxy/pac), `ProfilePreset` (system/direct), `ProfileAutoSwitch` (rule-based routing). System profiles `DIRECT` and `SYSTEM` have fixed IDs `"direct"` and `"system"`. `onProfileUpdate(cb)` fires in every extension context with the new and old profiles.
 
 UI pickers share one reactive profile list (`src/components/configs/useProfileOptions.ts`) and the `ProfileOptions` dropdown items.
 
@@ -74,6 +78,15 @@ UI pickers share one reactive profile list (`src/components/configs/useProfileOp
 - **`proxyverse/`** -- Our format (`version: "2025-01"`), validated with `zod/mini`. Unknown keys are stripped (so runtime fields like `pacScript.lastFetched` never reach a file), and each profile is validated on its own: an invalid one, a reserved ID (`direct`/`system`) or a repeated ID is skipped with a warning.
 - **`switchyomega/`** -- SwitchyOmega / ZeroOmega backups (`schemaVersion` 2, `+<name>` profiles, maybe base64). Profile IDs are UUIDv5 of the name, so re-importing overwrites. An enabled attached rule list (`__ruleListOf_<switch>`) is merged into its switch profile; rule lists become static auto profiles. Conditions map to our rules only where the match is exact (`conditions.ts`): URL regexes to `regex` rules, host regexes to `domain` wildcards via `regex.ts` (only those with an exact wildcard equivalent). Anything else (time, weekday, host levels, other regexes) becomes a `disabled` rule holding the condition text, plus a warning. Wildcards never carry `+()[]{}^$|\`: Firefox's PAC `shExpMatch` treats them as regex syntax.
 
+### Cloud sync (`src/services/sync/`)
+
+Profiles and four preferences (`theme.darkmode`, `ui.language`, `ui.actionBehavior`, `ui.failureBadge`) sync through byte.vet. Protocol: `docs/sync-protocol.md`; wire contract: `docs/sync-api.openapi.yaml`, vendored from the byte.vet webapp repo (which owns it), with types generated into `api.gen.ts` (a unit test fails when they drift).
+- **`buildEnv.ts`** -- `PROXYVERSE_ENV` at build time (`build/env.ts`): `production` (byte.vet, client `proxyverse-extension`), `northernlight` (preview server, client `proxyverse-extension-dev`), `e2e` (origin overridable at runtime via `sync.devOrigin`). Chrome dev/e2e builds carry `build/dev-extension-key.pub` as `manifest.key`, fixing the extension ID and so the OAuth redirect URI (`build/README.md`); store builds never do. `identity` is a required permission everywhere: browsers show no warning for it.
+- **`auth.ts`** -- OAuth 2.1 code + PKCE via `Host.launchWebAuthFlow`, run from the Preference page (sign-in can outlive an idle service worker). Refresh tokens are single use and reuse revokes the sign-in, so refreshes hold the `proxyverse-sync-auth` lock and store the new refresh token first.
+- **`engine.ts`** -- `SyncEngine.run()`: per-record last-writer-wins by HLC (`hlc.ts`). `state.local` (this device's version) vs `state.base` (last agreed with the server) decides what's dirty; there's no outbox, so pending changes survive service worker restarts. Remote changes are written storage first, then state. `records.ts` normalizes profiles with the export schema (no PAC script for URL profiles, upper-case color) and hashes canonical JSON. The engine takes a `SyncDevice` (storage) and an API factory, so unit tests run two devices against the mock store.
+- **`index.ts` / `background.ts`** -- the real engine and `sync:*` messages (Sync now, first-sync choice, snapshots); the background syncs at start, after sign-in, 2 s after a local change and on the `sync:poll` alarm (15 min).
+- State lives in `sync.auth`, `sync.state`, `sync.settings` (local storage, never synced). "This device only" profiles are `sync.settings.localOnly`, not a profile field. UI: `src/components/sync/` (Preference section) and the switch in `ProfileConfig.vue`.
+
 ## Critical Gotcha: `deepClone()` must use JSON round-trip
 
 `deepClone()` in `src/services/utils.ts` uses `JSON.parse(JSON.stringify(obj))`. **Do not replace with `structuredClone()`** -- Vue's reactive Proxy objects throw `DataCloneError` under `structuredClone()`. The JSON round-trip serializes through Vue's Proxy traps and produces plain objects safe for `chrome.storage`.
@@ -89,6 +102,7 @@ UI pickers share one reactive profile list (`src/components/configs/useProfileOp
 - Use `data-testid` selectors; Arco dropdowns, selects and toasts render in `<body>` (helpers in `e2e/pages/arco.ts`).
 - `sw.restart()` stops the service worker via CDP `Target.closeTarget`. PAC refresh alarms are fired with `sw.fireAlarm()`.
 - Use `freshHost()` for site hosts in tests that inspect the request log (late requests of an earlier test can land in the shared log), and `freshProxyHost()` for proxy hostnames (Chrome caches proxy credentials per proxy host for the browser's lifetime).
+- **Sync** runs against a fake byte.vet (`e2e/servers/syncStore.ts` logic, `syncApi.ts` HTTP), on loopback so it's never proxied: `servers.sync` (seed records, `failNext`, `issueTokens`, request log) and `servers.syncOrigin`. Helpers in `e2e/pages/sync.ts`: `useFakeByteVet` before a UI sign-in (Chromium's auth window works headless against it), `signInDirectly` to skip it. The same store is the fake server of the engine's unit tests.
 - Firefox smoke (`e2e/firefox/`): temporary add-ons don't get private-browsing access (required by `proxy.settings`), and WebDriver can't open `moz-extension://` URLs. The driver grants the permission and opens pages from Marionette's chrome context, which needs geckodriver ≥ 0.36 with `--allow-system-access`.
 
 ## Firefox build differences
@@ -98,6 +112,8 @@ The `vite.config.ts` `TRANSFORMER_CONFIG` rewrites `manifest.json` at build time
 - Removes `version_name`
 - Adds `browser_specific_settings.gecko` (minimum version 115, for `storage.session`)
 
+The manifest transform lives in `build/manifest.ts` (pure, unit-tested in `tests/build/`).
+
 ## Path alias
 
 `@/` maps to `src/` (configured in both `vite.config.ts` and `tsconfig.json`).

build/README.md

@@ -0,0 +1,37 @@
+# build/
+
+Build-time helpers imported by `vite.config.ts`:
+
+- `env.ts` -- the build environments chosen by `PROXYVERSE_ENV`
+  (`production`, the default; `northernlight`; `e2e`) and the sync settings
+  each one compiles in.
+- `manifest.ts` -- turns the root `manifest.json` into a build's manifest.
+
+## Dev extension key
+
+`dev-extension-key.pub` is the public half of an RSA key (base64 DER
+SubjectPublicKeyInfo). Chrome builds with `PROXYVERSE_ENV=northernlight` or
+`e2e` put it in `manifest.key`, so that unpacked builds always get the same
+extension ID, and so the same OAuth redirect URI, registered on the
+`proxyverse-extension-dev` client. **Dev only**: production builds never
+carry a key (the stores assign the ID), and the private key is not kept
+anywhere -- Chrome doesn't need it for unpacked extensions.
+
+| | |
+|---|---|
+| Chrome/Edge dev extension ID | `ohbdfegnjlfoohcaiedlpbmcacohacmg` |
+| Chrome/Edge dev redirect URI | `https://ohbdfegnjlfoohcaiedlpbmcacohacmg.chromiumapp.org/oauth` |
+| Firefox redirect URI | `https://1e603ec5914c06cbcc280eb4c99adc6034a2cdff.extensions.allizom.org/oauth` |
+
+The Chrome ID is the first 32 hex digits of SHA-256 of the DER key, with
+`0-f` mapped to `a-p`. Firefox derives its redirect URL from the add-on ID
+(`proxyverse@byte.vet`, the same in every build): the hex SHA-1 of the ID,
+under `extensions.allizom.org` (`identity.getRedirectURL()`, see
+`toolkit/components/extensions/child/ext-identity.js`).
+
+To recompute them:
+
+```sh
+base64 -d -i build/dev-extension-key.pub | shasum -a 256 | cut -c1-32 | tr 0-9a-f a-p
+printf %s proxyverse@byte.vet | shasum -a 1
+```

build/dev-extension-key.pub

@@ -0,0 +1 @@
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtCGr3XRCvuGwix6pIJHWaB6w7bJDoQ5PgNkg84XTEwOSExGa82atNHKCvfES2zWy+k6nmj9oHnsmcknqcb0HeDr7P9lKTArhVwlV8zD0OlJAdZ6epZz8TqNgpv/rYOSkBcuXhGeOvHM2KsWFrp2ayn8xsDDUPvu+eqAq5aetG5WfX2lPfGP7kO4GOW4+sf8QsDAi5RMiVCNOnJllJfceKPeBUTrAhZM/uoJFkEISbK8TQb42FUS2JDyuNu5wQ1Do+xYoyhEwi5QFGP+vuaNZFksT6StYrbkLbQjQQCaUGhCL8riwp7nyPdUQjo0jumsr7KXuKGY/1SwerdEDJzIaVwIDAQAB

build/env.ts

@@ -0,0 +1,55 @@
+// Build environments, chosen by PROXYVERSE_ENV when the extension is built.
+// They set the byte.vet origin and OAuth client the sync feature talks to
+// (read at runtime by src/services/sync/buildEnv.ts) and a few manifest
+// details (build/manifest.ts). Store builds are always "production".
+
+export type BuildEnv = "production" | "northernlight" | "e2e";
+
+/** The sync server and OAuth client a build talks to. */
+export type SyncTarget = {
+  /** The byte.vet origin serving OAuth and the sync API, no trailing slash. */
+  origin: string;
+  clientID: string;
+  /** Whether the origin may be overridden at runtime (e2e only). */
+  originOverride: boolean;
+};
+
+export const BUILD_ENVS: Record<BuildEnv, SyncTarget> = {
+  production: {
+    origin: "https://byte.vet",
+    clientID: "proxyverse-extension",
+    originOverride: false,
+  },
+  northernlight: {
+    origin: "https://northernlight.byte.vet",
+    clientID: "proxyverse-extension-dev",
+    originOverride: false,
+  },
+  e2e: {
+    // placeholder: the e2e suite sets the real origin at runtime
+    origin: "http://127.0.0.1",
+    clientID: "proxyverse-extension-dev",
+    originOverride: true,
+  },
+};
+
+/** Parse PROXYVERSE_ENV; unset or empty means "production". */
+export function parseBuildEnv(value: string | undefined): BuildEnv {
+  if (!value) return "production";
+  if (Object.hasOwn(BUILD_ENVS, value)) return value as BuildEnv;
+  throw new Error(
+    `Unknown PROXYVERSE_ENV "${value}"; expected one of: ${Object.keys(
+      BUILD_ENVS
+    ).join(", ")}`
+  );
+}
+
+/** Vite `define` entries for src/services/sync/buildEnv.ts. */
+export function syncDefines(env: BuildEnv): Record<string, string> {
+  const { origin, clientID, originOverride } = BUILD_ENVS[env];
+  return {
+    __SYNC_ORIGIN__: JSON.stringify(origin),
+    __SYNC_CLIENT_ID__: JSON.stringify(clientID),
+    __SYNC_ORIGIN_OVERRIDE__: JSON.stringify(originOverride),
+  };
+}

build/manifest.ts

@@ -0,0 +1,93 @@
+// The manifest.json of a build, from the one at the repo root. Pure, so it
+// can be unit-tested (tests/build/manifest.test.ts).
+//
+// Rolldown (Vite 8) panics on nested member expressions such as
+// `manifest.background.service_worker` while transforming the config and
+// the modules it imports, so this module only reads one level deep at a
+// time, hoisting nested objects into locals.
+
+import type { BuildEnv } from "./env.ts";
+
+export type Manifest = Record<string, unknown>;
+
+export type BuildManifestOptions = {
+  mode: "chrome" | "firefox";
+  env: BuildEnv;
+  /** The CRX version, e.g. "1.2.3" or "1.2.3-beta" (see `crxVersion`). */
+  version: string;
+  /** The built background script's file name. */
+  serviceWorker: string;
+  /**
+   * The dev public key (base64 DER), giving unpacked Chrome builds a fixed
+   * extension ID. Required for Chrome northernlight and e2e builds.
+   */
+  devKey?: string;
+};
+
+const GECKO_ID = "proxyverse@byte.vet";
+
+/** The CRX version from CRX_VER (e.g. a "v1.2.3" tag). */
+export function crxVersion(crxVer: string | undefined): string {
+  if (!crxVer) return "0.0.0-dev";
+  const ver = crxVer.startsWith("v") ? crxVer.slice(1) : crxVer;
+  return ver.slice(0, 14);
+}
+
+/** Envs whose Chrome builds carry the dev key. */
+export function needsDevKey(mode: "chrome" | "firefox", env: BuildEnv) {
+  return mode === "chrome" && env !== "production";
+}
+
+const strings = (v: unknown): string[] =>
+  Array.isArray(v) ? v.map(String) : [];
+
+export function buildManifest(
+  base: Manifest,
+  opts: BuildManifestOptions
+): Manifest {
+  const { mode, env, version, serviceWorker, devKey } = opts;
+  const manifest: Manifest = JSON.parse(JSON.stringify(base));
+  const background = { ...(manifest.background as Record<string, unknown>) };
+  let permissions = strings(manifest.permissions);
+
+  manifest.version = version.split("-", 1)[0];
+  manifest.version_name = version;
+  // the hashed file name avoids cache issues
+  if (mode === "firefox") background.scripts = [serviceWorker];
+  else background.service_worker = serviceWorker;
+
+  if (mode === "firefox") {
+    delete background.service_worker;
+
+    delete manifest.version_name;
+
+    // Firefox has a sidebar instead of Chrome's side panel
+    delete manifest.side_panel;
+    permissions = permissions.filter((p) => p !== "sidePanel");
+    manifest.sidebar_action = {
+      default_panel: "index.html#/sidebar",
+      default_icon: "mini-logo.png",
+      open_at_install: false,
+    };
+
+    manifest.browser_specific_settings = {
+      gecko: {
+        id: GECKO_ID,
+        strict_min_version: "115.0",
+      },
+    };
+  }
+
+  delete manifest.key;
+  if (needsDevKey(mode, env)) {
+    if (!devKey) {
+      throw new Error(`A ${env} Chrome build needs the dev key (devKey)`);
+    }
+    manifest.key = devKey;
+  }
+
+  manifest.background = background;
+  manifest.permissions = permissions;
+
+  return manifest;
+}