diff --git a/AGENTS.md b/AGENTS.md index efa106d..4d39c7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,17 +26,20 @@ deno run --allow-net --allow-read=. --allow-write=. --allow-sys --allow-env --al ./build/main.js # after building ``` -Available CLI commands: +Available CLI commands (all documented in `docs/commands.md`): +- `chel init` - Generate a template `chel.toml` - `chel keygen` - Generate cryptographic key pairs - `chel manifest` - Create signed contract manifests +- `chel verifySignature` - Check a manifest's signature and file hashes - `chel deploy` - Deploy contracts to a server or directory - `chel serve` - Start the server with contract preloading - `chel pin` - Pin contracts to specific versions - `chel upload` - Upload files to a server or directory +- `chel get` - Print the entry stored under a key - `chel hash` - Compute CID hash of a file - `chel migrate` - Migrate data between database backends -- `chel latestState` - Get latest contract state -- `chel eventsAfter` / `chel eventsBefore` - Query contract events +- `chel eventsAfter` - Query contract events +- `chel version` - Print the version ## Project Structure @@ -187,4 +190,4 @@ Manifests are JSON with signed body: 3. `git diff --exit-code -- build` to confirm the rebuild is reproducible 4. `npm publish --access public` -See the Packaging section of README.md for the full procedure. +See `docs/distributing.md` for the full procedure. diff --git a/README.md b/README.md index 34f69da..eadfb79 100644 --- a/README.md +++ b/README.md @@ -6,502 +6,49 @@ # Chel: Chelonia Command-line Interface -Modern CLI for Chelonia contract development, deployment, and management. - -## All Available Commands - -``` -chel -chel help [command] -chel version -chel pin [--dir ] [--overwrite] -chel test -chel init [--force] [--config ] -chel keygen [--out=] -chel manifest [-k|--key [-k|--key ...]] [--out=] [-s|--slim ] [-V|--contract-version ] -chel deploy [ [ ...]] -chel upload [ [ ...]] -chel serve [options] -chel latestState -chel eventsAfter [--limit N] [--url ] [--keys ] -chel hash -chel migrate --from --to [--from-config ] [--to-config ] -``` - -Note: in many (if not all) instances, the `` parameter can refer to a local folder path, in which case the command will operate without making a network connection, and will instead use the folder's contents to perform its operations. - -Every command also takes `--config `, to use another config file than `./chel.toml` (see [`chel.toml`](#cheltoml--runtime-cli-configuration)). - -### `chel pin` - Per-Contract Versioning System - -🎯 Pin individual contracts to specific versions independently! - -**Key Features:** -- ✅ **Per-contract versioning** using `chelonia.json` configuration -- ✅ **Individual contract pinning** by specifying manifest file path -- ✅ **New directory structure**: `contracts///` -- ✅ **Manifest-based workflow** - requires existing manifest files -- ✅ **Ecosystem-agnostic** - no coupling to Node.js/npm - -**Workflow:** -1. **Generate keys**: Use `chel keygen` to create cryptographic key files (required for production) -2. **Pin from manifest**: Use `chel pin` with the manifest file path -3. **Contract files copied**: Contract files (main/slim) and manifest are copied to new structure - -**Usage Examples:** -```bash -# First, generate cryptographic keys (required for production) -chel keygen - -# Then pin contracts with re-signing` -chel pin - -# Pin specific contract to a version using its manifest (from dist/contracts) -chel pin 2.0.5 dist/contracts/2.0.5/chatroom.2.0.5.manifest.json -chel pin 2.0.0 dist/contracts/2.0.0/group.2.0.0.manifest.json - -# Note: Contracts are pinned to the contracts/ output directory -``` - -**Configuration (`chelonia.json`):** -```json -{ - "contracts": { - "gi.contracts/chatroom": { - "version": "2.0.6", - "path": "contracts/gi.contracts_chatroom/2.0.6/chatroom.2.0.6.manifest.json" - }, - "gi.contracts/group": { - "version": "2.0.0", - "path": "contracts/gi.contracts_group/2.0.0/group.2.0.0.manifest.json" - } - } -} -``` - -**Directory Structure Created:** -``` -contracts/ -├── gi.contracts_chatroom/ -│ ├── 2.0.5/ -│ │ ├── chatroom.js -│ │ └── chatroom-slim.js -│ └── 2.0.6/ -│ ├── chatroom.js -│ └── chatroom-slim.js -└── gi.contracts_group/ - └── 2.0.0/ - ├── group.js - └── group-slim.js -``` - -**Command Options:** -- **`--overwrite`**: Force overwrite existing versions -- **Default**: Create new version by copying from source - -### `chel init` - Generate `chel.toml` - -Generates a template `chel.toml` in the current directory, pre-populated with -sensible defaults and a freshly-generated `server_id`. This is the recommended -way to bootstrap a new server instance. - -``` -chel init [--force] [--config ] -``` - -**Options:** -- `--force`: Overwrite an existing `chel.toml`. Without this flag, `chel init` - refuses to clobber an existing file — a safety against accidentally rotating - `server_id`, which would orphan push subscriptions tagged with the old id - (see [`chel.toml`](#cheltoml--runtime-cli-configuration)). -- `--config `: Write the file there instead of `./chel.toml`. The other - commands then need the same `--config ` to read it. - -**What it writes:** -- A top-level `server_id` (a random UUID). Keep this stable for the lifetime of - the database; do **not** reuse it across environments (prod / staging / dev) - that share a database. -- `[server]` / `[database]` blocks mirroring the defaults from `parseConfig.ts`, - with less-common options left as commented-out lines for guidance. - -**Example:** -```bash -$ chel init -wrote: chel.toml -server_id: 7b3f2c1a-... keep this stable; do not reuse across environments, - and do not rotate it on an existing DB. -``` - -### `chel serve` - Development Server with Contract Preloading - -``` -chel serve [options] - -OPTIONS - ---dashboard-port set dashboard port (default: 8888) ---port set application port (default: 8000) ---dev start in development mode (watch and redeploy contract manifests) -``` - -> [!IMPORTANT] -> **Prerequisites:** Ensure your application directory contains a `contracts/` directory with the correct contract structure before running `chel serve`. The server automatically preloads all contract manifests found in `contracts///` directories into the database on startup in development mode. - -**Example Output:** -```bash -$ chel serve -🚀 Starting Chelonia app server... -📦 Step 1: Preloading contracts... -📋 Found 4 contract manifest(s) to deploy -contracts/gi.contracts_chatroom/2.0.6/chatroom.js: /data/zLAeVmpcc88g... -contracts/gi.contracts_group/2.0.0/group.js: /data/zLAeVmpcc88g... -✅ Successfully preloaded 4 contract(s) into database -🚀 Step 2: Starting dashboard server... -📊 Dashboard server running at: http://localhost:8888 -🚀 Step 3: Starting application server... -``` - -**Usage Examples:** -```bash -# Start with automatic contract preloading -chel serve - -# Serve Group Income app from extracted directory -chel serve ./gi-v2.0.0 - -# Serve with custom ports and SQLite database -chel serve --dashboard-port 8888 --port 8000 ./my-app -``` - -**What happens during startup:** -1. **Contract Discovery** - Scans `contracts///` directories -2. **Manifest Collection** - Finds all `.manifest.json` files -3. **Database Preloading** - Deploys all contracts with content-addressed storage -4. **Server Startup** - Starts dashboard and application servers -5. **Ready for Development** - All historical contracts available for message processing - -### `chel keygen` - -``` -{ - "version": "1.0.0", - "cipher": "algo", - "pubkey": "...", - "privkey": "...", - "encrypted": "algo" -} -``` - -If `"encrypted"` doesn't exist - it means the `"privkey"` was saved in the clear. - -### `chel serve` - -Starts a local development server for Chelonia applications. - -``` -chel serve [options] - -OPTIONS - ---dashboard-port set dashboard port (default: 8888) ---port set application port (default: 8000) -``` - -**Example:** -```bash -# Serve Group Income app from extracted directory -chel serve ./gi-v2.0.0 - -# Serve with custom ports and SQLite database -chel serve --dashboard-port 8888 --port 8000 ./my-app -``` - -The serve command will: -- Start a dashboard server (default: http://localhost:8888, see below) -- Start an application server (default: http://localhost:8000) -- Serve static assets and handle API routes -- Support different database backends (memory, filesystem, SQLite, Redis) - -The dashboard shows the server's registered usernames, the contracts it -stores, how much space each account uses and each account's credits. It never -shows what is inside a contract, since that is encrypted. Because of the -usernames, it has two settings under `[server]` in `chel.toml`: - -- `dashboardListenIP`: the address it listens on, `127.0.0.1` by default, so - only this machine can open it. -- `dashboardAdminPassword`: until this is set, the dashboard shows no data. - Once it is, the browser asks for it (any user name works). To keep it out of - the file, set `server__dashboardAdminPassword` in the environment instead. - The password travels with every request, so put the dashboard behind HTTPS - before opening it to other machines. - -### `chel manifest` - -Let's say you have the following files: - -- `contract-bundle.js` -- `contract-slim.js` - -Running `chel manifest --add-key alex.json --slim contract-slim.js deploy-key.json contract-bundle.js` will generate the following `contract-bundle.manifest.json`: - -``` -{ - "head": { - "manifestVersion": "1.0.0" - }, - "body": JSON.stringify({ - "version": " by default", - "contract": { "hash": "", "file": "contract-bundle.js" }, - "contractSlim": { "hash": "", "file": "contract-slim.js" }, - "authors": [ - {"cipher": "algo", "key": ""}, - {"cipher": "algo", "key": ""} - ] - }), - "signature": { - "key": "", - "signature": "" - } -} -``` - -It will upload both versions of the contracts, and this JSON. - -This format makes it as efficient as possible for using the contract system from both in-app and from the commandline. - -The CLI tool will always use the self-contained contract bundle, whereas apps can load less code by loading the slim version of the contract. You just need to make sure that none of the external dependencies that you're referencing ever change if you do this, as otherwise you will get different state between the two versions of the contracts. - -Note also that Chelonia is fundamentally language agnostic. We started out using Chelonia to build JS apps, but you can use this protocol with any programming language that supports source evaluation at runtime. - -Some commands of this CLI tool (like `latestState`), only support JavaScript, but that is a limitation of resources on our side, and not a fundamental limitation of the protocol. - -### `chel deploy` - -Deploys manifest(s) generated with `chel manifest`. - -Automatically uploads any corresponding contract files. - -Outputs the hash(es) corresponding to the manifest(s). - -Useful command: - -``` -cp -r path/to/contracts/* test/assets/ && ls ./test/assets/*-slim.js | sed -En 's/.*\/(.*)-slim.js/\1/p' | xargs -I {} ./src/main.ts manifest --out=test/assets/{}.manifest.json --slim test/assets/{}-slim.js key.json test/assets/{}.js && ls ./test/assets/*.manifest.json | xargs ./src/main.ts deploy http://127.0.0.1:8888 -``` - -### `chel eventsAfter` - -Displays a JSON array of events that happened in a given contract since a given -height. Older events come first. The output is parseable with tools like `jq`. - -``` -chel eventsAfter [--limit N] [--url ] [--keys ] -``` - -If `--url` is supplied, the remote server's `/eventsAfter` endpoint is queried. -Otherwise, the local database (configured via `chel.toml`) is read directly. - -#### Decrypting events with `--keys` - -By default, encrypted events (`OP_ACTION_ENCRYPTED` and friends) are returned -as opaque envelopes — useful for forensics, but unreadable. Pass `--keys` with -a JSON file of secret keys to decrypt them in place. - -**Current limitation:** decryption only works when the returned event list -starts with the contract's `OP_CONTRACT` event (for example, by querying from -height `0`). If you query from a later height, `decryptOne` has no prior -contract/key state, so encrypted events are returned as raw envelopes with -`no prior contract state` rather than being decrypted. - -**Warning:** this file contains secret key material and grants decryption -capability for any events encrypted to those keys. Do **not** commit it to -version control, include it in logs, or share it casually. Store it with -restrictive file permissions, use it only for local debugging, and delete or -otherwise protect it when you are done. - -The file is a flat object mapping each `keyId` to its serialized secret key: - -```json -{ - "z9brR...sigKeyId": "", - "z9brR...encKeyId": "" -} -``` - -This is exactly the shape Chelonia stores at `rootState.secretKeys` in any app -built with [`libcheloniajs`](https://github.com/okTurtles/libcheloniajs). The -easiest way to generate this file is from a running app: - -**In a browser DevTools console (with the app open):** - -```js -copy(JSON.stringify(await sbp('chelonia/rootState').secretKeys, null, 2)) -// then paste into keys.json -``` - -**Example:** +Chel is the command-line tool for +[Chelonia](https://github.com/okTurtles/libcheloniajs), which implements +[Shelter Protocol](https://shelterprotocol.net) for end-to-end encrypted apps. +It signs and pins your app's contracts, runs the server that stores and relays +their messages, and reads and writes the server's data. + +## Features + +- **One tool, from contract to server.** Make a signing key, sign each version + of a contract, pin it, and serve your app: `chel serve` uploads the pinned + contracts when it starts. +- **A native binary from npm.** `npm install` picks the build for your + platform, so you don't need Deno to run it. +- **Your choice of database:** in memory, files, SQLite, Redis, or a mix of + them by key prefix. +- **Ready for a public server.** Signups are limited per IP address, new + accounts have a size cap, and storage is counted per account. +- **A dashboard for your server.** `chel serve` also starts a dashboard with + the registered users, the stored contracts, and each account's storage and + credits. It shows this only behind a password, and by default only the + server's own machine can open it. See + [docs/server.md](docs/server.md#the-dashboard). +- **One config file.** `chel.toml` is checked for typos and wrong values when + chel starts, and any setting can also come from an environment variable. + +![The chel dashboard, with the number of users and contracts, the storage used, the newest users and the space usage](docs/images/dashboard.png) + +## Install + +Add it to your project, and run it with `npx chel`: ```bash -chel eventsAfter --keys ./keys.json 0 --limit 10 -``` - -When `--keys` is supplied each event is emitted as: - -```jsonc -{ - "height": 1, - "hash": "z9brR...", - "contractID": "z9brR...", - "op": "ae", - "signingKeyId": "z9brR...", - "innerSigningKeyId": "z9brR...", - "decryptedValue": { /* the inner contract action payload */ }, - "raw": { - "serverMeta": { - "date": "2025-06-09T20:35:43.305Z", - "isKeyOp": true - }, - "message": "{...}" - } -} -``` - -Decryption is best-effort: if the required key is missing or a signature -cannot be verified, the event is still emitted with the original envelope -under `raw`, and `decryptedValue` may be absent or `undefined`. Some failure -cases may also include a per-event `error` and/or print a one-line warning to -`stderr`, but that is not guaranteed for every decryption failure. - -### `chel migrate` - -Performs a non-destructive migration from one backend (`--from`) to another -one (`--to`). For example, this can be used to migrate from the `fs` backend to -the `sqlite` backend. - -Since some backends may require additional configuration, `chel migrate` allows -for specifying additional options for these backends with the `--from-config` -and `--to-config` options. - -By default, the general `chel` configuration will be used both for the `--from` -and `--to` backends (such as the values given in `chel.toml`). However, if -`--from-config` or `--to-config` are specified, those will take precedence. - -The configuration files for `--from-config` and `--to-config` follow the same -syntax and structure as `chel.toml`. This makes it easy to migrate to a new -backend by writing a new `chel.toml` for that backend. - -#### Examples - -```sh -# Migrate from sqlite to redis using default values (possibly overridden in chel.toml) -chel migrate --from sqlite --to redis +npm install --save-dev @chelonia/cli ``` -```sh -# Migrate from sqlite to redis using default values (possibly overridden in chel.toml) -# for redis and the configuration in sqlite.toml for sqlite -chel migrate --from sqlite --from-config sqlite.toml --to redis -``` - -```sh -# Migrate from sqlite to redis using default values (possibly overridden in chel.toml) -# for sqlite and the configuration in redis.toml for redis -chel migrate --from sqlite --to redis --to-config redis.toml -``` - -```sh -# Migrate from sqlite to redis using the configuration in sqlite.toml for sqlite -# and the configuration in redis.toml for redis -chel migrate --from sqlite --from-config sqlite.toml --to redis --to-config redis.toml -``` - -## Configuration Files - -The project uses two separate configuration files for different purposes: - -### `chel.toml` — Runtime CLI Configuration - -`chel.toml` configures the `chel` command itself at runtime. It is read by `nconf` with priority: environment variables > CLI arguments > `chel.toml` > defaults. It controls things like server host/port, database backend selection, and other operational settings. - -By default, `chel` reads `./chel.toml`. To use another file, pass its path with `--config`: +Or install it for your user: ```bash -chel serve --config data/chel.toml dist -``` - -```toml -# Example chel.toml -[server] -host = "0.0.0.0" -port = 8000 -dashboardPort = 8888 -# The dashboard listens on this machine only, unless you change this -dashboardListenIP = "127.0.0.1" -# The dashboard shows no data until this is set to a long, random password -# dashboardAdminPassword = "..." - -[server.vapid] -email = "admin@example.com" - -[database] -backend = "sqlite" - -# Signup size cap and the billing free tier; see -# docs/signup-and-billing.md. -# [server.signup] -# maxFirstMessageBytes = 8192 -# -# [server.billing] -# freeAllowanceBytes = 10485760 +npm install --global @chelonia/cli ``` -### `chelonia.json` — App Properties - -`chelonia.json` is an **app-level properties file** that describes which -contracts make up the application and their pinned versions. It is **not** -runtime configuration for `chel`. Instead, it is: - -- **Created and updated** by `chel pin` when pinning contracts to specific - versions. -- **Read by the server** (`chel serve`) at startup to provide version - information and other app-specific settings to clients. - -The file contains a `contracts` object keyed by the full contract name -(e.g. `gi.contracts/chatroom`), where each entry has: - -| Field | Description | -|-------|-------------| -| `version` | The pinned version string for this contract | -| `path` | Relative path to the manifest file in the `contracts/` directory | - -It may also contain an `appVersion` field for the overall application version. - -**Example:** -```json -{ - "appVersion": "2.0.0", - "contracts": { - "gi.contracts/chatroom": { - "version": "2.0.6", - "path": "contracts/gi.contracts_chatroom/2.0.6/chatroom.2.0.6.manifest.json" - }, - "gi.contracts/group": { - "version": "2.0.0", - "path": "contracts/gi.contracts_group/2.0.0/group.2.0.0.manifest.json" - } - } -} -``` - -**Summary of differences:** - -| | `chel.toml` | `chelonia.json` | -|---|---|---| -| **Purpose** | Runtime configuration of the `chel` CLI | App properties (e.g., contract versions) | -| **Managed by** | Manually by the developer / system administrator | Automatically by `chel pin` | -| **Read by** | `chel` commands (via `nconf`) | The server (at startup), values exposed to clients | -| **Format** | TOML | JSON | -| **Example content** | Database configuration | Contract versions | +`chel --help` lists the commands, and `chel --help` shows the +options of one. ## Supported Platforms @@ -515,144 +62,47 @@ Pre-compiled native binaries are published for the following platforms: | macOS | x64 (Intel) | `@chelonia/cli-x64-darwin` | | macOS | arm64 (Apple Silicon) | `@chelonia/cli-arm64-darwin` | -When you install `@chelonia/cli`, npm automatically selects the correct -sub-package for your platform via `optionalDependencies`. Windows arm64 is -currently **not** supported. - -The Linux binaries are linked against glibc, so Linux distributions built on -musl libc (Alpine, for example) are **not** covered: the native SQLite addon -each Linux binary carries is the glibc build for its platform. Running `chel` -from source with Deno works there instead. Only the SQLite backend is affected, -and selecting it on such a system fails with a message saying so rather than -with a raw loader or missing-module error. +When you install `@chelonia/cli`, npm selects the sub-package for your platform +through `optionalDependencies`. Windows on arm64 is **not** supported. -From source, nothing has to be compiled locally: the SQLite driver ships musl -prebuilds (`prebuilds/linuxmusl-.node`) next to the glibc ones, and its -resolver chooses between them by asking Node whether the process is running -against glibc. Deno hardcoded a glibc version into that answer until Deno -2.8.0 ([denoland/deno#33948](https://github.com/denoland/deno/issues/33948)), -so older versions pick the glibc prebuild on musl and then fail inside the -dynamic loader. +The Linux binaries are linked against glibc, so on musl-based distributions +(Alpine, for example) their SQLite backend does not work. Run chel from source +with Deno 2.8.0 or newer there, or use another database backend. The details +are in [docs/distributing.md](docs/distributing.md#linux-on-musl). -On Deno 2.8.0 or newer this works out of the box. On anything older, either -upgrade Deno or patch `node_modules/better-sqlite3/lib/binding.js` so that -`isLinuxMusl` always returns `true`. Compiling the addon from its C sources -(`npm run build-release`) does **not** help: the resolver returns a matching -prebuild before it ever looks at a locally built copy. +If `chel` cannot start, the launcher explains why rather than printing a stack +trace: exit code 127 means no sub-package for your platform is installed, and +126 means one was found but its binary is missing or not executable (reinstall +with `npm install --force @chelonia/cli`). -The `chel` command itself is always provided by the main `@chelonia/cli` -package, which ships a small launcher that spawns the native binary from the -sub-package npm selected. The sub-packages intentionally declare no command of -their own: if they did, they would compete with the main package for the same -`chel` entry in `node_modules/.bin`, and npm would end up wiring `chel` to a -bare platform binary (or to nothing at all when the main package is the project -root). Because npm therefore never adjusts the binary's permissions at install -time, the executable bit is baked into the published sub-package instead. +## Getting started -This layout is verified against npm 10, pnpm (both its isolated and hoisted -layouts), Yarn 1, and Yarn 4 (both Plug'n'Play and `node-modules`), for local -and global installs. If `chel` cannot start, the launcher explains why rather -than printing a stack trace: exit code 127 means no sub-package for your -platform is installed, and 126 means one was found but its binary is missing or -not executable (reinstall with `npm install --force @chelonia/cli`). - -## Packaging - -Steps to publish a new release of `@chelonia/cli` to npm: - -1. **Login to npm** (prerequisite): - - ```bash - npm login - ``` - - If your account uses 2FA, `npm publish` may prompt for authentication. - Do not rely on `--otp=`: a TOTP code entered at the start of the - release may well have expired by the time the first sub-package is - published. Complete the browser-based authentication flow when prompted, - or use a granular automation token instead. - -2. **Bump the version:** - - ```bash - npm version patch # or minor / major - ``` - - This triggers the `version` lifecycle hook, which runs *after* the version - is bumped but *before* `npm version` commits and tags. The hook: - - Runs `deno task sync-versions` to update `optionalDependencies` in - `package.json` so they match the new version (and stages the change) - - Runs `deno task build` to rebuild the JS bundle (stamped with the new - version in `build/version.json`) and the dashboard - - Stages the rebuilt `build/` artifacts - - As a result, the version commit and tag contain the up-to-date bundle and - the working directory stays clean. The publish step verifies that what it - ships is exactly this committed bundle and refuses to run otherwise. - This also guarantees the maintainer rebuilds the bundle on `master` - (merged PRs can each produce a correct bundle individually, yet the - combined result on `master` may still differ). - -3. **Build the release artifacts** (the tarballs for the GitHub release): - - ```bash - deno task dist - git diff --exit-code -- build # fails if the rebuild diverged from the tag - ``` - - This lints, rebuilds the bundle, compiles one native binary per supported - platform, and packs each one into `dist/chel-v-.tar.gz`, - printing SHA256 checksums. - - The `git diff --exit-code -- build` afterwards confirms the rebuild is - byte-identical to the version commit, i.e. that the build is reproducible. - It also has to pass before publishing, since the next step requires a clean - `build/`. If it fails, do not publish: investigate the divergence first. - -4. **Publish:** - - ```bash - npm publish --access public - ``` - - This triggers the `prepublishOnly` hook, which runs `deno task publish`. - That script: - - Verifies that the committed bundle matches `package.json` (via - `build/version.json`) and that the working tree is clean for `build/` - and `package.json`, refusing to publish otherwise - - Reuses the native binaries from step 3 instead of compiling its own - copies, so a release compiles each binary once and npm and GitHub are - guaranteed to ship identical bytes. If a binary is missing, or was built - from a different bundle, it is compiled here instead — publishing without - step 3 works, it just takes several minutes longer - - Creates and publishes a platform sub-package (`@chelonia/cli--`) for each target - - Reconciles `optionalDependencies` as a final safety check - - After the sub-packages are published, `npm publish` proceeds to publish the - main `@chelonia/cli` package, whose `optionalDependencies` now point to the - freshly published sub-packages. - -5. **Push the version commit and tag:** +```bash +chel init # writes chel.toml, with a fresh server_id +chel keygen # makes a key pair to sign contracts with - ```bash - git push && git push --tags - ``` +# Sign a contract, then pin it into contracts/ and chelonia.json +chel manifest --name my.app/list --contract-version 1.0.0 list.js +chel pin list.1.0.0.manifest.json -### Build caching +# Serve the app in dist/, upload contracts/, and watch it for new versions +chel serve --dev --app-manifest chelonia.json dist +``` -Compiling the five native binaries is by far the slowest part of a release, so -the tarballs and the npm sub-packages share one set of them, kept under the -gitignored `dist/` directory. `deno task dist` remains usable on its own, at -any time, to produce the tarballs without publishing anything. +By default the server keeps everything in memory. Before running it for other +people, read [docs/server.md](docs/server.md#in-production). -The cache operates on content, not on timestamps. When the bundle, the Deno -version, or a compile flag changes, `deno task dist` compiles the binaries -again. When nothing changes, it does no compile work. To compile all binaries -again, set `CHEL_FORCE_COMPILE=1` or delete `dist/`. +## Documentation -To check that a compiled binary loads its SQLite addon, run -`CHEL_SMOKE_COMPILE=1 deno task smoke`. This test compiles a binary and is -therefore not part of `deno task test`. +- [Commands](docs/commands.md): every command and its options. +- [Running a server](docs/server.md): what `chel serve` does, the dashboard, + and production. +- [Configuration](docs/configuration.md): `chel.toml`, its database backends, + and `chelonia.json`. +- [Signup and billing](docs/signup-and-billing.md): the checks on new + accounts, and how storage is charged. +- [Distributing](docs/distributing.md): how chel is built, packaged and + published. ## History diff --git a/build/main.js b/build/main.js index 36c1cb3..d1efef5 100644 --- a/build/main.js +++ b/build/main.js @@ -70343,7 +70343,7 @@ var module6 = { }); }, command: "hash ", - describe: "Computes and logs the content identifier (CID) for the given file.\nFile contents will be interpreted as raw binary data, unless the file extension is '.json'.", + describe: "Computes and logs the content identifier (CID) for the given file.\nFile contents are hashed as raw binary data.", postHandler: (argv) => { return void hash2(argv); } diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..732d993 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,373 @@ +# Commands + +`chel --help` lists the commands, and `chel --help` shows the +options of one. + +Every command also takes `--config `, to use another config file than +`./chel.toml`. These commands read it: `deploy`, `eventsAfter`, `get`, +`migrate`, `pin`, `serve` and `upload`. See [configuration.md](configuration.md). + +| command | what it does | +| --- | --- | +| [`chel init`](#chel-init) | writes a new `chel.toml` | +| [`chel serve`](#chel-serve) | runs the server | +| [`chel keygen`](#chel-keygen) | makes a key pair for signing contracts | +| [`chel manifest`](#chel-manifest) | signs a contract | +| [`chel pin`](#chel-pin) | pins a contract version into `contracts/` | +| [`chel verifySignature`](#chel-verifysignature) | checks a manifest | +| [`chel deploy`](#chel-deploy) | uploads contracts and their manifests | +| [`chel upload`](#chel-upload) | uploads files | +| [`chel get`](#chel-get) | prints the entry stored under a key | +| [`chel eventsAfter`](#chel-eventsafter) | prints a contract's events as JSON | +| [`chel hash`](#chel-hash) | prints the CID of a file | +| [`chel migrate`](#chel-migrate) | copies all data from one database backend to another | +| [`chel version`](#chel-version) | prints the version | + +A contract usually goes through these steps: + +1. `chel keygen` makes a key pair, once. Keep it for as long as the app lives: + clients trust the signing keys of the first manifest they see for a + contract name, and refuse later manifests signed with other keys. +2. `chel manifest` signs each version of each contract. +3. `chel pin` copies that version into `contracts/` and records it in + `chelonia.json`. +4. `chel serve` uploads everything in `contracts/` when it starts. + +## chel init + +``` +chel init [--force] [--config ] +``` + +Writes a template `chel.toml` in the current directory, or at the `--config` +path, with a freshly generated `server_id` and the default settings. The less +common ones are left as comments. + +- `--force`: overwrite an existing file. Without it, `chel init` refuses to, + as a safety against changing `server_id` by accident: push subscriptions are + tagged with it, and the ones tagged with the old id would be orphaned. + +Keep `server_id` the same for the life of the database, and do not reuse it +across environments (production, staging, development) that share a database. + +```bash +$ chel init +wrote: chel.toml +server_id: 7b3f2c1a-... keep this stable; do not reuse across environments, + and do not rotate it on an existing DB. +``` + +## chel serve + +``` +chel serve [options] [directory] +``` + +Runs the server for the app in `directory` (`.` by default). See +[server.md](server.md) for what it does at startup, the dashboard and running +it in production. + +| option | default | | +| --- | --- | --- | +| `-p`, `--port` | `8000` | port of the app and the API | +| `-a`, `--dashboard-port` | `8888` | port of the dashboard | +| `-d`, `--dev` | off | development mode: also watch the manifests folder and upload new versions as they appear | +| `-m`, `--manifests-dir` | `contracts` | where to find the contract manifests to upload | +| `-i`, `--app-manifest` | `chelonia.json` in the app directory | the app's [`chelonia.json`](configuration.md#cheloniajson) | + +## chel keygen + +``` +chel keygen [--out ] [--pubout ] +``` + +Makes a key pair for signing contracts. `--out` is the file for the secret key +and `--pubout` the one for the public key. By default they are named after the +key, as `edwards25519sha512batch-.json` and +`edwards25519sha512batch-.pub.json`, where `` is the last 12 +characters of the key ID. + +The secret key file looks like this, and the public one is the same without +`privkey`: + +```json +{ + "version": "1.0.0", + "pubkey": "...", + "privkey": "..." +} +``` + +The private key is saved unencrypted, so keep the file out of version control +and off the server. + +## chel manifest + +``` +chel manifest [options] +``` + +Signs `contractBundle` with the secret key file `signingKey`, and writes its +manifest. + +| option | default | | +| --- | --- | --- | +| `-n`, `--name ` | the file name, without extension | contract name | +| `-V`, `--contract-version ` | `x` | contract version | +| `--out ` | `..manifest.json`, next to the contract | where to write the manifest. `-` prints it | +| `-s`, `--slim ` | | a slim bundle of the same contract, hashed into the manifest | +| `-k`, `--key ` | | another public key file allowed to sign this contract. Can be repeated | + +For example, with a key `deploy-key.json`, a second public key `alex.pub.json`, +and the two builds `contract-bundle.js` and `contract-slim.js`: + +```bash +chel manifest --name my.app/list --contract-version 1.0.0 \ + -k alex.pub.json --slim contract-slim.js deploy-key.json contract-bundle.js +``` + +writes `contract-bundle.1.0.0.manifest.json`. Its `head` and `body` are JSON +strings, and the signature covers both: + +``` +{ + "head": "{\"manifestVersion\":\"1.0.0\"}", + "body": JSON.stringify({ + "name": "my.app/list", + "version": "1.0.0", + "contract": { "hash": "", "file": "contract-bundle.js" }, + "signingKeys": ["", ""], + "contractSlim": { "file": "contract-slim.js", "hash": "" } + }), + "signature": { + "keyId": "", + "value": "" + } +} +``` + +The manifest records each file by its name only, and `chel deploy` looks for +them next to the manifest, so keep them in the same folder. + +The CLI always uses the self-contained bundle, but apps can load less code by +loading the slim version. That is only safe if the external code the slim +version relies on never changes, or the two versions would build different +state. + +Chelonia is language agnostic. It started out for JavaScript apps, but the +protocol works with any language that can evaluate source at runtime. + +## chel pin + +``` +chel pin [options] [] +``` + +Copies the contract files and the manifest into +`contracts///`, and records the version in +`chelonia.json`. A `/` in the contract name becomes `_` in the folder name. + +| option | | +| --- | --- | +| `-o`, `--overwrite` | replace a version folder that is already there | +| `-d`, `--dir ` | the project folder to pin into, instead of the current one | +| `-i`, `--app-manifest ` | the `chelonia.json` to update | + +If you give `manifest-version`, it must match the version in the manifest. If +`chelonia.json` already pins that version, `chel pin` does nothing. If the +version's folder is there but `chelonia.json` pins another version, it stops +unless you pass `--overwrite`. + +```bash +chel pin dist/contracts/chatroom.2.0.6.manifest.json 2.0.6 +``` + +pins `gi.contracts/chatroom` like this, next to the versions pinned before: + +``` +contracts/ +├── gi.contracts_chatroom/ +│ ├── 2.0.5/ +│ │ ├── chatroom.js +│ │ ├── chatroom-slim.js +│ │ └── chatroom.2.0.5.manifest.json +│ └── 2.0.6/ +│ ├── chatroom.js +│ ├── chatroom-slim.js +│ └── chatroom.2.0.6.manifest.json +└── gi.contracts_group/ + └── 2.0.0/ + └── ... +``` + +## chel verifySignature + +``` +chel verifySignature [-k ] +``` + +Checks a manifest: its signature, that the key it was signed with is listed in +its `signingKeys`, and that the contract files next to it match their hashes. +With `-k`, it also checks that the manifest was signed with that key. + +## chel deploy + +``` +chel deploy [--url ] +``` + +Uploads each manifest and the contract files it points to, and prints where +each one went. A folder means every manifest in it and in its subfolders. + +Without `--url` it writes into the database set in `chel.toml`. With `--url` +it sends them to that server, which only accepts them when it runs with +`NODE_ENV=development`. + +## chel upload + +``` +chel upload [--url ] +``` + +Uploads files, to the database set in `chel.toml` or, with `--url`, to a server +running with `NODE_ENV=development`. + +## chel get + +``` +chel get [--url ] +``` + +Prints the entry stored under `key`, from the database set in `chel.toml` or, +with `--url`, from that server's `/file/`. The output can go to a file: + +```bash +chel get --url https://example.com > file.png +``` + +## chel eventsAfter + +``` +chel eventsAfter [--limit ] [--url ] [--keys ] +``` + +Prints a JSON array of up to `--limit` (default 50) events of a contract, +starting at `height`. Older events come first, and the output works with tools +like `jq`. + +Without `--url` it reads the database set in `chel.toml`. With `--url` it calls +that server's `/eventsAfter` endpoint. + +### Decrypting events with `--keys` + +By default, encrypted events (`OP_ACTION_ENCRYPTED` and friends) are returned +as they are stored: useful for forensics, but unreadable. Pass `--keys` with a +JSON file of secret keys to decrypt them in place. + +**Current limitation:** decryption only works when the returned list starts +with the contract's `OP_CONTRACT` event, for example by querying from height +`0`. From a later height there is no prior contract and key state, so +encrypted events come back as they are, with a `no prior contract state` +error. + +**Warning:** this file holds secret keys, and can decrypt every event +encrypted to them. Do **not** commit it to version control, put it in logs or +share it. Store it with restrictive file permissions, use it only for local +debugging, and delete it when you are done. + +The file maps each key ID to its serialized secret key: + +```json +{ + "z9brR...sigKeyId": "", + "z9brR...encKeyId": "" +} +``` + +This is the shape Chelonia keeps at `rootState.secretKeys` in any app built +with [`@chelonia/lib`](https://github.com/okTurtles/libcheloniajs). The easiest +way to get it is from the browser's developer console, with the app open: + +```js +copy(JSON.stringify(await sbp('chelonia/rootState').secretKeys, null, 2)) +// then paste it into keys.json +``` + +```bash +chel eventsAfter --keys ./keys.json 0 --limit 10 +``` + +With `--keys`, each event comes out like this: + +```jsonc +{ + "height": 1, + "hash": "z9brR...", + "contractID": "z9brR...", + "op": "ae", + "signingKeyId": "z9brR...", + "innerSigningKeyId": "z9brR...", + "decryptedValue": { /* the inner contract action payload */ }, + "raw": { + "serverMeta": { + "date": "2025-06-09T20:35:43.305Z", + "isKeyOp": true + }, + "message": "{...}" + } +} +``` + +An `OP_ATOMIC` event also has `ops`, with one entry per operation in it. + +Decryption is best-effort: if a key is missing or a signature cannot be +verified, the event still comes out with the original message under `raw`, and +`decryptedValue` may be missing. Some failures also add an `error` to the event +or print a warning to `stderr`, but not every one does. + +## chel hash + +``` +chel hash +``` + +Prints the content identifier (CID) of a file, hashing its contents as raw +binary data. + +## chel migrate + +``` +chel migrate --from --to [--from-config ] [--to-config ] +``` + +Copies every key-value pair from one database backend (`--from`) to another +(`--to`), for example from `fs` to `sqlite`. The source is not changed or +deleted, the destination is created if needed, and invalid entries are +skipped. It needs read and write access to the source. + +Both backends use the settings in `chel.toml`, unless `--from-config` or +`--to-config` gives another file. Those files have the same format as +`chel.toml`, so migrating to a new backend can start with writing a +`chel.toml` for it. + +```sh +# From sqlite to redis, both with the settings in chel.toml +chel migrate --from sqlite --to redis + +# sqlite with the settings in sqlite.toml, redis with those in chel.toml +chel migrate --from sqlite --from-config sqlite.toml --to redis + +# sqlite with the settings in chel.toml, redis with those in redis.toml +chel migrate --from sqlite --to redis --to-config redis.toml + +# Each with its own file +chel migrate --from sqlite --from-config sqlite.toml --to redis --to-config redis.toml +``` + +## chel version + +``` +chel version +``` + +Prints the version of chel. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..e1d72fa --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,138 @@ +# Configuration + +chel uses two files for two different things. `chel.toml` configures the +`chel` command itself, and `chelonia.json` describes your app's contracts. + +## chel.toml + +`chel.toml` configures the `chel` command at runtime: things like the +server's host and ports, the database backend and the signup limits. + +By default, `chel` reads `./chel.toml`. To use another file, pass its path +with `--config`: + +```bash +chel serve --config data/chel.toml dist +``` + +Each setting comes from the first of these that has it: + +1. an environment variable +2. a command-line flag +3. `chel.toml` +4. the built-in default + +Environment variables use the setting's path with `__` between the parts. +Their values are parsed, so numbers and booleans work: + +```bash +server__port=9000 server__signup__disabled=true chel serve dist +``` + +`chel init` writes a starting file with a fresh `server_id` and the default +settings, the less common ones as comments. + +The commands that read the file (`deploy`, `eventsAfter`, `get`, `migrate`, +`pin`, `serve` and `upload`) check it when they start. An unknown key is a +warning, often with the name it looks like, since it is usually a typo. A +value of the wrong type or shape is an error. A missing `./chel.toml` is fine, +and the defaults apply, but a missing `--config` file is an error. + +```toml +# Example chel.toml +server_id = "..." # written by chel init + +[server] +host = "0.0.0.0" +port = 8000 +dashboardPort = 8888 +# The dashboard listens on this machine only, unless you change this +dashboardListenIP = "127.0.0.1" +# The dashboard shows no data until this is set to a long, random password +# dashboardAdminPassword = "..." + +[server.vapid] +email = "admin@example.com" + +[database] +backend = "sqlite" + +# Signup size cap and the billing free tier; see +# signup-and-billing.md. +# [server.signup] +# maxFirstMessageBytes = 8192 +# +# [server.billing] +# freeAllowanceBytes = 10485760 +``` + +### Database backends + +`database.backend` sets where chel keeps its data. Each backend takes its +options from its own table, `[database.backendOptions.]`. + +| backend | options | +| --- | --- | +| `mem` (default) | none. Everything is kept in memory, and lost when chel stops | +| `fs` | `dirname` (default `data`): the folder with one file per entry. `depth` and `keyChunkLength` spread the files over subfolders named after parts of their keys, and `skipFsCaseSensitivityCheck` skips the startup check that the folder is on a case-sensitive filesystem | +| `sqlite` | `filepath` (default `data/chelonia.db`) | +| `redis` | `url`, starting with `redis://` or `rediss://` | +| `router` | sends keys to other backends by prefix: each prefix maps to `{ name, options }`, and an entry for `*` takes every other key | + +With any backend but `mem`, `database.lruNumItems` (default `10000`) sets how +many entries the in-memory cache in front of it holds. + +## chelonia.json + +`chelonia.json` is an **app-level properties file** that describes which +contracts make up the application and their pinned versions. It is **not** +runtime configuration for `chel`. Instead, it is: + +- **Created and updated** by `chel pin` when pinning contracts to specific + versions. +- **Read by the server** (`chel serve`) at startup, which sends every client + that connects the app's version and the version of each contract. + +The file contains a `contracts` object keyed by the full contract name +(e.g. `gi.contracts/chatroom`), where each entry has: + +| Field | Description | +|-------|-------------| +| `version` | The pinned version string for this contract | +| `path` | Relative path to the manifest file in the `contracts/` directory | + +It may also contain an `appVersion` field for the overall application version. +`chel pin` does not write it. + +**Example:** +```json +{ + "appVersion": "2.0.0", + "contracts": { + "gi.contracts/chatroom": { + "version": "2.0.6", + "path": "contracts/gi.contracts_chatroom/2.0.6/chatroom.2.0.6.manifest.json" + }, + "gi.contracts/group": { + "version": "2.0.0", + "path": "contracts/gi.contracts_group/2.0.0/group.2.0.0.manifest.json" + } + } +} +``` + +`chel pin` writes `chelonia.json` in the current folder, but `chel serve` +looks for it in the app folder, unless `--app-manifest` (or `appManifest` in +`chel.toml`) points somewhere else. So when the app is in a subfolder, as in +`chel serve dist`, pass `--app-manifest chelonia.json`. Without it, the server +logs a warning and clients get no version information. + +**Summary of differences:** + +| | `chel.toml` | `chelonia.json` | +|---|---|---| +| **Purpose** | Runtime configuration of the `chel` CLI | App properties (e.g., contract versions) | +| **Managed by** | Manually by the developer / system administrator | Automatically by `chel pin` | +| **Read by** | `chel` commands (via `nconf`) | The server (at startup), values sent to clients | +| **Format** | TOML | JSON | +| **Example content** | Database configuration | Contract versions | diff --git a/docs/distributing.md b/docs/distributing.md new file mode 100644 index 0000000..07c13dc --- /dev/null +++ b/docs/distributing.md @@ -0,0 +1,141 @@ +# Distributing chel + +How chel is built, packaged and published to npm. The list of supported +platforms is in the [README](../README.md#supported-platforms). + +## Publishing a release + +Steps to publish a new release of `@chelonia/cli` to npm: + +1. **Login to npm** (prerequisite): + + ```bash + npm login + ``` + + If your account uses 2FA, `npm publish` may prompt for authentication. + Do not rely on `--otp=`: a TOTP code entered at the start of the + release may well have expired by the time the first sub-package is + published. Complete the browser-based authentication flow when prompted, + or use a granular automation token instead. + +2. **Bump the version:** + + ```bash + npm version patch # or minor / major + ``` + + This triggers the `version` lifecycle hook, which runs *after* the version + is bumped but *before* `npm version` commits and tags. The hook: + - Runs `deno task sync-versions` to update `optionalDependencies` in + `package.json` so they match the new version (and stages the change) + - Runs `deno task build` to rebuild the JS bundle (stamped with the new + version in `build/version.json`) and the dashboard + - Stages the rebuilt `build/` artifacts + + As a result, the version commit and tag contain the up-to-date bundle and + the working directory stays clean. The publish step verifies that what it + ships is exactly this committed bundle and refuses to run otherwise. + This also guarantees the maintainer rebuilds the bundle on `master` + (merged PRs can each produce a correct bundle individually, yet the + combined result on `master` may still differ). + +3. **Build the release artifacts** (the tarballs for the GitHub release): + + ```bash + deno task dist + git diff --exit-code -- build # fails if the rebuild diverged from the tag + ``` + + This lints, rebuilds the bundle, compiles one native binary per supported + platform, and packs each one into `dist/chel-v-.tar.gz`, + printing SHA256 checksums. + + The `git diff --exit-code -- build` afterwards confirms the rebuild is + byte-identical to the version commit, i.e. that the build is reproducible. + It also has to pass before publishing, since the next step requires a clean + `build/`. If it fails, do not publish: investigate the divergence first. + +4. **Publish:** + + ```bash + npm publish --access public + ``` + + This triggers the `prepublishOnly` hook, which runs `deno task publish`. + That script: + - Verifies that the committed bundle matches `package.json` (via + `build/version.json`) and that the working tree is clean for `build/` + and `package.json`, refusing to publish otherwise + - Reuses the native binaries from step 3 instead of compiling its own + copies, so a release compiles each binary once and npm and GitHub are + guaranteed to ship identical bytes. If a binary is missing, or was built + from a different bundle, it is compiled here instead. Publishing without + step 3 works, it just takes several minutes longer + - Creates and publishes a platform sub-package (`@chelonia/cli--`) for each target + - Reconciles `optionalDependencies` as a final safety check + + After the sub-packages are published, `npm publish` proceeds to publish the + main `@chelonia/cli` package, whose `optionalDependencies` now point to the + freshly published sub-packages. + +5. **Push the version commit and tag:** + + ```bash + git push && git push --tags + ``` + +## Build caching + +Compiling the five native binaries is by far the slowest part of a release, so +the tarballs and the npm sub-packages share one set of them, kept under the +gitignored `dist/` directory. `deno task dist` remains usable on its own, at +any time, to produce the tarballs without publishing anything. + +The cache operates on content, not on timestamps. When the bundle, the Deno +version, or a compile flag changes, `deno task dist` compiles the binaries +again. When nothing changes, it does no compile work. To compile all binaries +again, set `CHEL_FORCE_COMPILE=1` or delete `dist/`. + +To check that a compiled binary loads its SQLite addon, run +`CHEL_SMOKE_COMPILE=1 deno task smoke`. This test compiles a binary and is +therefore not part of `deno task test`. + +## How the npm packages work + +The `chel` command itself is always provided by the main `@chelonia/cli` +package, which ships a small launcher that spawns the native binary from the +sub-package npm selected. The sub-packages intentionally declare no command of +their own: if they did, they would compete with the main package for the same +`chel` entry in `node_modules/.bin`, and npm would end up wiring `chel` to a +bare platform binary (or to nothing at all when the main package is the project +root). Because npm therefore never adjusts the binary's permissions at install +time, the executable bit is baked into the published sub-package instead. + +This layout is verified against npm 10, pnpm (both its isolated and hoisted +layouts), Yarn 1, and Yarn 4 (both Plug'n'Play and `node-modules`), for local +and global installs. The launcher's exit codes are listed in the +[README](../README.md#supported-platforms). + +## Linux on musl + +The Linux binaries are linked against glibc, so Linux distributions built on +musl libc (Alpine, for example) are **not** covered: the native SQLite addon +each Linux binary carries is the glibc build for its platform. Running `chel` +from source with Deno works there instead. Only the SQLite backend is affected, +and selecting it on such a system fails with a message saying so rather than +with a raw loader or missing-module error. + +From source, nothing has to be compiled locally: the SQLite driver ships musl +prebuilds (`prebuilds/linuxmusl-.node`) next to the glibc ones, and its +resolver chooses between them by asking Node whether the process is running +against glibc. Deno hardcoded a glibc version into that answer until Deno +2.8.0 ([denoland/deno#33948](https://github.com/denoland/deno/issues/33948)), +so older versions pick the glibc prebuild on musl and then fail inside the +dynamic loader. + +On Deno 2.8.0 or newer this works out of the box. On anything older, either +upgrade Deno or patch `node_modules/better-sqlite3/lib/binding.js` so that +`isLinuxMusl` always returns `true`. Compiling the addon from its C sources +(`npm run build-release`) does **not** help: the resolver returns a matching +prebuild before it ever looks at a locally built copy. diff --git a/docs/images/dashboard.png b/docs/images/dashboard.png new file mode 100644 index 0000000..a0bc355 Binary files /dev/null and b/docs/images/dashboard.png differ diff --git a/docs/server.md b/docs/server.md new file mode 100644 index 0000000..c5ba46a --- /dev/null +++ b/docs/server.md @@ -0,0 +1,79 @@ +# Running a server + +`chel serve` is the whole server: it serves your built app, answers the API, +and runs the websocket that pushes updates to the clients. It also starts a +dashboard on a second port. Its options are in +[commands.md](commands.md#chel-serve). + +## What it serves + +- **The app**, at `/app/`: `index.html` from the app directory, and that + directory's `assets/` folder at `/assets/`. +- **The API and the websocket**, on the same port, `8000` by default. +- **The dashboard**, on its own port, `8888` by default. See + [the dashboard](#the-dashboard). + +## What happens at startup + +1. It checks that `server_id` is set, and refuses to start without it. + `chel init` writes one. +2. It starts the dashboard. +3. It uploads every manifest in the manifests folder (`contracts/` by default, + laid out as `contracts///`) to its database, together with + the contract files they point to. With `--dev` it also keeps watching the + folder, and uploads new versions as they appear. Without `--dev`, a new + version needs a restart or a `chel deploy`. +4. It starts the app server. + +```bash +# The app and its contracts in the current directory +chel serve + +# Other ports +chel serve --port 8001 --dashboard-port 8889 ./my-app + +# Development: also upload contract versions as they are pinned +chel serve --dev ./dist +``` + +## The dashboard + +The dashboard shows the server's registered usernames, the contracts it +stores, how much space each account uses and each account's credits. It never +shows what is inside a contract, since that is encrypted. Because of the +usernames, it has two settings under `[server]` in `chel.toml`: + +- `dashboardListenIP`: the address it listens on, `127.0.0.1` by default, so + only this machine can open it. +- `dashboardAdminPassword`: until this is set, the dashboard shows no data. + Once it is, the browser asks for it (any user name works). To keep it out of + the file, set `server__dashboardAdminPassword` in the environment instead. + The password travels with every request, so put the dashboard behind HTTPS + before opening it to other machines. + +## In production + +```bash +NODE_ENV=production chel serve --app-manifest chelonia.json dist +``` + +- **`NODE_ENV=production`** turns on the signup limits per IP address (see + [signup-and-billing.md](signup-and-billing.md)). Two routes for development + are only there when `NODE_ENV` is exactly `development`: the one + `chel deploy --url` and `chel upload --url` upload through, and one that + prints logs sent by clients. +- **A database that persists.** The default backend, `mem`, loses every + account when the server stops. See + [configuration.md](configuration.md#database-backends). +- **`chelonia.json`.** `contracts/` is read from the current folder, and so is + `chel.toml` unless `--config` says otherwise. But `chelonia.json` is read + from the app folder, unless `--app-manifest` points somewhere else. `chel pin` writes it in the current folder, hence the + flag when the app is in `dist/`. Without it, the server only logs a warning, + and clients get no version information. +- **Behind a reverse proxy.** chel limits signups per client address, and + takes that address from the `X-Real-IP` header when a request has one. So + the proxy must set `X-Real-IP` and pass websocket upgrades through, and + clients must not be able to reach chel directly: they could send their own + `X-Real-IP` and get around the limits. +- **Backups.** Every account is in the database, so back it up, together with + `chel.toml` and its `server_id`. diff --git a/docs/signup-and-billing.md b/docs/signup-and-billing.md index 48d802a..3e9d485 100644 --- a/docs/signup-and-billing.md +++ b/docs/signup-and-billing.md @@ -43,5 +43,5 @@ contract plus everything it owns. `server.billing.freeAllowanceBytes` tier. Storage beyond the allowance is charged by the credits worker, which runs a billing cycle in the background and debits the entity's balance for the bytes it keeps stored. Balances are recorded but not yet enforced: an -entity whose balance drops below zero is not frozen and can keep writing -(see okTurtles/chel#160). +entity whose balance drops below zero is not frozen and can keep writing. +Freezing it is the plan agreed in the discussion of okTurtles/chel#160. diff --git a/scripts/publish.ts b/scripts/publish.ts index 5c4e3ea..56bee72 100644 --- a/scripts/publish.ts +++ b/scripts/publish.ts @@ -118,7 +118,7 @@ async function assertFreshBundle (): Promise { throw new Error( `Untracked files under ${SERVE_DIR}/ or ${DASHBOARD_DIR}/ would be embedded ` + 'into the compiled binaries but are not committed; remove them, or rebuild ' + - 'and amend the version commit (see the release steps in README.md)' + 'and amend the version commit (see the release steps in docs/distributing.md)' ) } } diff --git a/src/hash.ts b/src/hash.ts index da67443..5b06730 100644 --- a/src/hash.ts +++ b/src/hash.ts @@ -32,7 +32,7 @@ export const module = { command: 'hash ', describe: 'Computes and logs the content identifier (CID) for the given file.\n' + - 'File contents will be interpreted as raw binary data, unless the file extension is \'.json\'.', + 'File contents are hashed as raw binary data.', postHandler: (argv) => { return void hash(argv) } diff --git a/src/parseConfig.test.ts b/src/parseConfig.test.ts index a64b6d6..041adaa 100644 --- a/src/parseConfig.test.ts +++ b/src/parseConfig.test.ts @@ -116,7 +116,7 @@ Deno.test({ }) }) - // What the README says the order is. nconf adds the env store first, so it + // What docs/configuration.md says the order is. nconf adds the env store first, so it // wins over a flag, which surprises most people. await t.step('come from the environment even when a flag is given', async () => { const args = ['serve', '--port', '7777'] diff --git a/test/docs-defaults.test.ts b/test/docs-defaults.test.ts new file mode 100644 index 0000000..d62a840 --- /dev/null +++ b/test/docs-defaults.test.ts @@ -0,0 +1,74 @@ +// Drift guard for the operator-facing documentation of the defaults. +// +// `chel init` interpolates every default into the generated `chel.toml` and +// `src/init.test.ts` pins that, so the template cannot go stale. The example +// and the backends table in docs/configuration.md and the prose in +// docs/signup-and-billing.md are written by hand, which is where the numbers an +// operator actually reads live, so this test pins those numbers too: a default +// that changes without the documentation following leaves the docs quietly +// wrong. Only the values are checked, not the wording around them, so the prose +// can be edited freely. +import { assert } from 'jsr:@std/assert' +import { nconfDefaults } from '../src/config-defaults.ts' +import { MAX_EVENT_BODY_BYTES } from '../src/serve/constants.ts' + +const config = await Deno.readTextFile('docs/configuration.md') +const doc = await Deno.readTextFile('docs/signup-and-billing.md') + +const assertDocumented = (text: string, description: string, snippet: string) => { + assert( + text.includes(snippet), + `the documentation should cover ${description}. Expected to find: ${snippet}` + ) +} + +Deno.test({ + name: 'the docs document the current defaults', + fn () { + const { server, database } = nconfDefaults + const { signup, billing } = server + + // The example chel.toml in docs/configuration.md + assertDocumented(config, 'the example app port', `port = ${server.port}`) + assertDocumented( + config, + 'the example dashboard port', + `dashboardPort = ${server.dashboardPort}` + ) + assertDocumented( + config, + 'the example dashboard address', + `dashboardListenIP = "${server.dashboardListenIP}"` + ) + assertDocumented( + config, + 'the example first-message cap', + `maxFirstMessageBytes = ${signup.maxFirstMessageBytes}` + ) + assertDocumented( + config, + 'the example free allowance', + `freeAllowanceBytes = ${billing.freeAllowanceBytes}` + ) + + // The backends table in docs/configuration.md + assertDocumented(config, 'the default backend', `\`${database.backend}\` (default)`) + assertDocumented(config, 'the cache size default', `(default \`${database.lruNumItems}\`)`) + + // The prose in docs/signup-and-billing.md. The three rate limits are + // checked together, in the order the doc lists them, so a changed default + // can't pass by matching another number in the doc. + const { minute, hour, day } = signup.limit + assertDocumented( + doc, + 'the per-minute/hour/day rate limit defaults', + `\`${minute}\`/\`${hour}\`/\`${day}\`` + ) + assertDocumented(doc, 'the first-message cap default', `\`${signup.maxFirstMessageBytes}\``) + assertDocumented(doc, 'the free allowance default', `\`${billing.freeAllowanceBytes}\``) + + // A limit that is enforced elsewhere but bounds these settings in + // practice, and which operators cannot discover from the settings + assertDocumented(doc, 'the POST /event request body limit', `(${MAX_EVENT_BODY_BYTES} bytes)`) + } +}) diff --git a/test/readme-config.test.ts b/test/readme-config.test.ts deleted file mode 100644 index e78e5da..0000000 --- a/test/readme-config.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Drift guard for the operator-facing documentation of the signup and billing -// settings. -// -// `chel init` interpolates every default into the generated `chel.toml` and -// `src/init.test.ts` pins that, so the template cannot go stale. The -// commented-out example in README.md and the prose in docs/signup-and-billing.md -// are written by hand, which is where the numbers an operator actually reads -// live, so this test pins those numbers too: a default that changes without the -// documentation following leaves the docs quietly wrong. Only the values are -// checked, not the wording around them, so the prose can be edited freely. -import { assert } from 'jsr:@std/assert' -import { nconfDefaults } from '../src/config-defaults.ts' -import { MAX_EVENT_BODY_BYTES } from '../src/serve/constants.ts' - -const readme = await Deno.readTextFile('README.md') -const doc = await Deno.readTextFile('docs/signup-and-billing.md') - -const assertDocumented = (text: string, description: string, snippet: string) => { - assert( - text.includes(snippet), - `the documentation should cover ${description}. Expected to find: ${snippet}` - ) -} - -Deno.test({ - name: 'README and docs document the current signup and billing defaults', - fn () { - const { signup, billing } = nconfDefaults.server - - // The commented-out example block near "Runtime CLI Configuration" - assertDocumented(readme, 'the example first-message cap', `maxFirstMessageBytes = ${signup.maxFirstMessageBytes}`) - assertDocumented(readme, 'the example free allowance', `freeAllowanceBytes = ${billing.freeAllowanceBytes}`) - - // The prose in docs/signup-and-billing.md - for (const [window, value] of Object.entries(signup.limit)) { - if (typeof value !== 'number') continue - assertDocumented(doc, `the per-${window} rate limit default`, `\`${value}\``) - } - assertDocumented(doc, 'the first-message cap default', `\`${signup.maxFirstMessageBytes}\``) - assertDocumented(doc, 'the free allowance default', `\`${billing.freeAllowanceBytes}\``) - - // A limit that is enforced elsewhere but bounds these settings in - // practice, and which operators cannot discover from the settings - assertDocumented(doc, 'the POST /event request body limit', `(${MAX_EVENT_BODY_BYTES} bytes)`) - } -})