mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-21 01:51:46 +01:00
Merge storytold main into next (17 Nov sync)
storytold main 83cb1133dd (291 commits) into the integration head bab1a99b85; 47 conflicted files.
- Domain test files (metafile, prepress, pdf plates, settings store, heif, media-testkit, atomic): tests next cut stay cut; tests new on main are kept, as trials where the crate's test target runs trials.
- atomic: main's group-bits test and Faulty::set_group go to src/tests.rs. audio: main's GSM and SBC oracles join tests/main.rs. cms constructors, pdf knockout_backdrop and testing_pdf join their crates' one test binary; timeline's preflight tests become src/preflight_tests.rs trials.
- update: main's changes to detect, install (linux, macos) and the tests land in next's split modules. The app's own offer is inspected on a thread of its own, picked up by tick's polling; settle_for_tests settles offers too. Later on a ready update drops the close it asked for. updater_pin_tests.rs stays deleted; its new pin test is in tests/it/update.rs.
- craft-color: main added a different crate at crates/color (the colour tags of compositor and display frames); it is next's craft-color module `tags`, re-exported at the root.
- Old crates/ui (combo, docking, testing, README) and keymap's ui/view.rs stay deleted; main's new crates/ui/src/testing files are dropped with them.
- New crates in the craft facade: ai, control, dds, frame, ipc, media-source, palette, press-print, score, ssip. testkit and xtask are listed outside it.
- Cargo.lock: the union of both locks, resolved by cargo metadata. Workflow: main's nextest Windows leg, with next's --keep-going clippy.
This commit is contained in:
commit
ac3dfd1f53
581 files changed
+77998
-3002
No files matched your search
@@ -1,2 +0,0 @@
|
||||
[lints]
|
||||
cargo.unused_workspace_dependencies = "allow"
|
||||
@@ -0,0 +1,44 @@
|
||||
# cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test
|
||||
# binaries. The Windows leg of the Workspace gate uses it (.github/workflows/workspace.yml); every
|
||||
# other leg, and local runs, keep `cargo test`. Nextest does not run doctests, so the Windows leg
|
||||
# also runs `cargo test --doc`.
|
||||
#
|
||||
# cargo nextest run --workspace --profile ci
|
||||
# cargo test --workspace --doc
|
||||
#
|
||||
# Refuse an older nextest: CI installs exactly this version (GHSA-vc25-qg58-w57c is fixed in it).
|
||||
nextest-version = "0.9.150"
|
||||
|
||||
# Some tests share a resource that an in-process mutex keeps to one test at a time. With one
|
||||
# process per test those mutexes no longer serialize anything, so these groups do it instead, one
|
||||
# test at a time, as under `cargo test`.
|
||||
[test-groups]
|
||||
# craft-ui: `gpu_test_lock()` / `gpu_lock()`. Each renderer creates its own wgpu instance and
|
||||
# device, and creating or destroying one while another test uses its device can crash.
|
||||
gpu = { max-threads = 1 }
|
||||
# craft-video h264_decode: `FIXTURE_GENERATION` keeps a second producer from replacing a fixture
|
||||
# file another test is reading.
|
||||
video-fixtures = { max-threads = 1 }
|
||||
# craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests
|
||||
# share the one test print queue the gate installs.
|
||||
print = { max-threads = 1 }
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))'
|
||||
test-group = 'gpu'
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(craft-video) & binary(h264_decode)'
|
||||
test-group = 'video-fixtures'
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(craft-print)'
|
||||
test-group = 'print'
|
||||
|
||||
# The gate (`--profile ci`). Run every test and list all failures at the end, as `cargo test`
|
||||
# reports a whole binary. Slow tests are named every 60 seconds; none is stopped for being slow,
|
||||
# so a hang is still ended by the gate's own timeout, as before.
|
||||
[profile.ci]
|
||||
fail-fast = false
|
||||
slow-timeout = { period = "60s" }
|
||||
failure-output = "immediate-final"
|
||||
@@ -25,7 +25,8 @@ jobs:
|
||||
with:
|
||||
os: ${{ github.ref == 'refs/heads/next' && '["ubuntu-latest"]' || '["ubuntu-latest","windows-latest","macos-latest"]' }}
|
||||
timeout-minutes: 120
|
||||
tools: ""
|
||||
# The Windows leg runs its tests with nextest (.config/nextest.toml).
|
||||
tools: cargo-nextest@0.9.150
|
||||
packages: mesa-vulkan-drivers libvulkan1
|
||||
craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83
|
||||
linux-env: WGPU_BACKEND=vulkan
|
||||
@@ -58,8 +59,12 @@ jobs:
|
||||
cargo fmt --check
|
||||
cargo clippy --workspace --all-targets --locked --keep-going -- -D warnings
|
||||
cargo clippy --workspace --all-targets --all-features --locked --keep-going -- -D warnings
|
||||
cargo test --workspace --locked
|
||||
cargo test --workspace --all-features --locked
|
||||
# The same tests as `cargo test`, run in parallel across test binaries instead of one
|
||||
# binary at a time; nextest skips doctests, so `cargo test --doc` runs them.
|
||||
cargo nextest run --workspace --locked --profile ci
|
||||
cargo test --workspace --doc --locked
|
||||
cargo nextest run --workspace --all-features --locked --profile ci
|
||||
cargo test --workspace --doc --all-features --locked
|
||||
|
||||
msrv:
|
||||
name: Rust 1.95
|
||||
|
||||
+23
-1
@@ -93,7 +93,7 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso
|
||||
| `crates/psd/tests/fixtures/atn/*.atn` (seven synthetic action sets, encoded field by field; `crates/psd/tests/fixtures/atn/README.md`) | PhotoCraft contributors | [PhotoCraft `crates/psd/tests/fixtures/atn`](https://github.com/storytold/photocraft/tree/63b641b55e54bf8f5c83efd977fdb16a2c255431/crates/psd/tests/fixtures/atn) | MIT OR Apache-2.0 |
|
||||
| `crates/lottie/tests/fixtures/` | EffectCraft contributors (original work: four hand-written Lottie JSON documents, written from the public Lottie format description; not exported from any Adobe product or third-party file) | [EffectCraft `crates/lottie/tests/fixtures`](https://github.com/storytold/effectcraft/tree/e18f48fb/crates/lottie/tests/fixtures) | MIT OR Apache-2.0 |
|
||||
| `crates/pkcs7/tests/data/` | PdfCraft contributors (original work: throw-away test keys and certificates made with OpenSSL 3 for these tests, as seven PKCS #12 files with the password `test` and two PEM certificates; they protect nothing; commands in `crates/pkcs7/tests/data/README.md`) | [PdfCraft `crates/sign/tests/data`](https://github.com/storytold/pdfcraft/tree/1d7a9f9f/crates/sign/tests/data) | MIT OR Apache-2.0 |
|
||||
| `crates/affinity/tests/support/corpus.tsv`, `crates/affinity/tests/support/affinity3-signatures.txt` (manifest and measured signatures only; the 51 documents are not bundled, `scripts/fetch-affinity-corpus.sh` downloads them outside the checkout) | The authors of each document: samuel-etver (vector-art), Seth A. Robinson (Patchy fixtures), Nick Beeuwsaert (AFDesignLoad), Jeremy Cantu (Branding), eviltwo (AssetStoreTemplate), Square (leakcanary) | each file's pinned upstream URL and SHA-256 in the manifest; list carried over from [VectorCraft `xtask/src/affinity_corpus.rs`](https://github.com/storytold/vectorcraft/blob/6e873b806c9d5b3f2b55f651df57bd97933b579a/xtask/src/affinity_corpus.rs) | CC0-1.0, MIT or Apache-2.0 (per file, in the manifest) |
|
||||
| `crates/affinity/tests/support/corpus.tsv`, `crates/affinity/tests/support/affinity3-signatures.txt` (manifest and measured signatures only; the 113 files, 90 of them documents, are not bundled, `scripts/fetch-affinity-corpus.sh` downloads them outside the checkout) | The authors of each document: samuel-etver (vector-art), satoshoe-dev (affinity-samples), Seth A. Robinson (Patchy fixtures), Nick Beeuwsaert (AFDesignLoad), Jeremy Cantu (Branding), eviltwo (AssetStoreTemplate), Square (leakcanary) | each file's pinned upstream URL and SHA-256 in the manifest; every name from [VectorCraft `xtask/src/affinity_corpus.rs`](https://github.com/storytold/vectorcraft/blob/bada8e17103db9917de44e40a1cead5a7fc71c3c/xtask/src/affinity_corpus.rs) and [PhotoCraft `xtask/affinity-corpus.sha256`](https://github.com/storytold/photocraft/blob/629751503165118d5fec5f70e64fe2d6592f7b53/xtask/affinity-corpus.sha256) | CC0-1.0, MIT or Apache-2.0 (per file, in the manifest) |
|
||||
| `crates/raw/docs/coverage/adobe-cameras.tsv` | Facts (maker, model, extension, profile, minimum Camera Raw version) parsed by LightCraft contributors from Adobe's published list | [Adobe Camera Raw supported cameras](https://helpx.adobe.com/camera-raw/desktop/dng-and-file-formats/camera-raw-plug-supported-cameras.html), Wayback capture 20261003133011; moved from [LightCraft `docs/raw`](https://github.com/storytold/lightcraft/tree/5ff567a4a2e067bef1b6bce152c11e7f727ccbaf/docs/raw) | Facts, not copyrightable expression; the file is MIT OR Apache-2.0 |
|
||||
| `crates/raw/docs/coverage/{coverage,coverage-sweep,evidence,usage-weights}.tsv`, `coverage.py` | LightCraft contributors | [LightCraft `docs/raw`](https://github.com/storytold/lightcraft/tree/5ff567a4a2e067bef1b6bce152c11e7f727ccbaf/docs/raw) | MIT OR Apache-2.0 |
|
||||
| `crates/raw/docs/raf-corpus.sha256` (hashes only; the files are not bundled) | The photographers who contributed each file to raw.pixls.us | [raw.pixls.us](https://raw.pixls.us/) | CC0-1.0 |
|
||||
@@ -106,3 +106,25 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso
|
||||
|
||||
`crates/tga/src/metadata_tests.rs` and metadata-boundary helpers in `crates/tga/src/lib.rs` are adapted from nuclearspike's FilmCraft #1932 at `e7842f76f887005c460c64a8ab51d59d086e6f27` (MIT OR Apache-2.0). Synthetic fixtures are original contributor test data; primary Git author retained.
|
||||
The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper are ported from Craft Anything `8f3b775157d331dd31cfa4da911e26d8856b8c77` (same MIT OR Apache-2.0 source licenses). Brandon Thomas is the primary import author; retained hardening credits include Afloat, BFlat, Rahul, RRFZ and NetterSilver. The finite-real fix preserves gmrmad1 as author from Craft Anything #1905 (`d85e104f`); quoted-attribute/line-ending regressions preserve RRFZ. Bounds/adaptation and bounded entity scanning are subsequent Dylan M. Taylor changes. App-specific JSON projections and keyed-archive interpretation remain outside this crate.
|
||||
|
||||
## Score serializers
|
||||
|
||||
`crates/score` extracts SoundCraft MusicXML and SVG score writing from `crates/engine/src/score.rs` at main `648ed9db9a0e98cc55643babffac381521a556c8`, with per-pitch ties adapted from [SoundCraft #182](https://github.com/storytold/soundcraft/pull/182) at `21cb618d93ec0e8d4771ac534994de8d5996ff10`. Source is MIT OR Apache-2.0. Import commits preserve Brandon Thomas, NetterSilver, Samuel Bacaro and Vikram Bhaskaran as primary authors of their respective changes. The consumer retains session mapping and layout. Shared API wiring, validation and focused tests are subsequent Dylan M. Taylor changes.
|
||||
|
||||
## Source-timed MIDI compatibility
|
||||
|
||||
`crates/midi/src/source_timed.rs` is extracted from LoopCraft `crates/midi/src/lib.rs` at `884e67c24b9b68badf6a4a4f7d62492f3673d354` (MIT OR Apache-2.0). The recorded source history for the extracted file is Dylan M. Taylor (`0d150c1f`, `440bc7ed`, `9e50523d`); Git authorship is preserved. It includes the original hostile-file, SMPTE, timing and round-trip tests. UTF-8 truncation and event-budget suffix repairs are subsequent changes. The module is an application-independent source-timed model; it does not depend on LoopCraft.
|
||||
## Non-seeking WAV primitives
|
||||
|
||||
`crates/audio/src/wav_stream.rs` is extracted from AniCraft `crates/audio/src/wav.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0). Tanish Manku is retained as the primary import author, from original commit `30c121d269a49d9abe54a3834ed4aa0201542535`. Odd-data padding accounting, shared API documentation and regression tests are subsequent Dylan M. Taylor changes.
|
||||
|
||||
## GIMP palette reader
|
||||
|
||||
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
|
||||
|
||||
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
|
||||
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
|
||||
Sanitization, animation fitting and numeric output policy remain caller-owned.
|
||||
## Screen Video key frames
|
||||
|
||||
`crates/video/src/screen_video.rs` extracts AniCraft `crates/xfl/src/video.rs` codec-3 decoding and its synthetic partial-block fixture, introduced by Tanish Manku in `1f3e603b7ff53a3da364e41ec4da8d43dc83aab0` with pixel-loop cleanup `2b7571363b4b647266df758f95b6e397021cfd4d` (MIT OR Apache-2.0). Import commits preserve Tanish as primary author. Caller-owned bounds and neutral RGBA output separate codec work from the consuming app's FLV first-frame/report policy. Boundary tests and integration are subsequent Dylan M. Taylor changes.
|
||||
Generated
+999
-17
File diff suppressed because it is too large.
Load diff
+16
-1
@@ -11,10 +11,13 @@ rust-version = "1.95"
|
||||
repository = "https://github.com/storytold/craft-libs"
|
||||
|
||||
[workspace.dependencies]
|
||||
craft-palette = { path = "crates/palette" }
|
||||
craft-score = { path = "crates/score" }
|
||||
craft-shx = { path = "crates/shx" }
|
||||
craft-http-url = { path = "crates/http-url" }
|
||||
craft-usd = { path = "crates/usd" }
|
||||
craft-smime = { path = "crates/smime" }
|
||||
craft-ssip = { path = "crates/ssip" }
|
||||
craft-ber = { path = "crates/ber" }
|
||||
craft-proto = { path = "crates/proto" }
|
||||
craft-render-parity = { path = "crates/render-parity" }
|
||||
@@ -26,6 +29,7 @@ craft-platform = { path = "crates/platform" }
|
||||
craft-aep = { path = "crates/aep" }
|
||||
craft-affinity = { path = "crates/affinity" }
|
||||
craft-win32 = { path = "crates/win32" }
|
||||
craft-ai = { path = "crates/ai" }
|
||||
craft-ai-core = { path = "crates/ai-core" }
|
||||
craft-anicraft-format = { path = "crates/anicraft-format" }
|
||||
craft-pdf = { path = "crates/pdf", version = "=0.1.0" }
|
||||
@@ -49,10 +53,13 @@ craft-gridcraft-format = { path = "crates/gridcraft-format" }
|
||||
craft-aaf = { path = "crates/aaf" }
|
||||
craft-behavior = { path = "crates/behavior" }
|
||||
craft-atomic = { path = "crates/atomic" }
|
||||
craft-xtask = { path = "crates/xtask" }
|
||||
craft-captions = { path = "crates/captions" }
|
||||
craft-cfb = { path = "crates/cfb" }
|
||||
craft-codepage = { path = "crates/codepage" }
|
||||
craft-cms = { path = "crates/cms" }
|
||||
craft-color = { path = "crates/color" }
|
||||
craft-frame = { path = "crates/frame" }
|
||||
craft-crs = { path = "crates/crs" }
|
||||
craft-crw = { path = "crates/crw" }
|
||||
craft-csv = { path = "crates/csv" }
|
||||
@@ -60,7 +67,6 @@ craft-dav = { path = "crates/dav" }
|
||||
craft-automation = { path = "crates/automation" }
|
||||
craft-browse = { path = "crates/browse" }
|
||||
craft = { path = "crates/craft" }
|
||||
craft-color = { path = "crates/color" }
|
||||
craft-design-lint = { path = "crates/design-lint" }
|
||||
craft-dialogs = { path = "crates/dialogs" }
|
||||
craft-doc-binary = { path = "crates/doc-binary" }
|
||||
@@ -109,12 +115,14 @@ craft-font-subset = { path = "crates/font-subset" }
|
||||
harfbuzz-subset-sys = { git = "https://github.com/storytold/harfbuzz.git", rev = "2659dd807f9a48f75604ee7c76cb709f7b70b950" }
|
||||
craft-host = { path = "crates/host" }
|
||||
craft-i18n = { path = "crates/i18n" }
|
||||
craft-ipc = { path = "crates/ipc" }
|
||||
craft-idml = { path = "crates/idml" }
|
||||
craft-indd = { path = "crates/indd" }
|
||||
craft-jmap = { path = "crates/jmap" }
|
||||
craft-ical = { path = "crates/ical" }
|
||||
craft-imaging = { path = "crates/imaging" }
|
||||
craft-jpeg = { path = "crates/jpeg" }
|
||||
craft-control = { path = "crates/control" }
|
||||
craft-control-protocol = { path = "crates/control-protocol" }
|
||||
craft-jsonrpc = { path = "crates/jsonrpc" }
|
||||
craft-mcp = { path = "crates/mcp" }
|
||||
@@ -131,6 +139,7 @@ craft-macho = { path = "crates/macho" }
|
||||
craft-math = { path = "crates/math" }
|
||||
# No default features: craft-audio turns on the craft-media modules each of its features needs.
|
||||
craft-media = { path = "crates/media", default-features = false }
|
||||
craft-media-source = { path = "crates/media-source" }
|
||||
craft-materialx = { path = "crates/materialx" }
|
||||
craft-media-testkit = { path = "crates/media-testkit" }
|
||||
craft-midi = { path = "crates/midi" }
|
||||
@@ -154,10 +163,12 @@ craft-pkcs7 = { path = "crates/pkcs7" }
|
||||
craft-plotter = { path = "crates/plotter" }
|
||||
craft-prepress = { path = "crates/prepress" }
|
||||
craft-pages = { path = "crates/pages" }
|
||||
craft-press-print = { path = "crates/press-print" }
|
||||
craft-print = { path = "crates/print" }
|
||||
craft-proof = { path = "crates/proof" }
|
||||
craft-raster-codecs = { path = "crates/raster-codecs" }
|
||||
craft-image-codecs = { path = "crates/image-codecs" }
|
||||
craft-dds = { path = "crates/dds" }
|
||||
craft-psd = { path = "crates/psd" }
|
||||
craft-pst = { path = "crates/pst" }
|
||||
craft-quit = { path = "crates/quit" }
|
||||
@@ -176,6 +187,7 @@ craft-stencilcraft-format = { path = "crates/stencilcraft-format" }
|
||||
craft-stencilcraft-document = { path = "crates/stencilcraft-document" }
|
||||
craft-svg-paths = { path = "crates/svg-paths" }
|
||||
craft-stream-codecs = { path = "crates/stream-codecs" }
|
||||
craft-testkit = { path = "crates/testkit" }
|
||||
craft-text = { path = "crates/text" }
|
||||
craft-tcraft = { path = "crates/tcraft" }
|
||||
craft-texture = { path = "crates/texture" }
|
||||
@@ -240,6 +252,9 @@ chrono = { version = "0.4.45", default-features = false, features = ["std"] }
|
||||
unsafe_code = "forbid"
|
||||
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(craft_pdf_unicode_fixtures)"] }
|
||||
|
||||
[workspace.lints.cargo]
|
||||
unused_workspace_dependencies = "allow"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
cognitive_complexity = "deny"
|
||||
too_many_arguments = "deny"
|
||||
|
||||
@@ -260,11 +260,10 @@ applications retain their own document-font selection and licensing policy.
|
||||
linesweeper
|
||||
|
||||
linesweeper 0.5.0 (https://crates.io/crates/linesweeper/0.5.0), MIT OR Apache-2.0, is stored in
|
||||
vendor/linesweeper with its upstream license files and local patches, to src/sweep/sweep_line.rs and
|
||||
docs/sweep.typ. Its manifest names Joe Neeman as author, and its LICENSE-MIT reads "Copyright (c) 2018
|
||||
Raph Levien". See vendor/linesweeper/README.craft.md and vendor/linesweeper/SOURCE.json for
|
||||
provenance, the local patches
|
||||
and the files left out.
|
||||
vendor/linesweeper with its upstream license files and local patches, to src/sweep/sweep_line.rs,
|
||||
src/curve/mod.rs, src/curve/line.rs and docs/sweep.typ. Its manifest names Joe Neeman as author, and
|
||||
its LICENSE-MIT reads "Copyright (c) 2018 Raph Levien". See vendor/linesweeper/README.craft.md and
|
||||
vendor/linesweeper/SOURCE.json for provenance, the local patches and the files left out.
|
||||
|
||||
Platform lifecycle source provenance
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
of compositions, layers, properties, keyframes, effects, masks, shapes, text and footage.
|
||||
Clean room, from EffectCraft. Mapping onto an app's document stays in the app.
|
||||
|
||||
- [`craft-ai`](crates/ai/README.md) *(spec)*: AI provider abstraction — BYO API keys (OpenAI-compatible) and one-click curated local models with capability probing, resumable checksummed downloads, and document segmentation that protects markup while translating prose;
|
||||
- [`craft-ai-core`](crates/ai-core/README.md): the data model and contract for AI in the Craft Apps (connections, models, tools, requests, jobs, consent, history, route spec). Pure, no I/O, wasm-safe. Contract in [docs/ai-shared-layer.md](docs/ai-shared-layer.md).
|
||||
- [`craft-pe`](crates/pe/src/lib.rs): PE/COFF headers (`.exe`, `.dll`, `.sys`, `.efi`) from untrusted bytes, from Craft Anything: the COFF header, PE32/PE32+ optional header with its data directories bounded by the header's real size, the section table, RVA-to-file mapping and the decoded Rich header.
|
||||
- [`craft-pdf`](crates/pdf/README.md): independently optional PDF parsing, interpretation, rasterization and guarded authoring. Apps retain layout, fonts, editing and export policy; writer-only consumers do not enable the shared reader.
|
||||
@@ -70,6 +71,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
|
||||
- [`craft-automation`](crates/automation/README.md): drive a running app from tests, agents and the CLI: control channel, screenshot queue, frame profiler and trace (native only); recorded actions, their playback and the Actions panel (every target).
|
||||
- [`craft-atomic`](crates/atomic/README.md): Crash-safe atomic file writing and replacement with same-directory staging, directory fsync, Apple `F_FULLFSYNC` fallback, transient lock backoff, and fault injection for the Craft suite.
|
||||
- [`craft-xtask`](crates/xtask/README.md): a library for the apps' `cargo xtask` binaries: the gate's step runner (steps in order, stop at the first failure, the ok, FAIL and skip summary, workspace tests that exclude the running xtask package), an optional slot lock held across processes and an optional per-step JSON report. Each app keeps its step list and its own policy. Planned for DesignCraft and LightCraft.
|
||||
- [`craft-captions`](crates/captions/README.md): caption and subtitle files behind one neutral
|
||||
model (cues with exact tick times): SubRip, WebVTT, Scenarist SCC, MacCaption MCC, EBU STL and
|
||||
TTML / IMSC1 / DFXP readers and writers, and CEA-608 / CEA-708 byte-pair coding. Carved from
|
||||
@@ -96,6 +98,11 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
black and white levels and as-shot white balance. The complete derivation record (plan, sources
|
||||
log, scripts, verification) is in `crates/crw/clean-room/`. Colour stays in each app.
|
||||
- [`craft-csv`](crates/csv/README.md): delimited text (CSV, TSV) and SYLK: encoding detection (UTF-8, UTF-16, Windows-1252 or Windows-1256), delimiter sniffing, a record parser with a cap on fields per record, field quoting for writers, and a SYLK (`.slk`) reader with R1C1 formulas rewritten to A1.
|
||||
- [`craft-dds`](crates/dds/README.md): DirectDraw Surface (DDS) textures, read and written: the
|
||||
top mip level of uncompressed, BC1 to BC3 (DXT1/3/5) and BC4/BC5 (grayscale data and normal
|
||||
maps) textures, legacy and DX10 headers, and our own block encoder with optional mip chains. No dependencies (rayon behind `parallel`), so an
|
||||
app that only exports textures needn't take the whole `craft-image-codecs` codec set and its
|
||||
licences; `craft-image-codecs` reads and writes DDS through it.
|
||||
- [`craft-dialogs`](crates/dialogs/README.md): native file and message dialogs that never block
|
||||
the event loop: queued requests shown at the end of the frame and answered on a later frame,
|
||||
free-standing Open and Folder panels and Save sheets on macOS, case-insensitive filters. What
|
||||
@@ -249,6 +256,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
primitives (`.abr`, `.asl`, `.aco`, `.act`, `.ase`, `.grd`, `.pat`, `.atn`, `.csh`, `.shc`) and
|
||||
the Photoshop blocks of a layered TIFF. Bounded and panic-free on hostile input. Compositing
|
||||
and the mapping onto a document stay in each app.
|
||||
- [`craft-ipc`](crates/ipc/README.md): Craft apps talking on one machine: per-user local sockets (Unix socket, Windows named pipe), token auth, instance discovery, launch on demand, focus hand-off, frame transfer and linked-file watching. Native only: it does not build for `wasm32`. Protocol in [PROTOCOL.md](crates/ipc/PROTOCOL.md).
|
||||
- [`craft-jmap`](crates/jmap/README.md): the JMAP protocol core (RFC 8620), from MailCraft, without
|
||||
sockets: session, requests of method calls with back-references, responses, ids, address
|
||||
templates, credentials and the judgement of HTTP answers, over any HTTP implementation. Bounded
|
||||
@@ -258,7 +266,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
encoder, from PhotoCraft; includes a frame probe for routing. Metadata interpretation stays in
|
||||
each app.
|
||||
- [`craft-raster-codecs`](crates/raster-codecs/README.md): small raster formats, one module each,
|
||||
from PhotoCraft: DDS (BC1 to BC3, read and write), DICOM, FITS (read and write), IFF/ILBM (read
|
||||
from PhotoCraft: DDS (BC1 to BC3 read and write, BC4 and BC5 read), DICOM, FITS (read and write), IFF/ILBM (read
|
||||
and write), PCX, PNM/PAM/PFM (read and write) and XPM, into one flat image type. Bounded and
|
||||
panic-free on hostile input. Colour, orientation and the document mapping stay in each app.
|
||||
- [`craft-kiwi`](crates/kiwi/README.md): clean-room decoder and encoder for the kiwi binary schema
|
||||
@@ -293,9 +301,8 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
stay in each app.
|
||||
- [`craft-mesh`](crates/mesh/README.md): 3D mesh file formats behind one neutral model (meshes
|
||||
of polygons with per-corner indices, materials, a node hierarchy): STL, OBJ with MTL, PLY,
|
||||
glTF 2.0 / GLB and 3MF, one feature each. Readers are bounded by `Limits`, never open a file themselves
|
||||
glTF 2.0 / GLB and FBX, one feature each. Readers are bounded by `Limits`, never open a file themselves
|
||||
and never panic. Carved from EffectCraft, SolidCraft, TextureCraft and MaterialCraft.
|
||||
glTF 2.0 / GLB, FBX and 3MF, one feature each. Readers are bounded by `Limits`, never open a file
|
||||
themselves and never panic. Carved from EffectCraft, SolidCraft, TextureCraft and MaterialCraft.
|
||||
- [`craft-midi`](crates/midi/README.md): Standard MIDI Files: a bounded reader for formats 0, 1 and
|
||||
2 and a format-1 writer, over a neutral model of notes and channel events in ticks at 960 per
|
||||
quarter note. From SoundCraft.
|
||||
@@ -337,6 +344,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
- [`craft-pptx`](crates/pptx/README.md): PresentationML, the model of a `.pptx` presentation:
|
||||
masters, layouts, slides, shapes, placeholders and their inheritance, transitions and animations,
|
||||
comments, sections, media and kept parts. From DeckCraft. DrawingML comes from `craft-drawingml`.
|
||||
- [`craft-press-print`](crates/press-print/README.md): press-accurate printing: a PDF page separated
|
||||
into plates by `craft-prepress`, composited into the printer's RGB profile and printed through
|
||||
`craft-print`'s RGB transport, application-managed.
|
||||
- [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery,
|
||||
duplex/copies/colour and driver preferences. PDF generation stays in each app.
|
||||
- [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar,
|
||||
@@ -381,6 +391,13 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
- [`craft-table`](crates/table/README.md): spreadsheet widgets: the cell grid with Excel's
|
||||
keys, outline bands, the PivotTable field list, the condition builder, chart painting, slicers,
|
||||
reference picking, selection statistics, error tags and filter buttons (`ui` feature).
|
||||
- [`craft-testkit`](crates/testkit/README.md): test support for the apps' own tests. Its `guards` module
|
||||
reports source patterns that broke platforms before: muda's predefined Quit item, WGSL float literals
|
||||
out of range, and `std::time::Instant` in code the web build runs. Its `deadline` module fails a test
|
||||
whose body runs past a limit, or whose condition does not hold in time; a variable the caller names
|
||||
replaces each limit. Its `hostile` module runs a parser on every truncation and on seeded damaged
|
||||
copies of a valid input, and fails the test with the case that panicked or did not finish. Not
|
||||
published.
|
||||
- [`craft-text`](crates/text/README.md): text processing below the composer. Its `hyphen` module finds
|
||||
hyphenation points, with a bundled US English dictionary and patterns and optional hypher patterns
|
||||
for 47 more languages (one cargo feature each). With `ui`, the canvas text editor, Character and
|
||||
|
||||
+17
-10
@@ -115,11 +115,15 @@ text-frame fixture.
|
||||
against the limits below: grids of 16 million cells, grids and stories shared by many layers,
|
||||
200 000 paragraphs, a document embedded hundreds of times, compounds that fan out or contain
|
||||
themselves, lists of a million null slots linked many times.
|
||||
* `scripts/fetch-affinity-corpus.sh` fetches 51 public CC0/MIT/Apache-2.0 documents at pinned
|
||||
commits, SHA-256 verified against `tests/support/corpus.tsv` (which names each file's source and
|
||||
licence), with the upstream licence notices. Of these, 33 are current `.af`: four CC0 vector
|
||||
illustrations and 28 original Affinity 3.2.3 feature probes plus one separately identified
|
||||
derived regression input from [SethRobinson/Patchy](https://github.com/SethRobinson/Patchy)
|
||||
* `scripts/fetch-affinity-corpus.sh` fetches 113 files at pinned commits, SHA-256 verified
|
||||
against `tests/support/corpus.tsv` (which names each file's source and license): 90 public
|
||||
CC0/MIT/Apache-2.0 documents, the 21 PNG files published with the
|
||||
[satoshoe-dev/affinity-samples](https://github.com/satoshoe-dev/affinity-samples) documents, and
|
||||
two upstream license notices. The manifest holds every name from the PhotoCraft and VectorCraft
|
||||
corpus lists, so 21 files are fetched under two names. Of the documents, 55 are current `.af`:
|
||||
four CC0 vector illustrations (each under two names), 18 CC0 Affinity 3.1.0 samples from
|
||||
satoshoe-dev/affinity-samples, and 28 original Affinity 3.2.3 feature probes plus one separately
|
||||
identified derived regression input from [SethRobinson/Patchy](https://github.com/SethRobinson/Patchy)
|
||||
(MIT). Only document data and provenance notices were inspected, never that project's importer
|
||||
or authoring scripts. The files are never committed.
|
||||
|
||||
@@ -129,11 +133,14 @@ text-frame fixture.
|
||||
```
|
||||
|
||||
Without `AFFINITY_CORPUS` the corpus tests are skipped. `AFFINITY_SAMPLES` names any other
|
||||
folder of documents to parse.
|
||||
* `tests/it/real_files.rs` parses every file and freezes the modelled features of all 33 current
|
||||
`.af` files (`tests/support/affinity3-signatures.txt`: counts, fonts, artboards, warnings and a
|
||||
CRC of the whole model). Native raster tests compare decoded pixels with independent embedded
|
||||
images; text tests assert Unicode run boundaries, fallback fonts and feature retention.
|
||||
folder of documents to parse. A corpus folder fetched for an older manifest fails the tests
|
||||
until the script is run on it again.
|
||||
* `tests/it/real_files.rs` checks the shape of every manifest row (four fields, a SHA-256, a
|
||||
relative name listed once, a license and an `https` URL), parses every file and freezes the
|
||||
modeled features of all 55 `.af` files (`tests/support/affinity3-signatures.txt`: counts,
|
||||
fonts, artboards, warnings and a CRC of the whole model). Native raster tests compare decoded
|
||||
pixels with independent embedded images; text tests assert Unicode run boundaries, fallback
|
||||
fonts and feature retention.
|
||||
* `fuzz/` has `cargo-fuzz` targets for the whole reader (`container`) and the object stream
|
||||
(`stream`):
|
||||
|
||||
|
||||
@@ -22,8 +22,55 @@ fn files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
}
|
||||
|
||||
fn samples() -> Option<PathBuf> {
|
||||
let pinned = corpus::pinned();
|
||||
std::env::var_os("AFFINITY_SAMPLES").map(PathBuf::from).or(pinned)
|
||||
folder(std::env::var_os("AFFINITY_SAMPLES"), corpus::pinned)
|
||||
}
|
||||
|
||||
/// The `AFFINITY_SAMPLES` folder when one is given, without reading the pinned corpus; otherwise
|
||||
/// the pinned corpus.
|
||||
fn folder(samples: Option<std::ffi::OsString>, pinned: impl FnOnce() -> Option<PathBuf>) -> Option<PathBuf> {
|
||||
samples.map(PathBuf::from).or_else(pinned)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_samples_folder_is_used_without_the_pinned_corpus() {
|
||||
assert_eq!(folder(Some("samples".into()), || panic!("the pinned corpus was read")), Some(PathBuf::from("samples")));
|
||||
assert_eq!(folder(None, || Some(PathBuf::from("pinned"))), Some(PathBuf::from("pinned")));
|
||||
assert_eq!(folder(None, || None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_pinned_manifest_is_well_formed() {
|
||||
let rows = corpus::rows(corpus::MANIFEST).unwrap();
|
||||
assert!(rows.iter().filter(|r| r.name.ends_with(".af")).count() >= 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_manifest_check_rejects_malformed_rows() {
|
||||
let digest = "0".repeat(64);
|
||||
let url = "https://example.com/a.af";
|
||||
let row = |name: &str| format!("{digest}\t{name}\tMIT\t{url}\n");
|
||||
// Two names may share a digest.
|
||||
assert!(corpus::rows(&(row("a.af") + &row("dir/b.af"))).is_ok());
|
||||
for (bad, why) in [
|
||||
(row("a.af") + &row("a.af"), "listed twice"),
|
||||
(format!("{digest}\ta.af\tMIT\n"), "four tab-separated fields"),
|
||||
(format!("{digest}\ta.af\tMIT\t{url}\textra\n"), "four tab-separated fields"),
|
||||
(format!("{}\ta.af\tMIT\t{url}\n", "0".repeat(63)), "invalid SHA-256"),
|
||||
(format!("{}\ta.af\tMIT\t{url}\n", "A".repeat(64)), "invalid SHA-256"),
|
||||
(row("../a.af"), "relative path"),
|
||||
(row("/a.af"), "relative path"),
|
||||
(row("dir//a.af"), "relative path"),
|
||||
(row("dir\\a.af"), "relative path"),
|
||||
(row("C:a.af"), "relative path"),
|
||||
(format!("{digest}\ta.af\t \t{url}\n"), "no license"),
|
||||
(format!("{digest}\ta.af\tMIT\thttp://example.com/a.af\n"), "https"),
|
||||
(format!("{digest}\ta.af\tMIT\thttps://\n"), "https"),
|
||||
] {
|
||||
match corpus::rows(&bad) {
|
||||
Ok(_) => panic!("accepted {bad:?}"),
|
||||
Err(e) => assert!(e.contains(why), "{bad:?}: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,3 +1,21 @@
|
||||
affinity-samples/01-layer-effects.af dpi=72;spreads=1;model_crc32=9a2e65d6;counts={"depth": 2, "fills": 13, "gradients": 0, "layers": 12, "nodes": 38, "segments": 51, "shapes": 13, "strokes": 0, "subpaths": 13, "text_chars": 166, "text_runs": 13, "texts": 13};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["blur effects (left out)", "layer effects of other kinds (left out) (2×)", "paragraph leading of an unknown kind (imported as automatic) (13×)", "unknown layers attached to other layers (imported without them) (2×)"]
|
||||
affinity-samples/02-adjustment-layers.af dpi=72;spreads=1;model_crc32=fc8b239f;counts={"depth": 2, "fills": 17, "gradients": 16, "layers": 8, "nodes": 42, "segments": 51, "shapes": 17, "strokes": 0, "subpaths": 17, "text_chars": 96, "text_runs": 9, "texts": 9, "unsupported": 8};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["adjustment layers (8×)", "paragraph leading of an unknown kind (imported as automatic) (9×)"]
|
||||
affinity-samples/03-brush-strokes.af dpi=72;spreads=1;model_crc32=a492c413;counts={"depth": 1, "fills": 1, "gradients": 0, "hidden": 4, "images": 2, "nodes": 14, "segments": 15, "shapes": 7, "strokes": 6, "subpaths": 7, "text_chars": 107, "text_runs": 5, "texts": 5};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (5×)"]
|
||||
affinity-samples/04-master-page.af dpi=300;spreads=3;model_crc32=ba260d8d;counts={"depth": 3, "fills": 1, "gradients": 0, "layers": 6, "master_instances": 3, "nodes": 30, "segments": 18, "shapes": 9, "strokes": 0, "subpaths": 9, "text_chars": 31, "text_runs": 3, "texts": 3, "unsupported": 9};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (3×)", "text fields such as page numbers (left out of the text)", "text whose story could not be read (master-page text, or an unknown layout) (9×)"]
|
||||
affinity-samples/05-symbols.af dpi=72;spreads=1;model_crc32=fe74faa6;counts={"depth": 2, "fills": 5, "gradients": 0, "groups": 1, "layers": 3, "nodes": 40, "segments": 129, "shapes": 32, "strokes": 1, "subpaths": 32, "text_chars": 179, "text_runs": 4, "texts": 4};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (4×)"]
|
||||
affinity-samples/06-special-shapes.af dpi=72;spreads=1;model_crc32=7e9a9eb3;counts={"depth": 1, "fills": 17, "gradients": 0, "nodes": 50, "segments": 268, "shapes": 33, "strokes": 16, "subpaths": 34, "text_chars": 182, "text_runs": 17, "texts": 17};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (17×)"]
|
||||
affinity-samples/07-transparency.af dpi=72;spreads=1;model_crc32=6595042f;counts={"depth": 1, "fills": 18, "gradients": 1, "nodes": 26, "opacity": 1, "segments": 57, "shapes": 19, "strokes": 0, "subpaths": 19, "text_chars": 151, "text_runs": 7, "texts": 7};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["bitmap and pattern fills (left empty)", "paragraph leading of an unknown kind (imported as automatic) (7×)", "transparency gradients (imported without them) (2×)"]
|
||||
affinity-samples/08-gradients.af dpi=72;spreads=1;model_crc32=5638db4d;counts={"depth": 1, "fills": 7, "gradients": 6, "nodes": 14, "segments": 27, "shapes": 7, "strokes": 0, "subpaths": 7, "text_chars": 97, "text_runs": 7, "texts": 7};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (7×)"]
|
||||
affinity-samples/09-corners-stars.af dpi=72;spreads=1;model_crc32=e3528065;counts={"depth": 2, "fills": 9, "gradients": 0, "layers": 8, "nodes": 26, "segments": 81, "shapes": 9, "strokes": 0, "subpaths": 9, "text_chars": 122, "text_runs": 9, "texts": 9};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (9×)"]
|
||||
affinity-samples/10-multiple-fills.af dpi=72;spreads=1;model_crc32=07d2e434;counts={"depth": 2, "fills": 7, "gradients": 3, "layers": 6, "nodes": 20, "segments": 21, "shapes": 7, "strokes": 3, "subpaths": 7, "text_chars": 140, "text_runs": 7, "texts": 7};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (7×)", "several fills or strokes on one object (only the active one is imported) (6×)"]
|
||||
affinity-samples/11-text.af dpi=72;spreads=1;model_crc32=de077247;counts={"depth": 2, "fills": 13, "gradients": 0, "groups": 1, "layers": 1, "nodes": 28, "segments": 198, "shapes": 13, "strokes": 0, "subpaths": 18, "text_chars": 452, "text_runs": 11, "texts": 11, "unsupported": 2};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)", "text fields such as page numbers (left out of the text) (2×)", "text frames with several columns (imported as one column)", "unknown layers (2×)"]
|
||||
affinity-samples/12-cmyk8.af dpi=72;spreads=1;model_crc32=f40c0d8e;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/12-grey8.af dpi=72;spreads=1;model_crc32=f40c0d8e;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/12-lab16.af dpi=72;spreads=1;model_crc32=f40c0d8e;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/12-rgb16.af dpi=72;spreads=1;model_crc32=f40c0d8e;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/12-rgb32.af dpi=72;spreads=1;model_crc32=a9aeb9ab;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/12-rgb8.af dpi=72;spreads=1;model_crc32=f40c0d8e;counts={"depth": 1, "fills": 31, "gradients": 1, "nodes": 41, "segments": 93, "shapes": 31, "strokes": 8, "subpaths": 31, "text_chars": 148, "text_runs": 10, "texts": 10};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (10×)"]
|
||||
affinity-samples/13-blend-modes.af dpi=72;spreads=1;model_crc32=13f9c6d7;counts={"depth": 2, "fills": 13, "gradients": 6, "layers": 6, "nodes": 26, "segments": 45, "shapes": 13, "strokes": 0, "subpaths": 13, "text_chars": 169, "text_runs": 7, "texts": 7};boards=[];fonts={"Fira Sans:FiraSans-Regular:400:false"};warnings=["paragraph leading of an unknown kind (imported as automatic) (7×)"]
|
||||
affinity3-mexican-guy.af dpi=300;spreads=1;model_crc32=32b7cf3f;counts={"depth": 8, "fills": 169, "gradients": 0, "groups": 23, "nodes": 283, "opacity": 1, "segments": 1491, "shapes": 260, "strokes": 114, "subpaths": 264};boards=[];fonts={};warnings=["blur effects (left out) (156×)"]
|
||||
affinity3-mexican-man.af dpi=300;spreads=1;model_crc32=c10daaea;counts={"depth": 7, "fills": 277, "gradients": 2, "groups": 29, "nodes": 903, "segments": 3225, "shapes": 874, "strokes": 540, "subpaths": 881};boards=[];fonts={};warnings=["blur effects (left out) (244×)"]
|
||||
affinity3-mexican-woman.af dpi=300;spreads=1;model_crc32=3bc13374;counts={"depth": 9, "fills": 527, "gradients": 0, "groups": 28, "nodes": 595, "segments": 3562, "shapes": 567, "strokes": 60, "subpaths": 570};boards=[];fonts={};warnings=["blur effects (left out) (278×)"]
|
||||
@@ -31,3 +49,7 @@ patchy-text-runs.af dpi=72;spreads=1;model_crc32=7820b881;counts={"depth": 1, "n
|
||||
patchy-transform.af dpi=72;spreads=1;model_crc32=8d56df84;counts={"depth": 1, "images": 1, "nodes": 1};boards=[];fonts={};warnings=[]
|
||||
patchy-vector-mask.af dpi=72;spreads=1;model_crc32=80ca946a;counts={"depth": 1, "fills": 2, "gradients": 0, "nodes": 2, "segments": 7, "shapes": 2, "strokes": 0, "subpaths": 2, "vector_masks": 1};boards=[];fonts={};warnings=["unknown layers attached to other layers (imported without them)"]
|
||||
patchy-vector.af dpi=72;spreads=1;model_crc32=3b16c7ed;counts={"depth": 1, "fills": 3, "gradients": 0, "nodes": 4, "segments": 17, "shapes": 4, "strokes": 2, "subpaths": 5};boards=[];fonts={};warnings=[]
|
||||
vector-art/mexican-guy.af dpi=300;spreads=1;model_crc32=32b7cf3f;counts={"depth": 8, "fills": 169, "gradients": 0, "groups": 23, "nodes": 283, "opacity": 1, "segments": 1491, "shapes": 260, "strokes": 114, "subpaths": 264};boards=[];fonts={};warnings=["blur effects (left out) (156×)"]
|
||||
vector-art/mexican-man.af dpi=300;spreads=1;model_crc32=c10daaea;counts={"depth": 7, "fills": 277, "gradients": 2, "groups": 29, "nodes": 903, "segments": 3225, "shapes": 874, "strokes": 540, "subpaths": 881};boards=[];fonts={};warnings=["blur effects (left out) (244×)"]
|
||||
vector-art/mexican-woman.af dpi=300;spreads=1;model_crc32=3bc13374;counts={"depth": 9, "fills": 527, "gradients": 0, "groups": 28, "nodes": 595, "segments": 3562, "shapes": 567, "strokes": 60, "subpaths": 570};boards=[];fonts={};warnings=["blur effects (left out) (278×)"]
|
||||
vector-art/playing-cards.af dpi=300;spreads=1;model_crc32=bc0c838d;counts={"depth": 5, "fills": 363, "gradients": 0, "groups": 758, "nodes": 1350, "segments": 4864, "shapes": 592, "strokes": 17, "subpaths": 656};boards=[];fonts={};warnings=["blur effects (left out) (309×)", "brush strokes (imported as plain strokes)"]
|
||||
@@ -10,9 +10,56 @@ use sha2::{Digest, Sha256};
|
||||
/// `sha256 name licence url` rows.
|
||||
pub const MANIFEST: &str = include_str!("corpus.tsv");
|
||||
|
||||
/// One manifest row.
|
||||
pub struct Row<'a> {
|
||||
pub digest: &'a str,
|
||||
pub name: &'a str,
|
||||
pub license: &'a str,
|
||||
pub url: &'a str,
|
||||
}
|
||||
|
||||
/// The rows of `manifest`. A row has exactly four tab-separated fields: a lowercase hexadecimal
|
||||
/// SHA-256, a relative `/`-separated name without empty, `.` or `..` parts, a license and an
|
||||
/// `https://` URL. Each name is listed once; several names may share a digest.
|
||||
pub fn rows(manifest: &str) -> Result<Vec<Row<'_>>, String> {
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
let mut rows = Vec::new();
|
||||
for line in manifest.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let cols: Vec<&str> = line.split('\t').collect();
|
||||
let &[digest, name, license, url] = cols.as_slice() else {
|
||||
return Err(format!("corpus row needs four tab-separated fields: {line}"));
|
||||
};
|
||||
if digest.len() != 64 || !digest.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) {
|
||||
return Err(format!("{name}: invalid SHA-256 {digest:?}"));
|
||||
}
|
||||
if name.starts_with('/')
|
||||
|| name.contains(['\\', ':'])
|
||||
|| name.chars().any(char::is_control)
|
||||
|| name.split('/').any(|part| part.is_empty() || part == "." || part == "..")
|
||||
{
|
||||
return Err(format!("{name:?}: the name must be a relative path inside the corpus folder"));
|
||||
}
|
||||
if license.trim().is_empty() {
|
||||
return Err(format!("{name}: no license"));
|
||||
}
|
||||
if !url.starts_with("https://") || url.len() == "https://".len() {
|
||||
return Err(format!("{name}: the URL must use https: {url:?}"));
|
||||
}
|
||||
if !seen.insert(name) {
|
||||
return Err(format!("{name}: listed twice"));
|
||||
}
|
||||
rows.push(Row { digest, name, license, url });
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
fn manifest() -> Vec<Row<'static>> {
|
||||
rows(MANIFEST).unwrap_or_else(|e| panic!("malformed corpus.tsv: {e}"))
|
||||
}
|
||||
|
||||
/// The names of the pinned files.
|
||||
pub fn names() -> impl Iterator<Item = &'static str> {
|
||||
MANIFEST.lines().filter_map(|l| l.split('\t').nth(1))
|
||||
manifest().into_iter().map(|row| row.name)
|
||||
}
|
||||
|
||||
pub fn pinned() -> Option<PathBuf> {
|
||||
@@ -21,10 +68,7 @@ pub fn pinned() -> Option<PathBuf> {
|
||||
return None;
|
||||
};
|
||||
let mut count = 0;
|
||||
for line in MANIFEST.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let mut cols = line.split('\t');
|
||||
let (Some(digest), Some(name)) = (cols.next(), cols.next()) else { panic!("malformed corpus row: {line}") };
|
||||
assert_eq!(digest.len(), 64, "invalid SHA256 for {name}");
|
||||
for Row { digest, name, .. } in manifest() {
|
||||
let bytes = std::fs::read(dir.join(name)).unwrap_or_else(|e| panic!("missing pinned Affinity file {name}: {e}"));
|
||||
let hex: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect();
|
||||
assert_eq!(hex, digest, "{name}: pinned corpus modified; run scripts/fetch-affinity-corpus.sh");
|
||||
|
||||
@@ -7,10 +7,59 @@ e56c35f876277b101790ff4384cfde103a093ed6c81790a2d62176de21da339f afdesignload-re
|
||||
08946c60c0da1c8aeea92d65acfb244721eeb3b480ed358609f4a48bbecfc5b9 afdesignload-shape_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/shape_test.afdesign
|
||||
7306ab4fcbdc225d816f32a0d2aca15ac887053f60170459ac947e70f4ef343d afdesignload-slice_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/slice_test.afdesign
|
||||
e5d04bac0235af9386aea251cdd6903ed7810d335fdb6194d0483ea30b14e24a afdesignload-test_path.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/test_path.afdesign
|
||||
cc8ca0db0d8fc545eb29153be900f153fa6b7b7eac068d05e54a502aba1264fe afdesignload/color.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/color.afdesign
|
||||
9a17f500d09fb3b1a70d24aabdc2882955ec3eac3700a474549e6100be3a8484 afdesignload/layer_mode.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/layer_mode.afdesign
|
||||
77ae60c02749224fe2c52691c951568138eaed5f8584ebc5da83715fae0234dd afdesignload/layer_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/layer_test.afdesign
|
||||
04944918ff8e29c5c1b466b7e43c232f7ab80db15fb934080bbcd7baaf093e22 afdesignload/margins.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/margins.afdesign
|
||||
35afa31b89f2b9970eb3701ee64e2a19476383637e8b07689334d6ffe556b593 afdesignload/raster_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/raster_test.afdesign
|
||||
e56c35f876277b101790ff4384cfde103a093ed6c81790a2d62176de21da339f afdesignload/revision_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/revision_test.afdesign
|
||||
08946c60c0da1c8aeea92d65acfb244721eeb3b480ed358609f4a48bbecfc5b9 afdesignload/shape_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/shape_test.afdesign
|
||||
7306ab4fcbdc225d816f32a0d2aca15ac887053f60170459ac947e70f4ef343d afdesignload/slice_test.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/slice_test.afdesign
|
||||
e5d04bac0235af9386aea251cdd6903ed7810d335fdb6194d0483ea30b14e24a afdesignload/test_path.afdesign MIT https://raw.githubusercontent.com/NickBeeuwsaert/AFDesignLoad/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns/test_path.afdesign
|
||||
5f5b98f0ab2121f75f24362792c00cd5b610767820eb754b5660668f478b0c86 affinity-samples/01-layer-effects.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/01-layer-effects.af
|
||||
33c1ad5022abc14e6136f4cb9fa0751c9778a0a564bc3abc2e0e6b051f7e00bb affinity-samples/01-layer-effects.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/01-layer-effects.png
|
||||
4d19e95abe3423692f5bdfd0ede5bf3aebdd4bf16c32eaa304cead91a7a2e9b7 affinity-samples/02-adjustment-layers.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/02-adjustment-layers.af
|
||||
726b7939e23a745f7578fa84188b56c2eb845cde28493499f1b18b4ede9b8a6e affinity-samples/02-adjustment-layers.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/02-adjustment-layers.png
|
||||
c6ca86439c89d83ba43c76e9d783865ac09ca0ff524d5b347caa77f473caac98 affinity-samples/03-brush-strokes.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/03-brush-strokes.af
|
||||
e57e36ffffdd88d3a62ebdb142435d7d3056286b22130f459c404a3ef529f61f affinity-samples/03-brush-strokes.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/03-brush-strokes.png
|
||||
e3bf04f7cc02e1d49ec79503f2b4044f385de1e890bd461013a5a6fca09b5223 affinity-samples/04-master-page.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/04-master-page.af
|
||||
246279f00d1f08917409b34e60359bd3bba158bcf4e8d5847a806f575d8eb98f affinity-samples/04-master-page_1.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/04-master-page_1.png
|
||||
0e023792ce89f724b1dfedb6360556348309e483c23e32b91d87afab5d64c9cc affinity-samples/04-master-page_2.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/04-master-page_2.png
|
||||
f431b15027e20ab760c4d241aa6981130560409cb270bbc6d4dd9241d1883f0a affinity-samples/04-master-page_3.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/04-master-page_3.png
|
||||
ef779b58a6721001ee961b42eacb96fb47dcaa35ccb7e6ccf2ec6f33106070b2 affinity-samples/05-symbols.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/05-symbols.af
|
||||
6675fc4a1563f61b7fcaa64978447e0fa6e1b0bb8073b4e63f958db1cf726fef affinity-samples/05-symbols.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/05-symbols.png
|
||||
9e463b4d1ca8e7a1b67b3c30351a28af8ed05b9cd11f3cea2159319c9974bc67 affinity-samples/06-special-shapes.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/06-special-shapes.af
|
||||
a897c607631acbea31c5a926af6ca20c0acb2744636003b8c9ef07861752d947 affinity-samples/06-special-shapes.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/06-special-shapes.png
|
||||
3aba0d83055f09da4dba465a3abf66c77d4362dc6060548e54db943029b3996e affinity-samples/07-transparency.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/07-transparency.af
|
||||
3b73945b7302627caf679a961053f30a9593bab7d709e27acb1df54dadc53968 affinity-samples/07-transparency.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/07-transparency.png
|
||||
4292c1571b6c8c19b80c530602ca21d5996df7b99221639a4cde76d8307ffbcb affinity-samples/08-gradients.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/08-gradients.af
|
||||
f6fb427c9715ea193e595d0c441b462224971fcda2aa2b4d8b71631d0f59f992 affinity-samples/08-gradients.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/08-gradients.png
|
||||
c913882b3bea7a4218aa88e96890181cdc08ce37f0859196a1ad1828be78b347 affinity-samples/09-corners-stars.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/09-corners-stars.af
|
||||
068bcc7599336a174d7ce87962da414ccf429d549484ccbe8f68f669779d84d0 affinity-samples/09-corners-stars.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/09-corners-stars.png
|
||||
f03e797d6dbf8a31eab9d0b9cf9a8bddcc8440c6003863f31d97fe23c195c4b2 affinity-samples/10-multiple-fills.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/10-multiple-fills.af
|
||||
e1951aa6d8acdbeccc281b6a20a03df9b257c0d739aa5f6aec27c105151fef3f affinity-samples/10-multiple-fills.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/10-multiple-fills.png
|
||||
6da8b59b40d0e082435c5a2004c50fa90e3c4f15c3a02763b8edf84b8b9f4506 affinity-samples/11-text.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/11-text.af
|
||||
bb2040cda017cb753d0594416ac0501a1049bc333a7cf2c1c832356e2bbe1a45 affinity-samples/11-text.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/11-text.png
|
||||
0c13cfe0f716902199775251b0983f28d60a64c1f79d3c6f496f2c019365bdb5 affinity-samples/12-cmyk8.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-cmyk8.af
|
||||
cfef257c50201968121332e8ae12a7fe2ecdcc3bce1faeb5e581751662098c05 affinity-samples/12-cmyk8.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-cmyk8.png
|
||||
03d4cfe5ce4b80833d83de89848e88639056e8984c049865c8f29b9ef41b40e5 affinity-samples/12-grey8.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-grey8.af
|
||||
855e4f08f21c60f1d2080c9533b98f99c7a635c5804abf8ec9d6ee1a29029cef affinity-samples/12-grey8.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-grey8.png
|
||||
9adc0d4242b5448ac84aef886bd7afc165e1959015125ce21bf402ddb65caba1 affinity-samples/12-lab16.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-lab16.af
|
||||
6c30cef8304fd08539d631988280a6b3bc8570db3f1425fa771d83c1591dfc8d affinity-samples/12-lab16.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-lab16.png
|
||||
26b47861275b2f83517e7d42ec051efc3cd22956440784b6df0bc016ecb0e008 affinity-samples/12-rgb16.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb16.af
|
||||
967ba0cb7d93496a7e454dcc37013c1b19089cc283c7e08f2dea7023ad330e80 affinity-samples/12-rgb16.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb16.png
|
||||
36e62b0de8488712c04831ba3ef466da68bf080ec3caf9fbe6646905d3fb87e8 affinity-samples/12-rgb32.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb32.af
|
||||
0ebcf7211447d64158c21c27eeb19b8737cb2cdcc708dcdf3b4fed5041b449e9 affinity-samples/12-rgb32.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb32.png
|
||||
4d210ce4724409fb6c9715046ad9efa0049442f8324a21174068a02c6df10632 affinity-samples/12-rgb8.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb8.af
|
||||
4e4dda966c337119165ba5d88cb3d4af10bf6507b9a69986d33060a8eeb37c21 affinity-samples/12-rgb8.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/12-rgb8.png
|
||||
b84815dbb5e3a7f7842d860acb888f93dbd1f949bcb6a3bf858515b85ea3c3b7 affinity-samples/13-blend-modes.af CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/13-blend-modes.af
|
||||
a49d2fe6043002aef4f4c6b8cf64a04838d21cf29795324c02fb1772d5519a21 affinity-samples/13-blend-modes.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/13-blend-modes.png
|
||||
b73eceb73a45df1a424d60164b7e1b521b9f01f56fe86159e477b49c9cd90f78 affinity-samples/source/07-texture.png CC0-1.0 https://raw.githubusercontent.com/satoshoe-dev/affinity-samples/96aca6bd86bbe7a677f1cf0630bd832d23943be8/source/07-texture.png
|
||||
c499cf3ca73cf7cd3e2e65bed048d1b8ecf03e7167b74d54f0ae6db31868cab8 affinity3-mexican-guy.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-guy.af
|
||||
730e3fae51d9044ada58f51c7b5f9327d99e8d62376453bc251d0d26cda47743 affinity3-mexican-man.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-man.af
|
||||
5f5fccc0bd8be2bb3b2a07461ec77a012a61eef1883485d7c80f72d4383231ce affinity3-mexican-woman.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-woman.af
|
||||
0a52bb923c949c566a4339b996bce1a67b683f61def1a73dfefb910effaa450d affinity3-playing-cards.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/playing-cards.af
|
||||
e1b121d7c3d02356baa5dea512f5aa1fe77249c28d7b227e29cab2bc67ec323a asset-store-template/AssetStore.aftemplate MIT https://raw.githubusercontent.com/eviltwo/AssetStoreTemplate/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate/AssetStore.aftemplate
|
||||
87a7bd93de16c404f6c6dd9cdebff96ec47832a3b342d2ce0af73d9a87e209e0 designer-beer.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/beer.afdesign
|
||||
09ba790c428e8e09dc0b9ddebb4f3db6daefaa2af1da492ee34e73fba1b8af69 designer-cactus.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/cactus.afdesign
|
||||
c3bcd0395b407493bc9d2877d424029b70f0e4c817a56e893f7ac74973df52e5 designer-car.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/car.afdesign
|
||||
@@ -19,6 +68,8 @@ c3bcd0395b407493bc9d2877d424029b70f0e4c817a56e893f7ac74973df52e5 designer-car.af
|
||||
e1b121d7c3d02356baa5dea512f5aa1fe77249c28d7b227e29cab2bc67ec323a eviltwo-AssetStore.aftemplate MIT https://raw.githubusercontent.com/eviltwo/AssetStoreTemplate/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate/AssetStore.aftemplate
|
||||
fbb196e918e302c16f8de9f2e6dffaef70ab3c317b32a9f44607fd4b2ead0206 jac21-SimpleLogo.afdesign MIT https://raw.githubusercontent.com/Jac21/Branding/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity/SimpleLogo.afdesign
|
||||
9d874245936483de3ec04cca48b383028d33666801dff1814246a44e695e29b8 jac21-SimpleLogoBanner.afdesign MIT https://raw.githubusercontent.com/Jac21/Branding/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity/SimpleLogoBanner.afdesign
|
||||
fbb196e918e302c16f8de9f2e6dffaef70ab3c317b32a9f44607fd4b2ead0206 jac21/SimpleLogo.afdesign MIT https://raw.githubusercontent.com/Jac21/Branding/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity/SimpleLogo.afdesign
|
||||
9d874245936483de3ec04cca48b383028d33666801dff1814246a44e695e29b8 jac21/SimpleLogoBanner.afdesign MIT https://raw.githubusercontent.com/Jac21/Branding/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity/SimpleLogoBanner.afdesign
|
||||
ec1072cb03fcd0ed7ac1dfc7e087f4920df545fe9c84a386a7dac2fd33fbbe1f leakcanary-vector_icon.afdesign Apache-2.0 https://raw.githubusercontent.com/square/leakcanary/00880a8fb66eb4046a0124bff8303a04874bc07e/docs/assets/vector_icon.afdesign
|
||||
bbc50c8c376e0e5980939be7df6769feed1a30289c7efc6391b204dfb15de88d patchy-LICENSE.txt MIT https://raw.githubusercontent.com/SethRobinson/Patchy/de84eab550758b30fa062e479f5778cce7693b73/LICENSE
|
||||
150e423be0c5eb22f252a38c2de68ab3ccec7e20bfba166f7ddd622ac243d8f4 patchy-NOTICE.txt MIT https://raw.githubusercontent.com/SethRobinson/Patchy/de84eab550758b30fa062e479f5778cce7693b73/NOTICE-THIRD-PARTY.md
|
||||
@@ -51,3 +102,12 @@ a5ccb62ee8d7e36d1a19a037aba989a5ba0232767e9f3305f7d9f7d4603291dd patchy-text-rot
|
||||
87e1dee23769932b7e414116233063a8d68726cbffdebb975b3cd0ef447c602a patchy-transform.af MIT https://raw.githubusercontent.com/SethRobinson/Patchy/de84eab550758b30fa062e479f5778cce7693b73/test-fixtures/af/tiny-transform.af
|
||||
fa5dd80f44ae0712a9888c62953fd442a28119034d343db93b7892d5d2a61328 patchy-vector-mask.af MIT https://raw.githubusercontent.com/SethRobinson/Patchy/de84eab550758b30fa062e479f5778cce7693b73/test-fixtures/af/tiny-vector-mask.af
|
||||
8ab1995ecbd9306ebcb7a4d2cd81701f34401e67f08b242f14dc3e3d04c94803 patchy-vector.af MIT https://raw.githubusercontent.com/SethRobinson/Patchy/de84eab550758b30fa062e479f5778cce7693b73/test-fixtures/af/tiny-vector.af
|
||||
87a7bd93de16c404f6c6dd9cdebff96ec47832a3b342d2ce0af73d9a87e209e0 vector-art/beer.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/beer.afdesign
|
||||
09ba790c428e8e09dc0b9ddebb4f3db6daefaa2af1da492ee34e73fba1b8af69 vector-art/cactus.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/cactus.afdesign
|
||||
c3bcd0395b407493bc9d2877d424029b70f0e4c817a56e893f7ac74973df52e5 vector-art/car.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/car.afdesign
|
||||
112562bc91e7cccf003d34364430544208373c4fb4bb4ab9acdaf27cf0125047 vector-art/flowers.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/flowers.afdesign
|
||||
023531b8216f2d4da24ab715e3b313506712e50d5f5bf7da927f434def1d0ea2 vector-art/lion-track.afdesign CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/lion-track.afdesign
|
||||
c499cf3ca73cf7cd3e2e65bed048d1b8ecf03e7167b74d54f0ae6db31868cab8 vector-art/mexican-guy.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-guy.af
|
||||
730e3fae51d9044ada58f51c7b5f9327d99e8d62376453bc251d0d26cda47743 vector-art/mexican-man.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-man.af
|
||||
5f5fccc0bd8be2bb3b2a07461ec77a012a61eef1883485d7c80f72d4383231ce vector-art/mexican-woman.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/mexican-woman.af
|
||||
0a52bb923c949c566a4339b996bce1a67b683f61def1a73dfefb910effaa450d vector-art/playing-cards.af CC0-1.0 https://raw.githubusercontent.com/samuel-etver/vector-art/255f8add3c8f0740196e22bd59502b811b532f0b/simple/playing-cards.af
|
||||
|
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "craft-ai"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "AI provider abstraction for the Craft suite — BYO API keys and one-click local models"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
dirs = "7"
|
||||
sysinfo = "0.39"
|
||||
ureq = { version = "3", features = ["json"] }
|
||||
sha2 = "0.11"
|
||||
candle-core = { version = "0.9", optional = true }
|
||||
candle-nn = { version = "0.9", optional = true }
|
||||
candle-transformers = { version = "0.9", optional = true }
|
||||
tokenizers = { version = "0.22", optional = true, default-features = false, features = ["onig"] }
|
||||
|
||||
[features]
|
||||
# Local GGUF inference on candle. Heavy dependency tree; apps opt in.
|
||||
local = ["dep:candle-core", "dep:candle-nn", "dep:candle-transformers", "dep:tokenizers"]
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,145 @@
|
||||
# craft-ai — AI providers for the Craft suite
|
||||
|
||||
Shared AI abstraction for every Craft app: translation, rewriting, summaries,
|
||||
vision captions — one crate, one settings store, one model cache.
|
||||
|
||||
Status: **spec** — this file is the contract; implementation follows.
|
||||
|
||||
## The feature this powers
|
||||
|
||||
A per-app **AI tab** offering document-wide operations — the flagship being
|
||||
"translate this whole document" — backed by either:
|
||||
|
||||
1. **BYOT** — the user's own API token against an OpenAI-compatible endpoint
|
||||
(OpenAI, Azure, Ollama, LM Studio, any compatible proxy), or
|
||||
2. **Local model** — a one-click download of a curated open model, chosen for
|
||||
the machine it's running on.
|
||||
|
||||
The user experience is deliberately *informed*: every step reports what is
|
||||
happening, why this model was picked, what it will cost in disk/RAM, and that
|
||||
nothing leaves the machine in local mode.
|
||||
|
||||
## Modules
|
||||
|
||||
### `probe` — capability detection
|
||||
|
||||
Runs once, caches the result:
|
||||
|
||||
| Signal | Source |
|
||||
| --- | --- |
|
||||
| Total/free RAM | `sysinfo` or `std::alloc`-safe OS calls |
|
||||
| CPU threads + features | `std::thread`, `is_x86_feature_detected!` (AVX2 gates Q4 speed) |
|
||||
| GPU + VRAM | `wgpu` adapter info (apps already link wgpu — reuse, don't re-probe) |
|
||||
| Free disk at the model cache dir | `fs2`/`std::fs` |
|
||||
| Network reachability | optional HEAD to the model host, skipped offline |
|
||||
|
||||
Output: `MachineCaps { ram_gb, vram_gb, threads, avx2, disk_free_gb }`.
|
||||
|
||||
### `registry` — the curated model list
|
||||
|
||||
A small hand-maintained table (Rust consts, not runtime-fetched — supply
|
||||
chain stays auditable). Each entry: `id`, display name, HF repo + file,
|
||||
`sha256`, size, min RAM, recommended RAM, license id, quality tier,
|
||||
languages covered.
|
||||
|
||||
Curation rule — **permissive licenses only** (the suite is MIT OR Apache-2.0
|
||||
and commercial users run it):
|
||||
|
||||
- ✅ Qwen2.5-0.5B / 1.5B / 3B Instruct (Apache-2.0)
|
||||
- ✅ SmolLM2-1.7B (Apache-2.0)
|
||||
- ✅ Gemma-3 (Gemma Terms — allowed, flag in UI)
|
||||
- ✅ Helsinki-NLP opus-mt-* per-language pairs (CC-BY/MIT)
|
||||
- ❌ NLLB-200, M2M-100 — CC-BY-NC, excluded
|
||||
- ❌ Llama-* — custom license acceptable? default: exclude until reviewed
|
||||
|
||||
`registry::suggest(&caps)` picks the best fitting model: prefers 3B when
|
||||
VRAM ≥ 6 GB or RAM ≥ 16 GB, falls to 1.5B at 8 GB, 0.5B below. Always shows
|
||||
alternatives — the suggestion is a default, not a cage.
|
||||
|
||||
### `provider` — inference backends
|
||||
|
||||
```rust
|
||||
trait Provider {
|
||||
fn name(&self) -> &str;
|
||||
fn complete(&self, prompt: &str, opts: &GenOpts) -> Result<String, AiError>;
|
||||
fn test(&self) -> Result<ProviderInfo, AiError>;
|
||||
}
|
||||
```
|
||||
|
||||
- `OpenAiCompat` — POST `/v1/chat/completions`, configurable endpoint+token+
|
||||
model. Covers OpenAI, Ollama (`http://localhost:11434/v1`), LM Studio,
|
||||
Groq, OpenRouter… Token stored via the OS keyring (`keyring` crate) or a
|
||||
0600-permissioned config file — never in project files, never logged.
|
||||
- `LocalGguf` — GGUF inference. First implementation target: **candle**
|
||||
(pure Rust, Apache/MIT — keeps the no-C++ toolchain rule). `llama-cpp-2`
|
||||
stays a possible perf upgrade later, gated behind a feature flag.
|
||||
|
||||
### `fetch` — one-click download
|
||||
|
||||
- Resumable HTTP GET (range requests), streamed to `*.part` then renamed —
|
||||
a half-downloaded GGUF can never be mistaken for a valid one.
|
||||
- SHA-256 checked against the registry entry; mismatch = hard fail, delete.
|
||||
- Progress events (`Download { bytes_done, bytes_total, bytes_per_sec }`)
|
||||
feed the UI's live status line.
|
||||
- Cache dir is **shared across the suite**:
|
||||
`%LOCALAPPDATA%\artcraft\models` (Windows), `~/Library/Application
|
||||
Support/artcraft/models` (macOS), `~/.cache/artcraft/models` (Linux).
|
||||
|
||||
### `translate` — document segmentation
|
||||
|
||||
The model is the easy part; the hard part is *what not to touch*. The
|
||||
translator walks the document as segments, not raw text:
|
||||
|
||||
- **Protected**: LaTeX commands/envs/math (`\cite{…}`, `$…$`), code fences,
|
||||
placeholders (`{count}`), URLs, citation keys, markup tags.
|
||||
- **Translated**: prose runs, captions, headings, table cells.
|
||||
- Per-app adapters declare the grammar: TexCraft protects `\command{…}` and
|
||||
math mode; WordCraft protects field codes; MatCraft protects code.
|
||||
- Batch-friendly: segments → prompt with numbered items → parse numbered
|
||||
output; verify count matches, retry split halves on mismatch.
|
||||
- Deterministic options: temperature 0-ish, glossary hook (`HashMap<term,
|
||||
translation>`) applied post-hoc.
|
||||
|
||||
## The AI tab (per app, `ui-egui/src/ai.rs`)
|
||||
|
||||
First-open flow — the "smooth informed" sequence:
|
||||
|
||||
1. Two cards: **Use an API provider** / **Run on this PC**.
|
||||
2. BYOT card: endpoint, token, model fields + **Test connection** → green
|
||||
check with model list echo.
|
||||
3. Local card: **Detect this PC** → probe runs →
|
||||
`Suggested: Qwen2.5-1.5B-Instruct (Q4_K_M, 1.1 GB) — fits your 16 GB RAM;
|
||||
est. ~8 tok/s` with an "Other models ▸" disclosure showing the registry.
|
||||
4. **Download** → progress bar with MB/%/speed → `Verifying checksum…` →
|
||||
`Loading…` → `Ready — local, offline, nothing leaves this PC`.
|
||||
5. Then the operations: "Translate document to [language ▾]", "Summarize
|
||||
selection", custom prompt box. Status line narrates each step;
|
||||
expandable activity log for the curious.
|
||||
6. Language detected vs document language → gentle offer chip:
|
||||
`This document is French — translate to English?`
|
||||
|
||||
## Non-negotiables
|
||||
|
||||
- **Honesty**: the UI always says which provider is active, what model,
|
||||
and "cloud — your document text is sent to the endpoint" vs "local —
|
||||
nothing leaves this PC". No silent cloud calls; AI is strictly opt-in.
|
||||
- **No crash**: model load failures, corrupt GGUFs, OOM — all `Result`,
|
||||
all surfaced as messages.
|
||||
- **Rust only**: candle over llama.cpp bindings preserves the wasm story
|
||||
(local inference on wasm is future-possible with `candle` + WGPU backend).
|
||||
- **Licensing**: registry carries SPDX ids; the UI shows them; CC-BY-NC and
|
||||
unreviewed custom licenses are excluded by policy.
|
||||
- **Everything is a command**: `ai.translate_doc`, `ai.status`,
|
||||
`ai.models.list`, `ai.model.download`, `ai.provider.test` — reachable
|
||||
from the app's control channel and MCP, same as every Craft convention.
|
||||
|
||||
## Milestones
|
||||
|
||||
| # | Deliverable | Done when |
|
||||
| --- | --- | --- |
|
||||
| A1 | `probe` + `registry` + `fetch` | CLI-free test: probe reports real caps; `suggest` picks sanely per fixture; a tiny model downloads + sha256 verifies |
|
||||
| A2 | `OpenAiCompat` provider | `translate` a string through Ollama/OpenAI with mocked HTTP tests |
|
||||
| A3 | `LocalGguf` on candle | Qwen2.5-0.5B loads and answers on CPU in a test (model fixture fetched in-test, skipped without network) |
|
||||
| A4 | `translate` segmentation | LaTeX/code/placeholder round-trip tests — protected spans byte-identical |
|
||||
| A5 | AI tab in one pilot app (suggest TexCraft or WordCraft) | download → translate document end-to-end, live status UX |
|
||||
| A6 | Commands + MCP surface | `ai.*` commands reachable via control channel |
|
||||
@@ -0,0 +1,52 @@
|
||||
//! The crate's error type: everything fails with `Result`, never a panic.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// Every failure a caller can act on: the variant says what went wrong,
|
||||
/// the message says why.
|
||||
#[derive(Debug)]
|
||||
pub enum AiError {
|
||||
/// The HTTP layer failed (DNS, TLS, status code, body).
|
||||
Http(String),
|
||||
/// A downloaded file's SHA-256 did not match the expected digest.
|
||||
Checksum { expected: String, got: String },
|
||||
/// The local disk refused us (permissions, full, vanished mid-write).
|
||||
Io(std::io::Error),
|
||||
/// The provider returned a response we could not parse.
|
||||
BadResponse(String),
|
||||
/// The provider rejected the request (bad token, quota, model name).
|
||||
Rejected(String),
|
||||
/// No provider is configured — the UI should route to setup.
|
||||
NotConfigured,
|
||||
/// Local model inference failed (load, OOM, bad GGUF).
|
||||
Inference(String),
|
||||
/// The operation was cancelled by the user.
|
||||
Cancelled,
|
||||
/// The same cache entry is already being downloaded by another
|
||||
/// caller — only one may hold the `.part` file at a time.
|
||||
Busy,
|
||||
}
|
||||
|
||||
impl fmt::Display for AiError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Http(m) => write!(f, "network error: {m}"),
|
||||
Self::Checksum { expected, got } => write!(f, "checksum mismatch: expected {expected}, got {got}"),
|
||||
Self::Io(e) => write!(f, "disk error: {e}"),
|
||||
Self::BadResponse(m) => write!(f, "unreadable response: {m}"),
|
||||
Self::Rejected(m) => write!(f, "provider rejected the request: {m}"),
|
||||
Self::NotConfigured => write!(f, "no AI provider configured"),
|
||||
Self::Inference(m) => write!(f, "local model error: {m}"),
|
||||
Self::Cancelled => write!(f, "cancelled"),
|
||||
Self::Busy => write!(f, "a download of this model is already in progress"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AiError {}
|
||||
|
||||
impl From<std::io::Error> for AiError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Self::Io(e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
//! One-click model download: resumable, `.part`-suffixed while in flight
|
||||
//! (a half-downloaded GGUF is never mistaken for a valid one), SHA-256
|
||||
//! verified, then atomically renamed.
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::Digest;
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::registry::ModelEntry;
|
||||
|
||||
/// Progress + lifecycle events, fed to the UI's status line.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Event {
|
||||
/// We know the total; download is starting or resuming.
|
||||
Started { total: u64, resumed_from: u64 },
|
||||
/// Bytes on disk so far. `bytes_per_sec` is a running average.
|
||||
Progress { done: u64, total: u64, bytes_per_sec: u64 },
|
||||
/// Body finished; hashing the file before we trust it.
|
||||
Verifying,
|
||||
/// Download is complete. `verified` is false when no reference digest
|
||||
/// existed anywhere (registry table and the host's ETag) — the file is
|
||||
/// kept but the UI should say "unverified" rather than pretend.
|
||||
Done { verified: bool },
|
||||
}
|
||||
|
||||
/// A handle to abort a download: `cancel` returns false → `AiError::Cancelled`.
|
||||
pub type Cancel<'a> = &'a dyn Fn() -> bool;
|
||||
|
||||
/// ureq's `timeout_recv_body` is a *total* budget, not a stall timer, so a
|
||||
/// single fixed cap would kill slow-but-alive downloads of multi-GB models.
|
||||
/// Instead the GET's body deadline scales with the registry's declared
|
||||
/// size: a floor rate of 256 KiB/s, between 5 min and 6 h. A dead
|
||||
/// connection can therefore never block the read loop indefinitely, and a
|
||||
/// retry still resumes from the `.part` file.
|
||||
const MIN_BODY_RATE: u64 = 256 * 1024;
|
||||
const BODY_BUDGET_MIN: Duration = Duration::from_secs(5 * 60);
|
||||
const BODY_BUDGET_MAX: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
|
||||
fn body_agent(remaining_bytes: u64) -> ureq::Agent {
|
||||
let budget = Duration::from_secs(remaining_bytes / MIN_BODY_RATE).clamp(BODY_BUDGET_MIN, BODY_BUDGET_MAX);
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(Duration::from_secs(15)))
|
||||
.timeout_send_request(Some(Duration::from_secs(30)))
|
||||
.timeout_recv_response(Some(Duration::from_secs(60)))
|
||||
.timeout_recv_body(Some(budget))
|
||||
.build()
|
||||
.into()
|
||||
}
|
||||
|
||||
/// OS-level lock guarding one cache entry against concurrent downloads;
|
||||
/// released automatically on close/process exit, so no stale lock files.
|
||||
fn lock_entry(part_path: &Path) -> Result<File, AiError> {
|
||||
let mut lock_name = part_path.as_os_str().to_os_string();
|
||||
lock_name.push(".lock");
|
||||
let lock = OpenOptions::new().write(true).create(true).truncate(false).open(PathBuf::from(lock_name))?;
|
||||
match lock.try_lock() {
|
||||
Ok(()) => Ok(lock),
|
||||
Err(std::fs::TryLockError::WouldBlock) => Err(AiError::Busy),
|
||||
Err(std::fs::TryLockError::Error(e)) => Err(AiError::Io(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Download `entry` into `cache_dir`, calling `on` with progress events.
|
||||
/// Resumes an existing `*.part` via HTTP Range. Returns the final path.
|
||||
pub fn download(entry: &ModelEntry, cache_dir: &Path, on: impl Fn(&Event), cancel: Option<Cancel<'_>>) -> Result<PathBuf, AiError> {
|
||||
if cancel.is_some_and(|c| !c()) {
|
||||
return Err(AiError::Cancelled);
|
||||
}
|
||||
fs::create_dir_all(cache_dir)?;
|
||||
let final_path = cache_dir.join(entry.file);
|
||||
if final_path.is_file() {
|
||||
// Already on disk — verify once, reuse if good.
|
||||
on(&Event::Verifying);
|
||||
if let Some(expected) = expected_sha256(entry)? {
|
||||
let got = sha256_file(&final_path)?;
|
||||
if got.eq_ignore_ascii_case(&expected) {
|
||||
on(&Event::Done { verified: true });
|
||||
return Ok(final_path);
|
||||
}
|
||||
return Err(AiError::Checksum { expected, got });
|
||||
}
|
||||
on(&Event::Done { verified: false });
|
||||
return Ok(final_path);
|
||||
}
|
||||
|
||||
let part_path = cache_dir.join(format!("{}.part", entry.file));
|
||||
// One downloader per cache entry: two concurrent calls would read the
|
||||
// same resume offset and append the same bytes twice, corrupting the
|
||||
// `.part`. The lock is held through the final rename.
|
||||
let _entry_lock = lock_entry(&part_path)?;
|
||||
let mut resumed_from = part_path.metadata().map_or(0, |m| m.len());
|
||||
|
||||
let expected = expected_sha256(entry)?;
|
||||
|
||||
let get = |range_from: u64| {
|
||||
// Budget for the *full* size, not the remainder: a 200 that
|
||||
// ignores Range means we re-download the whole model.
|
||||
let mut req = body_agent(entry.size_mb.saturating_mul(1 << 20)).get(entry.url);
|
||||
if range_from > 0 {
|
||||
req = req.header("Range", &format!("bytes={range_from}-"));
|
||||
}
|
||||
req.call()
|
||||
};
|
||||
let mut resp = match get(resumed_from) {
|
||||
Ok(r) => r,
|
||||
// A 416 means the server found our resume offset unsatisfiable —
|
||||
// the common cause is a `.part` that already holds the complete
|
||||
// file (cancelled between the last byte and the rename). If it
|
||||
// verifies, finish the job; otherwise restart from scratch.
|
||||
Err(ureq::Error::StatusCode(416)) if resumed_from > 0 => {
|
||||
if let Some(expected) = &expected
|
||||
&& sha256_file(&part_path).is_ok_and(|got| got.eq_ignore_ascii_case(expected))
|
||||
{
|
||||
on(&Event::Verifying);
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
on(&Event::Done { verified: true });
|
||||
return Ok(final_path);
|
||||
}
|
||||
fs::remove_file(&part_path)?;
|
||||
resumed_from = 0;
|
||||
get(0).map_err(http_err)?
|
||||
}
|
||||
Err(e) => return Err(http_err(e)),
|
||||
};
|
||||
|
||||
// A 200 to a Range request means the server ignored resume — truncate.
|
||||
let honored_resume = resumed_from > 0 && resp.status() == 206;
|
||||
let total = content_length(&resp).map(|len| if honored_resume { len + resumed_from } else { len });
|
||||
|
||||
on(&Event::Started { total: total.unwrap_or(0), resumed_from: if honored_resume { resumed_from } else { 0 } });
|
||||
|
||||
let mut file = if honored_resume { OpenOptions::new().append(true).open(&part_path)? } else { File::create(&part_path)? };
|
||||
|
||||
let mut body = resp.body_mut().as_reader();
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
let mut done = if honored_resume { resumed_from } else { 0 };
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
if cancel.is_some_and(|c| !c()) {
|
||||
return Err(AiError::Cancelled);
|
||||
}
|
||||
let n = match body.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => n,
|
||||
Err(e) => return Err(AiError::Io(e)),
|
||||
};
|
||||
file.write_all(&buf[..n])?;
|
||||
done += n as u64;
|
||||
let secs = started.elapsed().as_secs_f64().max(0.001);
|
||||
on(&Event::Progress { done, total: total.unwrap_or(0), bytes_per_sec: (done.saturating_sub(resumed_from) as f64 / secs) as u64 });
|
||||
}
|
||||
file.flush()?;
|
||||
drop(file);
|
||||
|
||||
// A short read means the connection dropped mid-body — keep the .part
|
||||
// for resume and report it as an HTTP failure, not a success.
|
||||
if let Some(t) = total
|
||||
&& done < t
|
||||
{
|
||||
return Err(AiError::Http(format!("connection dropped at {done}/{t} bytes — retry to resume")));
|
||||
}
|
||||
|
||||
on(&Event::Verifying);
|
||||
let verified = if let Some(expected) = expected {
|
||||
let got = sha256_file(&part_path)?;
|
||||
if !got.eq_ignore_ascii_case(&expected) {
|
||||
// A corrupt download is deleted — it can never silently pass.
|
||||
let _ = fs::remove_file(&part_path);
|
||||
return Err(AiError::Checksum { expected, got });
|
||||
}
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
on(&Event::Done { verified });
|
||||
Ok(final_path)
|
||||
}
|
||||
|
||||
/// The digest we hold this file to: our registry hash if pinned, else the
|
||||
/// ETag of the resolved URL (Hugging Face serves the LFS sha256 there).
|
||||
fn expected_sha256(entry: &ModelEntry) -> Result<Option<String>, AiError> {
|
||||
if let Some(s) = entry.sha256 {
|
||||
return Ok(Some(s.to_string()));
|
||||
}
|
||||
// No redirects: HF serves the LFS sha256 in `x-linked-etag` on the
|
||||
// 302 itself; following it would return the CDN's own etag instead.
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.max_redirects(0)
|
||||
.timeout_connect(Some(Duration::from_secs(15)))
|
||||
.timeout_recv_response(Some(Duration::from_secs(30)))
|
||||
.build()
|
||||
.into();
|
||||
match agent.head(entry.url).call() {
|
||||
Ok(resp) => Ok(etag(&resp)),
|
||||
// HEAD unsupported/unreachable → proceed without verification.
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn etag(resp: &ureq::http::Response<ureq::Body>) -> Option<String> {
|
||||
// Prefer the redirect target's etag (the LFS object), fall back to
|
||||
// the response's own. Quotes stripped; weak etags ("W/…") ignored.
|
||||
for name in ["x-linked-etag", "etag"] {
|
||||
if let Some(v) = resp.headers().get(name).and_then(|v| v.to_str().ok()) {
|
||||
let v = v.trim().trim_matches('"');
|
||||
if !v.starts_with("W/") && v.len() == 64 && v.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn content_length(resp: &ureq::http::Response<ureq::Body>) -> Option<u64> {
|
||||
resp.headers()
|
||||
.get("content-length")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.trim().parse().ok())
|
||||
// A redirect's own length is meaningless; only trust it after
|
||||
// ureq followed redirects (it does — so the final body length).
|
||||
.filter(|&n: &u64| n > 0)
|
||||
}
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, AiError> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
let mut buf = vec![0u8; 256 * 1024];
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(hasher.finalize().iter().map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
|
||||
fn http_err(e: ureq::Error) -> AiError {
|
||||
match e {
|
||||
ureq::Error::StatusCode(code) => AiError::Rejected(format!("HTTP {code}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a model would land once downloaded.
|
||||
pub fn model_path(entry: &ModelEntry, cache_dir: &Path) -> PathBuf {
|
||||
cache_dir.join(entry.file)
|
||||
}
|
||||
|
||||
/// Is `entry` already fully downloaded?
|
||||
pub fn is_downloaded(entry: &ModelEntry, cache_dir: &Path) -> bool {
|
||||
cache_dir.join(entry.file).is_file()
|
||||
}
|
||||
|
||||
/// `tokenizer.json` beside the GGUF: `<file>.tokenizer.json`.
|
||||
pub fn tokenizer_path(entry: &ModelEntry, cache_dir: &Path) -> PathBuf {
|
||||
cache_dir.join(format!("{}.tokenizer.json", entry.file))
|
||||
}
|
||||
|
||||
/// The companion tokenizer — small (a few MB), so no resume loop, but the
|
||||
/// same `.part`-then-rename rule applies.
|
||||
pub fn download_tokenizer(entry: &ModelEntry, cache_dir: &Path) -> Result<PathBuf, AiError> {
|
||||
let dest = tokenizer_path(entry, cache_dir);
|
||||
if dest.is_file() {
|
||||
return Ok(dest);
|
||||
}
|
||||
let tmp = dest.with_extension("part");
|
||||
let mut resp = body_agent(32 * 1024 * 1024).get(entry.tokenizer_url).call().map_err(http_err)?;
|
||||
let bytes = resp.body_mut().read_to_vec().map_err(|e| AiError::Http(e.to_string()))?;
|
||||
fs::write(&tmp, bytes)?;
|
||||
fs::rename(&tmp, &dest)?;
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Remove `entry`'s file (and any `.part` remnant). Missing is fine;
|
||||
/// other failures (permissions, locked) are surfaced to the caller.
|
||||
pub fn delete(entry: &ModelEntry, cache_dir: &Path) -> Result<(), AiError> {
|
||||
for p in [cache_dir.join(entry.file), cache_dir.join(format!("{}.part", entry.file))] {
|
||||
match fs::remove_file(&p) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => return Err(AiError::Io(e)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::registry::Arch;
|
||||
|
||||
fn entry() -> &'static ModelEntry {
|
||||
static E: ModelEntry = ModelEntry {
|
||||
id: "t",
|
||||
name: "t",
|
||||
url: "",
|
||||
file: "m.gguf",
|
||||
tokenizer_url: "",
|
||||
sha256: None,
|
||||
size_mb: 1,
|
||||
min_ram_gb: 0.0,
|
||||
rec_ram_gb: 0.0,
|
||||
license: "MIT",
|
||||
quality: crate::registry::Quality::Tiny,
|
||||
arch: Arch::Llama,
|
||||
langs: "",
|
||||
};
|
||||
&E
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn second_lock_on_same_entry_is_busy() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let part = dir.path().join("m.gguf.part");
|
||||
let _lock = lock_entry(&part).expect("first lock");
|
||||
assert!(matches!(lock_entry(&part), Err(AiError::Busy)));
|
||||
// Releasing lets the next caller in.
|
||||
drop(_lock);
|
||||
assert!(lock_entry(&part).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_reports_failures_beyond_not_found() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
// Absent paths are fine.
|
||||
delete(entry(), dir.path()).expect("missing ok");
|
||||
// A directory where the file would be fails removal on every OS.
|
||||
fs::create_dir(dir.path().join("m.gguf")).expect("mkdir");
|
||||
assert!(matches!(delete(entry(), dir.path()), Err(AiError::Io(_))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! craft-ai — AI provider abstraction for the Craft suite.
|
||||
//!
|
||||
//! See `README.md` for the design: BYO API provider (OpenAI-compatible) +
|
||||
//! one-click local models with capability detection.
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod error;
|
||||
mod provider;
|
||||
mod registry;
|
||||
mod translate;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod fetch;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[cfg(feature = "local")]
|
||||
pub mod local;
|
||||
// `MachineCaps` is portable data the registry needs on every target; only
|
||||
// the OS probe itself is native-only.
|
||||
pub mod probe;
|
||||
|
||||
pub use error::AiError;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use provider::OpenAiCompat;
|
||||
pub use provider::{GenOpts, Provider, ProviderInfo};
|
||||
pub use registry::{Arch, MODELS, ModelEntry, Quality, by_id, fits, suggest};
|
||||
pub use translate::{PdfGrammar, Segment, TranslateGrammar, is_protected_token, segment, translate_segments};
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Local GGUF inference on candle (pure Rust — keeps the no-C++ rule and
|
||||
//! the door open for wasm later). CPU only for now; GPU comes from the
|
||||
//! app's wgpu adapter in a later milestone.
|
||||
//!
|
||||
//! Both supported architectures speak ChatML — one prompt template covers
|
||||
//! Qwen2.5 and SmolLM2.
|
||||
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
|
||||
use candle_core::quantized::gguf_file;
|
||||
use candle_core::{Device, IndexOp, Tensor};
|
||||
use candle_transformers::generation::LogitsProcessor;
|
||||
use candle_transformers::models::{quantized_llama, quantized_qwen2};
|
||||
use tokenizers::Tokenizer;
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::provider::{GenOpts, Provider, ProviderInfo};
|
||||
use crate::registry::{Arch, ModelEntry};
|
||||
|
||||
enum Weights {
|
||||
Qwen2(quantized_qwen2::ModelWeights),
|
||||
Llama(quantized_llama::ModelWeights),
|
||||
}
|
||||
|
||||
impl Weights {
|
||||
fn forward(&mut self, ids: &Tensor, pos: usize) -> Result<Tensor, candle_core::Error> {
|
||||
match self {
|
||||
Self::Qwen2(w) => w.forward(ids, pos),
|
||||
Self::Llama(w) => w.forward(ids, pos),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A downloaded GGUF loaded and ready. Construction is the expensive
|
||||
/// part; `complete` is then just a forward loop.
|
||||
pub struct LocalGguf {
|
||||
weights: Weights,
|
||||
tokenizer: Tokenizer,
|
||||
device: Device,
|
||||
eos: u32,
|
||||
entry: &'static ModelEntry,
|
||||
}
|
||||
|
||||
const SEED: u64 = 0xC0FFEE;
|
||||
|
||||
impl LocalGguf {
|
||||
/// Load the GGUF + tokenizer.json of `entry` from the model cache.
|
||||
/// All failures are `Inference` errors — a corrupt file, an OOM, a
|
||||
/// wrong-arch GGUF, all land in the status line, never in a panic.
|
||||
pub fn load(entry: &'static ModelEntry, gguf_path: &Path, tokenizer_path: &Path) -> Result<Self, AiError> {
|
||||
let device = Device::Cpu;
|
||||
let mut file = File::open(gguf_path).map_err(AiError::Io)?;
|
||||
// Malformed GGUF metadata (e.g. `general.alignment = 0`) makes
|
||||
// candle's parser panic on a division — hostile input must come
|
||||
// back as an error, not a crash.
|
||||
// Malformed GGUF metadata (e.g. `general.alignment = 0`, or a
|
||||
// zero `attention.head_count` reaching a division in the weight
|
||||
// constructors) makes candle panic — hostile input must come
|
||||
// back as an error, not a crash. The guard covers parsing *and*
|
||||
// weight construction.
|
||||
let weights = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<Weights, AiError> {
|
||||
let content = gguf_file::Content::read(&mut file).map_err(|e| AiError::Inference(format!("bad GGUF: {e}")))?;
|
||||
match entry.arch {
|
||||
Arch::Qwen2 => Ok(Weights::Qwen2(
|
||||
quantized_qwen2::ModelWeights::from_gguf(content, &mut file, &device)
|
||||
.map_err(|e| AiError::Inference(format!("cannot load {}: {e}", entry.file)))?,
|
||||
)),
|
||||
Arch::Llama => Ok(Weights::Llama(
|
||||
quantized_llama::ModelWeights::from_gguf(content, &mut file, &device)
|
||||
.map_err(|e| AiError::Inference(format!("cannot load {}: {e}", entry.file)))?,
|
||||
)),
|
||||
}
|
||||
}))
|
||||
.map_err(|_| AiError::Inference("GGUF loader panicked on malformed metadata".to_owned()))??;
|
||||
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(|e| AiError::Inference(format!("bad tokenizer.json: {e}")))?;
|
||||
let eos = tokenizer.token_to_id("<|im_end|>").ok_or_else(|| AiError::Inference("tokenizer has no EOS token".into()))?;
|
||||
Ok(Self { weights, tokenizer, device, eos, entry })
|
||||
}
|
||||
|
||||
/// ChatML wrap — Qwen2.5 and SmolLM2 both ship this template.
|
||||
fn chat(prompt: &str) -> String {
|
||||
format!(
|
||||
"<|im_start|>system\nYou are a precise translation engine. Follow the instructions exactly.<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n",
|
||||
)
|
||||
}
|
||||
|
||||
fn generate(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
let text = Self::chat(prompt);
|
||||
let encoding = self.tokenizer.encode(text, true).map_err(|e| AiError::Inference(format!("encode failed: {e}")))?;
|
||||
let mut tokens: Vec<u32> = encoding.get_ids().to_vec();
|
||||
|
||||
let temp = if opts.temperature <= 0.02 { None } else { Some(f64::from(opts.temperature)) };
|
||||
let mut sampler = LogitsProcessor::new(SEED, temp, None);
|
||||
|
||||
let mut pos = 0usize;
|
||||
let mut generated: Vec<u32> = Vec::with_capacity(opts.max_tokens.min(4096) as usize);
|
||||
let mut hit_eos = false;
|
||||
for _ in 0..opts.max_tokens {
|
||||
let input_len = tokens.len() - pos;
|
||||
let input =
|
||||
Tensor::new(&tokens[pos..], &self.device).and_then(|t| t.reshape((1, t.dims1()?))).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
let logits = self.weights.forward(&input, pos).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
pos += input_len;
|
||||
// `forward` already returns the last sequence position's
|
||||
// logits [batch, vocab] — index the batch row only.
|
||||
let logits = logits.i(0).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
let next = sampler.sample(&logits).map_err(|e| AiError::Inference(e.to_string()))?;
|
||||
if next == self.eos {
|
||||
hit_eos = true;
|
||||
break;
|
||||
}
|
||||
tokens.push(next);
|
||||
generated.push(next);
|
||||
}
|
||||
// No EOS inside the budget → the reply was cut mid-translation;
|
||||
// partial output would silently truncate the document.
|
||||
if !hit_eos {
|
||||
return Err(AiError::Inference(format!("generation hit max_tokens ({}) without EOS", opts.max_tokens)));
|
||||
}
|
||||
// Decode the generated ids through the tokenizer: byte-level BPE
|
||||
// vocab strings can't be concatenated (spaces/newlines/non-ASCII
|
||||
// bytes stay encoded, and UTF-8 can split across tokens).
|
||||
self.tokenizer.decode(&generated, true).map_err(|e| AiError::Inference(format!("decode failed: {e}")))
|
||||
}
|
||||
}
|
||||
|
||||
impl Provider for LocalGguf {
|
||||
fn name(&self) -> String {
|
||||
format!("local: {}", self.entry.name)
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
self.generate(prompt, opts).map(|s| s.trim().to_string())
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError> {
|
||||
Ok(ProviderInfo { label: format!("loaded: {}", self.entry.name), models: vec![self.entry.id.to_string()] })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Machine capability detection — runs once, cache the result.
|
||||
//!
|
||||
//! GPU/VRAM is deliberately *not* probed here: apps already link wgpu, so
|
||||
//! they read their own adapter info and fill `MachineCaps::vram_gb` after
|
||||
//! calling [`probe`]. Re-probing the GPU would double the surface for a
|
||||
//! bug that reports nonsense VRAM on hybrid laptops.
|
||||
|
||||
/// What this machine can comfortably run.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct MachineCaps {
|
||||
/// Total physical RAM, in GiB.
|
||||
pub ram_gb: f32,
|
||||
/// Free space on the volume holding the model cache, in GiB.
|
||||
pub disk_free_gb: f32,
|
||||
/// Logical CPU threads.
|
||||
pub threads: usize,
|
||||
/// AVX2 detected — without it quantized GGUF inference crawls.
|
||||
pub avx2: bool,
|
||||
/// Dedicated VRAM in GiB, when the app supplied it from its wgpu adapter.
|
||||
/// `None` means "unknown or no discrete GPU" — never guessed.
|
||||
pub vram_gb: Option<f32>,
|
||||
}
|
||||
|
||||
impl MachineCaps {
|
||||
/// Rough throughput hint for a model of `size_mb`: tokens/sec on CPU.
|
||||
/// Deliberately conservative — an honest underestimate beats an
|
||||
/// optimistic one that makes the app look slow.
|
||||
pub fn est_tokens_per_sec(&self, size_mb: u64) -> f32 {
|
||||
let base = if self.avx2 { 6.0 } else { 1.2 };
|
||||
// ~0.5B params ≈ 350 MiB Q4; speed scales inversely with size.
|
||||
let scale = 350.0 / (size_mb.max(1) as f32);
|
||||
let thread_boost = (self.threads as f32 / 4.0).min(1.5);
|
||||
(base * scale * thread_boost).max(0.3)
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe the real machine once. `cache_dir` is the future model-cache
|
||||
/// location — disk space is measured on *its* volume, which may not be
|
||||
/// the OS volume.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn probe(cache_dir: &std::path::Path) -> MachineCaps {
|
||||
let mut sys = sysinfo::System::new();
|
||||
sys.refresh_memory();
|
||||
let ram_gb = sys.total_memory() as f32 / (1 << 30) as f32;
|
||||
|
||||
let disk_free_gb = disk_free(cache_dir);
|
||||
|
||||
MachineCaps { ram_gb, disk_free_gb, threads: std::thread::available_parallelism().map_or(1, |n| n.get()), avx2: avx2(), vram_gb: None }
|
||||
}
|
||||
|
||||
#[cfg(all(not(target_arch = "wasm32"), any(target_arch = "x86_64", target_arch = "x86")))]
|
||||
fn avx2() -> bool {
|
||||
std::arch::is_x86_feature_detected!("avx2")
|
||||
}
|
||||
|
||||
#[cfg(all(not(target_arch = "wasm32"), not(any(target_arch = "x86_64", target_arch = "x86"))))]
|
||||
fn avx2() -> bool {
|
||||
// AArch64 NEON is mandatory, so the honest answer is "fast enough".
|
||||
true
|
||||
}
|
||||
|
||||
/// Free bytes on the volume containing `dir`, walking up to the nearest
|
||||
/// existing ancestor if `dir` doesn't exist yet (the common case).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn disk_free(dir: &std::path::Path) -> f32 {
|
||||
// Canonicalize the existing ancestor: a relative or symlinked
|
||||
// `cache_dir` must still match an absolute mount point.
|
||||
let probe: std::path::PathBuf =
|
||||
dir.ancestors().find(|p| p.exists()).and_then(|p| p.canonicalize().ok()).unwrap_or_else(|| std::path::PathBuf::from("."));
|
||||
let disks = sysinfo::Disks::new_with_refreshed_list();
|
||||
// The longest mount point that prefixes our dir owns it.
|
||||
let best = disks.iter().filter(|d| probe.starts_with(d.mount_point())).max_by_key(|d| d.mount_point().as_os_str().len());
|
||||
best.map_or(0.0, |d| d.available_space() as f32 / (1 << 30) as f32)
|
||||
}
|
||||
|
||||
/// The suite-shared model cache (`%LOCALAPPDATA%\artcraft\models` etc.).
|
||||
/// `None` on platforms without a cache dir — the caller should refuse
|
||||
/// local models rather than splashing files into the cwd.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn models_dir() -> Option<std::path::PathBuf> {
|
||||
dirs::cache_dir().map(|d| d.join("artcraft").join("models"))
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
//! Inference backends behind one trait. Apps hold a `Box<dyn Provider>`
|
||||
//! and never care whether the bytes went to OpenAI, Ollama or a local GGUF.
|
||||
|
||||
use crate::error::AiError;
|
||||
|
||||
/// Generation parameters — kept minimal on purpose; the UI exposes
|
||||
/// "translate", not a sampler playground.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GenOpts {
|
||||
/// Max tokens to generate.
|
||||
pub max_tokens: u32,
|
||||
/// Sampling temperature; translation wants ~0.
|
||||
pub temperature: f32,
|
||||
}
|
||||
|
||||
impl Default for GenOpts {
|
||||
fn default() -> Self {
|
||||
Self { max_tokens: 2048, temperature: 0.1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// What `test()` learned about a provider — shown next to the green check.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ProviderInfo {
|
||||
/// Human label, e.g. "Ollama at http://localhost:11434".
|
||||
pub label: String,
|
||||
/// Model ids the endpoint advertised (may be empty if it doesn't say).
|
||||
pub models: Vec<String>,
|
||||
}
|
||||
|
||||
/// An AI backend the suite can use. `Send` so apps can move a loaded
|
||||
/// provider onto a worker thread for long generations.
|
||||
pub trait Provider: Send {
|
||||
/// Short name for the status line ("local: Qwen2.5 1.5B", "OpenAI-compatible").
|
||||
fn name(&self) -> String;
|
||||
/// One completion. Prompt construction is the caller's job — this
|
||||
/// trait is deliberately text-in/text-out so every consumer (translate,
|
||||
/// summarize, custom prompt) shares one path.
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError>;
|
||||
/// Can we reach it? Never sends document text — just a handshake.
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError>;
|
||||
/// True when document text stays on this machine.
|
||||
fn is_local(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Any OpenAI-compatible `/v1/chat/completions` endpoint: OpenAI, Azure,
|
||||
/// Ollama (`http://localhost:11434/v1`), LM Studio, Groq, OpenRouter…
|
||||
/// The token is a plain string — storage (OS keyring, settings file) is
|
||||
/// the app's choice, and it is never logged or echoed into errors.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub struct OpenAiCompat {
|
||||
/// Base URL ending at the API root, e.g. `https://api.openai.com/v1`.
|
||||
pub endpoint: String,
|
||||
/// Bearer token; `None` for unauthenticated local endpoints (Ollama).
|
||||
pub token: Option<String>,
|
||||
/// Model id sent in every request.
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl OpenAiCompat {
|
||||
pub fn new(endpoint: impl Into<String>, token: Option<String>, model: impl Into<String>) -> Self {
|
||||
Self { endpoint: endpoint.into(), token, model: model.into() }
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}/{}", self.endpoint.trim_end_matches('/'), path)
|
||||
}
|
||||
|
||||
/// ureq sets no timeouts by default — an endpoint that accepts the
|
||||
/// socket and never replies would block the caller forever, so every
|
||||
/// request goes through an agent with finite deadlines. The response
|
||||
/// budgets are generous: a non-streamed chat completion only sends
|
||||
/// headers once generation is done, which can take minutes.
|
||||
fn agent() -> ureq::Agent {
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(std::time::Duration::from_secs(15)))
|
||||
.timeout_send_request(Some(std::time::Duration::from_secs(30)))
|
||||
.timeout_send_body(Some(std::time::Duration::from_secs(60)))
|
||||
.timeout_recv_response(Some(std::time::Duration::from_secs(300)))
|
||||
.timeout_recv_body(Some(std::time::Duration::from_secs(300)))
|
||||
.build()
|
||||
.into()
|
||||
}
|
||||
|
||||
fn auth(&self, req: ureq::RequestBuilder<ureq::typestate::WithBody>) -> ureq::RequestBuilder<ureq::typestate::WithBody> {
|
||||
match &self.token {
|
||||
Some(t) if !t.is_empty() => req.header("Authorization", format!("Bearer {t}")),
|
||||
_ => req,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Provider for OpenAiCompat {
|
||||
fn name(&self) -> String {
|
||||
format!("{} ({})", self.model, self.endpoint)
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, opts: &GenOpts) -> Result<String, AiError> {
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err(AiError::NotConfigured);
|
||||
}
|
||||
let body = serde_json::json!({
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": opts.temperature,
|
||||
"max_tokens": opts.max_tokens,
|
||||
});
|
||||
let mut resp =
|
||||
self.auth(Self::agent().post(self.url("chat/completions"))).header("Content-Type", "application/json").send_json(body).map_err(|e| {
|
||||
match e {
|
||||
ureq::Error::StatusCode(401 | 403) => AiError::Rejected("authentication failed — check the API token".into()),
|
||||
ureq::Error::StatusCode(404) => AiError::Rejected("model or endpoint not found — check both fields".into()),
|
||||
ureq::Error::StatusCode(c) => AiError::Rejected(format!("HTTP {c}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
}
|
||||
})?;
|
||||
let v: serde_json::Value = resp.body_mut().read_json().map_err(|e| AiError::BadResponse(e.to_string()))?;
|
||||
// A completion cut off at max_tokens ("length") or aborted is not
|
||||
// a translation — returning it would silently truncate the doc.
|
||||
match v.pointer("/choices/0/finish_reason").and_then(serde_json::Value::as_str) {
|
||||
Some("stop") | Some("tool_calls") | None => {}
|
||||
Some(reason) => return Err(AiError::BadResponse(format!("incomplete completion (finish_reason={reason})"))),
|
||||
}
|
||||
v.pointer("/choices/0/message/content")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| AiError::BadResponse("no choices[0].message.content in response".into()))
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<ProviderInfo, AiError> {
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err(AiError::NotConfigured);
|
||||
}
|
||||
// GET /v1/models — a handshake only, no user text leaves the app.
|
||||
let mut req = Self::agent().get(self.url("models"));
|
||||
if let Some(t) = &self.token
|
||||
&& !t.is_empty()
|
||||
{
|
||||
req = req.header("Authorization", format!("Bearer {t}"));
|
||||
}
|
||||
let mut resp = req.call().map_err(|e| match e {
|
||||
ureq::Error::StatusCode(401 | 403) => AiError::Rejected("authentication failed — check the API token".into()),
|
||||
ureq::Error::StatusCode(c) => AiError::Rejected(format!("HTTP {c}")),
|
||||
other => AiError::Http(other.to_string()),
|
||||
})?;
|
||||
let v: serde_json::Value = resp.body_mut().read_json().map_err(|e| AiError::BadResponse(e.to_string()))?;
|
||||
let models = v
|
||||
.get("data")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.map(|arr| arr.iter().filter_map(|m| m.get("id").and_then(serde_json::Value::as_str).map(str::to_string)).collect())
|
||||
.unwrap_or_default();
|
||||
Ok(ProviderInfo { label: format!("reachable: {}", self.endpoint), models })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
// Local endpoints keep text on the machine; we can't verify a
|
||||
// remote URL's forwarding, so honesty means: loopback → local.
|
||||
// Parse the authority properly — a prefix match would call
|
||||
// `localhost.example.com` or `127.0.0.1.example.com` "local"
|
||||
// while document text leaves the machine.
|
||||
let e = self.endpoint.trim();
|
||||
let after_scheme = e.split_once("://").map_or(e, |(_, rest)| rest);
|
||||
let authority = after_scheme.split(['/', '?', '#']).next().unwrap_or("");
|
||||
let host_port = authority.rsplit('@').next().unwrap_or(""); // strip userinfo
|
||||
let host = if let Some(rest) = host_port.strip_prefix('[') {
|
||||
rest.split(']').next().unwrap_or("")
|
||||
} else {
|
||||
host_port.split(':').next().unwrap_or("")
|
||||
};
|
||||
host.eq_ignore_ascii_case("localhost") || host.parse::<std::net::IpAddr>().is_ok_and(|ip| ip.is_loopback())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(target_arch = "wasm32")))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn is_local_accepts_only_real_loopback() {
|
||||
let local = |ep: &str| OpenAiCompat::new(ep, None, "m").is_local();
|
||||
assert!(local("http://localhost:11434/v1"));
|
||||
assert!(local("https://localhost/v1"));
|
||||
assert!(local("http://127.0.0.1:8080/v1"));
|
||||
assert!(local("http://127.99.0.1/v1")); // whole 127/8 is loopback
|
||||
assert!(local("http://[::1]:11434/v1"));
|
||||
// Lookalike hostnames must NOT report local — document text would
|
||||
// leave the machine while the UI promises it stays.
|
||||
assert!(!local("https://localhost.example.com/v1"));
|
||||
assert!(!local("https://127.0.0.1.example.com/v1"));
|
||||
assert!(!local("https://localhost.evil.io/v1"));
|
||||
assert!(!local("https://api.openai.com/v1"));
|
||||
assert!(!local("https://example.com/localhost/v1"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! The curated model registry — a hand-maintained table, compiled in.
|
||||
//! Runtime-fetching this list would make our supply chain un-auditable.
|
||||
//!
|
||||
//! Curation rule: **permissive licenses only** (the suite is MIT OR
|
||||
//! Apache-2.0 and commercial users run it). CC-BY-NC models like NLLB-200
|
||||
//! are excluded by policy; their absence is deliberate.
|
||||
//!
|
||||
//! `sha256` may be `None`: Hugging Face serves the LFS digest in the
|
||||
//! `X-Linked-ETag` header of the resolve redirect, which [`crate::fetch`]
|
||||
//! uses as the expected checksum. We only hard-code a hash when we've
|
||||
//! verified it ourselves — never a guessed one.
|
||||
|
||||
use crate::probe::MachineCaps;
|
||||
|
||||
/// Model quality tier — higher is better output, bigger download.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Quality {
|
||||
Tiny,
|
||||
Small,
|
||||
Medium,
|
||||
}
|
||||
|
||||
/// Which candle architecture loads the GGUF.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Arch {
|
||||
Qwen2,
|
||||
Llama,
|
||||
}
|
||||
|
||||
/// One downloadable model.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ModelEntry {
|
||||
/// Stable identifier stored in settings (`"qwen2.5-1.5b-q4km"`).
|
||||
pub id: &'static str,
|
||||
/// Display name for the picker.
|
||||
pub name: &'static str,
|
||||
/// Canonical download URL (Hugging Face `resolve` endpoint).
|
||||
pub url: &'static str,
|
||||
/// Filename inside the model cache.
|
||||
pub file: &'static str,
|
||||
/// Companion tokenizer (needed by local inference; the GGUF ships
|
||||
/// vocab only, not the chat-time tokenizer.json).
|
||||
pub tokenizer_url: &'static str,
|
||||
/// Verified SHA-256 of the GGUF, when we have one.
|
||||
pub sha256: Option<&'static str>,
|
||||
/// Download size, MiB — shown to the user before they commit.
|
||||
pub size_mb: u64,
|
||||
/// Below this RAM the model is refused outright.
|
||||
pub min_ram_gb: f32,
|
||||
/// Recommended RAM for comfortable use.
|
||||
pub rec_ram_gb: f32,
|
||||
/// SPDX license id, displayed in the UI.
|
||||
pub license: &'static str,
|
||||
pub quality: Quality,
|
||||
pub arch: Arch,
|
||||
/// Human-readable language coverage for the picker.
|
||||
pub langs: &'static str,
|
||||
}
|
||||
|
||||
/// The registry, smallest first.
|
||||
pub static MODELS: &[ModelEntry] = &[
|
||||
ModelEntry {
|
||||
id: "qwen2.5-0.5b-q4km",
|
||||
name: "Qwen2.5 0.5B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf",
|
||||
file: "qwen2.5-0.5b-instruct-q4_k_m.gguf",
|
||||
tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 389,
|
||||
min_ram_gb: 2.0,
|
||||
rec_ram_gb: 4.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Tiny,
|
||||
arch: Arch::Qwen2,
|
||||
langs: "29+ languages incl. EN/ZH/FR/DE/ES/JA/KO/RU",
|
||||
},
|
||||
ModelEntry {
|
||||
id: "smollm2-1.7b-q4km",
|
||||
name: "SmolLM2 1.7B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/bartowski/SmolLM2-1.7B-Instruct-GGUF/resolve/main/SmolLM2-1.7B-Instruct-Q4_K_M.gguf",
|
||||
file: "SmolLM2-1.7B-Instruct-Q4_K_M.gguf",
|
||||
tokenizer_url: "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 1033,
|
||||
min_ram_gb: 4.0,
|
||||
rec_ram_gb: 8.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Small,
|
||||
arch: Arch::Llama,
|
||||
langs: "primarily EN/FR/DE/ES/IT",
|
||||
},
|
||||
ModelEntry {
|
||||
id: "qwen2.5-1.5b-q4km",
|
||||
name: "Qwen2.5 1.5B Instruct (Q4_K_M)",
|
||||
url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q4_k_m.gguf",
|
||||
file: "qwen2.5-1.5b-instruct-q4_k_m.gguf",
|
||||
tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/tokenizer.json",
|
||||
sha256: None,
|
||||
size_mb: 986,
|
||||
min_ram_gb: 4.0,
|
||||
rec_ram_gb: 8.0,
|
||||
license: "Apache-2.0",
|
||||
quality: Quality::Small,
|
||||
arch: Arch::Qwen2,
|
||||
langs: "29+ languages incl. EN/ZH/FR/DE/ES/JA/KO/RU",
|
||||
},
|
||||
// NOTE: Qwen2.5-3B was removed — its `qwen-research` license is
|
||||
// non-commercial and breaks the permissive-only rule above. No
|
||||
// permissively-licensed Medium-tier candidate is currently curated.
|
||||
];
|
||||
|
||||
/// Look up a model by stable id.
|
||||
pub fn by_id(id: &str) -> Option<&'static ModelEntry> {
|
||||
MODELS.iter().find(|m| m.id == id)
|
||||
}
|
||||
|
||||
/// Whether `entry` can run at all on this machine.
|
||||
pub fn fits(caps: &MachineCaps, entry: &ModelEntry) -> bool {
|
||||
let ram_ok = caps.ram_gb >= entry.min_ram_gb;
|
||||
let disk_ok = caps.disk_free_gb >= entry.size_mb as f32 / 1024.0 * 1.3; // file + headroom
|
||||
ram_ok && disk_ok
|
||||
}
|
||||
|
||||
/// The default suggestion for `caps`: best quality that fits, preferring
|
||||
/// 3B on roomy machines, 1.5B mid-range, 0.5B on small ones. Returns
|
||||
/// `None` when nothing fits — the UI then says so honestly instead of
|
||||
/// offering a download that can't run.
|
||||
pub fn suggest(caps: &MachineCaps) -> Option<&'static ModelEntry> {
|
||||
let capable: Vec<&ModelEntry> = MODELS.iter().filter(|m| fits(caps, m)).collect();
|
||||
let best_allowed = if caps.vram_gb.is_some_and(|v| v >= 6.0) || caps.ram_gb >= 16.0 {
|
||||
Quality::Medium
|
||||
} else if caps.ram_gb >= 8.0 {
|
||||
Quality::Small
|
||||
} else {
|
||||
Quality::Tiny
|
||||
};
|
||||
capable.into_iter().filter(|m| m.quality <= best_allowed).max_by_key(|m| m.quality)
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
//! Document translation as *segments*, never raw text: the model sees
|
||||
//! only translatable runs, and protected spans (URLs, placeholders,
|
||||
//! markup, numbers-only runs) come back byte-identical because we never
|
||||
//! sent them.
|
||||
//!
|
||||
//! Each app declares its grammar via [`TranslateGrammar`]; the default
|
||||
//! [`PdfGrammar`] suits extracted PDF text.
|
||||
|
||||
use crate::error::AiError;
|
||||
use crate::provider::{GenOpts, Provider};
|
||||
|
||||
/// A unit of text the translator decided on.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Segment {
|
||||
/// Sent to the model; replace with the translation.
|
||||
Text(String),
|
||||
/// Never sent; re-inserted verbatim.
|
||||
Protected(String),
|
||||
}
|
||||
|
||||
impl Segment {
|
||||
/// The source text, translated or not.
|
||||
pub fn source(&self) -> &str {
|
||||
match self {
|
||||
Self::Text(s) | Self::Protected(s) => s,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which runs of a document's text must never be translated.
|
||||
pub trait TranslateGrammar {
|
||||
/// True when `token` (already delimited — a word, symbol, or
|
||||
/// whitespace-free run) is structural and must survive verbatim.
|
||||
fn is_protected(&self, token: &str) -> bool;
|
||||
}
|
||||
|
||||
/// Grammar for text extracted from a PDF: protects URLs, e-mail
|
||||
/// addresses, `{placeholders}`, `<markup>`, and runs with no letters
|
||||
/// (page furniture, digits, punctuation).
|
||||
pub struct PdfGrammar;
|
||||
|
||||
impl TranslateGrammar for PdfGrammar {
|
||||
fn is_protected(&self, token: &str) -> bool {
|
||||
is_protected_token(token)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared protected-token test, useful to other apps' grammars.
|
||||
pub fn is_protected_token(token: &str) -> bool {
|
||||
let t = token.trim();
|
||||
if t.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if !t.chars().any(|c| c.is_alphabetic()) {
|
||||
return true; // numbers, punctuation, symbols
|
||||
}
|
||||
let lower = t.to_ascii_lowercase();
|
||||
lower.starts_with("http://")
|
||||
|| lower.starts_with("https://")
|
||||
|| lower.starts_with("www.")
|
||||
|| lower.contains('@') && lower.contains('.') && !lower.contains(' ')
|
||||
|| (t.starts_with('{') && t.ends_with('}'))
|
||||
|| (t.starts_with('<') && t.ends_with('>'))
|
||||
}
|
||||
|
||||
/// Split `text` into segments on whitespace boundaries, grouping
|
||||
/// consecutive protected or translatable tokens. `<markup>` spans are
|
||||
/// carved out first — a tag with attributes contains whitespace, and a
|
||||
/// whole `<em>Hello</em>` run must keep "Hello" translatable while the
|
||||
/// tags themselves stay byte-identical.
|
||||
pub fn segment(text: &str, grammar: &dyn TranslateGrammar) -> Vec<Segment> {
|
||||
fn flush(out: &mut Vec<Segment>, buf: &mut String, protected: bool) {
|
||||
if buf.is_empty() {
|
||||
return;
|
||||
}
|
||||
let seg = if protected { Segment::Protected(std::mem::take(buf)) } else { Segment::Text(std::mem::take(buf)) };
|
||||
out.push(seg);
|
||||
}
|
||||
|
||||
let mut out: Vec<Segment> = Vec::new();
|
||||
let mut buf = String::new();
|
||||
let mut buf_protected = false;
|
||||
let mut started = false;
|
||||
|
||||
for (token, is_tag) in split_tokens(text) {
|
||||
let ws = token.chars().all(char::is_whitespace);
|
||||
// Whitespace glues to whatever the current run is; markup tags
|
||||
// are always protected.
|
||||
let this_protected = if ws { buf_protected } else { is_tag || grammar.is_protected(token) };
|
||||
if started && this_protected != buf_protected {
|
||||
flush(&mut out, &mut buf, buf_protected);
|
||||
}
|
||||
buf_protected = this_protected;
|
||||
started = true;
|
||||
buf.push_str(token);
|
||||
}
|
||||
flush(&mut out, &mut buf, buf_protected);
|
||||
out
|
||||
}
|
||||
|
||||
/// Tokenize into `(slice, is_tag)`: markup tags are whole `<…>` spans
|
||||
/// (attributes included), everything else alternates between maximal
|
||||
/// whitespace and non-whitespace runs.
|
||||
fn split_tokens(text: &str) -> Vec<(&str, bool)> {
|
||||
let mut out = Vec::new();
|
||||
let mut pos = 0usize;
|
||||
let bytes = text.as_bytes();
|
||||
while pos < bytes.len() {
|
||||
if bytes[pos] == b'<'
|
||||
&& let Some(tag_len) = tag_len(&text[pos..])
|
||||
{
|
||||
out.push((&text[pos..pos + tag_len], true));
|
||||
pos += tag_len;
|
||||
continue;
|
||||
}
|
||||
// Advance a non-tag run up to the next '<' that starts a tag;
|
||||
// bare '<' characters just stay inside the run.
|
||||
let mut scan = pos;
|
||||
let mut run_end = bytes.len();
|
||||
while let Some(lt) = bytes[scan..].iter().position(|&b| b == b'<') {
|
||||
let at = scan + lt;
|
||||
if tag_len(&text[at..]).is_some() {
|
||||
run_end = at;
|
||||
break;
|
||||
}
|
||||
scan = at + 1;
|
||||
}
|
||||
for piece in split_words_ws(&text[pos..run_end]) {
|
||||
out.push((piece, false));
|
||||
}
|
||||
pos = run_end;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Length of a `<…>` tag starting at `s[0]`, or `None` when `<` does not
|
||||
/// begin markup (bare `<`, `<3`, `< hello`). Tags start `</`, `<?`, `<!`
|
||||
/// or `<`+letter and close at the first `>` before any `<` or newline;
|
||||
/// the scan is capped so pathological input can't stall it.
|
||||
fn tag_len(s: &str) -> Option<usize> {
|
||||
const MAX_TAG: usize = 512;
|
||||
let second = s.as_bytes().get(1)?;
|
||||
let opener_ok = second.is_ascii_alphabetic() || matches!(second, b'/' | b'!' | b'?');
|
||||
if !opener_ok {
|
||||
return None;
|
||||
}
|
||||
s.char_indices().skip(2).take(MAX_TAG).find(|(_, c)| matches!(c, '>' | '<' | '\n')).and_then(|(i, c)| (c == '>').then_some(i + 1))
|
||||
}
|
||||
|
||||
/// Split into alternating word/whitespace runs (like `split_inclusive`
|
||||
/// but grouping every maximal run of whitespace or non-whitespace).
|
||||
fn split_words_ws(text: &str) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0usize;
|
||||
let mut prev_ws: Option<bool> = None;
|
||||
for (i, c) in text.char_indices() {
|
||||
let ws = c.is_whitespace();
|
||||
if let Some(p) = prev_ws
|
||||
&& p != ws
|
||||
{
|
||||
out.push(&text[start..i]);
|
||||
start = i;
|
||||
}
|
||||
prev_ws = Some(ws);
|
||||
}
|
||||
if start < text.len() {
|
||||
out.push(&text[start..]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// How many segments one prompt carries — keeps responses snappy and
|
||||
/// makes a numbering slip cheap to retry.
|
||||
const BATCH: usize = 20;
|
||||
/// …and how many bytes one prompt carries. A document with no protected
|
||||
/// tokens is one `Text` segment of arbitrary length — without a size
|
||||
/// bound the request can exceed `max_tokens` and come back truncated
|
||||
/// (which providers must reject, not return as success).
|
||||
const BATCH_BYTES: usize = 4 * 1024;
|
||||
/// A single `Text` run larger than this is chopped into pieces before
|
||||
/// batching, so no one request is unbounded.
|
||||
const RUN_BYTES: usize = 3 * 1024;
|
||||
|
||||
/// A translatable piece of one segment: the whitespace-stripped `core`
|
||||
/// is sent to the model, `lead`/`trail` are restored verbatim around the
|
||||
/// translation so `Visitez` never glues to `https://a.io`.
|
||||
struct Run<'a> {
|
||||
seg: usize,
|
||||
lead: &'a str,
|
||||
core: &'a str,
|
||||
trail: &'a str,
|
||||
/// False for whitespace-only pieces: never sent to the model, but
|
||||
/// still reassembled into the segment.
|
||||
send: bool,
|
||||
}
|
||||
|
||||
/// `(leading ws, core, trailing ws)` of `s`, all on char boundaries.
|
||||
fn ws_edges(s: &str) -> (&str, &str, &str) {
|
||||
let lead = s.len() - s.trim_start().len();
|
||||
let tail = s.len() - s.trim_end().len();
|
||||
if lead + tail >= s.len() {
|
||||
return (s, "", ""); // all-whitespace run: nothing to translate
|
||||
}
|
||||
(&s[..lead], &s[lead..s.len() - tail], &s[s.len() - tail..])
|
||||
}
|
||||
|
||||
/// Chop `s` into ≤ `max`-byte pieces on whitespace boundaries (the last
|
||||
/// ws before the limit; a run with no ws is hard-cut at the limit). The
|
||||
/// boundary whitespace stays inside the pieces — nothing is lost.
|
||||
fn split_run(s: &str, max: usize) -> Vec<&str> {
|
||||
let mut out = Vec::new();
|
||||
let mut rest = s;
|
||||
while rest.len() > max {
|
||||
let mut ws_cut = None;
|
||||
let mut boundary = 0usize;
|
||||
for (i, c) in rest.char_indices() {
|
||||
if i >= max {
|
||||
break;
|
||||
}
|
||||
boundary = i + c.len_utf8();
|
||||
if c.is_whitespace() {
|
||||
ws_cut = Some(i); // cut *before* the ws — it leads the next piece
|
||||
}
|
||||
}
|
||||
let cut = match ws_cut.filter(|w| *w > max / 4) {
|
||||
Some(w) => w,
|
||||
None => boundary.max(1),
|
||||
};
|
||||
out.push(&rest[..cut]);
|
||||
rest = &rest[cut..];
|
||||
}
|
||||
if !rest.is_empty() {
|
||||
out.push(rest);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Translate every `Text` segment to `target_lang`, leaving `Protected`
|
||||
/// untouched. Batched with numbered items; a count mismatch retries the
|
||||
/// offending half rather than misaligning the whole document.
|
||||
pub fn translate_segments(provider: &mut dyn Provider, segs: &[Segment], target_lang: &str) -> Result<Vec<String>, AiError> {
|
||||
// Source whitespace never reaches the model: each Text segment is
|
||||
// split edge-to-edge, and oversized runs are chopped further.
|
||||
let mut runs: Vec<Run<'_>> = Vec::new();
|
||||
for (i, s) in segs.iter().enumerate() {
|
||||
let Segment::Text(src) = s else { continue };
|
||||
for piece in split_run(src, RUN_BYTES) {
|
||||
let (lead, core, trail) = ws_edges(piece);
|
||||
// Pure-whitespace pieces (possible after a long run is chopped)
|
||||
// keep their bytes in the output but skip the provider.
|
||||
runs.push(Run { seg: i, lead, core, trail, send: !core.is_empty() });
|
||||
}
|
||||
}
|
||||
let run_segs: std::collections::HashSet<usize> = runs.iter().map(|r| r.seg).collect();
|
||||
|
||||
let mut out: Vec<String> = segs.iter().map(|s| s.source().to_string()).collect();
|
||||
for &i in &run_segs {
|
||||
out[i].clear(); // rebuilt from translated runs below
|
||||
}
|
||||
let mut i = 0;
|
||||
while i < runs.len() {
|
||||
// Whitespace-only pieces restore verbatim, in order, and never
|
||||
// reach the model.
|
||||
while i < runs.len() && !runs[i].send {
|
||||
let r = &runs[i];
|
||||
out[r.seg].push_str(r.lead);
|
||||
out[r.seg].push_str(r.trail);
|
||||
i += 1;
|
||||
}
|
||||
// A batch is bounded by item count *and* total prompt bytes.
|
||||
let mut bytes = 0usize;
|
||||
let start = i;
|
||||
while i < runs.len() && runs[i].send && i - start < BATCH && bytes + runs[i].core.len() <= BATCH_BYTES {
|
||||
bytes += runs[i].core.len();
|
||||
i += 1;
|
||||
}
|
||||
if i == start {
|
||||
if i < runs.len() {
|
||||
i += 1; // never spin on an oversized single item
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let chunk = &runs[start..i];
|
||||
let translated = translate_batch(provider, chunk, target_lang)?;
|
||||
for (run, t) in chunk.iter().zip(translated) {
|
||||
out[run.seg].push_str(run.lead);
|
||||
out[run.seg].push_str(&t);
|
||||
out[run.seg].push_str(run.trail);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One numbered batch; splits recursively on count mismatch so a single
|
||||
/// malformed reply can't corrupt neighbors.
|
||||
fn translate_batch(provider: &mut dyn Provider, runs: &[Run<'_>], lang: &str) -> Result<Vec<String>, AiError> {
|
||||
if runs.len() == 1 {
|
||||
return Ok(vec![translate_one(provider, runs[0].core, lang)?]);
|
||||
}
|
||||
let prompt = numbered_prompt(&runs.iter().map(|r| r.core.to_string()).collect::<Vec<_>>(), lang);
|
||||
let reply = provider.complete(&prompt, &GenOpts::default())?;
|
||||
match parse_numbered(&reply, runs.len()) {
|
||||
Some(v) => Ok(v),
|
||||
None => {
|
||||
let mid = runs.len() / 2;
|
||||
let mut a = translate_batch(provider, &runs[..mid], lang)?;
|
||||
let b = translate_batch(provider, &runs[mid..], lang)?;
|
||||
a.extend(b);
|
||||
Ok(a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_one(provider: &mut dyn Provider, source: &str, lang: &str) -> Result<String, AiError> {
|
||||
let prompt = format!("Translate the following text to {lang}. Reply with ONLY the translation, no quotes, no notes.\n\n{source}");
|
||||
Ok(provider.complete(&prompt, &GenOpts::default())?.trim().to_string())
|
||||
}
|
||||
|
||||
/// `"1. …"`-style numbered prompt — models follow it far more reliably
|
||||
/// than delimiter schemes.
|
||||
fn numbered_prompt(sources: &[String], lang: &str) -> String {
|
||||
let mut p = format!(
|
||||
"Translate each numbered item to {lang}. Keep the numbering. \
|
||||
Preserve all formatting, placeholders like {{name}}, URLs and markup verbatim. \
|
||||
Reply with ONLY the numbered translations.\n\n"
|
||||
);
|
||||
for (i, s) in sources.iter().enumerate() {
|
||||
p.push_str(&format!("{}. {}\n", i + 1, s.replace('\n', " ")));
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Parse `"1. …\n2. …"` back into items; `None` when the reply is
|
||||
/// malformed — duplicates, out-of-range numbers, or a missing item all
|
||||
/// mean the numbering slipped, and the caller retries/splits rather
|
||||
/// than accepting corrupted text. Non-numbered lines after an item are
|
||||
/// continuations of it; non-numbered chatter before item 1 is rejected
|
||||
/// the same way, so nothing is silently dropped.
|
||||
fn parse_numbered(reply: &str, want: usize) -> Option<Vec<String>> {
|
||||
let mut out = vec![String::new(); want];
|
||||
let mut seen = vec![false; want];
|
||||
let mut current: Option<usize> = None;
|
||||
for line in reply.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match numbered_line(line) {
|
||||
Some(idx) if idx >= 1 && idx <= want => {
|
||||
if seen[idx - 1] {
|
||||
return None; // "1. un\n2. deux\n1. trois" overwrites — reject
|
||||
}
|
||||
seen[idx - 1] = true;
|
||||
let rest = line.split_once(['.', ')']).map_or("", |(_, r)| r).trim().to_string();
|
||||
out[idx - 1] = rest;
|
||||
current = Some(idx - 1);
|
||||
}
|
||||
Some(_) => return None, // numbered but out of range — malformed reply
|
||||
// A wrapped/multi-line item: keep the continuation, not
|
||||
// "1. Première phrase.\nSuite" dropping "Suite". Unnumbered
|
||||
// preamble is rejected — not our format.
|
||||
None => {
|
||||
let i = current?;
|
||||
out[i].push('\n');
|
||||
out[i].push_str(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
(seen.iter().all(|s| *s) && out.iter().all(|s| !s.is_empty())).then_some(out)
|
||||
}
|
||||
|
||||
/// `Some(idx)` when `line` opens as `"12."` / `"12)"` — a 1-3 digit
|
||||
/// number followed by `'.'` or `')'`. Anything else is `None`, so
|
||||
/// "Section 2. notes" is a continuation, not item 2.
|
||||
fn numbered_line(line: &str) -> Option<usize> {
|
||||
let line = line.trim_start();
|
||||
let digits = line.bytes().take_while(|b| b.is_ascii_digit()).count();
|
||||
if digits == 0 || digits > 3 || !matches!(line.as_bytes().get(digits), Some(b'.' | b')')) {
|
||||
return None;
|
||||
}
|
||||
line.get(..digits)?.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn split_words_ws_groups_runs() {
|
||||
assert_eq!(split_words_ws("a b\tc"), vec!["a", " ", "b", "\t", "c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pdf_grammar_protects() {
|
||||
let g = PdfGrammar;
|
||||
assert!(g.is_protected("https://x.io/a"));
|
||||
assert!(g.is_protected("{count}"));
|
||||
assert!(g.is_protected("42.5"));
|
||||
assert!(!g.is_protected("Bonjour"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segmentation_glues_whitespace_and_protects_spans() {
|
||||
let segs = segment("Visit https://a.io now {n} times", &PdfGrammar);
|
||||
// "Visit " text, protected url, " now " text, protected {n}, " times" text
|
||||
let texts: Vec<&Segment> = segs.iter().filter(|s| matches!(s, Segment::Text(_))).collect();
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "https://a.io")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "{n}")));
|
||||
assert!(texts.iter().any(|s| s.source().contains("Visit")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_roundtrip() {
|
||||
let r = "1. un\n2. deux\n3. trois";
|
||||
assert_eq!(parse_numbered(r, 3), Some(vec!["un".into(), "deux".into(), "trois".into()]));
|
||||
assert_eq!(parse_numbered("1. un\n3. trois", 3), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_rejects_duplicates_and_overflow() {
|
||||
assert_eq!(parse_numbered("1. un\n2. deux\n1. trois", 2), None);
|
||||
assert_eq!(parse_numbered("1. un\n2. deux\n3. trois", 2), None);
|
||||
assert_eq!(parse_numbered("0. zero\n1. un\n2. deux", 2), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_keeps_continuation_lines() {
|
||||
let r = "1. Première phrase.\nSuite du paragraphe\n2. Deuxième élément.";
|
||||
assert_eq!(parse_numbered(r, 2), Some(vec!["Première phrase.\nSuite du paragraphe".into(), "Deuxième élément.".into()]));
|
||||
// Preamble chatter that isn't a numbered item is rejected, not
|
||||
// silently dropped.
|
||||
assert_eq!(parse_numbered("Sure, here you go:\n1. un\n2. deux", 2), None);
|
||||
// "Section 2. notes" looks numbered but isn't → continuation.
|
||||
let r2 = "1. un\nSection 2. notes\n2. deux";
|
||||
assert_eq!(parse_numbered(r2, 2), Some(vec!["un\nSection 2. notes".into(), "deux".into()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markup_tags_split_from_their_text() {
|
||||
let segs = segment("<em>Hello</em> world", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p == "<em>")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.trim() == "</em>")));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("Hello"))));
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("world"))));
|
||||
// Tags containing spaces stay whole and protected.
|
||||
let segs = segment("<a href=\"x\">link</a>", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Protected(p) if p == "<a href=\"x\">")));
|
||||
// A bare '<' is text, not a tag opener — words around it stay
|
||||
// translatable (the lone '<' itself is protected as punctuation).
|
||||
let segs = segment("a < b and <3 hearts", &PdfGrammar);
|
||||
assert!(segs.iter().any(|s| matches!(s, Segment::Text(t) if t.contains("hearts"))));
|
||||
assert!(!segs.iter().any(|s| matches!(s, Segment::Protected(p) if p.contains('<') && p.contains('>'))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ws_edges_and_split_run_preserve_every_byte() {
|
||||
assert_eq!(ws_edges(" héllo \n"), (" ", "héllo", " \n"));
|
||||
assert_eq!(ws_edges(" "), (" ", "", ""));
|
||||
let s = "alpha beta gamma delta epsilon";
|
||||
let pieces = split_run(s, 12);
|
||||
assert!(pieces.iter().all(|p| p.len() <= 12));
|
||||
assert_eq!(pieces.concat(), s); // boundary whitespace survives
|
||||
// A run with no whitespace is hard-cut at a char boundary.
|
||||
let long = "x".repeat(50);
|
||||
let pieces = split_run(&long, 20);
|
||||
assert!(pieces.iter().all(|p| p.len() <= 20));
|
||||
assert_eq!(pieces.concat(), long);
|
||||
}
|
||||
|
||||
/// A provider that echoes numbered replies — exercises the batching,
|
||||
/// whitespace restore and reassembly end to end without a network.
|
||||
struct Echo;
|
||||
|
||||
impl Provider for Echo {
|
||||
fn name(&self) -> String {
|
||||
"echo".into()
|
||||
}
|
||||
|
||||
fn complete(&mut self, prompt: &str, _opts: &GenOpts) -> Result<String, AiError> {
|
||||
if prompt.starts_with("Translate each numbered item") {
|
||||
// Parse our own prompt's numbering and echo each item.
|
||||
let body = prompt.split("\n\n").nth(1).unwrap_or("");
|
||||
let mut out = String::new();
|
||||
for line in body.lines() {
|
||||
out.push_str(line);
|
||||
out.push('\n');
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
Ok(prompt.rsplit("\n\n").next().unwrap_or(prompt).to_string())
|
||||
}
|
||||
|
||||
fn test(&mut self) -> Result<crate::provider::ProviderInfo, AiError> {
|
||||
Ok(crate::provider::ProviderInfo { label: "echo".into(), models: vec![] })
|
||||
}
|
||||
|
||||
fn is_local(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbered_parse_accepts_paren_markers() {
|
||||
// "1) …" replies contain sentence-ending periods inside items —
|
||||
// the leading delimiter must be picked before inspecting text.
|
||||
let r = "1) Première phrase.\nSuite.\n2) Deuxième élément.";
|
||||
assert_eq!(parse_numbered(r, 2), Some(vec!["Première phrase.\nSuite.".into(), "Deuxième élément.".into()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_only_pieces_survive_reconstruction() {
|
||||
let segs = vec![Segment::Text(format!("{}Hello", " ".repeat(RUN_BYTES)))];
|
||||
let mut echo = Echo;
|
||||
let out = translate_segments(&mut echo, &segs, "fr").unwrap_or_default();
|
||||
assert_eq!(out.concat(), format!("{}Hello", " ".repeat(RUN_BYTES)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translate_restores_whitespace_around_protected_spans() {
|
||||
let segs = segment("Visit https://a.io now", &PdfGrammar);
|
||||
let mut echo = Echo;
|
||||
let out = translate_segments(&mut echo, &segs, "fr").unwrap_or_default();
|
||||
assert_eq!(out.concat(), "Visit https://a.io now");
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,9 @@ pub trait AtomicFs {
|
||||
fn sync_dir(&self, dir: &Path) -> io::Result<()>;
|
||||
/// Give the open temporary file its final permissions.
|
||||
fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()>;
|
||||
/// Put the open temporary file in group `gid` (Unix only).
|
||||
#[cfg(unix)]
|
||||
fn set_group(&self, file: &File, gid: u32) -> io::Result<()>;
|
||||
}
|
||||
|
||||
/// The production file system implementation.
|
||||
@@ -127,6 +130,11 @@ impl AtomicFs for RealFs {
|
||||
fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> {
|
||||
file.set_permissions(perms)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn set_group(&self, file: &File, gid: u32) -> io::Result<()> {
|
||||
std::os::unix::fs::fchown(file, None, Some(gid))
|
||||
}
|
||||
}
|
||||
|
||||
/// Flush `file` to stable storage (`sync_all`).
|
||||
@@ -343,10 +351,29 @@ fn split(target: &Path, path: &Path) -> io::Result<(PathBuf, String)> {
|
||||
Ok((dir, name.to_string_lossy().into_owned()))
|
||||
}
|
||||
|
||||
/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions of the file it
|
||||
/// replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when the mode
|
||||
/// already matches after `fill`. A leftover file with the same name is skipped. When any step
|
||||
/// fails, the temporary file is removed.
|
||||
/// Give `perms` the replaced file's group, or drop its group bits when the stage cannot join
|
||||
/// that group. A stage already in the group needs no fchown, which some filesystems reject even
|
||||
/// as a no-op, and dropping the bits there would narrow a mode the replaced file granted.
|
||||
#[cfg(unix)]
|
||||
fn adopt_group(fs: &dyn AtomicFs, file: &File, stage_gid: u32, replaced_gid: u32, perms: std::fs::Permissions) -> io::Result<std::fs::Permissions> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if perms.mode() & 0o070 == 0 || stage_gid == replaced_gid {
|
||||
return Ok(perms);
|
||||
}
|
||||
if fs.set_group(file, replaced_gid).is_ok() {
|
||||
return Ok(perms);
|
||||
}
|
||||
let mut narrowed = perms;
|
||||
narrowed.set_mode(narrowed.mode() & !0o070);
|
||||
Ok(narrowed)
|
||||
}
|
||||
|
||||
/// Create a temporary file in `dir`, fill it with `fill`, give it the permissions and group of
|
||||
/// the file it replaces (`replaced`, Unix) and flush it. Permission restoration is skipped when
|
||||
/// the mode already matches after `fill`; the group bits are dropped when the file cannot be put
|
||||
/// in the replaced file's group, so they never open the file to a group that was not granted
|
||||
/// them. A leftover file with the same name is skipped. When any step fails, the temporary file
|
||||
/// is removed.
|
||||
fn stage(
|
||||
fs: &dyn AtomicFs,
|
||||
dir: &Path,
|
||||
@@ -384,6 +411,14 @@ fn stage(
|
||||
if (file.metadata()?.permissions().mode() & 0o7777) == (p.mode() & 0o7777) {
|
||||
return Ok(());
|
||||
}
|
||||
// The group bits belong to the replaced file's group, which a new
|
||||
// inode need not be in (setgid directory, another primary group).
|
||||
if let Some(replaced) = replaced {
|
||||
let stage_gid = std::os::unix::fs::MetadataExt::gid(&file.metadata()?);
|
||||
let replaced_gid = std::os::unix::fs::MetadataExt::gid(replaced);
|
||||
let perms = adopt_group(fs, &file, stage_gid, replaced_gid, p)?;
|
||||
return fs.set_permissions(&file, perms);
|
||||
}
|
||||
}
|
||||
fs.set_permissions(&file, p)
|
||||
})
|
||||
|
||||
@@ -48,6 +48,9 @@ struct Faulty {
|
||||
/// Simulate a network filesystem exposing the same fixed mode on every new file.
|
||||
#[cfg(unix)]
|
||||
fixed_mode: Option<u32>,
|
||||
/// Simulate a filesystem that refuses the stage's group change.
|
||||
#[cfg(unix)]
|
||||
fail_set_group: bool,
|
||||
}
|
||||
|
||||
impl AtomicFs for Faulty {
|
||||
@@ -92,6 +95,14 @@ impl AtomicFs for Faulty {
|
||||
self.events.borrow_mut().push("sync_dir");
|
||||
RealFs.sync_dir(dir)
|
||||
}
|
||||
#[cfg(unix)]
|
||||
fn set_group(&self, file: &File, gid: u32) -> io::Result<()> {
|
||||
self.events.borrow_mut().push("set_group");
|
||||
if self.fail_set_group {
|
||||
return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected: set_group failed"));
|
||||
}
|
||||
RealFs.set_group(file, gid)
|
||||
}
|
||||
/// Record permission restoration, then inject a configured error or delegate the
|
||||
/// mode change to the real filesystem.
|
||||
fn set_permissions(&self, file: &File, perms: std::fs::Permissions) -> io::Result<()> {
|
||||
@@ -163,6 +174,36 @@ fn unchanged_replacement_modes_need_no_chmod() -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn group_bits_are_dropped_when_the_stage_cannot_join_the_replaced_group() -> io::Result<()> {
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
let d = TempDir::new("group-fallback");
|
||||
let file = File::create(d.0.join("stage"))?;
|
||||
let gid = file.metadata()?.gid();
|
||||
let elsewhere = gid.wrapping_add(1);
|
||||
|
||||
let fs = Faulty { fail_set_group: true, ..Default::default() };
|
||||
let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o640))?.mode();
|
||||
assert_eq!(mode & 0o777, 0o600, "group bits are dropped when the group cannot be joined");
|
||||
assert!(fs.events.borrow().contains(&"set_group"));
|
||||
|
||||
// No group bits: the group change is never attempted.
|
||||
let fs = Faulty { fail_set_group: true, ..Default::default() };
|
||||
let mode = adopt_group(&fs, &file, gid, elsewhere, std::fs::Permissions::from_mode(0o600))?.mode();
|
||||
assert_eq!(mode & 0o777, 0o600);
|
||||
assert!(!fs.events.borrow().contains(&"set_group"));
|
||||
|
||||
// A stage already in the replaced file's group keeps its bits untouched.
|
||||
let fs = Faulty { fail_set_group: true, ..Default::default() };
|
||||
let mode = adopt_group(&fs, &file, gid, gid, std::fs::Permissions::from_mode(0o640))?.mode();
|
||||
assert_eq!(mode & 0o777, 0o640);
|
||||
assert!(!fs.events.borrow().contains(&"set_group"));
|
||||
|
||||
assert_no_temp(&d.0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinguishes_atomicity_and_durability_order() {
|
||||
let d = TempDir::new("durability_order");
|
||||
|
||||
@@ -47,6 +47,10 @@ qoa = ["qoa-decode", "qoa-encode"]
|
||||
qoa-decode = []
|
||||
qoa-encode = []
|
||||
opus-encode = []
|
||||
# GSM 06.10 full-rate speech: decoder and encoder in the `gsm` module, each on its own.
|
||||
gsm = ["gsm-decode", "gsm-encode"]
|
||||
gsm-decode = []
|
||||
gsm-encode = []
|
||||
ac3-decode = ["craft-media/bitstream"]
|
||||
mpa-decode = ["craft-media/bitstream"]
|
||||
# MPEG-1/2 Layer III (MP3) decoder (`mp3::Decoder`).
|
||||
@@ -59,6 +63,8 @@ ac3-encode = ["craft-media/bitstream"]
|
||||
eac3-encode = ["craft-media/bitstream"]
|
||||
# MPEG-1/2 Layer III (MP3) encoder (`mp3`).
|
||||
mp3-encode = ["craft-media/bitstream"]
|
||||
# Bluetooth Low Complexity Subband Coding (SBC) audio codec (`sbc`).
|
||||
sbc = []
|
||||
# Multi-resolution waveform overviews (`peaks`).
|
||||
peaks = []
|
||||
# Encode AAC channels in parallel (with `aac-encode`).
|
||||
|
||||
@@ -44,6 +44,7 @@ No default features. The shared types (`AudioBuffer`, `AudioInfo`, `ChannelLayou
|
||||
| `mpa-decode` | `mpa::Decoder` | craft-media `bitstream` |
|
||||
| `mp3-decode` | `mp3::Decoder`, `mp3::InfoTag` | `mpa-decode` |
|
||||
| `mp3-encode` | `mp3::Encoder`; `encode`, `FileEncoder` for MP3 | craft-media `bitstream` |
|
||||
| `gsm-decode`, `gsm-encode` (`gsm`) | `gsm::Decoder`, `gsm::Encoder`, `gsm::decode`, `gsm::encode`: GSM 06.10 full-rate speech, bit-exact, raw `.gsm` and WAV (GSM 6.10) packings | none |
|
||||
| `peaks` | `peaks::Peaks` | none |
|
||||
| `rayon` | parallel AAC encoding (with `aac-encode`) | `rayon` |
|
||||
| `serde` | `Serialize`/`Deserialize` on the shared types | `serde` |
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+10
-1
@@ -30,7 +30,8 @@
|
||||
//! `aac-encode`), [`alac`] (`alac-decode`, `alac-encode`), [`opus`] (`opus-decode`, and
|
||||
//! [`opus::encoder`] with `opus-encode`), [`ac3`] (`ac3-decode`), [`mpa`] (`mpa-decode`), [`mp3`]
|
||||
//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-decode`: the frame and file decoder;
|
||||
//! `flac-encode`: the packet encoder), [`qoa`] (`qoa-decode`, `qoa-encode`: whole QOA files).
|
||||
//! `flac-encode`: the packet encoder), [`qoa`] (`qoa-decode`, `qoa-encode`: whole QOA files),
|
||||
//! [`gsm`] (`gsm-decode`, `gsm-encode`: GSM 06.10 full-rate speech).
|
||||
//!
|
||||
//! The shared types below build with no features.
|
||||
|
||||
@@ -41,6 +42,8 @@ pub mod ac3;
|
||||
#[cfg(any(feature = "alac-decode", feature = "alac-encode"))]
|
||||
pub mod alac;
|
||||
mod buffer;
|
||||
#[cfg(any(feature = "gsm-decode", feature = "gsm-encode"))]
|
||||
pub mod gsm;
|
||||
mod layout;
|
||||
#[cfg(any(feature = "mp3-encode", feature = "mp3-decode"))]
|
||||
pub mod mp3;
|
||||
@@ -56,6 +59,8 @@ pub mod peaks;
|
||||
pub mod qoa;
|
||||
#[cfg(feature = "decode-native")]
|
||||
mod read;
|
||||
#[cfg(feature = "sbc")]
|
||||
pub mod sbc;
|
||||
#[cfg(any(feature = "aac-decode", feature = "alac-decode", feature = "opus-decode", feature = "ac3-decode", feature = "mpa-decode"))]
|
||||
mod track;
|
||||
#[cfg(feature = "vorbis-encode")]
|
||||
@@ -782,3 +787,7 @@ mod tests {
|
||||
|
||||
/// DVD and Blu-ray LPCM packet decoding.
|
||||
pub mod lpcm;
|
||||
|
||||
/// Non-seeking canonical WAV streaming primitives.
|
||||
#[cfg(feature = "pcm-encode")]
|
||||
pub mod wav_stream;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,119 @@
|
||||
//! Canonical RIFF/WAVE header and interleaved sample blocks for non-seekable sinks.
|
||||
//!
|
||||
//! Extracted from AniCraft's WAV writer, preserving its byte-level compatibility.
|
||||
//! Unlike `FileEncoder`, these primitives do not require `Seek` and leave final lengths,
|
||||
//! chunk padding and sink error handling to the caller. The caller must supply a complete
|
||||
//! frame count and append exactly that many frames. For odd-length data, append a pad byte
|
||||
//! (the header includes that padding in the RIFF size); current AniCraft output is always stereo (even).
|
||||
//! Non-finite samples become silence, including float output. This deliberately differs
|
||||
//! from the general encoder's infinity handling. Prefer `FileEncoder` for new seekable sinks.
|
||||
|
||||
/// Sample formats written.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Bits {
|
||||
/// Signed 16-bit PCM.
|
||||
Int16,
|
||||
/// Signed 24-bit PCM.
|
||||
Int24,
|
||||
/// IEEE 754 32-bit float.
|
||||
Float32,
|
||||
}
|
||||
|
||||
impl Bits {
|
||||
fn bytes(self) -> u16 {
|
||||
match self {
|
||||
Bits::Int16 => 2,
|
||||
Bits::Int24 => 3,
|
||||
Bits::Float32 => 4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const PCM: u16 = 1;
|
||||
const FLOAT: u16 = 3;
|
||||
|
||||
/// A WAV header for `frames` frames of `channels` channels.
|
||||
/// Returns `None` for zero rate/channels or any size/byte-rate field overflow.
|
||||
pub fn header(rate: u32, channels: u16, bits: Bits, frames: u64) -> Option<Vec<u8>> {
|
||||
if rate == 0 || channels == 0 {
|
||||
return None;
|
||||
}
|
||||
let b = bits.bytes();
|
||||
let data = u32::try_from(frames.checked_mul(u64::from(b) * u64::from(channels))?).ok()?;
|
||||
let riff = data.checked_add(36)?.checked_add(data & 1)?;
|
||||
let block = channels.checked_mul(b)?;
|
||||
let byte_rate = rate.checked_mul(u32::from(block))?;
|
||||
let tag = if bits == Bits::Float32 { FLOAT } else { PCM };
|
||||
let mut h = Vec::with_capacity(44);
|
||||
h.extend_from_slice(b"RIFF");
|
||||
h.extend_from_slice(&riff.to_le_bytes());
|
||||
h.extend_from_slice(b"WAVEfmt ");
|
||||
h.extend_from_slice(&16u32.to_le_bytes());
|
||||
h.extend_from_slice(&tag.to_le_bytes());
|
||||
h.extend_from_slice(&channels.to_le_bytes());
|
||||
h.extend_from_slice(&rate.to_le_bytes());
|
||||
h.extend_from_slice(&byte_rate.to_le_bytes());
|
||||
h.extend_from_slice(&block.to_le_bytes());
|
||||
h.extend_from_slice(&(b * 8).to_le_bytes());
|
||||
h.extend_from_slice(b"data");
|
||||
h.extend_from_slice(&data.to_le_bytes());
|
||||
Some(h)
|
||||
}
|
||||
|
||||
/// Appends samples in `bits` format. Integers are rounded and clipped to range; `x · 32768`
|
||||
/// for 16-bit, so 16-bit sources read by [`crate::decode`] come back bit-identical.
|
||||
pub fn append(out: &mut Vec<u8>, samples: &[f32], bits: Bits) {
|
||||
out.reserve(samples.len() * usize::from(bits.bytes()));
|
||||
for &s in samples {
|
||||
let s = if s.is_finite() { s } else { 0.0 };
|
||||
match bits {
|
||||
Bits::Int16 => out.extend_from_slice(&((s * 32_768.0).round().clamp(-32_768.0, 32_767.0) as i16).to_le_bytes()),
|
||||
Bits::Int24 => {
|
||||
let v = (f64::from(s) * 8_388_608.0).round().clamp(-8_388_608.0, 8_388_607.0) as i32;
|
||||
out.extend_from_slice(v.to_le_bytes().get(..3).unwrap_or(&[0, 0, 0]));
|
||||
}
|
||||
Bits::Float32 => out.extend_from_slice(&s.to_le_bytes()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used, clippy::indexing_slicing)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn canonical_header_and_odd_padding() {
|
||||
let h = header(48_000, 2, Bits::Int16, 1).unwrap();
|
||||
assert_eq!(h.len(), 44);
|
||||
assert_eq!(&h[..12], b"RIFF\x28\0\0\0WAVE");
|
||||
assert_eq!(&h[28..36], &[0, 238, 2, 0, 4, 0, 16, 0]);
|
||||
let h = header(8_000, 1, Bits::Int24, 1).unwrap();
|
||||
assert_eq!(&h[4..8], &40u32.to_le_bytes());
|
||||
assert_eq!(&h[40..44], &3u32.to_le_bytes());
|
||||
assert!(header(0, 2, Bits::Int16, 1).is_none());
|
||||
assert!(header(48_000, 0, Bits::Int16, 1).is_none());
|
||||
assert!(header(u32::MAX, 2, Bits::Int16, 1).is_none());
|
||||
assert!(header(48_000, 2, Bits::Float32, u64::MAX).is_none());
|
||||
assert!(header(48_000, u16::MAX, Bits::Float32, 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn samples_preserve_rounding_clipping_silence_and_chunk_boundaries() {
|
||||
let samples = [-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0, f32::NAN, f32::INFINITY, f32::NEG_INFINITY];
|
||||
let mut pcm = Vec::new();
|
||||
append(&mut pcm, &samples, Bits::Int16);
|
||||
let expected = [-32768i16, -32768, -16384, 0, 16384, 32767, 32767, 0, 0, 0];
|
||||
assert_eq!(pcm, expected.iter().flat_map(|s| s.to_le_bytes()).collect::<Vec<_>>());
|
||||
for bits in [Bits::Int16, Bits::Int24, Bits::Float32] {
|
||||
let mut whole = Vec::new();
|
||||
append(&mut whole, &samples, bits);
|
||||
let mut blocks = Vec::new();
|
||||
for block in samples.chunks(3) {
|
||||
append(&mut blocks, block, bits);
|
||||
}
|
||||
assert_eq!(whole, blocks);
|
||||
assert_eq!(&whole[whole.len() - 3 * usize::from(bits.bytes())..], vec![0; 3 * usize::from(bits.bytes())]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! GSM 06.10 against ffmpeg's decoder as the reference (oracle; skipped when ffmpeg is absent, a
|
||||
//! failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`): ffmpeg decodes what our encoder writes, in both
|
||||
//! packings, to exactly the samples our decoder returns, and so does arbitrary frame data.
|
||||
|
||||
use craft_audio::gsm::{Packing, decode, encode};
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn xorshift(seed: &mut u32) -> u32 {
|
||||
*seed ^= *seed << 13;
|
||||
*seed ^= *seed >> 17;
|
||||
*seed ^= *seed << 5;
|
||||
*seed
|
||||
}
|
||||
|
||||
fn signal(frames: usize) -> Vec<i16> {
|
||||
let mut seed = 0x9E37_79B9u32;
|
||||
(0..frames * 160)
|
||||
.map(|i| {
|
||||
let t = i as f64 / 8000.0;
|
||||
let f0 = 110.0 + 40.0 * (t * 2.0).sin();
|
||||
let voiced = (t * f0 * std::f64::consts::TAU).sin() * 7000.0 + (t * 4.0 * f0 * std::f64::consts::TAU).sin() * 2000.0;
|
||||
let noise = f64::from(xorshift(&mut seed) % 3000) - 1500.0;
|
||||
// Voiced stretches, noise bursts, silence and full-scale clipping.
|
||||
match (i / 1600) % 5 {
|
||||
0 | 1 => voiced as i16,
|
||||
2 => noise as i16,
|
||||
3 => 0,
|
||||
_ => (voiced * 6.0).clamp(-32768.0, 32767.0) as i16,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn wav49_file(blocks: &[u8], samples: usize) -> Vec<u8> {
|
||||
let mut fmt = Vec::new();
|
||||
fmt.extend_from_slice(&0x31u16.to_le_bytes()); // GSM 6.10
|
||||
fmt.extend_from_slice(&1u16.to_le_bytes());
|
||||
fmt.extend_from_slice(&8000u32.to_le_bytes());
|
||||
fmt.extend_from_slice(&1625u32.to_le_bytes());
|
||||
fmt.extend_from_slice(&65u16.to_le_bytes());
|
||||
fmt.extend_from_slice(&0u16.to_le_bytes());
|
||||
fmt.extend_from_slice(&2u16.to_le_bytes());
|
||||
fmt.extend_from_slice(&320u16.to_le_bytes());
|
||||
let mut body = b"WAVE".to_vec();
|
||||
body.extend_from_slice(b"fmt ");
|
||||
body.extend_from_slice(&(fmt.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(&fmt);
|
||||
body.extend_from_slice(b"fact");
|
||||
body.extend_from_slice(&4u32.to_le_bytes());
|
||||
body.extend_from_slice(&(samples as u32).to_le_bytes());
|
||||
body.extend_from_slice(b"data");
|
||||
body.extend_from_slice(&(blocks.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(blocks);
|
||||
let mut out = b"RIFF".to_vec();
|
||||
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&body);
|
||||
out
|
||||
}
|
||||
|
||||
fn ffmpeg_decode(ffmpeg: &Path, file: &Path, forced_format: Option<&str>) -> Vec<i16> {
|
||||
let mut cmd = Command::new(ffmpeg);
|
||||
cmd.args(["-v", "error", "-y"]);
|
||||
if let Some(f) = forced_format {
|
||||
cmd.args(["-f", f]);
|
||||
}
|
||||
let out = cmd.arg("-i").arg(file).args(["-f", "s16le", "-acodec", "pcm_s16le", "-"]).output().expect("run ffmpeg");
|
||||
assert!(out.status.success(), "ffmpeg failed on {}: {}", file.display(), String::from_utf8_lossy(&out.stderr));
|
||||
out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect()
|
||||
}
|
||||
|
||||
/// Frames whose long-term and RPE fields are arbitrary, over the log-area ratios of real speech.
|
||||
/// Random log-area ratios give filters that overflow at every sample, where the specification's
|
||||
/// saturating arithmetic and other decoders' differ; here every field of every sub-frame still
|
||||
/// takes all of its values, with the amplitude and gain kept where the signal stays in range.
|
||||
fn random_frames(n: usize, seed: &mut u32) -> Vec<u8> {
|
||||
let speech = encode(&signal(n), Packing::Raw);
|
||||
let mut out = Vec::new();
|
||||
for b in speech.as_chunks::<33>().0 {
|
||||
let mut frame = craft_audio::gsm::Frame::from_raw(b).expect("frame");
|
||||
for s in &mut frame.sub {
|
||||
s.lag = 40 + (xorshift(seed) % 81) as u8;
|
||||
s.gain = (xorshift(seed) % 3) as u8;
|
||||
s.grid = (xorshift(seed) & 3) as u8;
|
||||
s.xmax = (xorshift(seed) % 26) as u8;
|
||||
for p in &mut s.pulses {
|
||||
*p = (xorshift(seed) & 7) as u8;
|
||||
}
|
||||
}
|
||||
out.extend_from_slice(&frame.to_raw());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ffmpeg_decodes_our_raw_streams_to_the_same_samples() {
|
||||
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
|
||||
let dir = craft_media_testkit::fixture_subdir("gsm");
|
||||
for frames in [1usize, 2, 7, 50, 120] {
|
||||
let bytes = encode(&signal(frames), Packing::Raw);
|
||||
let file = dir.join(format!("ours_{frames}.gsm"));
|
||||
std::fs::write(&file, &bytes).expect("write fixture");
|
||||
let theirs = ffmpeg_decode(&ffmpeg, &file, Some("gsm"));
|
||||
let ours = decode(&bytes, Packing::Raw).expect("decode");
|
||||
assert_eq!(theirs.len(), ours.len(), "{frames} frames: sample count");
|
||||
assert!(theirs == ours, "{frames} frames: samples differ");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ffmpeg_decodes_our_wav49_streams_to_the_same_samples() {
|
||||
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
|
||||
let dir = craft_media_testkit::fixture_subdir("gsm");
|
||||
for frames in [2usize, 4, 20, 100] {
|
||||
let pcm = signal(frames);
|
||||
let bytes = encode(&pcm, Packing::Wav49);
|
||||
let file = dir.join(format!("ours_{frames}.wav"));
|
||||
std::fs::write(&file, wav49_file(&bytes, pcm.len())).expect("write fixture");
|
||||
let theirs = ffmpeg_decode(&ffmpeg, &file, None);
|
||||
let ours = decode(&bytes, Packing::Wav49).expect("decode");
|
||||
assert_eq!(theirs.len(), ours.len(), "{frames} frames: sample count");
|
||||
assert!(theirs == ours, "{frames} frames: samples differ");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ffmpeg_and_we_decode_arbitrary_frames_identically() {
|
||||
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
|
||||
let dir = craft_media_testkit::fixture_subdir("gsm");
|
||||
let mut seed = 0xC0FF_EE11u32;
|
||||
let bytes = random_frames(300, &mut seed);
|
||||
let file = dir.join("random.gsm");
|
||||
std::fs::write(&file, &bytes).expect("write fixture");
|
||||
let theirs = ffmpeg_decode(&ffmpeg, &file, Some("gsm"));
|
||||
let ours = decode(&bytes, Packing::Raw).expect("decode");
|
||||
assert_eq!(theirs.len(), ours.len());
|
||||
assert!(theirs == ours, "arbitrary frames decode differently");
|
||||
}
|
||||
@@ -9,6 +9,12 @@ mod codecs;
|
||||
#[cfg(all(feature = "flac-decode", feature = "flac-encode"))]
|
||||
#[path = "flac_decode.rs"]
|
||||
mod flac_decode;
|
||||
#[cfg(feature = "gsm")]
|
||||
#[path = "gsm_oracle.rs"]
|
||||
mod gsm_oracle;
|
||||
#[cfg(feature = "qoa")]
|
||||
#[path = "qoa_oracle.rs"]
|
||||
mod qoa_oracle;
|
||||
#[cfg(feature = "sbc")]
|
||||
#[path = "sbc_oracle.rs"]
|
||||
mod sbc_oracle;
|
||||
@@ -0,0 +1,298 @@
|
||||
#![cfg(feature = "sbc")]
|
||||
|
||||
use craft_audio::sbc::{
|
||||
AllocationMethod, BlockCount, ChannelMode, Decoder, Encoder, Header, SYNCWORD_MSBC, SYNCWORD_SBC, SamplingFrequency, Subbands, decode, encode,
|
||||
};
|
||||
use std::process::Command;
|
||||
|
||||
fn pearson_correlation(x: &[i16], y: &[i16], lag: usize) -> f64 {
|
||||
let n = x.len().min(y.len().saturating_sub(lag));
|
||||
if n == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let x = &x[..n];
|
||||
let y = &y[lag..lag + n];
|
||||
let mean_x = x.iter().map(|&v| v as f64).sum::<f64>() / (n as f64);
|
||||
let mean_y = y.iter().map(|&v| v as f64).sum::<f64>() / (n as f64);
|
||||
let mut num = 0.0;
|
||||
let mut denom_x = 0.0;
|
||||
let mut denom_y = 0.0;
|
||||
for (&a, &b) in x.iter().zip(y) {
|
||||
let da = (a as f64) - mean_x;
|
||||
let db = (b as f64) - mean_y;
|
||||
num += da * db;
|
||||
denom_x += da * da;
|
||||
denom_y += db * db;
|
||||
}
|
||||
let denom = (denom_x * denom_y).sqrt();
|
||||
if denom > 0.0 { num / denom } else { 0.0 }
|
||||
}
|
||||
|
||||
fn generate_sine(sample_rate: u32, channels: usize, num_frames: usize) -> Vec<i16> {
|
||||
let mut pcm = Vec::with_capacity(num_frames * channels);
|
||||
for i in 0..num_frames {
|
||||
let t = (i as f64) / (sample_rate as f64);
|
||||
let s_l = (16000.0 * (2.0 * std::f64::consts::PI * 440.0 * t).sin()) as i16;
|
||||
pcm.push(s_l);
|
||||
if channels == 2 {
|
||||
let s_r = (16000.0 * (2.0 * std::f64::consts::PI * 880.0 * t).sin()) as i16;
|
||||
pcm.push(s_r);
|
||||
}
|
||||
}
|
||||
pcm
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_header_parse_and_frame_length() {
|
||||
// Standard SBC header: 0x9C, 44.1kHz, 16 blocks, Joint Stereo, Loudness, 8 subbands, bitpool 32
|
||||
let mut packet = vec![SYNCWORD_SBC, 0xBD, 32, 0x00];
|
||||
let header = Header::parse(&packet).expect("parse header");
|
||||
assert_eq!(header.sampling_frequency, SamplingFrequency::Freq44k1);
|
||||
assert_eq!(header.blocks, BlockCount::Blocks16);
|
||||
assert_eq!(header.channel_mode, ChannelMode::JointStereo);
|
||||
assert_eq!(header.allocation_method, AllocationMethod::Loudness);
|
||||
assert_eq!(header.subbands, Subbands::Subbands8);
|
||||
assert_eq!(header.bitpool, 32);
|
||||
assert_eq!(header.channels(), 2);
|
||||
assert_eq!(header.samples_per_frame(), 128);
|
||||
|
||||
// Frame length formula check
|
||||
let expected_len = 4 + (4 * 8 * 2) / 8 + (8usize + 16 * 32).div_ceil(8);
|
||||
assert_eq!(header.frame_length(), expected_len);
|
||||
assert_eq!(header.frame_length(), 77);
|
||||
|
||||
// mSBC header: 0xAD
|
||||
packet[0] = SYNCWORD_MSBC;
|
||||
let msbc_header = Header::parse(&packet).expect("parse msbc header");
|
||||
assert_eq!(msbc_header.frame_length(), 57);
|
||||
assert_eq!(msbc_header.sample_rate(), 16000);
|
||||
assert_eq!(msbc_header.channels(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_silence_roundtrip() {
|
||||
for &sample_rate in &[16_000u32, 32_000, 44_100, 48_000] {
|
||||
for &channels in &[1usize, 2] {
|
||||
for &subbands in &[4usize, 8] {
|
||||
for &blocks in &[4usize, 8, 12, 16] {
|
||||
let frame_samples = blocks * subbands * channels;
|
||||
let pcm = vec![0i16; frame_samples * 4];
|
||||
let mut encoder = Encoder::new(sample_rate, channels, blocks, subbands, 32, AllocationMethod::Loudness).expect("create encoder");
|
||||
|
||||
let mut sbc_bytes = Vec::new();
|
||||
encoder.encode(&pcm, &mut sbc_bytes).expect("encode silence");
|
||||
assert!(!sbc_bytes.is_empty());
|
||||
|
||||
let mut decoder = Decoder::new();
|
||||
let mut decoded = Vec::new();
|
||||
decoder.decode(&sbc_bytes, &mut decoded).expect("decode silence");
|
||||
|
||||
assert_eq!(decoded.len(), pcm.len());
|
||||
for &s in &decoded {
|
||||
assert!(s.abs() <= 16, "silence sample too large: {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sine_wave_roundtrip_fidelity() {
|
||||
let sample_rate = 44_100u32;
|
||||
let channels = 2usize;
|
||||
let num_frames = 128 * 20;
|
||||
let pcm = generate_sine(sample_rate, channels, num_frames);
|
||||
|
||||
let sbc_bytes = encode(&pcm, sample_rate, channels, 32).expect("encode sine");
|
||||
let (decoded, info) = decode(&sbc_bytes).expect("decode sine");
|
||||
|
||||
assert_eq!(info.sample_rate, sample_rate);
|
||||
assert_eq!(info.channels, channels);
|
||||
assert_eq!(decoded.len(), pcm.len());
|
||||
|
||||
// 8-subband filterbank total delay is 73 samples per channel
|
||||
let corr_l = pearson_correlation(&pcm[0..], &decoded[0..], 73 * 2);
|
||||
assert!(corr_l > 0.98, "left channel correlation too low: {corr_l}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_and_corrupt_inputs() {
|
||||
let mut decoder = Decoder::new();
|
||||
let mut dst = Vec::new();
|
||||
|
||||
// Empty packet
|
||||
assert!(decoder.decode_frame(&[], &mut dst).is_err());
|
||||
|
||||
// Too short header
|
||||
assert!(decoder.decode_frame(&[SYNCWORD_SBC, 0x00], &mut dst).is_err());
|
||||
|
||||
// Invalid syncword
|
||||
assert!(decoder.decode_frame(&[0x00, 0xBD, 32, 0x00], &mut dst).is_err());
|
||||
|
||||
// CRC mismatch
|
||||
let bad_crc_frame = vec![SYNCWORD_SBC, 0xBD, 32, 0x55, 0x00, 0x00, 0x00, 0x00];
|
||||
assert!(decoder.decode_frame(&bad_crc_frame, &mut dst).is_err());
|
||||
|
||||
// Bitpool capacity exceeded (Mono 4 subbands capacity is 64; 65 must fail)
|
||||
let bad_bitpool_frame = vec![SYNCWORD_SBC, 0x00, 65, 0x00];
|
||||
assert!(decoder.decode_frame(&bad_bitpool_frame, &mut dst).is_err());
|
||||
assert!(Encoder::new(16000, 1, 4, 4, 65, AllocationMethod::Loudness).is_err());
|
||||
|
||||
// Unsupported 15-block configuration (only 16kHz mono 8-subband 26-bitpool is valid mSBC)
|
||||
assert!(Encoder::new(44100, 2, 15, 8, 32, AllocationMethod::Loudness).is_err());
|
||||
|
||||
// Incomplete PCM frames rejected by encoder
|
||||
let mut enc = Encoder::new(44100, 2, 16, 8, 32, AllocationMethod::Loudness).expect("enc");
|
||||
let mut out = Vec::new();
|
||||
assert!(enc.encode(&[0i16; 100], &mut out).is_err());
|
||||
|
||||
// Valid mSBC encoding produces exact 57-byte mSBC frame
|
||||
let mut msbc_enc = Encoder::new(16000, 1, 15, 8, 26, AllocationMethod::Loudness).expect("msbc enc");
|
||||
let mut msbc_out = Vec::new();
|
||||
msbc_enc.encode(&vec![0i16; 120], &mut msbc_out).expect("encode msbc frame");
|
||||
assert_eq!(msbc_out.len(), 57);
|
||||
assert_eq!(msbc_out[0], SYNCWORD_MSBC);
|
||||
|
||||
// Hostile fuzzed input should never panic
|
||||
let mut seed = 0x1234_5678u32;
|
||||
for len in [10, 50, 100, 500] {
|
||||
let mut random_bytes = vec![0u8; len];
|
||||
for b in &mut random_bytes {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
*b = seed as u8;
|
||||
}
|
||||
let _ = decoder.decode(&random_bytes, &mut dst);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_oracle_ffmpeg_cross_verification() {
|
||||
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else {
|
||||
return;
|
||||
};
|
||||
let dir = craft_media_testkit::fixture_subdir("sbc");
|
||||
|
||||
// 1. FFmpeg decodes our encoded SBC files
|
||||
for &(sample_rate, channels) in &[(16_000u32, 1usize), (44_100, 2), (48_000, 2)] {
|
||||
let pcm = generate_sine(sample_rate, channels, 128 * 20);
|
||||
let sbc_bytes = encode(&pcm, sample_rate, channels, 32).expect("encode sbc");
|
||||
|
||||
let sbc_file = dir.join(format!("ours_{sample_rate}_{channels}.sbc"));
|
||||
std::fs::write(&sbc_file, &sbc_bytes).expect("write sbc fixture");
|
||||
|
||||
let out = Command::new(&ffmpeg)
|
||||
.args(["-v", "error", "-y", "-i"])
|
||||
.arg(&sbc_file)
|
||||
.args(["-f", "s16le", "-acodec", "pcm_s16le", "-"])
|
||||
.output()
|
||||
.expect("run ffmpeg");
|
||||
|
||||
assert!(out.status.success(), "ffmpeg failed to decode our SBC: {}", String::from_utf8_lossy(&out.stderr));
|
||||
|
||||
let theirs: Vec<i16> = out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect();
|
||||
|
||||
assert!(!theirs.is_empty());
|
||||
let corr = pearson_correlation(&pcm, &theirs, 73 * channels);
|
||||
assert!(corr > 0.95, "FFmpeg decoded output correlation against input too low ({sample_rate} Hz, {channels} ch): {corr}");
|
||||
}
|
||||
|
||||
// 2. Our decoder decodes FFmpeg-encoded SBC files
|
||||
for &(sample_rate, channels) in &[(16_000u32, 1usize), (44_100, 2), (48_000, 2)] {
|
||||
let ff_sbc_file = dir.join(format!("ff_{sample_rate}_{channels}.sbc"));
|
||||
let ff_enc = Command::new(&ffmpeg)
|
||||
.args([
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
&format!("sine=frequency=440:duration=0.2:sample_rate={sample_rate}"),
|
||||
"-c:a",
|
||||
"sbc",
|
||||
"-ac",
|
||||
&channels.to_string(),
|
||||
])
|
||||
.arg(&ff_sbc_file)
|
||||
.output()
|
||||
.expect("ffmpeg encode sbc");
|
||||
|
||||
assert!(ff_enc.status.success(), "ffmpeg failed to encode SBC: {}", String::from_utf8_lossy(&ff_enc.stderr));
|
||||
|
||||
let ff_sbc_bytes = std::fs::read(&ff_sbc_file).expect("read ff sbc fixture");
|
||||
let (decoded, info) = decode(&ff_sbc_bytes).expect("decode ff sbc");
|
||||
assert_eq!(info.sample_rate, sample_rate);
|
||||
assert_eq!(info.channels, channels);
|
||||
assert!(!decoded.is_empty());
|
||||
|
||||
// Decode ff_sbc_file with ffmpeg as well to compare
|
||||
let ff_raw = Command::new(&ffmpeg)
|
||||
.args(["-v", "error", "-y", "-i"])
|
||||
.arg(&ff_sbc_file)
|
||||
.args(["-f", "s16le", "-acodec", "pcm_s16le", "-"])
|
||||
.output()
|
||||
.expect("ffmpeg decode raw");
|
||||
|
||||
let ff_pcm: Vec<i16> = ff_raw.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect();
|
||||
|
||||
let corr = pearson_correlation(&decoded, &ff_pcm, 0);
|
||||
assert!(corr > 0.99, "Our decoder output correlation against FFmpeg decoder output too low: {corr}");
|
||||
}
|
||||
|
||||
// 3. Our decoder decodes FFmpeg-encoded mSBC wideband speech
|
||||
let msbc_file = dir.join("ff_msbc.sbc");
|
||||
let ff_msbc_enc = Command::new(&ffmpeg)
|
||||
.args(["-v", "error", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=0.2:sample_rate=16000", "-c:a", "sbc", "-msbc", "1"])
|
||||
.arg(&msbc_file)
|
||||
.output()
|
||||
.expect("ffmpeg encode msbc");
|
||||
|
||||
assert!(ff_msbc_enc.status.success(), "ffmpeg failed to encode mSBC: {}", String::from_utf8_lossy(&ff_msbc_enc.stderr));
|
||||
|
||||
let msbc_bytes = std::fs::read(&msbc_file).expect("read msbc fixture");
|
||||
let (decoded, info) = decode(&msbc_bytes).expect("decode msbc");
|
||||
assert_eq!(info.sample_rate, 16000);
|
||||
assert_eq!(info.channels, 1);
|
||||
assert_eq!(info.blocks, 15);
|
||||
assert_eq!(info.subbands, 8);
|
||||
assert!(!decoded.is_empty());
|
||||
|
||||
let ff_msbc_raw = Command::new(&ffmpeg)
|
||||
.args(["-v", "error", "-y", "-i"])
|
||||
.arg(&msbc_file)
|
||||
.args(["-f", "s16le", "-acodec", "pcm_s16le", "-"])
|
||||
.output()
|
||||
.expect("ffmpeg decode msbc raw");
|
||||
|
||||
let ff_msbc_pcm: Vec<i16> = ff_msbc_raw.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect();
|
||||
|
||||
let corr = pearson_correlation(&decoded, &ff_msbc_pcm, 0);
|
||||
assert!(corr > 0.99, "mSBC decoded output correlation against FFmpeg decoder too low: {corr}");
|
||||
|
||||
// 4. FFmpeg decodes our encoded mSBC file
|
||||
let msbc_sine = generate_sine(16000, 1, 120 * 20);
|
||||
let mut our_msbc_enc = Encoder::new(16000, 1, 15, 8, 26, AllocationMethod::Loudness).expect("msbc enc");
|
||||
let mut our_msbc_bytes = Vec::new();
|
||||
our_msbc_enc.encode(&msbc_sine, &mut our_msbc_bytes).expect("encode msbc");
|
||||
|
||||
let our_msbc_file = dir.join("ours_msbc.sbc");
|
||||
std::fs::write(&our_msbc_file, &our_msbc_bytes).expect("write our msbc fixture");
|
||||
|
||||
let our_msbc_out = Command::new(&ffmpeg)
|
||||
.args(["-v", "error", "-y", "-i"])
|
||||
.arg(&our_msbc_file)
|
||||
.args(["-f", "s16le", "-acodec", "pcm_s16le", "-"])
|
||||
.output()
|
||||
.expect("ffmpeg decode our msbc");
|
||||
|
||||
assert!(our_msbc_out.status.success(), "ffmpeg failed to decode our mSBC: {}", String::from_utf8_lossy(&our_msbc_out.stderr));
|
||||
|
||||
let our_msbc_theirs: Vec<i16> = our_msbc_out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect();
|
||||
|
||||
assert!(!our_msbc_theirs.is_empty());
|
||||
let corr_msbc = pearson_correlation(&msbc_sine, &our_msbc_theirs, 73);
|
||||
assert!(corr_msbc > 0.95, "FFmpeg decoded output correlation against our mSBC input too low: {corr_msbc}");
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
//! JACOsub (`.jss`, `.js`): multi-feature scriptable subtitle format.
|
||||
//!
|
||||
//! ```text
|
||||
//! # JACOsub file
|
||||
//! #TimeRes 100
|
||||
//! 0:00:01.00 0:00:03.00 D0 First line\nsecond line
|
||||
//! ```
|
||||
//!
|
||||
//! Lines beginning with `#` are comments or header directives. Timed caption lines begin with
|
||||
//! `START STOP [DIRECTIVE] Text`, where timestamps are `H:MM:SS.FF` (or `@units`) relative to
|
||||
//! the declared `#TimeRes` (default 30). Inside the text, `\n` represents a newline.
|
||||
|
||||
use crate::text::{escape_markup, ms_to_tick, plain_text, tick_to_ms};
|
||||
use crate::{Cue, Document, Error, Result};
|
||||
|
||||
/// Parse a JACOsub timestamp `H:MM:SS.FF` or `@units` into milliseconds.
|
||||
pub(crate) fn parse_time(s: &str, time_res: i64) -> Option<i64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(units_str) = s.strip_prefix('@') {
|
||||
let units: i64 = units_str.trim().parse().ok()?;
|
||||
let time_res = if time_res > 0 { time_res } else { 30 };
|
||||
return Some(units.saturating_mul(1000) / time_res);
|
||||
}
|
||||
|
||||
let (h_str, rest) = s.split_once(':')?;
|
||||
let (m_str, rest) = rest.split_once(':')?;
|
||||
let (s_str, f_str) = rest.split_once('.')?;
|
||||
|
||||
let h: i64 = h_str.trim().parse().ok()?;
|
||||
let m: i64 = m_str.trim().parse().ok()?;
|
||||
let sec: i64 = s_str.trim().parse().ok()?;
|
||||
let frac: i64 = f_str.trim().parse().ok()?;
|
||||
|
||||
let time_res = if time_res > 0 { time_res } else { 30 };
|
||||
let frac_ms = frac.saturating_mul(1000) / time_res;
|
||||
|
||||
let total_secs = h.saturating_mul(3600).saturating_add(m.saturating_mul(60)).saturating_add(sec);
|
||||
Some(total_secs.saturating_mul(1000).saturating_add(frac_ms))
|
||||
}
|
||||
|
||||
/// Format milliseconds into JACOsub timestamp string `H:MM:SS.cc` (at 100 centiseconds/sec).
|
||||
pub(crate) fn format_time(ms: i64) -> String {
|
||||
let ms = ms.max(0);
|
||||
let total_secs = ms / 1000;
|
||||
let centis = (ms % 1000) / 10;
|
||||
let hours = total_secs / 3600;
|
||||
let mins = (total_secs % 3600) / 60;
|
||||
let secs = total_secs % 60;
|
||||
|
||||
format!("{hours}:{mins:02}:{secs:02}.{centis:02}")
|
||||
}
|
||||
|
||||
/// Check if a line is a JACOsub timed caption line: `H:MM:SS.FF H:MM:SS.FF ...` or `@units @units ...`.
|
||||
pub(crate) fn is_timed_line(line: &str) -> bool {
|
||||
let line = line.trim();
|
||||
if line.starts_with('#') {
|
||||
return false;
|
||||
}
|
||||
let mut parts = line.split_whitespace();
|
||||
let Some(t0) = parts.next() else { return false };
|
||||
let Some(t1) = parts.next() else { return false };
|
||||
(t0.contains(':') && t0.contains('.') && t1.contains(':') && t1.contains('.')) || (t0.starts_with('@') && t1.starts_with('@'))
|
||||
}
|
||||
|
||||
pub fn parse(text: &str) -> Result<Document> {
|
||||
let mut doc = Document::default();
|
||||
let mut time_res: i64 = 30; // standard default
|
||||
let mut shift_ms: i64 = 0;
|
||||
let mut found = false;
|
||||
|
||||
for (n, line) in text.lines().enumerate() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(cmd) = line.strip_prefix('#') {
|
||||
let cmd_lower = cmd.to_ascii_lowercase();
|
||||
if cmd_lower.starts_with("timeres") || cmd_lower.starts_with('t') {
|
||||
let parts: Vec<&str> = cmd.split_whitespace().collect();
|
||||
if parts.len() >= 2
|
||||
&& let Ok(res) = parts[1].parse::<i64>()
|
||||
&& res > 0
|
||||
{
|
||||
time_res = res;
|
||||
}
|
||||
} else if cmd_lower.starts_with("shift") || cmd_lower.starts_with('s') {
|
||||
let parts: Vec<&str> = cmd.split_whitespace().collect();
|
||||
if parts.len() >= 2 {
|
||||
let s_val = parts[1].trim();
|
||||
let (neg, s_raw) = if let Some(stripped) = s_val.strip_prefix('-') {
|
||||
(true, stripped)
|
||||
} else if let Some(stripped) = s_val.strip_prefix('+') {
|
||||
(false, stripped)
|
||||
} else {
|
||||
(false, s_val)
|
||||
};
|
||||
if let Some(ms) = parse_time(s_raw, time_res) {
|
||||
shift_ms = if neg { -ms } else { ms };
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if !is_timed_line(line) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut parts = line.split_whitespace();
|
||||
let Some(start_str) = parts.next() else { continue };
|
||||
let Some(end_str) = parts.next() else { continue };
|
||||
|
||||
let start_ms = match parse_time(start_str, time_res) {
|
||||
Some(ms) => ms.saturating_add(shift_ms).max(0),
|
||||
None => {
|
||||
doc.warnings.push(format!("line {}: invalid start timestamp, skipped", n + 1));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let end_ms = match parse_time(end_str, time_res) {
|
||||
Some(ms) => ms.saturating_add(shift_ms).max(0),
|
||||
None => {
|
||||
doc.warnings.push(format!("line {}: invalid end timestamp, skipped", n + 1));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
found = true;
|
||||
|
||||
// Skip to after the end timestamp in the original line to preserve exact spaces
|
||||
let after_end_pos = match line.find(end_str) {
|
||||
Some(idx) => idx + end_str.len(),
|
||||
None => continue,
|
||||
};
|
||||
let remainder = &line[after_end_pos..];
|
||||
let trimmed_rem = remainder.trim_start();
|
||||
|
||||
// Check if there is a directive prefix: either [Directive] or a single non-space word starting with D
|
||||
let text_part = if let Some(stripped) = trimmed_rem.strip_prefix('[') {
|
||||
match stripped.find(']') {
|
||||
Some(end_bracket) => stripped[end_bracket + 1..].trim_start(),
|
||||
None => trimmed_rem,
|
||||
}
|
||||
} else if let Some(space_idx) = trimmed_rem.find(' ') {
|
||||
let first_token = &trimmed_rem[..space_idx];
|
||||
if first_token.starts_with('D') && first_token[1..].chars().all(|c| c.is_ascii_digit()) {
|
||||
trimmed_rem[space_idx + 1..].trim_start()
|
||||
} else {
|
||||
trimmed_rem
|
||||
}
|
||||
} else {
|
||||
trimmed_rem
|
||||
};
|
||||
|
||||
// Text parsing: `\n` represents newline
|
||||
let raw_lines: Vec<&str> = text_part.split(r"\n").map(str::trim).filter(|l| !l.is_empty()).collect();
|
||||
let lines: Vec<String> = raw_lines.into_iter().map(escape_markup).collect();
|
||||
|
||||
let start = ms_to_tick(start_ms);
|
||||
let end = ms_to_tick(end_ms);
|
||||
if end <= start || lines.is_empty() {
|
||||
doc.warnings.push(format!("line {}: empty or ends before it starts; skipped", n + 1));
|
||||
continue;
|
||||
}
|
||||
|
||||
doc.cues.push(Cue { start, end, text: lines.join("\n"), ..Default::default() });
|
||||
}
|
||||
|
||||
if !found && !text.trim().is_empty() {
|
||||
return Err(Error::NotFormat("JACOsub"));
|
||||
}
|
||||
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
pub fn write(doc: &Document) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str("# JACOsub file\n");
|
||||
out.push_str("#TimeRes 100\n\n");
|
||||
|
||||
for c in &doc.cues {
|
||||
let start_ms = tick_to_ms(c.start);
|
||||
let end_ms = tick_to_ms(c.end).max(start_ms + 10);
|
||||
let raw_lines = plain_text(&c.text);
|
||||
let lines: Vec<&str> = raw_lines.lines().map(str::trim).filter(|l| !l.is_empty()).collect();
|
||||
|
||||
if !lines.is_empty() {
|
||||
let text = lines.join(r"\n");
|
||||
out.push_str(&format!("{} {} D0 {}\n", format_time(start_ms), format_time(end_ms), text));
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_and_write_roundtrip() {
|
||||
let jss = "# JACOsub file\n#TimeRes 100\n\n0:00:01.00 0:00:03.00 D0 First line\\nsecond line\n0:00:04.50 0:00:06.00 D0 Second cue\n";
|
||||
let doc = parse(jss).expect("parse");
|
||||
assert_eq!(doc.cues.len(), 2);
|
||||
assert_eq!(doc.cues[0].start, ms_to_tick(1000));
|
||||
assert_eq!(doc.cues[0].end, ms_to_tick(3000));
|
||||
assert_eq!(doc.cues[0].text, "First line\nsecond line");
|
||||
assert_eq!(doc.cues[1].start, ms_to_tick(4500));
|
||||
assert_eq!(doc.cues[1].end, ms_to_tick(6000));
|
||||
assert_eq!(doc.cues[1].text, "Second cue");
|
||||
|
||||
let written = write(&doc);
|
||||
assert_eq!(written, jss);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bracketed_directive() {
|
||||
let jss = "#TimeRes 100\n0:00:01.00 0:00:03.00 [Style1] Bracketed style text\n";
|
||||
let doc = parse(jss).expect("parse");
|
||||
assert_eq!(doc.cues.len(), 1);
|
||||
assert_eq!(doc.cues[0].text, "Bracketed style text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_shift_under_one_hour() {
|
||||
let jss = "#TimeRes 100\n#Shift -0:00:01.00\n0:00:02.00 0:00:04.00 D0 Shifted line\n";
|
||||
let doc = parse(jss).expect("parse");
|
||||
assert_eq!(doc.cues.len(), 1);
|
||||
assert_eq!(doc.cues[0].start, ms_to_tick(1000));
|
||||
assert_eq!(doc.cues[0].end, ms_to_tick(3000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_jacosub() {
|
||||
assert!(parse("1\n00:00:01,000 --> 00:00:02,000\nHello\n").is_err());
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,10 @@ struct ReadmeExamples;
|
||||
|
||||
pub mod ass;
|
||||
pub mod cea608;
|
||||
pub mod jacosub;
|
||||
pub mod mcc;
|
||||
pub mod pjs;
|
||||
pub mod realtext;
|
||||
pub mod scc;
|
||||
pub mod srt;
|
||||
pub mod stl;
|
||||
@@ -72,10 +75,28 @@ pub enum Format {
|
||||
Dfxp,
|
||||
/// Advanced SubStation Alpha (also reads SubStation Alpha).
|
||||
Ass,
|
||||
/// Phoenix Japanimation Society (PJS) subtitles.
|
||||
Pjs,
|
||||
/// RealText streaming timed text markup.
|
||||
RealText,
|
||||
/// JACOsub scriptable subtitles.
|
||||
JacoSub,
|
||||
}
|
||||
|
||||
impl Format {
|
||||
pub const ALL: [Format; 8] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp, Format::Ass];
|
||||
pub const ALL: [Format; 11] = [
|
||||
Format::Srt,
|
||||
Format::WebVtt,
|
||||
Format::Scc,
|
||||
Format::Mcc,
|
||||
Format::Stl,
|
||||
Format::Ttml,
|
||||
Format::Dfxp,
|
||||
Format::Ass,
|
||||
Format::Pjs,
|
||||
Format::RealText,
|
||||
Format::JacoSub,
|
||||
];
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Format::Srt => "SubRip (SRT)",
|
||||
@@ -86,6 +107,9 @@ impl Format {
|
||||
Format::Ttml => "TTML (IMSC1)",
|
||||
Format::Dfxp => "DFXP (TTML1)",
|
||||
Format::Ass => "Advanced SubStation Alpha (ASS)",
|
||||
Format::Pjs => "PJS",
|
||||
Format::RealText => "RealText",
|
||||
Format::JacoSub => "JACOsub",
|
||||
}
|
||||
}
|
||||
pub fn extension(self) -> &'static str {
|
||||
@@ -98,6 +122,9 @@ impl Format {
|
||||
Format::Ttml => "ttml",
|
||||
Format::Dfxp => "dfxp",
|
||||
Format::Ass => "ass",
|
||||
Format::Pjs => "pjs",
|
||||
Format::RealText => "rt",
|
||||
Format::JacoSub => "jss",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<Format> {
|
||||
@@ -110,6 +137,9 @@ impl Format {
|
||||
"ttml" | "imsc" | "imsc1" | "xml" => Some(Format::Ttml),
|
||||
"dfxp" => Some(Format::Dfxp),
|
||||
"ass" | "ssa" => Some(Format::Ass),
|
||||
"pjs" => Some(Format::Pjs),
|
||||
"rt" | "realtext" => Some(Format::RealText),
|
||||
"jss" | "js" | "jacosub" => Some(Format::JacoSub),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -260,6 +290,16 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option<Format> {
|
||||
if head.starts_with("[Script Info]") {
|
||||
return Some(Format::Ass);
|
||||
}
|
||||
if realtext::is_realtext(head) {
|
||||
return Some(Format::RealText);
|
||||
}
|
||||
let first_lines: Vec<&str> = head.lines().map(str::trim).filter(|l| !l.is_empty()).take(3).collect();
|
||||
if first_lines.first().is_some_and(|l| pjs::split_cue(l).is_some()) {
|
||||
return Some(Format::Pjs);
|
||||
}
|
||||
if first_lines.iter().any(|l| jacosub::is_timed_line(l)) || (head.starts_with('#') && (head.contains("TimeRes") || head.contains("#T"))) {
|
||||
return Some(Format::JacoSub);
|
||||
}
|
||||
let ext = ext.map(|e| e.trim_start_matches('.').to_ascii_lowercase());
|
||||
if (head.starts_with('<') && (head.contains("<tt ") || head.contains("<tt>") || head.contains(":tt ")))
|
||||
&& (head.contains("ns/ttml") || head.contains("ttaf1"))
|
||||
@@ -271,7 +311,7 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option<Format> {
|
||||
match ext.as_deref() {
|
||||
Some("srt") if has_arrow_timing || head.is_empty() => Some(Format::Srt),
|
||||
Some("vtt") if has_arrow_timing => Some(Format::WebVtt),
|
||||
Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa") => None,
|
||||
Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa" | "pjs" | "rt" | "jss" | "js") => None,
|
||||
_ if has_arrow_timing => Some(Format::Srt),
|
||||
_ => None,
|
||||
}
|
||||
@@ -306,6 +346,9 @@ fn parse_unbounded(bytes: &[u8], format: Format) -> Result<Document> {
|
||||
Format::Mcc => mcc::parse(&text),
|
||||
Format::Ttml | Format::Dfxp => ttml::parse(&text),
|
||||
Format::Ass => ass::parse(&text),
|
||||
Format::Pjs => pjs::parse(&text),
|
||||
Format::RealText => realtext::parse(&text),
|
||||
Format::JacoSub => jacosub::parse(&text),
|
||||
Format::Stl => stl::parse(bytes),
|
||||
}
|
||||
}
|
||||
@@ -344,6 +387,9 @@ pub fn write(doc: &Document, format: Format, opts: WriteOptions) -> Vec<u8> {
|
||||
Format::Ttml => ttml::write(doc, ttml::Flavor::Imsc1, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(),
|
||||
Format::Dfxp => ttml::write(doc, ttml::Flavor::Dfxp, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(),
|
||||
Format::Ass => ass::write(doc).into_bytes(),
|
||||
Format::Pjs => pjs::write(doc).into_bytes(),
|
||||
Format::RealText => realtext::write(doc).into_bytes(),
|
||||
Format::JacoSub => jacosub::write(doc).into_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//! Phoenix Japanimation Society (PJS) subtitles (`.pjs`): times in tenths of a second.
|
||||
//!
|
||||
//! ```text
|
||||
//! 10,50,"First line"
|
||||
//! 60,120,"Second line|with break"
|
||||
//! ```
|
||||
//!
|
||||
//! Lines are comma-separated `start,end,"text"` with times in deciseconds (tenths of a second).
|
||||
//! Inside the quotes, `|` breaks a line.
|
||||
|
||||
use crate::text::{escape_markup, ms_to_tick, number, plain_lines, tick_to_ms};
|
||||
use crate::{Cue, Document, Error, Result};
|
||||
|
||||
/// `start,end,"text"` as its parts.
|
||||
pub(crate) fn split_cue(line: &str) -> Option<(i64, i64, &str)> {
|
||||
let (start_str, rest) = line.split_once(',')?;
|
||||
let (end_str, text_str) = rest.split_once(',')?;
|
||||
let start = number(start_str.trim())?;
|
||||
let end = number(end_str.trim())?;
|
||||
let text = text_str.trim();
|
||||
let text = if let Some(stripped) = text.strip_prefix('"').and_then(|t| t.strip_suffix('"')) { stripped } else { text };
|
||||
Some((start, end, text))
|
||||
}
|
||||
|
||||
/// Parse a PJS subtitle document.
|
||||
pub fn parse(text: &str) -> Result<Document> {
|
||||
let mut doc = Document::default();
|
||||
let mut found = false;
|
||||
for (n, line) in text.lines().enumerate() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some((start_ds, end_ds, raw)) = split_cue(line) else {
|
||||
doc.warnings.push(format!("line {}: not a PJS cue, skipped", n + 1));
|
||||
continue;
|
||||
};
|
||||
found = true;
|
||||
let mut lines = Vec::new();
|
||||
for l in raw.split('|') {
|
||||
let trimmed = l.trim();
|
||||
if !trimmed.is_empty() {
|
||||
let unquoted = trimmed.replace("\"\"", "\"");
|
||||
lines.push(escape_markup(&unquoted));
|
||||
}
|
||||
}
|
||||
let start = ms_to_tick(start_ds.saturating_mul(100));
|
||||
let end = ms_to_tick(end_ds.saturating_mul(100));
|
||||
if end <= start || lines.is_empty() {
|
||||
doc.warnings.push(format!("line {}: empty or ends before it starts; skipped", n + 1));
|
||||
continue;
|
||||
}
|
||||
doc.cues.push(Cue { start, end, text: lines.join("\n"), ..Default::default() });
|
||||
}
|
||||
if !found && !text.trim().is_empty() {
|
||||
return Err(Error::NotFormat("PJS"));
|
||||
}
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
/// Serialize a document into PJS format.
|
||||
pub fn write(doc: &Document) -> String {
|
||||
let mut out = String::new();
|
||||
let ds = |t| (tick_to_ms(t) + 50) / 100;
|
||||
for c in &doc.cues {
|
||||
let start = ds(c.start);
|
||||
let end = ds(c.end).max(start + 1);
|
||||
let lines: Vec<String> =
|
||||
plain_lines(&c.text).into_iter().map(|l| l.replace('"', "\"\"").replace('|', "|")).filter(|l| !l.is_empty()).collect();
|
||||
if !lines.is_empty() {
|
||||
out.push_str(&format!("{start},{end},\"{}\"\n", lines.join("|")));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_and_write_roundtrip() {
|
||||
let pjs = "10,50,\"First line\"\n60,120,\"Second line|with break\"\n130,150,\"Say \"\"Hi\"\"\"\n";
|
||||
let doc = parse(pjs).expect("parse");
|
||||
assert_eq!(doc.cues.len(), 3);
|
||||
assert_eq!(doc.cues[0].start, ms_to_tick(1000));
|
||||
assert_eq!(doc.cues[0].end, ms_to_tick(5000));
|
||||
assert_eq!(doc.cues[0].text, "First line");
|
||||
assert_eq!(doc.cues[1].start, ms_to_tick(6000));
|
||||
assert_eq!(doc.cues[1].end, ms_to_tick(12000));
|
||||
assert_eq!(doc.cues[1].text, "Second line\nwith break");
|
||||
assert_eq!(doc.cues[2].text, "Say \"Hi\"");
|
||||
|
||||
let written = write(&doc);
|
||||
assert_eq!(written, pjs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_pjs() {
|
||||
assert!(parse("1\n00:00:01,000 --> 00:00:02,000\nHello\n").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
//! RealText (`.rt`): streaming HTML-like timed text markup.
|
||||
//!
|
||||
//! ```text
|
||||
//! <window type="generic" width="320" height="60" wordwrap="true">
|
||||
//! <time begin="00:01.00" end="00:03.00"/>First line<br/>second line
|
||||
//! </window>
|
||||
//! ```
|
||||
//!
|
||||
//! RealText documents encapsulate timed text inside a `<window>` container with `<time begin="…" [end="…"]/>`
|
||||
//! markers. Inside each caption block, `<br/>` and `<br>` represent line breaks.
|
||||
|
||||
use crate::text::{escape_markup, last_cue_end, ms_to_tick, plain_lines, plain_text, tick_to_ms};
|
||||
use crate::{Cue, Document, Error, Result};
|
||||
|
||||
/// Parse a RealText timestamp string into milliseconds.
|
||||
/// Supports `SS[.fff]`, `MM:SS[.fff]`, and `HH:MM:SS[.fff]`.
|
||||
pub(crate) fn parse_time(s: &str) -> Option<i64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let parts: Vec<&str> = s.split(':').collect();
|
||||
let (hours, mins, secs_str) = match parts.len() {
|
||||
1 => (0i64, 0i64, parts[0]),
|
||||
2 => (0i64, parts[0].parse().ok()?, parts[1]),
|
||||
3 => (parts[0].parse().ok()?, parts[1].parse().ok()?, parts[2]),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let (secs, ms) = if let Some((sec_p, frac_p)) = secs_str.split_once('.') {
|
||||
let sec: i64 = sec_p.parse().ok()?;
|
||||
let frac_str = frac_p.trim_end();
|
||||
let ms = match frac_str.len() {
|
||||
0 => 0i64,
|
||||
1 => frac_str.parse::<i64>().ok()?.saturating_mul(100),
|
||||
2 => frac_str.parse::<i64>().ok()?.saturating_mul(10),
|
||||
3 => frac_str.parse::<i64>().ok()?,
|
||||
_ => {
|
||||
let first3 = frac_str.get(..3)?;
|
||||
first3.parse::<i64>().ok()?
|
||||
}
|
||||
};
|
||||
(sec, ms)
|
||||
} else {
|
||||
(secs_str.parse().ok()?, 0i64)
|
||||
};
|
||||
|
||||
let total_ms = hours.saturating_mul(3600).saturating_add(mins.saturating_mul(60)).saturating_add(secs).saturating_mul(1000).saturating_add(ms);
|
||||
|
||||
Some(total_ms)
|
||||
}
|
||||
|
||||
/// Format milliseconds into RealText timestamp string `MM:SS.ss` or `HH:MM:SS.ss`.
|
||||
pub(crate) fn format_time(ms: i64) -> String {
|
||||
let ms = ms.max(0);
|
||||
let total_secs = ms / 1000;
|
||||
let centis = (ms % 1000) / 10;
|
||||
let hours = total_secs / 3600;
|
||||
let mins = (total_secs % 3600) / 60;
|
||||
let secs = total_secs % 60;
|
||||
|
||||
if hours > 0 { format!("{hours:02}:{mins:02}:{secs:02}.{centis:02}") } else { format!("{mins:02}:{secs:02}.{centis:02}") }
|
||||
}
|
||||
|
||||
/// Extract attribute value from a tag (e.g., `begin="00:01.00"`).
|
||||
fn attr_value<'a>(tag: &'a str, attr_name: &str) -> Option<&'a str> {
|
||||
let mut rest = tag;
|
||||
while let Some(idx) = rest.find(attr_name) {
|
||||
let candidate = &rest[idx + attr_name.len()..];
|
||||
let trimmed = candidate.trim_start();
|
||||
if let Some(after_eq) = trimmed.strip_prefix('=') {
|
||||
let after_eq = after_eq.trim_start();
|
||||
let quote = after_eq.chars().next()?;
|
||||
if quote == '"' || quote == '\'' {
|
||||
let val_start = 1;
|
||||
let val_end = after_eq[1..].find(quote)?;
|
||||
return Some(&after_eq[val_start..1 + val_end]);
|
||||
}
|
||||
}
|
||||
rest = &rest[idx + attr_name.len()..];
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn find_case_insensitive(haystack: &str, needle: &str) -> Option<usize> {
|
||||
let needle_bytes = needle.as_bytes();
|
||||
if needle_bytes.is_empty() {
|
||||
return Some(0);
|
||||
}
|
||||
let haystack_bytes = haystack.as_bytes();
|
||||
if haystack_bytes.len() < needle_bytes.len() {
|
||||
return None;
|
||||
}
|
||||
haystack_bytes.windows(needle_bytes.len()).position(|window| window.iter().zip(needle_bytes).all(|(&h, &n)| h.eq_ignore_ascii_case(&n)))
|
||||
}
|
||||
|
||||
/// Check if text is a RealText file.
|
||||
pub(crate) fn is_realtext(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
let lower_prefix: String = trimmed.chars().take(100).flat_map(|c| c.to_lowercase()).collect();
|
||||
if lower_prefix.starts_with("<window") {
|
||||
return true;
|
||||
}
|
||||
find_case_insensitive(text, "<window").is_some() && find_case_insensitive(text, "<time").is_some()
|
||||
}
|
||||
|
||||
pub fn parse(text: &str) -> Result<Document> {
|
||||
if !is_realtext(text) {
|
||||
return Err(Error::NotFormat("RealText"));
|
||||
}
|
||||
|
||||
let mut doc = Document::default();
|
||||
struct RawCue<'a> {
|
||||
begin: i64,
|
||||
end: Option<i64>,
|
||||
body: &'a str,
|
||||
}
|
||||
|
||||
let mut raw_cues: Vec<RawCue> = Vec::new();
|
||||
let mut search_pos = 0;
|
||||
|
||||
while let Some(time_idx) = find_case_insensitive(&text[search_pos..], "<time") {
|
||||
let actual_idx = search_pos + time_idx;
|
||||
let tag_end = match text[actual_idx..].find('>') {
|
||||
Some(end) => actual_idx + end + 1,
|
||||
None => break,
|
||||
};
|
||||
|
||||
let tag_str = &text[actual_idx..tag_end];
|
||||
let begin = attr_value(tag_str, "begin").and_then(parse_time);
|
||||
let end = attr_value(tag_str, "end").and_then(parse_time);
|
||||
|
||||
if let Some(begin_ms) = begin {
|
||||
// Text runs from tag_end to next '<time' or '</window>' or EOF
|
||||
let remaining = &text[tag_end..];
|
||||
let body_end =
|
||||
find_case_insensitive(remaining, "<time").or_else(|| find_case_insensitive(remaining, "</window>")).unwrap_or(remaining.len());
|
||||
|
||||
let body = &text[tag_end..tag_end + body_end];
|
||||
raw_cues.push(RawCue { begin: begin_ms, end, body });
|
||||
search_pos = tag_end;
|
||||
} else {
|
||||
search_pos = tag_end;
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..raw_cues.len() {
|
||||
let rc = &raw_cues[i];
|
||||
let start = ms_to_tick(rc.begin);
|
||||
let end = if let Some(explicit_end) = rc.end {
|
||||
ms_to_tick(explicit_end)
|
||||
} else if i + 1 < raw_cues.len() {
|
||||
ms_to_tick(raw_cues[i + 1].begin)
|
||||
} else {
|
||||
last_cue_end(start)
|
||||
};
|
||||
|
||||
// Convert <br/> and <br> to newlines, decode markup to clean text
|
||||
let normalized = rc.body.replace("<br/>", "\n").replace("<br>", "\n").replace("<BR/>", "\n").replace("<BR>", "\n");
|
||||
let plain = plain_text(&normalized);
|
||||
let lines: Vec<&str> = plain.lines().map(str::trim).filter(|l| !l.is_empty()).collect();
|
||||
|
||||
if end <= start || lines.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let escaped_lines: Vec<String> = lines.into_iter().map(escape_markup).collect();
|
||||
doc.cues.push(Cue { start, end, text: escaped_lines.join("\n"), ..Default::default() });
|
||||
}
|
||||
|
||||
if doc.cues.is_empty() && !text.trim().is_empty() {
|
||||
return Err(Error::NotFormat("RealText"));
|
||||
}
|
||||
|
||||
Ok(doc)
|
||||
}
|
||||
|
||||
pub fn write(doc: &Document) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str("<window type=\"generic\" width=\"320\" height=\"60\" wordwrap=\"true\">\n");
|
||||
|
||||
for c in &doc.cues {
|
||||
let start_ms = tick_to_ms(c.start);
|
||||
let mut end_ms = tick_to_ms(c.end);
|
||||
if (end_ms / 10) <= (start_ms / 10) {
|
||||
end_ms = start_ms.saturating_add(10);
|
||||
}
|
||||
let lines: Vec<String> = plain_lines(&c.text).into_iter().map(|l| escape_markup(&l)).filter(|l| !l.is_empty()).collect();
|
||||
|
||||
if !lines.is_empty() {
|
||||
let body = lines.join("<br/>");
|
||||
out.push_str(&format!("<time begin=\"{}\" end=\"{}\"/>{}\n", format_time(start_ms), format_time(end_ms), body));
|
||||
}
|
||||
}
|
||||
|
||||
out.push_str("</window>\n");
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_and_write_roundtrip() {
|
||||
let rt = "<window type=\"generic\" width=\"320\" height=\"60\" wordwrap=\"true\">\n<time begin=\"00:01.00\" end=\"00:03.00\"/>Hello world\n<time begin=\"00:04.50\" end=\"00:06.25\"/>Second cue<br/>with newline\n</window>\n";
|
||||
let doc = parse(rt).expect("parse");
|
||||
assert_eq!(doc.cues.len(), 2);
|
||||
assert_eq!(doc.cues[0].start, ms_to_tick(1000));
|
||||
assert_eq!(doc.cues[0].end, ms_to_tick(3000));
|
||||
assert_eq!(doc.cues[0].text, "Hello world");
|
||||
assert_eq!(doc.cues[1].start, ms_to_tick(4500));
|
||||
assert_eq!(doc.cues[1].end, ms_to_tick(6250));
|
||||
assert_eq!(doc.cues[1].text, "Second cue\nwith newline");
|
||||
|
||||
let written = write(&doc);
|
||||
let doc2 = parse(&written).expect("parse written");
|
||||
assert_eq!(doc2.cues.len(), 2);
|
||||
assert_eq!(doc2.cues[0].text, doc.cues[0].text);
|
||||
assert_eq!(doc2.cues[1].text, doc.cues[1].text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_timestamps() {
|
||||
assert_eq!(parse_time("5"), Some(5000));
|
||||
assert_eq!(parse_time("5.5"), Some(5500));
|
||||
assert_eq!(parse_time("00:02.50"), Some(2500));
|
||||
assert_eq!(parse_time("01:02.500"), Some(62500));
|
||||
assert_eq!(parse_time("01:02:03.40"), Some(3723400));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_misidentify_embedded_window_in_pjs() {
|
||||
let pjs = "10,50,\"Use <window> here\"\n";
|
||||
assert!(!is_realtext(pjs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_cue_quantization_roundtrip() {
|
||||
let mut doc = Document::default();
|
||||
doc.cues.push(Cue { start: ms_to_tick(1000), end: ms_to_tick(1001), text: "Short".into(), ..Default::default() });
|
||||
let written = write(&doc);
|
||||
let parsed = parse(&written).expect("parse short cue document");
|
||||
assert_eq!(parsed.cues.len(), 1);
|
||||
assert_eq!(parsed.cues[0].text, "Short");
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,52 @@ pub fn plain_text(s: &str) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
use crate::TICKS_PER_MS;
|
||||
use crate::time::{TICKS_PER_SECOND, Tick};
|
||||
|
||||
/// How long the last cue of a format that gives only start times lasts: such a file says when a
|
||||
/// caption appears, and the next one ends it.
|
||||
pub(crate) const LAST_CUE: Tick = Tick(4 * TICKS_PER_SECOND);
|
||||
|
||||
/// `start + LAST_CUE`, not overflowing.
|
||||
pub(crate) fn last_cue_end(start: Tick) -> Tick {
|
||||
Tick(start.0.saturating_add(LAST_CUE.0))
|
||||
}
|
||||
|
||||
/// Milliseconds as a time.
|
||||
pub(crate) fn ms_to_tick(ms: i64) -> Tick {
|
||||
Tick(ms.saturating_mul(TICKS_PER_MS))
|
||||
}
|
||||
|
||||
/// A time in whole milliseconds, rounded to the nearest; negative times read as zero.
|
||||
pub(crate) fn tick_to_ms(t: Tick) -> i64 {
|
||||
let per = i128::from(TICKS_PER_MS);
|
||||
((i128::from(t.0.max(0)) + per / 2) / per) as i64
|
||||
}
|
||||
|
||||
/// A decimal number of at most nine digits (so that times built from it cannot overflow).
|
||||
pub(crate) fn number(s: &str) -> Option<i64> {
|
||||
if s.is_empty() || s.len() > 9 || !s.bytes().all(|b| b.is_ascii_digit()) { None } else { s.parse().ok() }
|
||||
}
|
||||
|
||||
/// Plain text as cue text: a literal `&` or `<` becomes an entity so that it is not read as markup.
|
||||
pub(crate) fn escape_markup(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The cue's text as plain lines (tags gone, entities decoded, empty lines dropped).
|
||||
pub(crate) fn plain_lines(text: &str) -> Vec<String> {
|
||||
text.lines().map(plain_text).map(|l| l.trim().to_string()).filter(|l| !l.is_empty()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -2,6 +2,7 @@ mod ass;
|
||||
mod limits;
|
||||
mod literal_roundtrips;
|
||||
mod mcc_708_timing;
|
||||
mod pjs_realtext_jacosub;
|
||||
mod roundtrip;
|
||||
mod ttml_body_styles;
|
||||
mod ttml_body_timing;
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
//! Integration and FFmpeg oracle tests for PJS, RealText, and JACOsub legacy caption formats.
|
||||
|
||||
use craft_captions::{Cue, Document, Format, TICKS_PER_SECOND, Tick, WriteOptions, detect, parse, write};
|
||||
|
||||
fn ms(millis: i64) -> Tick {
|
||||
Tick(millis.saturating_mul(TICKS_PER_SECOND / 1000))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_legacy_formats() {
|
||||
let pjs = b"10,50,\"Hello world\"\n60,120,\"Line one|Line two\"\n";
|
||||
assert_eq!(detect(pjs, Some("pjs")), Some(Format::Pjs));
|
||||
assert_eq!(detect(pjs, None), Some(Format::Pjs));
|
||||
|
||||
let rt = b"<window type=\"generic\" width=\"320\" height=\"60\">\n<time begin=\"00:01.00\" end=\"00:03.00\"/>Hello\n</window>\n";
|
||||
assert_eq!(detect(rt, Some("rt")), Some(Format::RealText));
|
||||
assert_eq!(detect(rt, None), Some(Format::RealText));
|
||||
|
||||
let jss = b"# JACOsub file\n#TimeRes 100\n0:00:01.00 0:00:03.00 D0 Hello\n";
|
||||
assert_eq!(detect(jss, Some("jss")), Some(Format::JacoSub));
|
||||
assert_eq!(detect(jss, Some("js")), Some(Format::JacoSub));
|
||||
assert_eq!(detect(jss, None), Some(Format::JacoSub));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pjs_roundtrip_and_edge_cases() {
|
||||
let doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(1000), end: ms(4500), text: "First caption".into(), ..Default::default() },
|
||||
Cue { start: ms(6000), end: ms(10200), text: "Line 1\nLine 2".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let bytes = write(&doc, Format::Pjs, WriteOptions::default());
|
||||
let parsed = parse(&bytes, Format::Pjs).expect("parse pjs");
|
||||
assert_eq!(parsed.cues.len(), 2);
|
||||
assert_eq!(parsed.cues[0].start, ms(1000));
|
||||
assert_eq!(parsed.cues[0].end, ms(4500));
|
||||
assert_eq!(parsed.cues[0].text, "First caption");
|
||||
assert_eq!(parsed.cues[1].start, ms(6000));
|
||||
assert_eq!(parsed.cues[1].end, ms(10200));
|
||||
assert_eq!(parsed.cues[1].text, "Line 1\nLine 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_realtext_roundtrip_and_edge_cases() {
|
||||
let doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(2000), end: ms(5500), text: "Welcome to RealText".into(), ..Default::default() },
|
||||
Cue { start: ms(7000), end: ms(12000), text: "Multi-line\nRealText caption".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let bytes = write(&doc, Format::RealText, WriteOptions::default());
|
||||
let parsed = parse(&bytes, Format::RealText).expect("parse realtext");
|
||||
assert_eq!(parsed.cues.len(), 2);
|
||||
assert_eq!(parsed.cues[0].start, ms(2000));
|
||||
assert_eq!(parsed.cues[0].end, ms(5500));
|
||||
assert_eq!(parsed.cues[0].text, "Welcome to RealText");
|
||||
assert_eq!(parsed.cues[1].start, ms(7000));
|
||||
assert_eq!(parsed.cues[1].end, ms(12000));
|
||||
assert_eq!(parsed.cues[1].text, "Multi-line\nRealText caption");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jacosub_roundtrip_and_edge_cases() {
|
||||
let doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(1000), end: ms(4000), text: "First JACOsub cue".into(), ..Default::default() },
|
||||
Cue { start: ms(5500), end: ms(9200), text: "Line A\nLine B".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let bytes = write(&doc, Format::JacoSub, WriteOptions::default());
|
||||
let parsed = parse(&bytes, Format::JacoSub).expect("parse jacosub");
|
||||
assert_eq!(parsed.cues.len(), 2);
|
||||
assert_eq!(parsed.cues[0].start, ms(1000));
|
||||
assert_eq!(parsed.cues[0].end, ms(4000));
|
||||
assert_eq!(parsed.cues[0].text, "First JACOsub cue");
|
||||
assert_eq!(parsed.cues[1].start, ms(5500));
|
||||
assert_eq!(parsed.cues[1].end, ms(9200));
|
||||
assert_eq!(parsed.cues[1].text, "Line A\nLine B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hostile_inputs_never_panic() {
|
||||
for format in [Format::Pjs, Format::RealText, Format::JacoSub] {
|
||||
let _ = parse(b"", format);
|
||||
let _ = parse(b" \n\r\n\t ", format);
|
||||
let _ = parse(b"invalid text without any headers or timestamps\nfoo bar baz", format);
|
||||
let _ = parse(b"9999999999999999999999999999999999,99999999999999999999,\"overflow\"", format);
|
||||
let _ = parse(b"<window><time begin=\"invalid\" end=\"NaN\"/>unclosed", format);
|
||||
let _ = parse(b"#TimeRes 0\n@9999999999999999999999 @0 D0 test", format);
|
||||
|
||||
for len in [1, 5, 20, 100, 500] {
|
||||
let garbage = vec![0xFE; len];
|
||||
let _ = parse(&garbage, format);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ffmpeg_oracle_pjs_realtext_jacosub() {
|
||||
let Some(ff) = craft_media_testkit::ffmpeg_or_skip("legacy text subtitles oracle") else { return };
|
||||
|
||||
let temp_dir = std::env::temp_dir().join("craft_captions_legacy_test");
|
||||
let _ = std::fs::create_dir_all(&temp_dir);
|
||||
|
||||
// 1. PJS Oracle: write PJS -> decode with FFmpeg -> compare cues
|
||||
let pjs_doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(1000), end: ms(4000), text: "PJS oracle cue 1".into(), ..Default::default() },
|
||||
Cue { start: ms(5000), end: ms(9000), text: "PJS cue 2 line 1\nPJS cue 2 line 2".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let pjs_path = temp_dir.join("test.pjs");
|
||||
std::fs::write(&pjs_path, write(&pjs_doc, Format::Pjs, WriteOptions::default())).expect("write pjs");
|
||||
|
||||
let ff_pjs_srt =
|
||||
std::process::Command::new(&ff).args(["-y", "-i", pjs_path.to_str().expect("path"), "-f", "srt", "-"]).output().expect("ffmpeg pjs").stdout;
|
||||
let ff_pjs_doc = parse(&ff_pjs_srt, Format::Srt).expect("parse ff pjs srt");
|
||||
assert_eq!(ff_pjs_doc.cues.len(), pjs_doc.cues.len());
|
||||
for i in 0..pjs_doc.cues.len() {
|
||||
assert_eq!(ff_pjs_doc.cues[i].start, pjs_doc.cues[i].start);
|
||||
assert_eq!(ff_pjs_doc.cues[i].end, pjs_doc.cues[i].end);
|
||||
assert_eq!(ff_pjs_doc.cues[i].text.trim(), pjs_doc.cues[i].text.trim());
|
||||
}
|
||||
|
||||
// 2. RealText Oracle: write RealText -> decode with FFmpeg -> compare cues
|
||||
let rt_doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(1000), end: ms(3000), text: "RealText oracle cue 1".into(), ..Default::default() },
|
||||
Cue { start: ms(4500), end: ms(8000), text: "RealText cue 2 line 1\nRealText cue 2 line 2".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let rt_path = temp_dir.join("test.rt");
|
||||
std::fs::write(&rt_path, write(&rt_doc, Format::RealText, WriteOptions::default())).expect("write rt");
|
||||
|
||||
let ff_rt_srt =
|
||||
std::process::Command::new(&ff).args(["-y", "-i", rt_path.to_str().expect("path"), "-f", "srt", "-"]).output().expect("ffmpeg rt").stdout;
|
||||
let ff_rt_doc = parse(&ff_rt_srt, Format::Srt).expect("parse ff rt srt");
|
||||
assert_eq!(ff_rt_doc.cues.len(), rt_doc.cues.len());
|
||||
for i in 0..rt_doc.cues.len() {
|
||||
assert_eq!(ff_rt_doc.cues[i].start, rt_doc.cues[i].start);
|
||||
assert_eq!(ff_rt_doc.cues[i].end, rt_doc.cues[i].end);
|
||||
assert_eq!(ff_rt_doc.cues[i].text.trim(), rt_doc.cues[i].text.trim());
|
||||
}
|
||||
|
||||
// 3. JACOsub Oracle: write JACOsub -> decode with FFmpeg -> compare cues
|
||||
let jss_doc = Document {
|
||||
cues: vec![
|
||||
Cue { start: ms(1000), end: ms(4000), text: "JACOsub oracle cue 1".into(), ..Default::default() },
|
||||
Cue { start: ms(5000), end: ms(9000), text: "JACOsub cue 2 line 1\nJACOsub cue 2 line 2".into(), ..Default::default() },
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let jss_path = temp_dir.join("test.jss");
|
||||
std::fs::write(&jss_path, write(&jss_doc, Format::JacoSub, WriteOptions::default())).expect("write jss");
|
||||
|
||||
let ff_jss_srt =
|
||||
std::process::Command::new(&ff).args(["-y", "-i", jss_path.to_str().expect("path"), "-f", "srt", "-"]).output().expect("ffmpeg jss").stdout;
|
||||
let ff_jss_doc = parse(&ff_jss_srt, Format::Srt).expect("parse ff jss srt");
|
||||
assert_eq!(ff_jss_doc.cues.len(), jss_doc.cues.len());
|
||||
for i in 0..jss_doc.cues.len() {
|
||||
assert_eq!(ff_jss_doc.cues[i].start, jss_doc.cues[i].start);
|
||||
assert_eq!(ff_jss_doc.cues[i].end, jss_doc.cues[i].end);
|
||||
assert_eq!(ff_jss_doc.cues[i].text.trim(), jss_doc.cues[i].text.trim());
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,47 @@ pub fn srgb_trc() -> Curve {
|
||||
}
|
||||
|
||||
impl Curve {
|
||||
/// Creates a gamma curve `y = x^g`. Returns `Identity` if `g == 1.0` (within 1e-6).
|
||||
pub fn gamma(g: f64) -> Result<Self, crate::CmsError> {
|
||||
if !crate::math::fits_s15f16(g) || crate::math::s15f16_round(g) <= 0.0 {
|
||||
return Err(crate::CmsError::InvalidArgument("gamma must remain positive and representable in ICC fixed point".into()));
|
||||
}
|
||||
if (g - 1.0).abs() < 1e-6 { Ok(Curve::Identity) } else { Ok(Curve::Gamma(g)) }
|
||||
}
|
||||
|
||||
/// Creates an ICC parametric curve (types 0 to 4).
|
||||
pub fn parametric(kind: u16, p: [f64; 7]) -> Result<Self, crate::CmsError> {
|
||||
if kind > 4 {
|
||||
return Err(crate::CmsError::InvalidArgument(format!("parametric curve kind {kind} must be <= 4")));
|
||||
}
|
||||
let count = Self::param_count(kind);
|
||||
if !p.iter().take(count).all(|&v| crate::math::fits_s15f16(v)) {
|
||||
return Err(crate::CmsError::InvalidArgument("parametric curve parameters must be representable in ICC fixed point".into()));
|
||||
}
|
||||
Ok(Curve::Parametric { kind, p })
|
||||
}
|
||||
|
||||
/// Creates a sampled lookup table curve over evenly spaced inputs in `[0, 1]`.
|
||||
pub fn from_table(samples: Vec<f32>) -> Result<Self, crate::CmsError> {
|
||||
if samples.len() < 2 {
|
||||
return Err(crate::CmsError::InvalidArgument("sampled curve must contain at least 2 entries".into()));
|
||||
}
|
||||
if !samples.iter().all(|s| s.is_finite() && (0.0..=1.0).contains(s)) {
|
||||
return Err(crate::CmsError::InvalidArgument("sampled curve values must be finite and within [0, 1]".into()));
|
||||
}
|
||||
Ok(Curve::Table(samples))
|
||||
}
|
||||
|
||||
/// Validate public enum variants before a profile stores them in ICC tags.
|
||||
pub(crate) fn validate_icc(&self) -> Result<(), crate::CmsError> {
|
||||
match self {
|
||||
Self::Identity => Ok(()),
|
||||
Self::Gamma(g) => Self::gamma(*g).map(|_| ()),
|
||||
Self::Parametric { kind, p } => Self::parametric(*kind, *p).map(|_| ()),
|
||||
Self::Table(t) if t.len() >= 2 && u32::try_from(t.len()).is_ok() && t.iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) => Ok(()),
|
||||
Self::Table(_) => Err(crate::CmsError::InvalidArgument("sampled curve must contain representable finite normalized samples".into())),
|
||||
}
|
||||
}
|
||||
/// Number of parameters stored for a parametric function type.
|
||||
pub fn param_count(kind: u16) -> usize {
|
||||
match kind {
|
||||
|
||||
@@ -46,6 +46,7 @@ pub use curve::Curve;
|
||||
pub use gamut::GamutCheck;
|
||||
pub use gpu_matrix_shaper::GpuMatrixShaper;
|
||||
pub use lut3d::Lut3d;
|
||||
pub use math::Mat3;
|
||||
pub use named::{NamedColor, NamedColors};
|
||||
pub use normalize::normalize_for_embedding;
|
||||
pub use profile::{ColorSpace, Pcs, Profile, ProfileClass};
|
||||
|
||||
@@ -221,6 +221,12 @@ pub fn d50_quantized() -> [f64; 3] {
|
||||
[s15f16_round(D50[0]), s15f16_round(D50[1]), s15f16_round(D50[2])]
|
||||
}
|
||||
|
||||
/// Whether ICC s15Fixed16 encoding can round this value without saturation.
|
||||
pub(crate) fn fits_s15f16(v: f64) -> bool {
|
||||
let scaled = (v * 65536.0).round();
|
||||
scaled.is_finite() && scaled >= f64::from(i32::MIN) && scaled <= f64::from(i32::MAX)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -228,6 +228,128 @@ pub struct Profile {
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
/// Builds a matrix/TRC display profile from an already PCS D50-adapted colorant matrix,
|
||||
/// a white point, and per-channel tone reproduction curves.
|
||||
///
|
||||
/// # Matrix Coordinate Space
|
||||
/// In accordance with ICC.1:2010 §9.2.35–37, the colorant matrix tags (`rXYZ`, `gXYZ`, `bXYZ`)
|
||||
/// stored in an ICC profile must be chromatically adapted to the D50 Profile Connection Space (PCS).
|
||||
/// `pcs_matrix` must therefore be expressed relative to D50 (e.g. as read directly from ICC profile tags).
|
||||
///
|
||||
/// If `white` is not D50, the profile's `chad` tag is populated with the Bradford chromatic adaptation
|
||||
/// matrix mapping `white` to D50.
|
||||
pub fn matrix_shaper(white: [f64; 3], pcs_matrix: Mat3, trc: [Curve; 3]) -> Result<Profile, CmsError> {
|
||||
for curve in &trc {
|
||||
curve.validate_icc()?;
|
||||
}
|
||||
if !white.iter().all(|&c| math::fits_s15f16(c) && math::s15f16_round(c) > 0.0) {
|
||||
return Err(CmsError::InvalidArgument("white point must remain positive and representable in ICC fixed point".into()));
|
||||
}
|
||||
if !pcs_matrix.iter().flatten().all(|&v| v.is_finite()) {
|
||||
return Err(CmsError::InvalidArgument("matrix elements must be finite".into()));
|
||||
}
|
||||
if math::invert(&pcs_matrix).is_none() {
|
||||
return Err(CmsError::InvalidArgument("colorant matrix must be invertible".into()));
|
||||
}
|
||||
|
||||
let d50 = math::d50_quantized();
|
||||
let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4;
|
||||
|
||||
let chad = if is_d50 { None } else { Some(math::bradford(white, math::D50)) };
|
||||
|
||||
// Quantize and normalize to ensure (1, 1, 1) maps exactly to PCS D50
|
||||
let mut q = pcs_matrix.map(|row| row.map(math::s15f16_round));
|
||||
for (i, row) in q.iter_mut().enumerate() {
|
||||
let err = d50[i] - row.iter().sum::<f64>();
|
||||
let k = (0..3).max_by(|a, b| row[*a].abs().total_cmp(&row[*b].abs())).unwrap_or(0);
|
||||
row[k] = math::s15f16_round(row[k] + err);
|
||||
}
|
||||
if !q.iter().flatten().all(|&v| math::fits_s15f16(v)) || math::invert(&q).is_none() {
|
||||
return Err(CmsError::InvalidArgument("colorant matrix is singular or unrepresentable after quantization/normalization".into()));
|
||||
}
|
||||
let chad_quantized = chad.map(|a| a.map(|row| row.map(math::s15f16_round)));
|
||||
// Positive finite XYZ inputs can still overflow Bradford cone ratios or quantization.
|
||||
// Refuse invalid stored adaptation rather than serializing nonfinite coefficients.
|
||||
if chad_quantized.as_ref().is_some_and(|a| !a.iter().flatten().all(|&v| math::fits_s15f16(v)) || math::invert(a).is_none()) {
|
||||
return Err(CmsError::InvalidArgument("chromatic adaptation must remain representable and invertible after quantization".into()));
|
||||
}
|
||||
|
||||
Ok(Profile {
|
||||
version: (4, 0x30),
|
||||
class: ProfileClass::Display,
|
||||
color_space: ColorSpace::Rgb,
|
||||
pcs: Pcs::Xyz,
|
||||
rendering_intent: Intent::RelativeColorimetric,
|
||||
description: "Parametric Matrix/TRC".into(),
|
||||
copyright: "".into(),
|
||||
white_point: if is_d50 { d50 } else { white },
|
||||
black_point: None,
|
||||
chad: chad_quantized,
|
||||
matrix: Some(q),
|
||||
trc: Some(trc),
|
||||
gray_trc: None,
|
||||
a2b: [None, None, None],
|
||||
b2a: [None, None, None],
|
||||
bytes: None,
|
||||
hash: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Builds a matrix/TRC display profile from unadapted source primaries relative to `white`.
|
||||
///
|
||||
/// The unadapted `source_matrix` (whose columns sum to `white`) is chromatically adapted to PCS D50
|
||||
/// using the Bradford transform before storage in the profile's matrix tags.
|
||||
pub fn from_source_matrix(white: [f64; 3], source_matrix: Mat3, trc: [Curve; 3]) -> Result<Profile, CmsError> {
|
||||
if !white.iter().all(|&c| math::fits_s15f16(c) && math::s15f16_round(c) > 0.0) {
|
||||
return Err(CmsError::InvalidArgument("white point must remain positive and representable in ICC fixed point".into()));
|
||||
}
|
||||
if !source_matrix.iter().flatten().all(|&v| v.is_finite()) {
|
||||
return Err(CmsError::InvalidArgument("matrix elements must be finite".into()));
|
||||
}
|
||||
if math::invert(&source_matrix).is_none() {
|
||||
return Err(CmsError::InvalidArgument("colorant matrix must be invertible".into()));
|
||||
}
|
||||
|
||||
let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4;
|
||||
let pcs_matrix = if is_d50 {
|
||||
source_matrix
|
||||
} else {
|
||||
let adapt = math::bradford(white, math::D50);
|
||||
math::mul(&adapt, &source_matrix)
|
||||
};
|
||||
|
||||
Self::matrix_shaper(white, pcs_matrix, trc)
|
||||
}
|
||||
|
||||
/// Builds a monochrome profile from a white point and tone reproduction curve.
|
||||
pub fn gray(white: [f64; 3], trc: Curve) -> Result<Profile, CmsError> {
|
||||
trc.validate_icc()?;
|
||||
if !white.iter().all(|&c| math::fits_s15f16(c) && math::s15f16_round(c) > 0.0) {
|
||||
return Err(CmsError::InvalidArgument("white point must remain positive and representable in ICC fixed point".into()));
|
||||
}
|
||||
let is_d50 = (white[0] - math::D50[0]).abs() < 1e-4 && (white[1] - math::D50[1]).abs() < 1e-4 && (white[2] - math::D50[2]).abs() < 1e-4;
|
||||
|
||||
Ok(Profile {
|
||||
version: (4, 0x30),
|
||||
class: ProfileClass::Display,
|
||||
color_space: ColorSpace::Gray,
|
||||
pcs: Pcs::Xyz,
|
||||
rendering_intent: Intent::RelativeColorimetric,
|
||||
description: "Parametric Gray".into(),
|
||||
copyright: "".into(),
|
||||
white_point: if is_d50 { math::d50_quantized() } else { white },
|
||||
black_point: None,
|
||||
chad: None,
|
||||
matrix: None,
|
||||
trc: None,
|
||||
gray_trc: Some(trc),
|
||||
a2b: [None, None, None],
|
||||
b2a: [None, None, None],
|
||||
bytes: None,
|
||||
hash: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Parses ICC profile bytes.
|
||||
pub fn parse(bytes: &[u8]) -> Result<Profile, CmsError> {
|
||||
parse::reset_budgets();
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
use craft_cms::{Builtin, Curve, Intent, Mat3, Profile, Transform};
|
||||
|
||||
#[test]
|
||||
fn test_curve_constructors() {
|
||||
// Gamma
|
||||
let g22 = Curve::gamma(2.2).expect("valid gamma");
|
||||
assert_eq!(g22, Curve::Gamma(2.2));
|
||||
|
||||
let g10 = Curve::gamma(1.0).expect("gamma 1.0 is identity");
|
||||
assert_eq!(g10, Curve::Identity);
|
||||
|
||||
let g_near = Curve::gamma(1.0 + 1e-7).expect("near-identity gamma normalized");
|
||||
assert_eq!(g_near, Curve::Identity);
|
||||
|
||||
assert!(Curve::gamma(0.0).is_err());
|
||||
assert!(Curve::gamma(-2.2).is_err());
|
||||
assert!(Curve::gamma(f64::NAN).is_err());
|
||||
assert!(Curve::gamma(f64::INFINITY).is_err());
|
||||
|
||||
// Parametric
|
||||
let p0 = Curve::parametric(0, [2.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).expect("valid parametric type 0");
|
||||
assert_eq!(p0, Curve::Parametric { kind: 0, p: [2.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] });
|
||||
|
||||
assert!(Curve::parametric(5, [0.0; 7]).is_err());
|
||||
assert!(Curve::parametric(0, [f64::NAN, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).is_err());
|
||||
|
||||
// Sampled table
|
||||
let table = Curve::from_table(vec![0.0, 0.5, 1.0]).expect("valid sampled table");
|
||||
assert_eq!(table, Curve::Table(vec![0.0, 0.5, 1.0]));
|
||||
|
||||
assert!(Curve::from_table(vec![0.5]).is_err());
|
||||
assert!(Curve::from_table(vec![0.0, f32::NAN]).is_err());
|
||||
assert!(Curve::from_table(vec![0.0, -0.1, 1.0]).is_err());
|
||||
assert!(Curve::from_table(vec![0.0, 2.0]).is_err());
|
||||
assert!(Curve::from_table(vec![0.0, 1.0001]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_profile_constructors() {
|
||||
let srgb = Builtin::Srgb.profile();
|
||||
let white_d65 = [0.95047, 1.0, 1.08883];
|
||||
|
||||
// 1. Matrix shaper with already PCS D50-adapted matrix (e.g. read from ICC tags)
|
||||
let pcs_d50_matrix: Mat3 = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]];
|
||||
let trc = [craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc()];
|
||||
let custom_from_pcs = Profile::matrix_shaper(white_d65, pcs_d50_matrix, trc.clone()).expect("matrix shaper from PCS D50");
|
||||
assert_eq!(custom_from_pcs.color_space, craft_cms::ColorSpace::Rgb);
|
||||
assert!(custom_from_pcs.same_colors(srgb), "custom profile from PCS D50 matrix should match reference sRGB");
|
||||
|
||||
// 2. Matrix shaper from unadapted D65 source primaries (undergoes single Bradford adaptation)
|
||||
let source_d65_matrix: Mat3 = [[0.4124564, 0.3575761, 0.1804375], [0.2126729, 0.7151522, 0.0721750], [0.0193339, 0.1191920, 0.9503041]];
|
||||
let custom_from_source = Profile::from_source_matrix(white_d65, source_d65_matrix, trc).expect("matrix shaper from source");
|
||||
assert!(custom_from_source.same_colors(srgb), "custom profile adapted from source primaries should match reference sRGB");
|
||||
|
||||
// Evaluate transform
|
||||
let xform = Transform::new(&custom_from_pcs, srgb, Intent::RelativeColorimetric, true).expect("transform from custom RGB to sRGB");
|
||||
let mut out = [0.0f32; 3];
|
||||
xform.eval(&[0.5, 0.5, 0.5], &mut out).expect("eval");
|
||||
assert!((out[0] - 0.5).abs() < 1e-2);
|
||||
assert!((out[1] - 0.5).abs() < 1e-2);
|
||||
assert!((out[2] - 0.5).abs() < 1e-2);
|
||||
|
||||
// Invalid matrix shaper inputs
|
||||
assert!(Profile::matrix_shaper([0.0, 1.0, 1.0], pcs_d50_matrix, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err());
|
||||
assert!(Profile::matrix_shaper([f64::NAN, 1.0, 1.0], pcs_d50_matrix, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err());
|
||||
let singular: Mat3 = [[0.0; 3]; 3];
|
||||
assert!(Profile::matrix_shaper(white_d65, singular, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err());
|
||||
|
||||
// Near-degenerate matrix that is invertible before quantization but becomes singular after
|
||||
let d50 = [0.9642, 1.0, 0.8249];
|
||||
let near_degenerate: Mat3 = [[d50[0] - 0.000001, 0.000001, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, d50[2]]];
|
||||
assert!(Profile::matrix_shaper(d50, near_degenerate, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err());
|
||||
|
||||
// Gray profile
|
||||
let gray = Profile::gray(white_d65, Curve::gamma(2.2).expect("curve")).expect("gray profile");
|
||||
assert_eq!(gray.color_space, craft_cms::ColorSpace::Gray);
|
||||
|
||||
let gray_to_srgb = Transform::new(&gray, srgb, Intent::RelativeColorimetric, true).expect("transform from gray to sRGB");
|
||||
let mut srgb_out = [0.0f32; 3];
|
||||
gray_to_srgb.eval(&[0.5], &mut srgb_out).expect("eval");
|
||||
assert!((srgb_out[0] - srgb_out[1]).abs() < 1e-3);
|
||||
assert!((srgb_out[1] - srgb_out[2]).abs() < 1e-3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matrix_constructors_against_independent_moxcms_oracle() {
|
||||
use moxcms::{ColorProfile, Layout, TransformOptions as MoxOpts};
|
||||
|
||||
let srgb = Builtin::Srgb.profile();
|
||||
let white_d65 = [0.95047, 1.0, 1.08883];
|
||||
|
||||
// 1. PCS D50-adapted matrix profile
|
||||
let pcs_d50_matrix: Mat3 = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]];
|
||||
let trc = [craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc(), craft_cms::curve::srgb_trc()];
|
||||
let custom_from_pcs = Profile::matrix_shaper(white_d65, pcs_d50_matrix, trc.clone()).expect("matrix shaper from PCS D50");
|
||||
|
||||
// 2. Unadapted source matrix profile
|
||||
let source_d65_matrix: Mat3 = [[0.4124564, 0.3575761, 0.1804375], [0.2126729, 0.7151522, 0.0721750], [0.0193339, 0.1191920, 0.9503041]];
|
||||
let custom_from_source = Profile::from_source_matrix(white_d65, source_d65_matrix, trc).expect("matrix shaper from source");
|
||||
|
||||
// Convert both to standard ICC bytes and parse with independent oracle moxcms
|
||||
let mox_from_pcs = ColorProfile::new_from_slice(&custom_from_pcs.to_bytes()).expect("moxcms parse custom PCS profile");
|
||||
let mox_from_source = ColorProfile::new_from_slice(&custom_from_source.to_bytes()).expect("moxcms parse custom source profile");
|
||||
let mox_dst_srgb = ColorProfile::new_from_slice(&srgb.to_bytes()).expect("moxcms parse builtin sRGB");
|
||||
|
||||
let test_pixels: Vec<u8> = (0..512u32).flat_map(|i| [(i * 17 % 256) as u8, (i * 53 % 256) as u8, (i * 97 % 256) as u8]).collect();
|
||||
|
||||
// Verify 1: moxcms transforming custom_from_pcs -> builtin sRGB produces identity (diff <= 1)
|
||||
let mox_xform_pcs = mox_from_pcs.create_transform_8bit(Layout::Rgb, &mox_dst_srgb, Layout::Rgb, MoxOpts::default()).unwrap();
|
||||
let mut mox_out_pcs = vec![0u8; test_pixels.len()];
|
||||
mox_xform_pcs.transform(&test_pixels, &mut mox_out_pcs).unwrap();
|
||||
let max_diff_pcs = test_pixels.iter().zip(&mox_out_pcs).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap();
|
||||
assert!(max_diff_pcs <= 1, "moxcms PCS-adapted profile -> sRGB max diff was {max_diff_pcs}");
|
||||
|
||||
// Verify 2: moxcms transforming custom_from_source -> builtin sRGB produces identity (diff <= 1)
|
||||
let mox_xform_src = mox_from_source.create_transform_8bit(Layout::Rgb, &mox_dst_srgb, Layout::Rgb, MoxOpts::default()).unwrap();
|
||||
let mut mox_out_src = vec![0u8; test_pixels.len()];
|
||||
mox_xform_src.transform(&test_pixels, &mut mox_out_src).unwrap();
|
||||
let max_diff_src = test_pixels.iter().zip(&mox_out_src).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap();
|
||||
assert!(max_diff_src <= 1, "moxcms source-adapted profile -> sRGB max diff was {max_diff_src}");
|
||||
|
||||
// Verify 3: craft_cms and moxcms agree when converting from custom_from_source to DisplayP3
|
||||
let p3 = Builtin::DisplayP3.profile();
|
||||
let mox_dst_p3 = ColorProfile::new_from_slice(&p3.to_bytes()).expect("moxcms parse Display P3");
|
||||
let mox_xform_p3 = mox_from_source.create_transform_8bit(Layout::Rgb, &mox_dst_p3, Layout::Rgb, MoxOpts::default()).unwrap();
|
||||
let mut mox_p3_out = vec![0u8; test_pixels.len()];
|
||||
mox_xform_p3.transform(&test_pixels, &mut mox_p3_out).unwrap();
|
||||
|
||||
let ours_xform_p3 = Transform::new(&custom_from_source, p3, Intent::RelativeColorimetric, false).unwrap();
|
||||
let mut ours_p3_out = vec![0u8; test_pixels.len()];
|
||||
ours_xform_p3.convert_u8(&test_pixels, 3, &mut ours_p3_out, 3, false).unwrap();
|
||||
|
||||
let max_p3_diff = ours_p3_out.iter().zip(&mox_p3_out).map(|(a, b)| (*a as i32 - *b as i32).abs()).max().unwrap();
|
||||
assert!(max_p3_diff <= 1, "craft_cms vs moxcms custom source profile -> P3 max diff was {max_p3_diff}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constructor_never_returns_a_nonfinite_adaptation() {
|
||||
let matrix = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]];
|
||||
for white in [[f64::MAX; 3], [f64::MIN_POSITIVE; 3], [1.0, 1.0, (0.8951 + 0.2664) / 0.1614]] {
|
||||
if let Ok(profile) = Profile::matrix_shaper(white, matrix, [Curve::Identity, Curve::Identity, Curve::Identity]) {
|
||||
assert!(profile.chad.unwrap().iter().flatten().all(|v| v.is_finite()), "nonfinite adaptation for {white:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constructors_reject_values_that_saturate_icc_fixed_point() {
|
||||
assert!(Profile::gray([f64::MAX; 3], Curve::Identity).is_err());
|
||||
assert!(Profile::gray([f64::MIN_POSITIVE; 3], Curve::Identity).is_err());
|
||||
assert!(Curve::gamma(32768.0).is_err());
|
||||
assert!(Curve::gamma(f64::MIN_POSITIVE).is_err());
|
||||
assert!(Curve::parametric(0, [32768.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).is_err());
|
||||
let matrix = [[40000.0, -39999.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
|
||||
assert!(Profile::matrix_shaper([0.9642, 1.0, 0.8249], matrix, [Curve::Identity, Curve::Identity, Curve::Identity]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_constructors_validate_direct_curve_variants() {
|
||||
let white = [0.9642, 1.0, 0.8249];
|
||||
let matrix = [[0.4360747, 0.3850649, 0.1430804], [0.2225045, 0.7168786, 0.0606169], [0.0139322, 0.0971045, 0.7141733]];
|
||||
for curve in [
|
||||
Curve::Gamma(f64::NAN),
|
||||
Curve::Gamma(32768.0),
|
||||
Curve::Parametric { kind: 5, p: [0.0; 7] },
|
||||
Curve::Table(vec![0.0, f32::NAN]),
|
||||
Curve::Table(vec![]),
|
||||
] {
|
||||
assert!(Profile::gray(white, curve.clone()).is_err());
|
||||
assert!(Profile::matrix_shaper(white, matrix, [curve.clone(), Curve::Identity, Curve::Identity]).is_err());
|
||||
assert!(Profile::from_source_matrix(white, matrix, [Curve::Identity, Curve::Identity, curve]).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn icc_fixed_point_boundaries_remain_accepted() {
|
||||
let max = f64::from(i32::MAX) / 65536.0;
|
||||
let min = f64::from(i32::MIN) / 65536.0;
|
||||
assert!(Curve::gamma(max).is_ok());
|
||||
assert!(Curve::gamma(1.0 / 65536.0).is_ok());
|
||||
assert!(Curve::parametric(4, [1.0, 1.0, min, max, 0.0, 0.0, 0.0]).is_ok());
|
||||
assert!(Profile::gray([max; 3], Curve::Identity).is_ok());
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod bench;
|
||||
mod cms;
|
||||
mod constructors;
|
||||
mod fogra39;
|
||||
mod named;
|
||||
mod profile_id;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "craft-color"
|
||||
description = "Colour for Craft apps: the colour picker, wells, swatch tables, ramps and the hex field."
|
||||
description = "Colour for Craft apps: the colour picker, wells, swatch tables, ramps and the hex field, and the colour tags, Y'CbCr matrices, transfer functions and HDR static metadata of compositor and display frames."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
@@ -6,4 +6,6 @@ Stories: `patterns/color/{picker-dialog, panel, swatches, eyedropper-bar, sample
|
||||
|
||||
Without `ui`: `CmykModel` (CMYK through a craft-cms profile), swatch files (`palette`: `.aco`, `.act`, `.ase`, `.gpl`, `.css`), the display transform and its caches (`display`), and colour harmonies (`harmony`).
|
||||
|
||||
The colour tags of compositor and display frames (`tags`, re-exported at the root; from FilmCraft's `filmcraft-color`): `ColorInfo` (matrix, transfer, primaries, range), `HdrMetadata`, Y'CbCr matrices, transfer functions (sRGB, PQ, HLG), sRGB tables, HSL and hex. craft-frame and craft-media-source use them.
|
||||
|
||||
The colour model and conversions are in `craft-theme`; profiles and transforms in `craft-cms`.
|
||||
+10
-2
@@ -1,9 +1,11 @@
|
||||
//! Colour for Craft apps: the colour picker, wells, swatch tables, ramps and the hex field.
|
||||
//! Colour for Craft apps: the colour picker, wells, swatch tables, ramps and the hex field, and the
|
||||
//! colour tags of compositor and display frames.
|
||||
//!
|
||||
//! The widgets are behind the `ui` feature; the colour model and conversions are in `craft-theme`.
|
||||
//! Without `ui`, the crate has the helpers that need no egui: [`AreaMode`], [`GrayModel`],
|
||||
//! [`CmykModel`], web-safe colours ([`web_safe`], [`not_web_safe`]), swatch files ([`palette`]), the display transform ([`display`]) and colour
|
||||
//! harmonies ([`harmony`]) and gradient stops ([`gradient`]).
|
||||
//! harmonies ([`harmony`]) and gradient stops ([`gradient`]). [`tags`] has the colour tags of compositor and
|
||||
//! display frames (matrix, transfer, primaries, range, HDR metadata) and their conversions, re-exported here.
|
||||
//! [`AreaMode`] (and `ui::RampKind`) serialize as their [`key`](AreaMode::key); the swatch and harmony settings types serialize too.
|
||||
#![warn(missing_docs)]
|
||||
|
||||
@@ -14,6 +16,7 @@ mod gray;
|
||||
pub mod harmony;
|
||||
mod mode;
|
||||
pub mod palette;
|
||||
pub mod tags;
|
||||
#[cfg(feature = "ui")]
|
||||
pub mod ui;
|
||||
mod web;
|
||||
@@ -21,6 +24,11 @@ mod web;
|
||||
pub use cmyk::CmykModel;
|
||||
pub use gray::GrayModel;
|
||||
pub use mode::AreaMode;
|
||||
pub use tags::{
|
||||
ColorInfo, HdrMetadata, InvalidBitDepth, Matrix, Primaries, REFERENCE_WHITE_NITS, Range, Transfer, from_linear, hlg_inverse_oetf, hlg_oetf,
|
||||
hsl_to_rgb, linear_to_srgb, linear_to_srgb_u8, linear_to_srgb_u8_table, luma709, normalize_c, normalize_y, parse_hex, pq_eotf, pq_inverse_eotf,
|
||||
rgb_to_hsl, rgb_to_ycbcr, srgb_to_linear, srgb_u8_to_linear_table, to_hex, to_linear, ycbcr_to_rgb,
|
||||
};
|
||||
pub use web::{not_web_safe, web_safe};
|
||||
|
||||
#[cfg(all(test, feature = "screen-testing"))]
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
//! Colour tags and the conversions compositor and display frames need: [`ColorInfo`] (matrix,
|
||||
//! transfer, primaries, range), [`HdrMetadata`], Y'CbCr matrices, transfer functions, and a few
|
||||
//! helpers (sRGB tables, HSL, hex). Used by craft-frame and by media `MediaInfo`. It is not a
|
||||
//! colour-managed pipeline: LUTs, camera log, grade and tone mapping stay out.
|
||||
//!
|
||||
//! Came from FilmCraft's `filmcraft-color` (the metadata and transfer part of that crate).
|
||||
//! Distinct from `craft-video`'s decoder colour tags (`ColorTags` in #1231 / #1768).
|
||||
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// HDR reference white in cd/m² (ITU-R BT.2408).
|
||||
pub const REFERENCE_WHITE_NITS: f64 = 203.0;
|
||||
|
||||
/// YUV ↔ RGB matrix coefficients.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Matrix {
|
||||
/// ITU-R BT.601 (standard definition).
|
||||
Bt601,
|
||||
/// ITU-R BT.709 (high definition).
|
||||
#[default]
|
||||
Bt709,
|
||||
/// ITU-R BT.2020, non-constant luminance.
|
||||
Bt2020Ncl,
|
||||
}
|
||||
|
||||
impl Matrix {
|
||||
/// (Kr, Kb) luma coefficients.
|
||||
pub fn kr_kb(self) -> (f32, f32) {
|
||||
match self {
|
||||
Matrix::Bt601 => (0.299, 0.114),
|
||||
Matrix::Bt709 => (0.2126, 0.0722),
|
||||
Matrix::Bt2020Ncl => (0.2627, 0.0593),
|
||||
}
|
||||
}
|
||||
/// From an ISO/IEC 23091-2 `matrix_coefficients` code.
|
||||
pub fn from_code(c: u8) -> Option<Matrix> {
|
||||
match c {
|
||||
1 => Some(Matrix::Bt709),
|
||||
5 | 6 => Some(Matrix::Bt601),
|
||||
// H.273 table 4: code 10 is constant luminance, not this matrix.
|
||||
9 => Some(Matrix::Bt2020Ncl),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Transfer characteristics.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Transfer {
|
||||
/// BT.709 display, approximated with the sRGB curve by this slim conversion layer.
|
||||
/// This is not a pure-gamma BT.1886 decode.
|
||||
#[default]
|
||||
Bt709,
|
||||
/// IEC 61966-2-1 sRGB.
|
||||
Srgb,
|
||||
/// Linear light.
|
||||
Linear,
|
||||
/// SMPTE ST 2084.
|
||||
Pq,
|
||||
/// ARIB STD-B67.
|
||||
Hlg,
|
||||
}
|
||||
|
||||
impl Transfer {
|
||||
/// From an ISO/IEC 23091-2 `transfer_characteristics` code.
|
||||
pub fn from_code(c: u8) -> Option<Transfer> {
|
||||
match c {
|
||||
1 | 6 | 14 | 15 => Some(Transfer::Bt709),
|
||||
13 => Some(Transfer::Srgb),
|
||||
8 => Some(Transfer::Linear),
|
||||
16 => Some(Transfer::Pq),
|
||||
18 => Some(Transfer::Hlg),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Chromaticity primaries.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Primaries {
|
||||
/// ITU-R BT.709 (and sRGB).
|
||||
#[default]
|
||||
Bt709,
|
||||
/// ITU-R BT.601, 625-line systems (EBU).
|
||||
Bt601_625,
|
||||
/// ITU-R BT.601, 525-line systems (SMPTE C).
|
||||
Bt601_525,
|
||||
/// ITU-R BT.2020.
|
||||
Bt2020,
|
||||
/// Display P3 (DCI-P3 primaries, D65 white).
|
||||
P3D65,
|
||||
}
|
||||
|
||||
/// Quantisation range of integer Y'CbCr samples.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Range {
|
||||
/// 16–235 (8-bit) "video" range.
|
||||
#[default]
|
||||
Limited,
|
||||
/// 0–255 (8-bit) "full" range.
|
||||
Full,
|
||||
}
|
||||
|
||||
/// Colour metadata attached to a compositor or display frame.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ColorInfo {
|
||||
/// The Y'CbCr ↔ R'G'B' matrix.
|
||||
pub matrix: Matrix,
|
||||
/// The transfer characteristics.
|
||||
pub transfer: Transfer,
|
||||
/// The chromaticity primaries.
|
||||
pub primaries: Primaries,
|
||||
/// The quantisation range of integer samples.
|
||||
pub range: Range,
|
||||
}
|
||||
|
||||
impl ColorInfo {
|
||||
/// sRGB in full range.
|
||||
pub const SRGB_FULL: ColorInfo = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Srgb, primaries: Primaries::Bt709, range: Range::Full };
|
||||
/// BT.709 video in limited range.
|
||||
pub const REC709: ColorInfo = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Bt709, primaries: Primaries::Bt709, range: Range::Limited };
|
||||
}
|
||||
|
||||
/// HDR static metadata of a stream: SMPTE ST 2086 mastering display luminance and the CTA-861.3
|
||||
/// content light levels (MaxCLL / MaxFALL), in cd/m². Read from `mdcv` / `clli` (MP4 / MOV) and
|
||||
/// Matroska `MasteringMetadata` / `MaxCLL` / `MaxFALL`.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HdrMetadata {
|
||||
/// The mastering display's peak luminance.
|
||||
pub mastering_max_nits: Option<f32>,
|
||||
/// The mastering display's black level.
|
||||
pub mastering_min_nits: Option<f32>,
|
||||
/// Maximum content light level: the brightest pixel in the content.
|
||||
pub max_cll: Option<f32>,
|
||||
/// Maximum frame-average light level.
|
||||
pub max_fall: Option<f32>,
|
||||
}
|
||||
|
||||
impl HdrMetadata {
|
||||
/// The content's peak for tone mapping: MaxCLL when signalled (the brightest pixel actually
|
||||
/// in the content), else the mastering display's peak; `None` when neither is (0 means
|
||||
/// unknown in both). Never below HDR reference white.
|
||||
pub fn peak_nits(&self) -> Option<f32> {
|
||||
let known = |v: Option<f32>| v.filter(|n| *n > 0.0);
|
||||
let p = match (known(self.max_cll), known(self.mastering_max_nits)) {
|
||||
(Some(c), _) => c,
|
||||
(None, m) => m?,
|
||||
};
|
||||
Some(p.clamp(REFERENCE_WHITE_NITS as f32, 10_000.0))
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert normalised Y'CbCr (Y in 0..1, Cb/Cr in -0.5..0.5) to non-linear R'G'B'.
|
||||
#[inline]
|
||||
pub fn ycbcr_to_rgb(y: f32, cb: f32, cr: f32, m: Matrix) -> [f32; 3] {
|
||||
let (kr, kb) = m.kr_kb();
|
||||
let kg = 1.0 - kr - kb;
|
||||
let r = y + 2.0 * (1.0 - kr) * cr;
|
||||
let b = y + 2.0 * (1.0 - kb) * cb;
|
||||
let g = (y - kr * r - kb * b) / kg;
|
||||
[r, g, b]
|
||||
}
|
||||
|
||||
/// Convert non-linear R'G'B' to normalised Y'CbCr.
|
||||
#[inline]
|
||||
pub fn rgb_to_ycbcr(r: f32, g: f32, b: f32, m: Matrix) -> [f32; 3] {
|
||||
let (kr, kb) = m.kr_kb();
|
||||
let kg = 1.0 - kr - kb;
|
||||
let y = kr * r + kg * g + kb * b;
|
||||
[y, (b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr))]
|
||||
}
|
||||
|
||||
/// Unsupported integer sample depth. Normalization supports 8 through 16 bits.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct InvalidBitDepth(pub u32);
|
||||
|
||||
impl std::fmt::Display for InvalidBitDepth {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "unsupported sample bit depth {}; expected 8..=16", self.0)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for InvalidBitDepth {}
|
||||
|
||||
/// Normalise an integer luma code value to 0..1 given bit depth and range.
|
||||
#[inline]
|
||||
pub fn normalize_y(v: u32, bits: u32, range: Range) -> Result<f32, InvalidBitDepth> {
|
||||
if !(8..=16).contains(&bits) {
|
||||
return Err(InvalidBitDepth(bits));
|
||||
}
|
||||
let scale = (1u32 << (bits - 8)) as f32;
|
||||
Ok(match range {
|
||||
Range::Limited => (v as f32 - 16.0 * scale) / (219.0 * scale),
|
||||
Range::Full => v as f32 / ((1u32 << bits) - 1) as f32,
|
||||
})
|
||||
}
|
||||
|
||||
/// Normalise an integer chroma code value to −0.5..0.5 given bit depth and range.
|
||||
#[inline]
|
||||
pub fn normalize_c(v: u32, bits: u32, range: Range) -> Result<f32, InvalidBitDepth> {
|
||||
if !(8..=16).contains(&bits) {
|
||||
return Err(InvalidBitDepth(bits));
|
||||
}
|
||||
let scale = (1u32 << (bits - 8)) as f32;
|
||||
Ok(match range {
|
||||
Range::Limited => (v as f32 - 128.0 * scale) / (224.0 * scale),
|
||||
Range::Full => (v as f32 - (1u32 << (bits - 1)) as f32) / ((1u32 << bits) - 1) as f32,
|
||||
})
|
||||
}
|
||||
|
||||
/// sRGB electro-optical transfer (encoded → linear).
|
||||
#[inline]
|
||||
pub fn srgb_to_linear(v: f32) -> f32 {
|
||||
if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) }
|
||||
}
|
||||
|
||||
/// sRGB inverse EOTF (linear → encoded).
|
||||
#[inline]
|
||||
pub fn linear_to_srgb(v: f32) -> f32 {
|
||||
if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 }
|
||||
}
|
||||
|
||||
/// Encoded → linear for a transfer function (display-referred; PQ normalised to 100 nits = 1.0).
|
||||
pub fn to_linear(v: f32, t: Transfer) -> f32 {
|
||||
match t {
|
||||
Transfer::Srgb => srgb_to_linear(v),
|
||||
// Rec.709 is treated as display gamma 2.4; the sRGB curve is visually identical for
|
||||
// previews and keeps round-trips exact.
|
||||
Transfer::Bt709 => srgb_to_linear(v),
|
||||
Transfer::Linear => v,
|
||||
Transfer::Pq => pq_eotf(v) * 100.0,
|
||||
Transfer::Hlg => hlg_inverse_oetf(v),
|
||||
}
|
||||
}
|
||||
|
||||
/// Linear → encoded for a transfer function (the inverse of [`to_linear`]).
|
||||
pub fn from_linear(v: f32, t: Transfer) -> f32 {
|
||||
match t {
|
||||
Transfer::Srgb | Transfer::Bt709 => linear_to_srgb(v),
|
||||
Transfer::Linear => v,
|
||||
Transfer::Pq => pq_inverse_eotf(v / 100.0),
|
||||
Transfer::Hlg => hlg_oetf(v),
|
||||
}
|
||||
}
|
||||
|
||||
/// SMPTE ST 2084 EOTF, output normalised to 10 000 nits = 1.0.
|
||||
pub fn pq_eotf(e: f32) -> f32 {
|
||||
let (m1, m2) = (0.159_301_76_f32, 78.84375_f32);
|
||||
let (c1, c2, c3) = (0.835_937_5_f32, 18.851_563_f32, 18.6875_f32);
|
||||
let p = e.max(0.0).powf(1.0 / m2);
|
||||
((p - c1).max(0.0) / (c2 - c3 * p)).powf(1.0 / m1)
|
||||
}
|
||||
|
||||
/// Inverse of [`pq_eotf`].
|
||||
pub fn pq_inverse_eotf(y: f32) -> f32 {
|
||||
let (m1, m2) = (0.159_301_76_f32, 78.84375_f32);
|
||||
let (c1, c2, c3) = (0.835_937_5_f32, 18.851_563_f32, 18.6875_f32);
|
||||
let p = y.max(0.0).powf(m1);
|
||||
((c1 + c2 * p) / (1.0 + c3 * p)).powf(m2)
|
||||
}
|
||||
|
||||
/// ARIB STD-B67 HLG OETF.
|
||||
pub fn hlg_oetf(l: f32) -> f32 {
|
||||
let (a, b, c) = (0.178_832_77_f32, 0.284_668_92_f32, 0.559_910_7_f32);
|
||||
if l <= 1.0 / 12.0 { (3.0 * l.max(0.0)).sqrt() } else { a * (12.0 * l - b).ln() + c }
|
||||
}
|
||||
|
||||
/// Inverse of [`hlg_oetf`].
|
||||
pub fn hlg_inverse_oetf(e: f32) -> f32 {
|
||||
let (a, b, c) = (0.178_832_77_f32, 0.284_668_92_f32, 0.559_910_7_f32);
|
||||
if e <= 0.5 { e * e / 3.0 } else { (((e - c) / a).exp() + b) / 12.0 }
|
||||
}
|
||||
|
||||
/// 256-entry table: sRGB-encoded u8 → linear f32.
|
||||
pub fn srgb_u8_to_linear_table() -> &'static [f32; 256] {
|
||||
static T: OnceLock<[f32; 256]> = OnceLock::new();
|
||||
T.get_or_init(|| std::array::from_fn(|i| srgb_to_linear(i as f32 / 255.0)))
|
||||
}
|
||||
|
||||
/// 4096-entry table: linear (0..1, quantised to 12 bits) → sRGB-encoded u8.
|
||||
pub fn linear_to_srgb_u8_table() -> &'static [u8; 4096] {
|
||||
static T: OnceLock<[u8; 4096]> = OnceLock::new();
|
||||
T.get_or_init(|| std::array::from_fn(|i| (linear_to_srgb(i as f32 / 4095.0) * 255.0 + 0.5).clamp(0.0, 255.0) as u8))
|
||||
}
|
||||
|
||||
/// Linear 0..1 to an sRGB-encoded u8, via [`linear_to_srgb_u8_table`].
|
||||
#[inline]
|
||||
pub fn linear_to_srgb_u8(v: f32) -> u8 {
|
||||
let i = (v.clamp(0.0, 1.0) * 4095.0 + 0.5) as usize;
|
||||
linear_to_srgb_u8_table()[i]
|
||||
}
|
||||
|
||||
/// Rec.709 luma of a linear or encoded RGB triple.
|
||||
#[inline]
|
||||
pub fn luma709(r: f32, g: f32, b: f32) -> f32 {
|
||||
0.2126 * r + 0.7152 * g + 0.0722 * b
|
||||
}
|
||||
|
||||
/// RGB (0..1) → HSL (h in 0..1).
|
||||
pub fn rgb_to_hsl(r: f32, g: f32, b: f32) -> [f32; 3] {
|
||||
let max = r.max(g).max(b);
|
||||
let min = r.min(g).min(b);
|
||||
let l = (max + min) / 2.0;
|
||||
if (max - min).abs() < 1e-7 {
|
||||
return [0.0, 0.0, l];
|
||||
}
|
||||
let d = max - min;
|
||||
let s = if l > 0.5 { d / (2.0 - max - min) } else { d / (max + min) };
|
||||
let h = if max == r {
|
||||
(g - b) / d + if g < b { 6.0 } else { 0.0 }
|
||||
} else if max == g {
|
||||
(b - r) / d + 2.0
|
||||
} else {
|
||||
(r - g) / d + 4.0
|
||||
};
|
||||
[h / 6.0, s, l]
|
||||
}
|
||||
|
||||
/// Inverse of [`rgb_to_hsl`].
|
||||
pub fn hsl_to_rgb(h: f32, s: f32, l: f32) -> [f32; 3] {
|
||||
if s <= 0.0 {
|
||||
return [l, l, l];
|
||||
}
|
||||
let q = if l < 0.5 { l * (1.0 + s) } else { l + s - l * s };
|
||||
let p = 2.0 * l - q;
|
||||
let f = |mut t: f32| {
|
||||
t = t.rem_euclid(1.0);
|
||||
if t < 1.0 / 6.0 {
|
||||
p + (q - p) * 6.0 * t
|
||||
} else if t < 0.5 {
|
||||
q
|
||||
} else if t < 2.0 / 3.0 {
|
||||
p + (q - p) * (2.0 / 3.0 - t) * 6.0
|
||||
} else {
|
||||
p
|
||||
}
|
||||
};
|
||||
[f(h + 1.0 / 3.0), f(h), f(h - 1.0 / 3.0)]
|
||||
}
|
||||
|
||||
/// Parse `#rrggbb` / `#rrggbbaa` into 0..1 floats.
|
||||
pub fn parse_hex(s: &str) -> Option<[f32; 4]> {
|
||||
let s = s.trim();
|
||||
let s = s.strip_prefix('#').unwrap_or(s);
|
||||
if !s.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return None;
|
||||
}
|
||||
let b = |i: usize| u8::from_str_radix(s.get(i..i + 2)?, 16).ok().map(|v| v as f32 / 255.0);
|
||||
match s.len() {
|
||||
6 => Some([b(0)?, b(2)?, b(4)?, 1.0]),
|
||||
8 => Some([b(0)?, b(2)?, b(4)?, b(6)?]),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The inverse of [`parse_hex`], lowercase, omitting alpha when it is 1.
|
||||
pub fn to_hex(c: [f32; 4]) -> String {
|
||||
let q = |v: f32| (v.clamp(0.0, 1.0) * 255.0).round() as u8;
|
||||
if c[3] >= 0.999 {
|
||||
format!("#{:02x}{:02x}{:02x}", q(c[0]), q(c[1]), q(c[2]))
|
||||
} else {
|
||||
format!("#{:02x}{:02x}{:02x}{:02x}", q(c[0]), q(c[1]), q(c[2]), q(c[3]))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use craft_ui::testing::trial;
|
||||
|
||||
#[trial]
|
||||
fn parse_hex_rejects_signs() {
|
||||
assert_eq!(parse_hex("#+f+f+f"), None);
|
||||
assert_eq!(parse_hex("#-f-f-f"), None);
|
||||
assert_eq!(parse_hex("##0f0f0f"), None);
|
||||
assert_eq!(parse_hex("#0f0f0"), None);
|
||||
assert!(parse_hex("#0f0f0f").is_some());
|
||||
assert!(parse_hex("0f0f0f80").is_some());
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn ycbcr_roundtrip() {
|
||||
for m in [Matrix::Bt601, Matrix::Bt709, Matrix::Bt2020Ncl] {
|
||||
let rgb = [0.8, 0.3, 0.1];
|
||||
let y = rgb_to_ycbcr(rgb[0], rgb[1], rgb[2], m);
|
||||
let back = ycbcr_to_rgb(y[0], y[1], y[2], m);
|
||||
for k in 0..3 {
|
||||
assert!((rgb[k] - back[k]).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn invalid_bit_depths_are_errors_not_shifts() {
|
||||
for bits in [0, 1, 7, 17, 31, 32, u32::MAX] {
|
||||
assert_eq!(normalize_y(0, bits, Range::Full), Err(InvalidBitDepth(bits)));
|
||||
assert_eq!(normalize_c(0, bits, Range::Limited), Err(InvalidBitDepth(bits)));
|
||||
}
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn limited_range_levels() {
|
||||
assert_eq!(normalize_y(16, 8, Range::Limited).unwrap(), 0.0);
|
||||
assert_eq!(normalize_y(235, 8, Range::Limited).unwrap(), 1.0);
|
||||
assert_eq!(normalize_y(940, 10, Range::Limited).unwrap(), 1.0);
|
||||
assert_eq!(normalize_c(128, 8, Range::Limited).unwrap(), 0.0);
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn transfer_roundtrips() {
|
||||
for t in [Transfer::Srgb, Transfer::Pq, Transfer::Hlg, Transfer::Linear] {
|
||||
for v in [0.0, 0.1, 0.5, 0.9] {
|
||||
let l = to_linear(v, t);
|
||||
assert!((from_linear(l, t) - v).abs() < 1e-3, "{t:?} {v}");
|
||||
}
|
||||
}
|
||||
assert_eq!(linear_to_srgb_u8(1.0), 255);
|
||||
assert_eq!(linear_to_srgb_u8(0.0), 0);
|
||||
assert_eq!(linear_to_srgb_u8(srgb_u8_to_linear_table()[128]), 128);
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn hsl_roundtrip() {
|
||||
let c = [0.2, 0.6, 0.9];
|
||||
let h = rgb_to_hsl(c[0], c[1], c[2]);
|
||||
let b = hsl_to_rgb(h[0], h[1], h[2]);
|
||||
for k in 0..3 {
|
||||
assert!((c[k] - b[k]).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn hex() {
|
||||
assert_eq!(parse_hex("#ff8000"), Some([1.0, 128.0 / 255.0, 0.0, 1.0]));
|
||||
assert_eq!(to_hex([1.0, 0.0, 0.0, 1.0]), "#ff0000");
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn hdr_peak_nits() {
|
||||
assert_eq!(HdrMetadata::default().peak_nits(), None);
|
||||
assert_eq!(HdrMetadata { max_cll: Some(4000.0), mastering_max_nits: Some(1000.0), ..HdrMetadata::default() }.peak_nits(), Some(4000.0));
|
||||
assert_eq!(HdrMetadata { max_cll: Some(1000.0), mastering_max_nits: Some(4000.0), ..HdrMetadata::default() }.peak_nits(), Some(1000.0));
|
||||
assert_eq!(HdrMetadata { max_cll: Some(0.0), mastering_max_nits: Some(400.0), ..HdrMetadata::default() }.peak_nits(), Some(400.0));
|
||||
// never below reference white
|
||||
assert_eq!(HdrMetadata { max_cll: Some(100.0), ..HdrMetadata::default() }.peak_nits(), Some(REFERENCE_WHITE_NITS as f32));
|
||||
}
|
||||
|
||||
#[trial]
|
||||
fn codes() {
|
||||
assert_eq!(Matrix::from_code(9), Some(Matrix::Bt2020Ncl));
|
||||
assert_eq!(Matrix::from_code(10), None);
|
||||
assert_eq!(Matrix::from_code(1), Some(Matrix::Bt709));
|
||||
assert_eq!(Matrix::from_code(6), Some(Matrix::Bt601));
|
||||
assert_eq!(Transfer::from_code(16), Some(Transfer::Pq));
|
||||
assert_eq!(Transfer::from_code(99), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "craft-control"
|
||||
description = "The control channel any app may share: token files, the loopback server, and its requests; no UI toolkit"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
# The loopback server is for desktop apps; the web app is driven through the page instead.
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
# The per-launch control token: 32 random bytes from the operating system (MIT OR Apache-2.0).
|
||||
getrandom = "0.3"
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,673 @@
|
||||
//! The control channel's per-launch token (docs/control-protocol.md). An app listening for
|
||||
//! control makes a random token, writes it to a file only the user can read
|
||||
//! ([`App::token_path`]) and refuses every connection whose first request doesn't carry it, so
|
||||
//! other users on the same machine (loopback is shared, on Windows across remote-desktop sessions
|
||||
//! too) can't drive the app. Clients find it with [`App::client_token`]. Next to it, `<port>.pid`
|
||||
//! names the process that wrote it, so a client refused with a leftover token can say the file is
|
||||
//! stale.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::io::{Error, ErrorKind, Read, Result, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Shortest token accepted from the app's `CONTROL_TOKEN` variable (hex characters).
|
||||
pub const MIN_TOKEN_LEN: usize = 32;
|
||||
|
||||
/// Ports tried from the app's default port upward when it is taken (a second window, another
|
||||
/// program).
|
||||
pub const PORT_TRIES: u16 = 16;
|
||||
|
||||
/// Which app a control channel belongs to: where its token files live, its environment and its
|
||||
/// name in messages.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct App {
|
||||
/// The app's name in messages ("restart SolidCraft").
|
||||
pub name: &'static str,
|
||||
/// The per-user data folder; the token files go in its `control` folder.
|
||||
pub data_dir: fn() -> Option<PathBuf>,
|
||||
/// The app's environment variable `name` (`CONTROL_DIR`, `CONTROL_TOKEN`), with the app's prefix.
|
||||
pub env: fn(&str) -> Option<OsString>,
|
||||
/// A time in seconds since the Unix epoch, as the app writes UTC times in messages.
|
||||
pub utc: fn(u64) -> String,
|
||||
/// The port the control channel tries first when the launch gives none. Below the ephemeral
|
||||
/// ranges of Linux (32768+) and Windows/macOS (49152+), so the system never hands it to an
|
||||
/// outgoing connection.
|
||||
pub default_port: u16,
|
||||
}
|
||||
|
||||
/// Where the control channel listens, from the launch's `--control`/environment port and the app's
|
||||
/// allow-agents setting.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Listen {
|
||||
/// Exactly this port (`--control PORT`, the app's `CONTROL_PORT`); no fallback.
|
||||
Exact(u16),
|
||||
/// The app's [`App::default_port`], or the next free one of [`PORT_TRIES`].
|
||||
Default,
|
||||
}
|
||||
|
||||
/// What a launch listens on. An explicit port wins over the app's allow-agents setting
|
||||
/// (`enabled`), so a scripted launch works with the setting off; with the setting off and no port,
|
||||
/// nothing listens.
|
||||
pub fn plan(explicit: Option<u16>, enabled: bool) -> Option<Listen> {
|
||||
match explicit {
|
||||
Some(port) => Some(Listen::Exact(port)),
|
||||
None if enabled => Some(Listen::Default),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind a loopback-only listener on `first`, or on the next free port when it is taken, up to
|
||||
/// `tries` ports in all.
|
||||
pub fn bind_with_fallback(first: u16, tries: u16) -> Result<std::net::TcpListener> {
|
||||
let mut last = Error::new(ErrorKind::AddrInUse, "no port to try");
|
||||
for port in (first..=u16::MAX).take(usize::from(tries.max(1))) {
|
||||
match std::net::TcpListener::bind(("127.0.0.1", port)) {
|
||||
Ok(l) => return Ok(l),
|
||||
// In use; Windows reports a port another program holds exclusively as denied.
|
||||
Err(e) if matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) => last = e,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
impl App {
|
||||
/// The folder of the token files: `<data folder>/control` (the app's `CONTROL_DIR` replaces it).
|
||||
fn control_dir(&self) -> Option<PathBuf> {
|
||||
match self.custom_control_dir() {
|
||||
Some(d) => Some(PathBuf::from(d)),
|
||||
None => Some((self.data_dir)()?.join("control")),
|
||||
}
|
||||
}
|
||||
|
||||
/// The folder the app's `CONTROL_DIR` names in place of the default one.
|
||||
fn custom_control_dir(&self) -> Option<OsString> {
|
||||
(self.env)("CONTROL_DIR").filter(|d| !d.is_empty())
|
||||
}
|
||||
|
||||
/// The token file of the app listening on `port`: `<data folder>/control/<port>.token`
|
||||
/// (the app's `CONTROL_DIR` replaces the folder).
|
||||
pub fn token_path(&self, port: u16) -> Option<PathBuf> {
|
||||
Some(self.control_dir()?.join(format!("{port}.token")))
|
||||
}
|
||||
|
||||
/// The discovery file: the port of the most recently started app, so a client with no port
|
||||
/// finds it. It holds only the port; the token stays in its own file.
|
||||
pub fn current_port_path(&self) -> Option<PathBuf> {
|
||||
Some(self.control_dir()?.join("current.port"))
|
||||
}
|
||||
|
||||
/// Record `port` as the running app's, readable only by the user like the token files.
|
||||
pub fn write_current_port(&self, port: u16) -> Result<PathBuf> {
|
||||
let path = self.current_port_path().ok_or_else(|| Error::new(ErrorKind::NotFound, "no data folder"))?;
|
||||
self.write_token_file(&path, &port.to_string())?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// The port the discovery file names, if there is one.
|
||||
pub fn read_current_port(&self) -> Option<u16> {
|
||||
parse_port(&std::fs::read_to_string(self.current_port_path()?).ok().filter(|t| t.len() <= 64)?)
|
||||
}
|
||||
|
||||
/// Remove the discovery file (and its process file) at exit, unless a later app has taken it
|
||||
/// over: it names another port, or another launch now holds this port's token file. Under
|
||||
/// Flatpak every instance has its own loopback, so two windows can listen on the same port
|
||||
/// number and write the same file names; the one closing first must leave the other's (#8794).
|
||||
/// Run it after [`remove_token_file`], which takes this launch's own token file away first.
|
||||
pub fn remove_current_port(&self, port: u16, token: &str) {
|
||||
if let (Some(current), Some(file)) = (self.current_port_path(), self.token_path(port)) {
|
||||
remove_discovery(¤t, &file, port, token);
|
||||
}
|
||||
}
|
||||
|
||||
/// A token set on purpose with the app's `CONTROL_TOKEN` (test harnesses, sandboxed packages
|
||||
/// whose data folder the client can't see), if it is long enough.
|
||||
pub fn env_token(&self) -> Option<String> {
|
||||
(self.env)("CONTROL_TOKEN").and_then(|v| v.into_string().ok()).map(|t| t.trim().to_string()).filter(|t| t.len() >= MIN_TOKEN_LEN)
|
||||
}
|
||||
|
||||
/// The token a client sends to the app on `port`: `explicit` (`--token`), else the app's
|
||||
/// `CONTROL_TOKEN`, else the app's token file.
|
||||
pub fn client_token(&self, port: u16, explicit: Option<&str>) -> Option<ClientToken> {
|
||||
if let Some(t) = explicit.map(str::trim).filter(|t| !t.is_empty()) {
|
||||
return Some(ClientToken { token: t.to_string(), file: None, app: *self });
|
||||
}
|
||||
if let Some(t) = self.env_token() {
|
||||
return Some(ClientToken { token: t, file: None, app: *self });
|
||||
}
|
||||
// Read directly, not through any agent's file scope: the app's own file.
|
||||
read_token_file(&self.token_path(port)?, *self)
|
||||
}
|
||||
|
||||
/// Write `token` to `path`, readable and writable only by the user: the folder is made `0700`
|
||||
/// and the file `0600` on Unix; on Windows the file's inherited permissions are replaced by one
|
||||
/// entry for the user (`icacls`) before the token goes in; when that fails, the still empty file
|
||||
/// is removed and no token is written. A file left by an earlier launch is replaced. The process
|
||||
/// file beside it ([`pid_path`]) records this process, best effort.
|
||||
pub fn write_token_file(&self, path: &Path, token: &str) -> Result<()> {
|
||||
#[cfg(windows)]
|
||||
let restrict = |p: &Path| restrict_to_user(p, self.custom_control_dir().is_some());
|
||||
#[cfg(not(windows))]
|
||||
let restrict = |_: &Path| Ok(());
|
||||
write_token_file_with(path, token, restrict)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_port(text: &str) -> Option<u16> {
|
||||
text.trim().parse::<u16>().ok().filter(|p| *p != 0)
|
||||
}
|
||||
|
||||
/// [`App::remove_current_port`] on given files: `current` is the discovery file, `file` the token
|
||||
/// file of `port`.
|
||||
fn remove_discovery(current: &Path, file: &Path, port: u16, token: &str) {
|
||||
let named = std::fs::read_to_string(current).ok().filter(|t| t.len() <= 64).and_then(|t| parse_port(&t));
|
||||
if named != Some(port) || held_by_another(file, token) {
|
||||
return;
|
||||
}
|
||||
let _ = std::fs::remove_file(current);
|
||||
let _ = std::fs::remove_file(pid_path(current));
|
||||
}
|
||||
|
||||
/// A client token and, when it came from a token file, where that file is.
|
||||
pub struct ClientToken {
|
||||
pub token: String,
|
||||
pub file: Option<TokenSource>,
|
||||
/// The app it is for, named in an explanation.
|
||||
pub app: App,
|
||||
}
|
||||
|
||||
/// The token file a client read: its path, when it was written and the process that wrote it.
|
||||
pub struct TokenSource {
|
||||
pub path: PathBuf,
|
||||
pub modified: Option<SystemTime>,
|
||||
/// From `<port>.pid`: the writer's process id and, on Linux, its start time.
|
||||
pub writer: Option<(u32, Option<u64>)>,
|
||||
}
|
||||
|
||||
impl ClientToken {
|
||||
/// The app's `error`, and for a refused token read from a file, which file it was, when it
|
||||
/// was written and whether the instance that wrote it has exited. Never the token itself.
|
||||
pub fn explain_error(&self, error: &str) -> String {
|
||||
let Some(src) = &self.file else { return error.to_string() };
|
||||
if !error.starts_with("unauthorized") {
|
||||
return error.to_string();
|
||||
}
|
||||
let written = src
|
||||
.modified
|
||||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|t| (self.app.utc)(t.as_secs()))
|
||||
.unwrap_or_else(|| "at an unknown time".into());
|
||||
let age = src.modified.and_then(|t| t.elapsed().ok()).map(|d| format!(" ({} s ago)", d.as_secs())).unwrap_or_default();
|
||||
let path = src.path.display();
|
||||
let name = self.app.name;
|
||||
match src.writer.filter(|(pid, start)| writer_exited(*pid, *start)) {
|
||||
Some((pid, _)) => format!(
|
||||
"{error}; stale token file from an instance that exited (process {pid}): {path}, written {written}{age}; delete that file or restart {name}"
|
||||
),
|
||||
None => format!("{error}; token from {path}, written {written}{age}; if {name} was force-quit, restart it or delete that file"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Files bigger than this are not token files.
|
||||
const MAX_TOKEN_FILE: u64 = 4096;
|
||||
|
||||
fn read_token_file(path: &Path, app: App) -> Option<ClientToken> {
|
||||
let f = std::fs::File::open(path).ok()?;
|
||||
let modified = f.metadata().ok().and_then(|m| m.modified().ok());
|
||||
let mut text = String::new();
|
||||
f.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?;
|
||||
if text.len() as u64 > MAX_TOKEN_FILE {
|
||||
return None;
|
||||
}
|
||||
let token = text.trim();
|
||||
if token.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
|
||||
let writer = read_pid_file(&pid_path(&path));
|
||||
Some(ClientToken { token: token.to_string(), file: Some(TokenSource { path, modified, writer }), app })
|
||||
}
|
||||
|
||||
/// The process file beside the token file `path`: `7878.token` → `7878.pid`.
|
||||
pub fn pid_path(path: &Path) -> PathBuf {
|
||||
path.with_extension("pid")
|
||||
}
|
||||
|
||||
/// `<pid> [<start time>]`; anything else (or no file) is no information.
|
||||
fn read_pid_file(path: &Path) -> Option<(u32, Option<u64>)> {
|
||||
let mut text = String::new();
|
||||
std::fs::File::open(path).ok()?.take(256).read_to_string(&mut text).ok()?;
|
||||
let mut fields = text.split_whitespace();
|
||||
let pid = fields.next()?.parse::<u32>().ok().filter(|&pid| pid > 0)?;
|
||||
Some((pid, fields.next().and_then(|s| s.parse().ok())))
|
||||
}
|
||||
|
||||
/// Whether the process that wrote a token file is known to be gone: not running, or its id now
|
||||
/// belongs to a process that started at another time. Unknown counts as running.
|
||||
fn writer_exited(pid: u32, start: Option<u64>) -> bool {
|
||||
process_running(pid) == Some(false) || start.zip(process_start(pid)).is_some_and(|(saved, now)| saved != now)
|
||||
}
|
||||
|
||||
/// Whether process `pid` runs; `None` when that can't be found out.
|
||||
fn process_running(pid: u32) -> Option<bool> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if !Path::new("/proc/self").exists() {
|
||||
return None;
|
||||
}
|
||||
match std::fs::metadata(format!("/proc/{pid}")) {
|
||||
Ok(_) => Some(true),
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => Some(false),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
{
|
||||
let output = std::process::Command::new("ps").args(["-p", &pid.to_string(), "-o", "pid="]).output().ok()?;
|
||||
if output.status.success() {
|
||||
return Some(true);
|
||||
}
|
||||
(output.status.code() == Some(1) && output.stdout.iter().all(u8::is_ascii_whitespace)).then_some(false)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::process::CommandExt;
|
||||
let output = std::process::Command::new("tasklist")
|
||||
.args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"])
|
||||
.creation_flags(0x0800_0000)
|
||||
.output()
|
||||
.ok()?;
|
||||
if !output.status.success() || output.stdout.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// `"image","pid",…` per process; with no match, a message line without a pid.
|
||||
let text = String::from_utf8_lossy(&output.stdout);
|
||||
Some(text.lines().any(|line| line.split("\",\"").nth(1).and_then(|s| s.trim_matches('"').parse::<u32>().ok()) == Some(pid)))
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
let _ = pid;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Linux: the process's start time (clock ticks after boot), which tells a reused id apart.
|
||||
fn process_start(pid: u32) -> Option<u64> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let mut stat = String::new();
|
||||
std::fs::File::open(format!("/proc/{pid}/stat")).ok()?.take(4096).read_to_string(&mut stat).ok()?;
|
||||
stat.rsplit_once(')')?.1.split_whitespace().nth(19)?.parse().ok()
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = pid;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// `a == b`, taking the same time wherever they differ.
|
||||
pub fn tokens_match(a: &str, b: &str) -> bool {
|
||||
let (a, b) = (a.as_bytes(), b.as_bytes());
|
||||
a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
|
||||
}
|
||||
|
||||
/// [`App::write_token_file`], with `restrict` limiting the new file to the user (Windows).
|
||||
fn write_token_file_with(path: &Path, token: &str, restrict: impl FnOnce(&Path) -> Result<()>) -> Result<()> {
|
||||
let dir = path.parent().ok_or_else(|| Error::new(ErrorKind::InvalidInput, "no folder"))?;
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
let mut opts = std::fs::OpenOptions::new();
|
||||
opts.write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
|
||||
std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?;
|
||||
opts.mode(0o600);
|
||||
}
|
||||
let opened = opts.open(path)?;
|
||||
// On a refusal the file goes with the closure, so it is closed before it is removed.
|
||||
let mut f = match restrict(path).map(|()| opened) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
let _ = std::fs::remove_file(path);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
f.write_all(token.as_bytes())?;
|
||||
f.write_all(b"\n")?;
|
||||
f.sync_all()?;
|
||||
let pid = std::process::id();
|
||||
let line = process_start(pid).map_or_else(|| format!("{pid}\n"), |start| format!("{pid} {start}\n"));
|
||||
let _ = std::fs::write(pid_path(path), line);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a token file this launch wrote, and its process file, while it still holds `token`.
|
||||
/// Another launch may have written the same file since (a second window on the same port,
|
||||
/// #8794); its file stays. The process id can't tell the two apart, as every Flatpak instance
|
||||
/// has the same one, but each launch's token is its own.
|
||||
pub fn remove_token_file(path: &Path, token: &str) {
|
||||
if read_token(path).is_some_and(|held| tokens_match(&held, token.trim())) {
|
||||
let _ = std::fs::remove_file(path);
|
||||
let _ = std::fs::remove_file(pid_path(path));
|
||||
}
|
||||
}
|
||||
|
||||
/// The token a token file holds, trimmed; `None` when there is no readable token file.
|
||||
fn read_token(path: &Path) -> Option<String> {
|
||||
let mut text = String::new();
|
||||
std::fs::File::open(path).ok()?.take(MAX_TOKEN_FILE + 1).read_to_string(&mut text).ok()?;
|
||||
let token = text.trim();
|
||||
(text.len() as u64 <= MAX_TOKEN_FILE && !token.is_empty()).then(|| token.to_string())
|
||||
}
|
||||
|
||||
/// Whether the token file at `path` holds a token other than `token`: another launch's.
|
||||
fn held_by_another(path: &Path, token: &str) -> bool {
|
||||
read_token(path).is_some_and(|held| !tokens_match(&held, token.trim()))
|
||||
}
|
||||
|
||||
/// How long `icacls` may take (#6145). It finishes at once; on some computers the new process
|
||||
/// is left suspended (security software holding it), and waiting on it without a limit froze
|
||||
/// the app.
|
||||
#[cfg(windows)]
|
||||
const ICACLS_LIMIT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// How a run of `icacls` ended.
|
||||
#[cfg(any(windows, test))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum IcaclsEnd {
|
||||
Succeeded,
|
||||
Failed,
|
||||
/// It didn't finish within [`ICACLS_LIMIT`] and was ended.
|
||||
TimedOut,
|
||||
}
|
||||
|
||||
/// Whether a token may go into a file after `icacls` ended this way. `custom_folder`: the file is
|
||||
/// in the folder the app's `CONTROL_DIR` names. A timeout is accepted only in the default folder,
|
||||
/// the user's own local app data, which other users can't read by default; nothing is known about
|
||||
/// the permissions of any other folder.
|
||||
#[cfg(any(windows, test))]
|
||||
fn restriction_result(end: IcaclsEnd, custom_folder: bool, path: &Path) -> Result<()> {
|
||||
match end {
|
||||
IcaclsEnd::Succeeded => Ok(()),
|
||||
IcaclsEnd::TimedOut if !custom_folder => Ok(()),
|
||||
IcaclsEnd::TimedOut => Err(Error::new(ErrorKind::TimedOut, format!("icacls did not finish in time to restrict {}", path.display()))),
|
||||
IcaclsEnd::Failed => Err(Error::other(format!("icacls failed to restrict {}", path.display()))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Windows: drop inherited entries and grant only the current user full access. An `icacls`
|
||||
/// that doesn't finish in time is ended, and [`restriction_result`] decides what that means.
|
||||
#[cfg(windows)]
|
||||
fn restrict_to_user(path: &Path, custom_folder: bool) -> Result<()> {
|
||||
use std::os::windows::process::CommandExt;
|
||||
let user = std::env::var("USERNAME").map_err(|_| Error::other("USERNAME is not set"))?;
|
||||
let domain = std::env::var("USERDOMAIN").ok().filter(|d| !d.is_empty());
|
||||
let who = domain.map_or(user.clone(), |d| format!("{d}\\{user}"));
|
||||
let child = std::process::Command::new("icacls")
|
||||
.arg(path)
|
||||
.args(["/inheritance:r", "/grant:r", &format!("{who}:F")])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.creation_flags(0x0800_0000)
|
||||
.spawn()?;
|
||||
let end = match wait_within(child, ICACLS_LIMIT)? {
|
||||
Some(status) if status.success() => IcaclsEnd::Succeeded,
|
||||
Some(_) => IcaclsEnd::Failed,
|
||||
None => IcaclsEnd::TimedOut,
|
||||
};
|
||||
restriction_result(end, custom_folder, path)
|
||||
}
|
||||
|
||||
/// Wait for `child` at most `limit`: its exit status, or `None` (and the child ended) when it
|
||||
/// hasn't finished by then.
|
||||
#[cfg(any(windows, test))]
|
||||
fn wait_within(mut child: std::process::Child, limit: std::time::Duration) -> Result<Option<std::process::ExitStatus>> {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Ok(Some(status));
|
||||
}
|
||||
if start.elapsed() >= limit {
|
||||
// Not waited for afterwards: a suspended process may not end at once.
|
||||
let _ = child.kill();
|
||||
return Ok(None);
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the token file (path, token) when the app ends, while it still holds that token.
|
||||
pub struct TokenFile(pub PathBuf, pub String);
|
||||
|
||||
impl Drop for TokenFile {
|
||||
fn drop(&mut self) {
|
||||
remove_token_file(&self.0, &self.1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// An app with no data folder and no environment, whose times read `t<seconds>`.
|
||||
const TEST: App = App { name: "TestCraft", data_dir: || None, env: |_| None, utc: |secs| format!("t{secs}"), default_port: 27878 };
|
||||
|
||||
/// #6145: a child that doesn't finish in time is given up on, not waited for forever.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_slow_child_is_waited_for_only_so_long() {
|
||||
use std::time::{Duration, Instant};
|
||||
let start = Instant::now();
|
||||
let slow = std::process::Command::new("sleep").arg("30").spawn().unwrap();
|
||||
assert!(wait_within(slow, Duration::from_millis(200)).unwrap().is_none());
|
||||
assert!(start.elapsed() < Duration::from_secs(5), "{:?}", start.elapsed());
|
||||
let quick = std::process::Command::new("true").spawn().unwrap();
|
||||
assert!(wait_within(quick, Duration::from_secs(5)).unwrap().is_some_and(|s| s.success()));
|
||||
let failing = std::process::Command::new("false").spawn().unwrap();
|
||||
assert!(wait_within(failing, Duration::from_secs(5)).unwrap().is_some_and(|s| !s.success()));
|
||||
}
|
||||
|
||||
/// #6145's freeze fix stays in the default folder, the user's own; a folder picked with
|
||||
/// the app's `CONTROL_DIR` gets no token unless the file was really restricted.
|
||||
#[test]
|
||||
fn only_the_default_folder_accepts_an_icacls_that_timed_out() {
|
||||
let path = Path::new("control/7878.token");
|
||||
for custom in [false, true] {
|
||||
assert!(restriction_result(IcaclsEnd::Succeeded, custom, path).is_ok());
|
||||
assert!(restriction_result(IcaclsEnd::Failed, custom, path).is_err());
|
||||
}
|
||||
assert!(restriction_result(IcaclsEnd::TimedOut, false, path).is_ok());
|
||||
let e = restriction_result(IcaclsEnd::TimedOut, true, path).unwrap_err();
|
||||
assert_eq!(e.kind(), ErrorKind::TimedOut);
|
||||
assert!(e.to_string().contains("7878.token"), "{e}");
|
||||
}
|
||||
|
||||
/// A file that couldn't be restricted is removed, empty and with no process file beside it, and
|
||||
/// the error goes to the caller. A restricted one gets its token.
|
||||
#[test]
|
||||
fn a_token_file_that_could_not_be_restricted_is_removed() {
|
||||
let dir = std::env::temp_dir().join(format!("craft-control-restrict-{}", std::process::id()));
|
||||
let path = dir.join("7879.token");
|
||||
let e = write_token_file_with(&path, "secret-token", |p| {
|
||||
assert_eq!(std::fs::read(p).unwrap(), b"", "restricted before the token goes in");
|
||||
restriction_result(IcaclsEnd::TimedOut, true, p)
|
||||
})
|
||||
.unwrap_err();
|
||||
assert_eq!(e.kind(), ErrorKind::TimedOut);
|
||||
assert!(!path.exists() && !pid_path(&path).exists());
|
||||
write_token_file_with(&path, "secret-token", |p| restriction_result(IcaclsEnd::TimedOut, false, p)).unwrap();
|
||||
assert_eq!(read_token(&path).as_deref(), Some("secret-token"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_compare_whole() {
|
||||
assert!(tokens_match("abc123", "abc123"));
|
||||
assert!(!tokens_match("abc123", "abc124"));
|
||||
assert!(!tokens_match("abc123", "abc12"));
|
||||
assert!(!tokens_match("", "a"));
|
||||
}
|
||||
|
||||
/// The preference starts the listener by default, off it doesn't, and a port given on the
|
||||
/// command line or in the environment wins over it either way.
|
||||
#[test]
|
||||
fn the_preference_decides_unless_a_port_is_given() {
|
||||
assert_eq!(plan(None, true), Some(Listen::Default));
|
||||
assert_eq!(plan(None, false), None);
|
||||
assert_eq!(plan(Some(7878), false), Some(Listen::Exact(7878)));
|
||||
assert_eq!(plan(Some(7878), true), Some(Listen::Exact(7878)));
|
||||
}
|
||||
|
||||
/// A taken port falls through to the next free one, on loopback only; a run of taken ports
|
||||
/// is an error, not a hang.
|
||||
#[test]
|
||||
fn a_taken_port_falls_back_to_the_next_free_one() {
|
||||
let first = bind_with_fallback(TEST.default_port, PORT_TRIES).unwrap();
|
||||
let a = first.local_addr().unwrap();
|
||||
assert!(a.ip().is_loopback() && (TEST.default_port..TEST.default_port + PORT_TRIES).contains(&a.port()), "{a}");
|
||||
let second = bind_with_fallback(a.port(), PORT_TRIES).unwrap();
|
||||
assert!(second.local_addr().unwrap().port() > a.port());
|
||||
assert!(bind_with_fallback(a.port(), 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_discovery_file_holds_a_port() {
|
||||
assert_eq!(parse_port("27879\n"), Some(27879));
|
||||
assert_eq!(parse_port("0"), None);
|
||||
assert_eq!(parse_port("70000"), None);
|
||||
assert_eq!(parse_port("nope"), None);
|
||||
let dir = std::env::temp_dir().join(format!("solidcraft-current-port-{}", std::process::id()));
|
||||
let path = dir.join("current.port");
|
||||
TEST.write_token_file(&path, "27881").unwrap();
|
||||
assert_eq!(parse_port(&std::fs::read_to_string(&path).unwrap()), Some(27881));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A one-line token file (and one with no process file) still reads.
|
||||
#[test]
|
||||
fn a_token_file_reads_with_and_without_its_process_file() {
|
||||
let dir = std::env::temp_dir().join(format!("craft-control-read-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("7879.token");
|
||||
std::fs::write(&path, "legacy-token\n").unwrap();
|
||||
let client = read_token_file(&path, TEST).unwrap();
|
||||
assert_eq!(client.token, "legacy-token");
|
||||
let src = client.file.as_ref().unwrap();
|
||||
assert_eq!(src.path, std::fs::canonicalize(&path).unwrap());
|
||||
assert!(src.modified.is_some() && src.writer.is_none());
|
||||
std::fs::write(pid_path(&path), "not a pid\n").unwrap();
|
||||
assert!(read_token_file(&path, TEST).unwrap().file.unwrap().writer.is_none());
|
||||
std::fs::write(&path, "x".repeat(4097)).unwrap();
|
||||
assert!(read_token_file(&path, TEST).is_none());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A refused token's explanation names the app and writes the time as the app does.
|
||||
#[test]
|
||||
fn an_explanation_names_the_app() {
|
||||
let file = Some(TokenSource { path: PathBuf::from("7878.token"), modified: Some(UNIX_EPOCH), writer: None });
|
||||
let client = ClientToken { token: "secret-token".into(), file, app: TEST };
|
||||
let error = client.explain_error("unauthorized");
|
||||
assert!(error.contains("written t0 (") && error.contains("if TestCraft was force-quit"), "{error}");
|
||||
assert!(!error.contains("secret-token"), "{error}");
|
||||
let explicit = TEST.client_token(0, Some("explicit-token")).unwrap();
|
||||
assert!(explicit.token == "explicit-token" && explicit.file.is_none());
|
||||
assert_eq!(explicit.explain_error("unauthorized"), "unauthorized");
|
||||
}
|
||||
|
||||
/// A process file naming a process that has exited marks the token file stale.
|
||||
/// Where the process inspector cannot run (a sandboxed test runner), the exit
|
||||
/// is unknown, and the explanation must not claim it.
|
||||
#[cfg(any(unix, windows))]
|
||||
#[test]
|
||||
fn a_token_file_from_an_exited_process_is_stale() {
|
||||
let mut child = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "auth::tests::tokens_compare_whole"])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let pid = child.id();
|
||||
assert!(child.wait().unwrap().success());
|
||||
let source = |writer| Some(TokenSource { path: PathBuf::from("7878.token"), modified: None, writer });
|
||||
let stale = ClientToken { token: "secret-token".into(), file: source(Some((pid, None))), app: TEST };
|
||||
let error = stale.explain_error("unauthorized");
|
||||
match process_running(pid) {
|
||||
Some(false) => {
|
||||
assert!(error.contains(&format!("stale token file from an instance that exited (process {pid}): 7878.token")), "{error}");
|
||||
}
|
||||
None => assert!(!error.contains("stale") && error.contains("restart it or delete that file"), "{error}"),
|
||||
Some(true) => panic!("the reaped child {pid} is reported as running"),
|
||||
}
|
||||
assert!(!error.contains("secret-token"));
|
||||
let live = ClientToken { token: "t".into(), file: source(Some((std::process::id(), process_start(std::process::id())))), app: TEST };
|
||||
assert!(!live.explain_error("unauthorized").contains("stale"));
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let reused = process_start(std::process::id()).map(|s| s + 1);
|
||||
let reused = ClientToken { token: "t".into(), file: source(Some((std::process::id(), reused))), app: TEST };
|
||||
assert!(reused.explain_error("unauthorized").contains("stale token file"));
|
||||
}
|
||||
}
|
||||
|
||||
/// The token file is the user's alone, and a client reads it back.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn the_token_file_is_private_to_the_user() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let dir = std::env::temp_dir().join(format!("craft-control-auth-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let path = dir.join("control/7878.token");
|
||||
std::fs::create_dir_all(dir.join("control")).unwrap();
|
||||
std::fs::write(&path, "stale").unwrap();
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
|
||||
TEST.write_token_file(&path, "0123456789abcdef0123456789abcdef").unwrap();
|
||||
let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777;
|
||||
assert_eq!(mode(&path), 0o600);
|
||||
assert_eq!(mode(path.parent().unwrap()), 0o700);
|
||||
assert_eq!(std::fs::read_to_string(&path).unwrap().trim(), "0123456789abcdef0123456789abcdef");
|
||||
let writer = read_token_file(&path, TEST).unwrap().file.unwrap().writer;
|
||||
assert_eq!(writer.map(|(pid, _)| pid), Some(std::process::id()));
|
||||
drop(TokenFile(path.clone(), "0123456789abcdef0123456789abcdef".into()));
|
||||
assert!(!path.exists() && !pid_path(&path).exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// #8794: under Flatpak two windows can listen on the same port number (each has its own
|
||||
/// loopback) and write the same files. The one closing first leaves the other's token, process
|
||||
/// and discovery files; the last one takes them all away, `current.pid` too.
|
||||
#[test]
|
||||
fn a_closing_launch_leaves_another_launchs_files() {
|
||||
let dir = std::env::temp_dir().join(format!("craft-control-two-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let (file, current) = (dir.join("27878.token"), dir.join("current.port"));
|
||||
let (first, second) = ("a".repeat(64), "b".repeat(64));
|
||||
// The first window, then the second on the same port, each writing both files.
|
||||
for token in [&first, &second] {
|
||||
TEST.write_token_file(&file, token).unwrap();
|
||||
TEST.write_token_file(¤t, "27878").unwrap();
|
||||
}
|
||||
// The first closes: the second's files stay.
|
||||
remove_token_file(&file, &first);
|
||||
remove_discovery(¤t, &file, 27878, &first);
|
||||
assert_eq!(read_token(&file).as_deref(), Some(second.as_str()));
|
||||
assert!(pid_path(&file).exists() && current.exists() && pid_path(¤t).exists());
|
||||
// Another port named in the discovery file is another app's too.
|
||||
remove_discovery(¤t, &file, 27879, &second);
|
||||
assert!(current.exists());
|
||||
// The second closes: nothing of either is left.
|
||||
remove_token_file(&file, &format!(" {second}\n"));
|
||||
remove_discovery(¤t, &file, 27878, &second);
|
||||
assert!(!file.exists() && !pid_path(&file).exists() && !current.exists() && !pid_path(¤t).exists());
|
||||
// With no token file left (removed by hand, or never written), the discovery file of the
|
||||
// port still goes; a missing file is no error.
|
||||
TEST.write_token_file(¤t, "27878").unwrap();
|
||||
remove_token_file(&file, &second);
|
||||
remove_discovery(¤t, &file, 27878, &second);
|
||||
assert!(!current.exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//! Where a control request is, so the connection that sent it can cancel it (`control.cancel`,
|
||||
//! #1989): shared between the window, which moves it along, and the connection. Whoever changes
|
||||
//! it first wins.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
|
||||
/// A request's place in the app, and how to stop the work computing it.
|
||||
#[derive(Default)]
|
||||
pub struct Ticket {
|
||||
state: AtomicU8,
|
||||
/// It is an export computing on a copy, whose worker writes its file.
|
||||
writes_file: AtomicBool,
|
||||
/// Tells its worker to stop: set by the window when it hands the request to one.
|
||||
stopper: Mutex<Option<Box<dyn Fn() + Send + Sync>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Ticket {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Ticket").field("state", &self.state).field("writes_file", &self.writes_file).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Waiting its turn (sent, or held behind another request).
|
||||
const WAITING: u8 = 0;
|
||||
/// Running on the window thread (or handed to something that can't stop it, such as the file
|
||||
/// loader).
|
||||
const RUNNING: u8 = 1;
|
||||
/// Computing off the window thread.
|
||||
const COMPUTING: u8 = 2;
|
||||
const CANCELLED: u8 = 3;
|
||||
|
||||
/// What [`Ticket::cancel`] did.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Cancel {
|
||||
/// It was waiting its turn: it will not run.
|
||||
Removed,
|
||||
/// It was computing off the window thread: it is answered as stopped and nothing of it is
|
||||
/// applied. An export's worker may still write its file.
|
||||
Stopped { writes_file: bool },
|
||||
/// It runs on the window thread: it will finish.
|
||||
Running,
|
||||
/// It was cancelled or stopped before.
|
||||
Already,
|
||||
}
|
||||
|
||||
impl Ticket {
|
||||
/// Cancel the request, unless it already runs on the window thread.
|
||||
pub fn cancel(&self) -> Cancel {
|
||||
let mut state = self.state.load(Ordering::Acquire);
|
||||
loop {
|
||||
let done = match state {
|
||||
WAITING => Cancel::Removed,
|
||||
COMPUTING => Cancel::Stopped { writes_file: self.writes_file.load(Ordering::Acquire) },
|
||||
CANCELLED => return Cancel::Already,
|
||||
_ => return Cancel::Running,
|
||||
};
|
||||
match self.state.compare_exchange(state, CANCELLED, Ordering::AcqRel, Ordering::Acquire) {
|
||||
Ok(_) => {
|
||||
self.stop_worker();
|
||||
return done;
|
||||
}
|
||||
Err(now) => state = now,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.state.load(Ordering::Acquire) == CANCELLED
|
||||
}
|
||||
|
||||
/// The window takes the request to run it: false when it was cancelled.
|
||||
pub fn take(&self) -> bool {
|
||||
self.moves(WAITING, RUNNING)
|
||||
}
|
||||
|
||||
/// The work computing it stops on `stop` (a worker's own stop flag). Cancelled already, it is
|
||||
/// stopped at once.
|
||||
pub fn on_stop(&self, stop: impl Fn() + Send + Sync + 'static) {
|
||||
*self.stopper.lock().unwrap_or_else(PoisonError::into_inner) = Some(Box::new(stop));
|
||||
if self.is_cancelled() {
|
||||
self.stop_worker();
|
||||
}
|
||||
}
|
||||
|
||||
/// The window hands it to a worker (`writes_file`: an export on a copy).
|
||||
pub fn computing(&self, writes_file: bool) {
|
||||
self.writes_file.store(writes_file, Ordering::Release);
|
||||
let _ = self.moves(RUNNING, COMPUTING) || self.moves(WAITING, COMPUTING);
|
||||
}
|
||||
|
||||
/// Its worker is done and it runs on the window thread now (or it never left): false when it
|
||||
/// was cancelled meanwhile. Only the window moves it on from running, so running stays running.
|
||||
pub fn back(&self) -> bool {
|
||||
self.moves(COMPUTING, RUNNING) || self.state.load(Ordering::Acquire) == RUNNING
|
||||
}
|
||||
|
||||
/// Computed, but it waits its turn again (a file opening meanwhile): false when cancelled.
|
||||
pub fn wait_again(&self) -> bool {
|
||||
self.moves(COMPUTING, WAITING)
|
||||
}
|
||||
|
||||
/// The person at the app stopped it (Stop): its worker stops too.
|
||||
pub fn stop(&self) {
|
||||
self.state.store(CANCELLED, Ordering::Release);
|
||||
self.stop_worker();
|
||||
}
|
||||
|
||||
fn stop_worker(&self) {
|
||||
if let Some(stop) = self.stopper.lock().unwrap_or_else(PoisonError::into_inner).as_ref() {
|
||||
stop();
|
||||
}
|
||||
}
|
||||
|
||||
fn moves(&self, from: u8, to: u8) -> bool {
|
||||
self.state.compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire).is_ok()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,616 @@
|
||||
//! The control channel's client (docs/control-protocol.md): a connection to an app's loopback
|
||||
//! control port, with the app's token ([`App::client_token`]) on every request, one bounded
|
||||
//! round trip per request, a fresh connection after one is lost, and the wait for a request the
|
||||
//! app says is still running (#1618), through `control.pending`. Its errors tell an app that is
|
||||
//! not running from one that went away during a call ([`CallError`]). What a request means, and
|
||||
//! anything an app adds to its parameters, is the caller's.
|
||||
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::Effect;
|
||||
use crate::auth::App;
|
||||
|
||||
/// Longest reply line accepted from the app.
|
||||
pub const MAX_REPLY: u64 = 64 << 20;
|
||||
|
||||
/// How long a reply may take: kernel operations (booleans, fillets) on big parts can take a while.
|
||||
pub const READ_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// How long the app may take to answer a request sent [`Request::once`]: a little longer than its
|
||||
/// control server waits for a long request (3600 s), so its "still running" reply arrives first.
|
||||
pub const ONCE_TIMEOUT: Duration = Duration::from_secs(3720);
|
||||
|
||||
/// How long connecting to the app may take.
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_millis(800);
|
||||
|
||||
/// How often a long wait is reported ([`Connection::with_progress`]).
|
||||
const PROGRESS_EVERY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// How an error starts when a request is still running in the app after the wait: the request
|
||||
/// was not cancelled and its reply waits on the connection; [`Connection::pending`] gets it.
|
||||
pub use crate::STILL_RUNNING;
|
||||
|
||||
/// How long to wait for a request the app says is still running.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Waits {
|
||||
/// How long a call goes on waiting, asking `control.pending`, after the app said its request
|
||||
/// is still running (the app says so after 60 s), before it says so itself.
|
||||
pub pending: Duration,
|
||||
/// How long a later call, or [`Connection::pending`], waits for a request still running from
|
||||
/// an earlier call.
|
||||
pub settle: Duration,
|
||||
/// How often `control.pending` is asked while waiting.
|
||||
pub poll: Duration,
|
||||
}
|
||||
|
||||
impl Default for Waits {
|
||||
fn default() -> Self {
|
||||
Waits { pending: Duration::from_secs(240), settle: Duration::from_secs(60), poll: Duration::from_millis(500) }
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a call has no result. Each holds the message for the caller (an agent reads it).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum CallError {
|
||||
/// Nothing listens at the address: the app is not running, or it quit before the call was
|
||||
/// sent. The message says how to start it.
|
||||
NotRunning(String),
|
||||
/// The app went away or did not answer during the call, after running it or not.
|
||||
Lost(String),
|
||||
/// The request is still running in the app after the wait (it starts with [`STILL_RUNNING`]),
|
||||
/// or an earlier one is and this one was not sent.
|
||||
StillRunning(String),
|
||||
/// The app answered with an error (a refused token says which token file it came from).
|
||||
Refused(String),
|
||||
/// The reply was not a control reply.
|
||||
BadReply(String),
|
||||
}
|
||||
|
||||
impl CallError {
|
||||
/// The message.
|
||||
pub fn message(&self) -> &str {
|
||||
match self {
|
||||
CallError::NotRunning(m) | CallError::Lost(m) | CallError::StillRunning(m) | CallError::Refused(m) | CallError::BadReply(m) => m,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CallError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.message())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for CallError {}
|
||||
|
||||
impl From<CallError> for String {
|
||||
fn from(e: CallError) -> String {
|
||||
e.message().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// A request: its method and parameters, how messages name it and what it does if it is still
|
||||
/// running when its call returns.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Request {
|
||||
pub method: String,
|
||||
pub params: Value,
|
||||
/// The request in messages ("`engine.execute` (solid.fillet)").
|
||||
pub what: String,
|
||||
/// What it still does when it finishes after its wait.
|
||||
pub effect: Effect,
|
||||
/// Never sent twice: on a fresh connection, waiting up to [`ONCE_TIMEOUT`], and once it is
|
||||
/// being written any error is final, as the app may have run it. Otherwise a request that
|
||||
/// finds its connection lost is sent once more on a fresh one (the app may have restarted).
|
||||
pub once: bool,
|
||||
}
|
||||
|
||||
impl Request {
|
||||
/// `method` with `params`, named "`method`" in messages, taken to change the document, sent
|
||||
/// again once on a fresh connection when its connection was lost.
|
||||
pub fn new(method: &str, params: Value) -> Self {
|
||||
Request { method: method.to_string(), params, what: format!("`{method}`"), effect: Effect::Changes, once: false }
|
||||
}
|
||||
}
|
||||
|
||||
/// A control reply as a call's result: its `result`, or its `error`.
|
||||
pub fn outcome(v: &Value) -> Result<Value, String> {
|
||||
if v.get("ok").and_then(Value::as_bool) == Some(true) {
|
||||
Ok(v.get("result").cloned().unwrap_or(Value::Null))
|
||||
} else {
|
||||
Err(v.get("error").and_then(Value::as_str).unwrap_or("error").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// A request the app said is still running. Its reply will come on this connection, through
|
||||
/// `control.pending`, so the connection is kept for it.
|
||||
struct Outstanding {
|
||||
what: String,
|
||||
/// What it does when it finishes; `None`: not known (a request this client did not send).
|
||||
effect: Option<Effect>,
|
||||
since: Instant,
|
||||
}
|
||||
|
||||
impl Outstanding {
|
||||
fn secs(&self) -> u64 {
|
||||
self.since.elapsed().as_secs()
|
||||
}
|
||||
}
|
||||
|
||||
/// What became of a request still running when its call returned, once a later call waited it
|
||||
/// out: kept for [`Connection::pending`].
|
||||
enum Settled {
|
||||
Finished { what: String, reply: Value },
|
||||
Lost { what: String },
|
||||
}
|
||||
|
||||
/// The end of a wait for an outstanding request.
|
||||
enum Waited {
|
||||
/// It finished: its own reply.
|
||||
Done(Value),
|
||||
/// The app holds nothing (any more) for this connection.
|
||||
Nothing,
|
||||
/// Still running when the wait ran out.
|
||||
Running,
|
||||
}
|
||||
|
||||
/// A connection to the app listening at an address. It connects when a call needs it, so a
|
||||
/// client started before the app reaches it once it runs.
|
||||
pub struct Connection {
|
||||
app: App,
|
||||
/// What the app calls its document in messages ("design").
|
||||
document: &'static str,
|
||||
addr: String,
|
||||
port: u16,
|
||||
conn: Option<(BufReader<TcpStream>, TcpStream)>,
|
||||
/// Counts the connections made, so a caller can tell a fresh one from the one it asked.
|
||||
serial: u64,
|
||||
next_id: u64,
|
||||
/// The token given, when one was. Otherwise each request looks the token up again, since
|
||||
/// every launch of the app makes a new one.
|
||||
explicit_token: Option<String>,
|
||||
/// A request still running in the app; its reply waits on `conn`.
|
||||
outstanding: Option<Outstanding>,
|
||||
/// An outstanding request's end, once a later call waited for it (for `pending`).
|
||||
settled: Option<Settled>,
|
||||
waits: Waits,
|
||||
/// Where a long wait is reported.
|
||||
progress: fn(&str),
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
/// A connection to `app` at `addr` (`host:port`), with `token` when given, else the app's
|
||||
/// token found the usual way ([`App::client_token`]); not connected yet.
|
||||
pub fn new(app: App, addr: &str, token: Option<&str>) -> Self {
|
||||
let port = addr.rsplit(':').next().and_then(|p| p.parse::<u16>().ok()).unwrap_or(0);
|
||||
Self {
|
||||
app,
|
||||
document: "design",
|
||||
addr: addr.to_string(),
|
||||
port,
|
||||
conn: None,
|
||||
serial: 0,
|
||||
next_id: 1,
|
||||
explicit_token: token.map(str::to_string),
|
||||
outstanding: None,
|
||||
settled: None,
|
||||
waits: Waits::default(),
|
||||
progress: |_| {},
|
||||
}
|
||||
}
|
||||
|
||||
/// What the app calls its document in messages ("design", the default).
|
||||
#[must_use]
|
||||
pub fn with_document(mut self, document: &'static str) -> Self {
|
||||
self.document = document;
|
||||
self
|
||||
}
|
||||
|
||||
/// Report a long wait for a request still running to `progress`, every 30 s.
|
||||
#[must_use]
|
||||
pub fn with_progress(mut self, progress: fn(&str)) -> Self {
|
||||
self.progress = progress;
|
||||
self
|
||||
}
|
||||
|
||||
/// How long to wait for a request still running.
|
||||
pub fn set_waits(&mut self, waits: Waits) {
|
||||
self.waits = waits;
|
||||
}
|
||||
|
||||
/// The address it connects to.
|
||||
pub fn addr(&self) -> &str {
|
||||
&self.addr
|
||||
}
|
||||
|
||||
/// The port of the address (0 when it names none).
|
||||
pub fn port(&self) -> u16 {
|
||||
self.port
|
||||
}
|
||||
|
||||
/// The token given, if one was.
|
||||
pub fn explicit_token(&self) -> Option<&str> {
|
||||
self.explicit_token.as_deref()
|
||||
}
|
||||
|
||||
/// Whether a connection is open now.
|
||||
pub fn is_connected(&self) -> bool {
|
||||
self.conn.is_some()
|
||||
}
|
||||
|
||||
/// The number of connections made so far: the same number while the same connection stays
|
||||
/// open.
|
||||
pub fn serial(&self) -> u64 {
|
||||
self.serial
|
||||
}
|
||||
|
||||
/// Whether a request of this connection is still running in the app.
|
||||
pub fn has_outstanding(&self) -> bool {
|
||||
self.outstanding.is_some()
|
||||
}
|
||||
|
||||
/// Close the connection. A request still running loses its reply with it.
|
||||
pub fn disconnect(&mut self) {
|
||||
self.conn = None;
|
||||
if let Some(o) = self.outstanding.take() {
|
||||
self.settled = Some(Settled::Lost { what: o.what });
|
||||
}
|
||||
}
|
||||
|
||||
/// Connect afresh. Failing to reach the app at all is `NotConnected`, so [`Connection::explain`]
|
||||
/// can tell it from a connection lost during a call.
|
||||
///
|
||||
/// # Errors
|
||||
/// `NotConnected` when nothing listens at the address (or it does not resolve).
|
||||
pub fn connect(&mut self) -> std::io::Result<()> {
|
||||
self.disconnect();
|
||||
let not_connected = |e: std::io::Error| std::io::Error::new(std::io::ErrorKind::NotConnected, e);
|
||||
let mut last = std::io::Error::new(std::io::ErrorKind::NotFound, format!("cannot resolve {}", self.addr));
|
||||
for sa in self.addr.to_socket_addrs().map_err(not_connected)? {
|
||||
match TcpStream::connect_timeout(&sa, CONNECT_TIMEOUT) {
|
||||
Ok(s) => {
|
||||
s.set_nodelay(true).ok();
|
||||
s.set_read_timeout(Some(READ_TIMEOUT)).ok();
|
||||
let read = s.try_clone()?;
|
||||
self.conn = Some((BufReader::new(read), s));
|
||||
self.serial += 1;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => last = e,
|
||||
}
|
||||
}
|
||||
Err(not_connected(last))
|
||||
}
|
||||
|
||||
/// See whether the app is there now; the error says why not. The connection doesn't stay
|
||||
/// open: the app closes one that shows no token within a few seconds, and the first call may
|
||||
/// be long away. That call connects again.
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`Connection::connect`].
|
||||
pub fn try_connect(&mut self) -> std::io::Result<()> {
|
||||
self.connect()?;
|
||||
self.disconnect();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The error for a failed exchange: whether the app is not running (and how to start it) or
|
||||
/// went away during the call.
|
||||
pub fn explain(&self, e: &std::io::Error) -> CallError {
|
||||
let (addr, port, name, document) = (&self.addr, self.port, self.app.name, self.document);
|
||||
let start = format!("{} --control {port}", name.to_lowercase());
|
||||
if e.kind() == std::io::ErrorKind::NotConnected {
|
||||
CallError::NotRunning(format!(
|
||||
"no {name} app is listening at {addr} ({e}): it is not running, or it quit. Start it with `{start}` (or ask the user to); the \
|
||||
next call connects to it"
|
||||
))
|
||||
} else {
|
||||
CallError::Lost(format!(
|
||||
"the {name} app at {addr} did not answer ({e}): it may have quit or crashed during this call, after running it or not. Inspect \
|
||||
the {document} before sending a change again; if the app quit, start it with `{start}`"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// The request line for `method`: a fresh id and the app's token (looked up again for every
|
||||
/// request: an app restarted mid-session has a new one).
|
||||
pub fn request_line(&mut self, method: &str, params: Value) -> String {
|
||||
let id = self.next_id;
|
||||
self.next_id += 1;
|
||||
let mut msg = json!({"id": id, "method": method, "params": params});
|
||||
if let Some(t) = self.app.client_token(self.port, self.explicit_token.as_deref()) {
|
||||
msg["token"] = json!(t.token);
|
||||
}
|
||||
msg.to_string()
|
||||
}
|
||||
|
||||
/// One request line out and one reply line back, at most [`MAX_REPLY`] long, connecting
|
||||
/// first when not connected. Any failure closes the connection (a timed-out read leaves a
|
||||
/// late reply on the stream).
|
||||
fn roundtrip(&mut self, line: &str) -> std::io::Result<String> {
|
||||
if self.conn.is_none() {
|
||||
self.connect()?;
|
||||
}
|
||||
let Some((reader, writer)) = self.conn.as_mut() else { return Err(std::io::Error::other("not connected")) };
|
||||
// One write: a request that loses its newline still reaches the app as a request.
|
||||
let sent = writer.write_all(format!("{line}\n").as_bytes()).and_then(|()| writer.flush());
|
||||
if let Err(e) = sent {
|
||||
self.disconnect();
|
||||
return Err(e);
|
||||
}
|
||||
let mut reply = String::new();
|
||||
let n = match reader.by_ref().take(MAX_REPLY).read_line(&mut reply) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
self.disconnect();
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
if n == 0 || !reply.ends_with('\n') {
|
||||
self.disconnect();
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "connection closed"));
|
||||
}
|
||||
Ok(reply)
|
||||
}
|
||||
|
||||
fn set_read_timeout(&self, t: Duration) {
|
||||
if let Some((_, writer)) = &self.conn {
|
||||
writer.set_read_timeout(Some(t)).ok();
|
||||
}
|
||||
}
|
||||
|
||||
/// A reply line as JSON.
|
||||
fn parse(reply: &str) -> Result<Value, CallError> {
|
||||
serde_json::from_str(reply).map_err(|e| CallError::BadReply(format!("bad reply from the app: {e}")))
|
||||
}
|
||||
|
||||
/// A control reply as a call's result; a refused token says which token file it came from
|
||||
/// (and whether it is stale).
|
||||
///
|
||||
/// # Errors
|
||||
/// [`CallError::Refused`] with the app's error.
|
||||
pub fn outcome(&self, v: &Value) -> Result<Value, CallError> {
|
||||
outcome(v).map_err(|error| {
|
||||
CallError::Refused(
|
||||
self.app.client_token(self.port, self.explicit_token.as_deref()).map_or_else(|| error.clone(), |t| t.explain_error(&error)),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// `method` with `params`, at once and once: not behind a request still running on this
|
||||
/// connection, no second try. The app's whole reply (`ok`, `result` or `error`, and anything
|
||||
/// it adds), for a request the app answers at once on any connection
|
||||
/// (`control.capabilities`, `control.running`).
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`Connection::explain`]; [`CallError::BadReply`].
|
||||
pub fn ask(&mut self, method: &str, params: Value) -> Result<Value, CallError> {
|
||||
let line = self.request_line(method, params);
|
||||
let reply = self.roundtrip(&line).map_err(|e| self.explain(&e))?;
|
||||
Self::parse(&reply)
|
||||
}
|
||||
|
||||
/// `method` with `params`, waiting at most `timeout` for the reply and never behind a request
|
||||
/// still running: for a request the app answers within a frame, asked again later when the
|
||||
/// app is busy. One try.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`CallError::StillRunning`] when a request of this connection is still running; as
|
||||
/// [`Connection::ask`] and [`Connection::outcome`].
|
||||
pub fn quick(&mut self, method: &str, params: Value, timeout: Duration) -> Result<Value, CallError> {
|
||||
if self.outstanding.is_some() {
|
||||
return Err(CallError::StillRunning("a request is still running in the app".into()));
|
||||
}
|
||||
if self.conn.is_none() {
|
||||
self.connect().map_err(|e| self.explain(&e))?;
|
||||
}
|
||||
let line = self.request_line(method, params);
|
||||
self.set_read_timeout(timeout);
|
||||
let reply = self.roundtrip(&line);
|
||||
self.set_read_timeout(READ_TIMEOUT);
|
||||
let reply = reply.map_err(|e| self.explain(&e))?;
|
||||
let v = Self::parse(&reply)?;
|
||||
self.outcome(&v)
|
||||
}
|
||||
|
||||
/// Ask the app, on this connection only, about the request it still runs for it.
|
||||
fn ask_pending(&mut self) -> Result<Value, CallError> {
|
||||
if self.conn.is_none() {
|
||||
self.disconnect();
|
||||
return Err(CallError::Lost("the connection to the app was lost".into()));
|
||||
}
|
||||
let v = self.ask("control.pending", json!({}))?;
|
||||
outcome(&v).map_err(CallError::Refused).inspect_err(|_| self.disconnect())
|
||||
}
|
||||
|
||||
/// `control.cancel` for the request still running on this connection. Sent at once, without
|
||||
/// waiting for that request first, and never on a fresh connection, which has nothing to
|
||||
/// cancel. Once the app no longer holds it (removed, stopped or finished), later calls don't
|
||||
/// wait for it.
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`Connection::ask`]; [`CallError::Refused`].
|
||||
pub fn cancel(&mut self, params: Value) -> Result<Value, CallError> {
|
||||
if self.conn.is_none() {
|
||||
return Ok(json!({"cancelled": null, "note": "nothing to cancel: no request of this client is running in the app"}));
|
||||
}
|
||||
let v = self.ask("control.cancel", params)?;
|
||||
let r = outcome(&v).map_err(CallError::Refused);
|
||||
if r.is_ok() {
|
||||
self.outstanding = None;
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
/// Wait up to `limit` for the outstanding request, asking `control.pending` now and then and
|
||||
/// reporting how long it takes. An error: the connection, and so the reply, was lost.
|
||||
fn wait_outstanding(&mut self, limit: Duration) -> Result<Waited, CallError> {
|
||||
let start = Instant::now();
|
||||
let mut told = start;
|
||||
loop {
|
||||
let Some(o) = &self.outstanding else { return Ok(Waited::Nothing) };
|
||||
let what = o.what.clone();
|
||||
let r = self.ask_pending()?;
|
||||
if r.get("pending").is_none_or(Value::is_null) {
|
||||
self.outstanding = None;
|
||||
return Ok(Waited::Nothing);
|
||||
}
|
||||
if r.get("done").and_then(Value::as_bool) == Some(true) {
|
||||
self.outstanding = None;
|
||||
return Ok(Waited::Done(r.get("reply").cloned().unwrap_or(Value::Null)));
|
||||
}
|
||||
let waited = start.elapsed();
|
||||
if waited >= limit {
|
||||
return Ok(Waited::Running);
|
||||
}
|
||||
if told.elapsed() >= PROGRESS_EVERY {
|
||||
let secs = self.outstanding.as_ref().map_or(0, Outstanding::secs);
|
||||
(self.progress)(&format!("{what} is still running in the app ({secs} s); waiting for it"));
|
||||
told = Instant::now();
|
||||
}
|
||||
std::thread::sleep(self.waits.poll.min(limit.saturating_sub(waited)));
|
||||
}
|
||||
}
|
||||
|
||||
/// The error for the outstanding request still running after its wait.
|
||||
fn still_running(&self, earlier: bool) -> CallError {
|
||||
let Some(o) = &self.outstanding else { return CallError::StillRunning(format!("{STILL_RUNNING} a request is still running in the app")) };
|
||||
let document = self.document;
|
||||
let effect =
|
||||
o.effect.map_or_else(|| format!("it was not cancelled and may still change the {document} when it finishes"), |e| e.when_done(document));
|
||||
let settle = self.waits.settle.as_secs();
|
||||
CallError::StillRunning(if earlier {
|
||||
format!(
|
||||
"{STILL_RUNNING} an earlier request, {}, is still running in the app ({} s; {effect}), so this call was not sent. Call the \
|
||||
`pending` tool to wait for its result, then send this call again",
|
||||
o.what,
|
||||
o.secs()
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"{STILL_RUNNING} {} is still running in the app after {} s ({effect}); don't send it again. Call the `pending` tool to \
|
||||
wait for its result (up to {settle} s per call); other calls wait for it first",
|
||||
o.what,
|
||||
o.secs()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Before a new request: wait for one still running from an earlier call, keeping its end for
|
||||
/// `pending`. Still running after the wait: the new request is not sent.
|
||||
fn settle_earlier(&mut self) -> Result<(), CallError> {
|
||||
let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return Ok(()) };
|
||||
match self.wait_outstanding(self.waits.settle) {
|
||||
Ok(Waited::Done(reply)) => self.settled = Some(Settled::Finished { what, reply }),
|
||||
Ok(Waited::Nothing) => {}
|
||||
Ok(Waited::Running) => return Err(self.still_running(true)),
|
||||
// The connection was lost, and the earlier reply with it (`pending` says so): the new
|
||||
// request goes on a fresh connection.
|
||||
Err(_) => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `control.pending` for the client: the end of the request still running from an earlier
|
||||
/// call, waiting for it up to the settle wait: `{pending: what | null, done?, reply?, lost?,
|
||||
/// running_s?}`. Never sent on a fresh connection, which has nothing pending.
|
||||
pub fn pending(&mut self) -> Value {
|
||||
if let Some(s) = self.settled.take() {
|
||||
return match s {
|
||||
Settled::Finished { what, reply } => json!({"pending": what, "done": true, "reply": reply}),
|
||||
Settled::Lost { what } => json!({"pending": what, "lost": true}),
|
||||
};
|
||||
}
|
||||
let Some(what) = self.outstanding.as_ref().map(|o| o.what.clone()) else { return json!({"pending": null}) };
|
||||
match self.wait_outstanding(self.waits.settle) {
|
||||
Ok(Waited::Done(reply)) => json!({"pending": what, "done": true, "reply": reply}),
|
||||
Ok(Waited::Nothing) => json!({"pending": null}),
|
||||
Ok(Waited::Running) => json!({"pending": what, "done": false, "running_s": self.outstanding.as_ref().map_or(0, Outstanding::secs)}),
|
||||
Err(_) => {
|
||||
self.settled = None;
|
||||
json!({"pending": what, "lost": true})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Send `request` and wait for its reply: the app's whole reply (`ok`, `result` or `error`,
|
||||
/// and anything it adds). First a request still running from an earlier call is waited for
|
||||
/// (except for `control.running`, which the app answers at once); a request the app says is
|
||||
/// still running is waited for through `control.pending`; a request refused unrun as busy
|
||||
/// behind one this client did not know of is sent once more after that one, when
|
||||
/// `retry_busy`.
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`Connection::explain`]: a request whose connection is lost is sent once more on a fresh
|
||||
/// one (unless [`Request::once`]), and when nothing listens any more the app went away, maybe
|
||||
/// while running it ([`CallError::Lost`]); [`CallError::StillRunning`]; [`CallError::BadReply`].
|
||||
pub fn exchange(&mut self, request: Request, retry_busy: bool) -> Result<Value, CallError> {
|
||||
let Request { method, params, what, effect, once } = request;
|
||||
if method != "control.running" {
|
||||
self.settle_earlier()?;
|
||||
}
|
||||
let again = retry_busy.then(|| params.clone());
|
||||
let sent = Instant::now();
|
||||
let line = self.request_line(&method, params);
|
||||
let reply = if once {
|
||||
// A fresh connection (an idle one may have been closed by an app restart; connecting
|
||||
// again sends nothing).
|
||||
self.connect().or_else(|_| self.connect()).and_then(|()| {
|
||||
self.set_read_timeout(ONCE_TIMEOUT);
|
||||
let r = self.roundtrip(&line);
|
||||
self.set_read_timeout(READ_TIMEOUT);
|
||||
r
|
||||
})
|
||||
} else {
|
||||
self.roundtrip(&line).or_else(|lost| {
|
||||
self.roundtrip(&line).map_err(|e| if e.kind() == std::io::ErrorKind::NotConnected && lost.kind() != e.kind() { lost } else { e })
|
||||
})
|
||||
}
|
||||
.map_err(|e| self.explain(&e))?;
|
||||
let v = Self::parse(&reply)?;
|
||||
// Timed out in the app, still running: wait for its result on this connection.
|
||||
if v.get("running").and_then(Value::as_bool) == Some(true) {
|
||||
self.outstanding = Some(Outstanding { what: what.clone(), effect: Some(effect), since: sent });
|
||||
(self.progress)(&format!(
|
||||
"{what} is still running in the app after {} s; waiting up to {} s more for it",
|
||||
sent.elapsed().as_secs(),
|
||||
self.waits.pending.as_secs()
|
||||
));
|
||||
let document = self.document;
|
||||
return match self.wait_outstanding(self.waits.pending)? {
|
||||
Waited::Done(reply) => Ok(reply),
|
||||
Waited::Nothing => Err(CallError::Lost(format!(
|
||||
"{what} timed out in the app, which kept no result for it: inspect the {document} before sending it again"
|
||||
))),
|
||||
Waited::Running => Err(self.still_running(false)),
|
||||
};
|
||||
}
|
||||
// Refused unrun behind a request this client did not know of: wait for that one, then
|
||||
// send this one once more.
|
||||
if v.get("busy").and_then(Value::as_bool) == Some(true)
|
||||
&& let Some(params) = again
|
||||
{
|
||||
self.outstanding = Some(Outstanding { what: "an earlier request".into(), effect: None, since: Instant::now() });
|
||||
self.settle_earlier()?;
|
||||
return self.exchange(Request { method, params, what, effect, once }, false);
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// [`Connection::exchange`] (retrying a busy refusal once) and the reply's result.
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`Connection::exchange`] and [`Connection::outcome`].
|
||||
pub fn call(&mut self, request: Request) -> Result<Value, CallError> {
|
||||
let v = self.exchange(request, true)?;
|
||||
self.outcome(&v)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "client_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,190 @@
|
||||
//! The client against the shared server ([`crate::server::Hub`]) on an ephemeral loopback port.
|
||||
|
||||
use super::*;
|
||||
use crate::ControlRequest;
|
||||
use crate::server::{Hub, Protocol, Stop, Summary, Waits as ServerWaits};
|
||||
use std::ffi::OsString;
|
||||
use std::net::TcpListener;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::mpsc::channel;
|
||||
|
||||
const TOKEN: &str = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
fn no_dir() -> Option<PathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn no_env(_: &str) -> Option<OsString> {
|
||||
None
|
||||
}
|
||||
|
||||
fn utc(secs: u64) -> String {
|
||||
secs.to_string()
|
||||
}
|
||||
|
||||
/// An app with no token files and no environment: a client sends only the token it is given.
|
||||
const APP: App = App { name: "Probe", data_dir: no_dir, env: no_env, utc, default_port: 0 };
|
||||
|
||||
/// An app whose requests all may change its document.
|
||||
struct Probe;
|
||||
|
||||
impl Protocol for Probe {
|
||||
fn summary(&self, method: &str, _: &Value) -> Summary {
|
||||
Summary { what: format!("`{method}`"), effect: Effect::Changes, command: None, steps: None, long: false }
|
||||
}
|
||||
fn document(&self) -> &'static str {
|
||||
"design"
|
||||
}
|
||||
fn change_event(&self) -> &'static str {
|
||||
"design.changed"
|
||||
}
|
||||
fn revision_method(&self) -> &'static str {
|
||||
"document.revision"
|
||||
}
|
||||
fn docs(&self) -> &'static str {
|
||||
"docs/control-protocol.md"
|
||||
}
|
||||
fn capabilities(&self) -> Value {
|
||||
json!({})
|
||||
}
|
||||
}
|
||||
|
||||
/// The app on `port` (0: an ephemeral one) behind [`TOKEN`]: its window answers each request after
|
||||
/// `delay` with the request's method and parameters, and the server says a request is still
|
||||
/// running after `wait`.
|
||||
fn serve(port: u16, delay: Duration, wait: Duration) -> Stop {
|
||||
// A port just closed may take a moment to be free again on a loaded machine.
|
||||
let started = Instant::now();
|
||||
let mut bound = TcpListener::bind(("127.0.0.1", port));
|
||||
while bound.is_err() && started.elapsed() < Duration::from_secs(2) {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
bound = TcpListener::bind(("127.0.0.1", port));
|
||||
}
|
||||
let listener = bound.unwrap();
|
||||
let (tx, rx) = channel::<ControlRequest>();
|
||||
std::thread::spawn(move || {
|
||||
for req in rx {
|
||||
std::thread::sleep(delay);
|
||||
let _ = req.reply.send(json!({"ok": true, "result": {"method": req.method, "params": req.params}}));
|
||||
}
|
||||
});
|
||||
let waits = ServerWaits { short: wait, long: wait, token: Duration::from_secs(5) };
|
||||
let hub = Arc::new(Hub::new(Probe, || {}, 1 << 20).with_waits(waits));
|
||||
hub.listen_stoppable(listener, TOKEN.into(), tx, false).unwrap()
|
||||
}
|
||||
|
||||
fn addr(port: u16) -> String {
|
||||
format!("127.0.0.1:{port}")
|
||||
}
|
||||
|
||||
/// A port nothing listens on.
|
||||
fn free_port() -> u16 {
|
||||
TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_is_one_round_trip_with_the_token() {
|
||||
let app = serve(0, Duration::ZERO, Duration::from_secs(10));
|
||||
let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN));
|
||||
assert!(!c.is_connected());
|
||||
let r = c.call(Request::new("document.inspect", json!({"a": 1}))).unwrap();
|
||||
assert_eq!(r, json!({"method": "document.inspect", "params": {"a": 1}}));
|
||||
assert!(c.is_connected());
|
||||
let serial = c.serial();
|
||||
// the next call goes on the same connection
|
||||
c.call(Request::new("document.inspect", json!({}))).unwrap();
|
||||
assert_eq!(c.serial(), serial);
|
||||
// `try_connect` sees the app there and leaves no connection open
|
||||
let mut fresh = Connection::new(APP, &addr(app.port()), Some(TOKEN));
|
||||
fresh.try_connect().unwrap();
|
||||
assert!(!fresh.is_connected());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wrong_token_or_none_is_refused() {
|
||||
let app = serve(0, Duration::ZERO, Duration::from_secs(10));
|
||||
for token in [Some("fedcba9876543210fedcba9876543210"), None] {
|
||||
let mut c = Connection::new(APP, &addr(app.port()), token);
|
||||
match c.call(Request::new("document.inspect", json!({}))) {
|
||||
Err(CallError::Refused(e)) => assert!(e.starts_with("unauthorized"), "{token:?}: {e}"),
|
||||
other => panic!("{token:?}: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_app_not_running_says_how_to_start_it() {
|
||||
let port = free_port();
|
||||
let mut c = Connection::new(APP, &addr(port), Some(TOKEN));
|
||||
let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err();
|
||||
let CallError::NotRunning(m) = &e else { panic!("{e:?}") };
|
||||
assert!(m.starts_with(&format!("no Probe app is listening at 127.0.0.1:{port}")), "{m}");
|
||||
assert!(m.contains(&format!("`probe --control {port}`")), "{m}");
|
||||
assert_eq!(c.try_connect().unwrap_err().kind(), std::io::ErrorKind::NotConnected);
|
||||
}
|
||||
|
||||
/// An app that goes away during a call is lost, not "not running"; the next call says it is not
|
||||
/// running, and once it runs again the next call reaches it.
|
||||
#[test]
|
||||
fn an_app_gone_during_a_call_is_lost_and_reached_again_once_back() {
|
||||
let app = serve(0, Duration::ZERO, Duration::from_secs(10));
|
||||
let port = app.port();
|
||||
let mut c = Connection::new(APP, &addr(port), Some(TOKEN));
|
||||
c.call(Request::new("document.inspect", json!({}))).unwrap();
|
||||
app.stop();
|
||||
let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err();
|
||||
let CallError::Lost(m) = &e else { panic!("{e:?}") };
|
||||
assert!(m.contains("may have quit") && m.contains("Inspect the design") && m.contains(&format!("`probe --control {port}`")), "{m}");
|
||||
assert!(matches!(c.call(Request::new("document.inspect", json!({}))), Err(CallError::NotRunning(_))));
|
||||
let _back = serve(port, Duration::ZERO, Duration::from_secs(10));
|
||||
assert_eq!(c.call(Request::new("ui.view", json!({}))).unwrap()["method"], json!("ui.view"));
|
||||
}
|
||||
|
||||
/// A server that reads a request and closes the connection without a reply: the call is lost
|
||||
/// (after one more try on a fresh connection), and the reply is never taken for "not running".
|
||||
#[test]
|
||||
fn a_connection_closed_without_a_reply_is_lost() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming().flatten() {
|
||||
let mut line = String::new();
|
||||
let _ = BufReader::new(stream).read_line(&mut line);
|
||||
}
|
||||
});
|
||||
let mut c = Connection::new(APP, &addr(port), Some(TOKEN));
|
||||
let e = c.call(Request::new("document.inspect", json!({}))).unwrap_err();
|
||||
assert!(matches!(e, CallError::Lost(_)), "{e:?}");
|
||||
assert!(!c.is_connected());
|
||||
}
|
||||
|
||||
/// A request the app says is still running is waited for through `control.pending`, on the same
|
||||
/// connection, and its own reply is the call's result.
|
||||
#[test]
|
||||
fn a_request_still_running_is_waited_for_through_pending() {
|
||||
let app = serve(0, Duration::from_millis(400), Duration::from_millis(50));
|
||||
let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN));
|
||||
c.set_waits(Waits { pending: Duration::from_secs(10), settle: Duration::from_secs(10), poll: Duration::from_millis(5) });
|
||||
let r = c.call(Request::new("solid.slow", json!({"n": 2}))).unwrap();
|
||||
assert_eq!(r, json!({"method": "solid.slow", "params": {"n": 2}}));
|
||||
assert!(!c.has_outstanding());
|
||||
assert_eq!(c.pending(), json!({"pending": null}));
|
||||
}
|
||||
|
||||
/// Past the wait, the call says the request is still running, and `pending` later has its end.
|
||||
#[test]
|
||||
fn a_request_still_running_past_the_wait_is_reported_and_its_end_kept() {
|
||||
let app = serve(0, Duration::from_millis(400), Duration::from_millis(20));
|
||||
let mut c = Connection::new(APP, &addr(app.port()), Some(TOKEN));
|
||||
c.set_waits(Waits { pending: Duration::from_millis(30), settle: Duration::from_secs(10), poll: Duration::from_millis(5) });
|
||||
let mut slow = Request::new("solid.slow", json!({}));
|
||||
slow.what = "`solid.slow` (now)".into();
|
||||
let e = c.call(slow).unwrap_err();
|
||||
let CallError::StillRunning(m) = &e else { panic!("{e:?}") };
|
||||
assert!(m.starts_with(STILL_RUNNING) && m.contains("`solid.slow` (now)") && m.contains("will still apply"), "{m}");
|
||||
assert!(c.has_outstanding());
|
||||
let end = c.pending();
|
||||
assert_eq!(end["done"], json!(true), "{end}");
|
||||
assert_eq!(end["reply"]["result"]["method"], json!("solid.slow"), "{end}");
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! What a client is told about a control request that outlasts its wait (docs/control-protocol.md,
|
||||
//! #1618): the app's control server and a bridged client both say what the request still does when
|
||||
//! it finishes. Which requests do what is the app's to say.
|
||||
|
||||
/// What a request still running does when it finishes.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Effect {
|
||||
/// It only reads (the document, the window or the command list): nothing changes.
|
||||
ReadsOnly,
|
||||
/// It writes a file (an export, a save, a render to a path); the document stays as it is.
|
||||
WritesFile,
|
||||
/// It changes the document (or may: anything not known to read or only write a file).
|
||||
Changes,
|
||||
}
|
||||
|
||||
impl Effect {
|
||||
/// What happens when the request finishes, for a message: "it only reads: nothing changes",
|
||||
/// "it was not cancelled and will still write the file when it finishes; the design does not
|
||||
/// change", … `document` is what the app calls its document ("design").
|
||||
pub fn when_done(self, document: &str) -> String {
|
||||
match self {
|
||||
Effect::ReadsOnly => "it only reads: nothing changes".into(),
|
||||
Effect::WritesFile => format!("it was not cancelled and will still write the file when it finishes; the {document} does not change"),
|
||||
Effect::Changes => "it was not cancelled and will still apply when it finishes".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
//! The control channel any app may share (docs/control-protocol.md): the per-launch token and its
|
||||
//! files ([`auth`]), the loopback server that admits a connection by the token and hands each
|
||||
//! request to the window ([`server`]), the client that talks to it ([`client`]), the request the
|
||||
//! window answers ([`ControlRequest`]), the document watch subscribed connections wait on
|
||||
//! ([`DocumentWatch`]), and what a request still running does when it finishes ([`Effect`]). It
|
||||
//! needs no UI toolkit; the window's own requests over egui (`ui.screenshot`, `ui.resize`) are
|
||||
//! `craft-control-egui`.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod auth;
|
||||
mod cancel;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod client;
|
||||
mod effect;
|
||||
mod request;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod server;
|
||||
|
||||
pub use cancel::{Cancel, Ticket};
|
||||
pub use effect::Effect;
|
||||
pub use request::{ControlRequest, ControlResponse, DOCUMENT_WATCH, DocumentWatch, Subscriber, cancelled_chat_response, chat_turn_cancelled};
|
||||
|
||||
/// How an error from the client (`client`, native only) starts when a request is still running
|
||||
/// in the app after the wait. Outside the client so callers on every target can test for it.
|
||||
pub const STILL_RUNNING: &str = "still running:";
|
||||
@@ -0,0 +1,125 @@
|
||||
//! A control request as the window gets it, and the design watch subscribed connections wait on.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::sync::{Condvar, Mutex, MutexGuard, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub type ControlResponse = Value;
|
||||
|
||||
pub struct ControlRequest {
|
||||
pub method: String,
|
||||
pub params: Value,
|
||||
pub reply: Sender<ControlResponse>,
|
||||
/// Set once the window starts on it; until then it waits its turn (`control.running`, #3654).
|
||||
pub started: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||
/// Chat requests already admitted by the gate may outlive a tab switch. This turn's
|
||||
/// validity follows them through queues, computation and cursor animation.
|
||||
pub chat_turn: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
|
||||
/// Where the request is, shared with the connection that sent it so it can cancel it
|
||||
/// (`control.cancel`, #1989).
|
||||
pub ticket: std::sync::Arc<crate::Ticket>,
|
||||
/// Told apart from every other request of this run: names its row while it waits.
|
||||
pub serial: u64,
|
||||
}
|
||||
|
||||
impl ControlRequest {
|
||||
pub fn new(method: impl Into<String>, params: Value) -> (Self, std::sync::mpsc::Receiver<ControlResponse>) {
|
||||
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let serial = NEXT.fetch_add(1, Ordering::Relaxed);
|
||||
(Self { method: method.into(), params, reply: tx, started: Default::default(), chat_turn: None, ticket: Default::default(), serial }, rx)
|
||||
}
|
||||
|
||||
/// Answer a request whose agent chat turn has ended, and say so: such a request is not run.
|
||||
pub fn refuse_cancelled_chat(&self) -> bool {
|
||||
if chat_turn_cancelled(self.chat_turn.as_deref()) {
|
||||
let _ = self.reply.send(cancelled_chat_response());
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the agent chat turn a request belongs to has ended (`None`: it belongs to none).
|
||||
pub fn chat_turn_cancelled(turn: Option<&std::sync::atomic::AtomicBool>) -> bool {
|
||||
turn.is_some_and(|active| !active.load(std::sync::atomic::Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// The reply to a request whose agent chat turn has ended.
|
||||
pub fn cancelled_chat_response() -> Value {
|
||||
json!({"ok": false, "error": "not run: the agent chat turn ended or its document tab was left"})
|
||||
}
|
||||
|
||||
/// The active design's id and revision for control connections that sent `control.subscribe`
|
||||
/// (#6985): the window publishes them each frame while one is subscribed, and each such
|
||||
/// connection waits for them to move, then sends the app's change event (`Protocol::change_event`).
|
||||
#[derive(Default)]
|
||||
pub struct DocumentWatch {
|
||||
subscribers: AtomicUsize,
|
||||
now: Mutex<Option<(String, u64)>>,
|
||||
moved: Condvar,
|
||||
}
|
||||
|
||||
/// The app's [`DocumentWatch`].
|
||||
pub static DOCUMENT_WATCH: DocumentWatch = DocumentWatch::new();
|
||||
|
||||
impl DocumentWatch {
|
||||
pub const fn new() -> Self {
|
||||
DocumentWatch { subscribers: AtomicUsize::new(0), now: Mutex::new(None), moved: Condvar::new() }
|
||||
}
|
||||
|
||||
fn now(&self) -> MutexGuard<'_, Option<(String, u64)>> {
|
||||
self.now.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// The design as last published (`None` before anything was).
|
||||
pub fn current(&self) -> Option<(String, u64)> {
|
||||
self.now().clone()
|
||||
}
|
||||
|
||||
/// The design as it is now; waiters wake only when it moved. Only the latest is kept, so
|
||||
/// changes made faster than a connection sends them go out as one.
|
||||
pub fn publish(&self, document_id: &str, revision: u64) {
|
||||
let mut now = self.now();
|
||||
if now.as_ref().is_some_and(|(id, r)| id == document_id && *r == revision) {
|
||||
return;
|
||||
}
|
||||
*now = Some((document_id.to_string(), revision));
|
||||
self.moved.notify_all();
|
||||
}
|
||||
|
||||
/// Whether a connection is subscribed: the window publishes only then.
|
||||
pub fn subscribed(&self) -> bool {
|
||||
self.subscribers.load(Ordering::Relaxed) > 0
|
||||
}
|
||||
|
||||
/// Count a subscribed connection until the returned guard is dropped.
|
||||
pub fn join(&'static self) -> Subscriber {
|
||||
self.subscribers.fetch_add(1, Ordering::Relaxed);
|
||||
Subscriber(self)
|
||||
}
|
||||
|
||||
/// Wait up to `wait` for the design to differ from `seen`, or for `stop`; the design then
|
||||
/// (`None` before anything was published).
|
||||
pub fn wait(&self, seen: &(String, u64), stop: &AtomicBool, wait: Duration) -> Option<(String, u64)> {
|
||||
let unmoved = |now: &mut Option<(String, u64)>| now.as_ref() == Some(seen) && !stop.load(Ordering::Relaxed);
|
||||
let (now, _) = self.moved.wait_timeout_while(self.now(), wait, unmoved).unwrap_or_else(PoisonError::into_inner);
|
||||
now.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// A subscribed connection, counted by its [`DocumentWatch`] while this lives.
|
||||
pub struct Subscriber(&'static DocumentWatch);
|
||||
|
||||
impl Drop for Subscriber {
|
||||
fn drop(&mut self) {
|
||||
self.0.subscribers.fetch_sub(1, Ordering::Relaxed);
|
||||
// Under the lock, so a connection about to wait sees its `stop` set before it does.
|
||||
let _now = self.0.now();
|
||||
self.0.moved.notify_all();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -182,6 +182,16 @@ craft-mcp-server = { workspace = true, optional = true }
|
||||
craft-pe = { workspace = true, optional = true }
|
||||
craft-sniff = { workspace = true, optional = true }
|
||||
craft-tar = { workspace = true, optional = true }
|
||||
craft-ai = { workspace = true, optional = true }
|
||||
craft-control = { workspace = true, optional = true }
|
||||
craft-dds = { workspace = true, optional = true }
|
||||
craft-frame = { workspace = true, optional = true }
|
||||
craft-ipc = { workspace = true, optional = true }
|
||||
craft-media-source = { workspace = true, optional = true }
|
||||
craft-palette = { workspace = true, optional = true }
|
||||
craft-press-print = { workspace = true, optional = true }
|
||||
craft-score = { workspace = true, optional = true }
|
||||
craft-ssip = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
ui = [
|
||||
@@ -234,6 +244,9 @@ aep = ["dep:craft-aep"]
|
||||
affinity = ["dep:craft-affinity"]
|
||||
affinity-synth = ["affinity", "craft-affinity/synth"]
|
||||
|
||||
ai = ["dep:craft-ai"]
|
||||
ai-local = ["ai", "craft-ai/local"]
|
||||
|
||||
ai-core = ["dep:craft-ai-core"]
|
||||
|
||||
anicraft-format = ["dep:craft-anicraft-format"]
|
||||
@@ -288,6 +301,8 @@ codepage = ["dep:craft-codepage"]
|
||||
|
||||
color = ["dep:craft-color", "keymap", "settings", "window", "platform"]
|
||||
|
||||
control = ["dep:craft-control"]
|
||||
|
||||
control-protocol = ["dep:craft-control-protocol"]
|
||||
|
||||
crs = ["dep:craft-crs"]
|
||||
@@ -298,6 +313,9 @@ csv = ["dep:craft-csv"]
|
||||
|
||||
dav = ["dep:craft-dav"]
|
||||
|
||||
dds = ["dep:craft-dds"]
|
||||
dds-parallel = ["dds", "craft-dds/parallel"]
|
||||
|
||||
deckcraft-format = ["dep:craft-deckcraft-format"]
|
||||
|
||||
designcraft-format = ["dep:craft-designcraft-format"]
|
||||
@@ -386,6 +404,8 @@ font-subset = ["dep:craft-font-subset"] # native only: HarfBuzz's C++ builds wit
|
||||
|
||||
format = ["dep:craft-format"]
|
||||
|
||||
frame = ["dep:craft-frame"]
|
||||
|
||||
gainmap = ["dep:craft-gainmap"]
|
||||
|
||||
gcode = ["dep:craft-gcode"]
|
||||
@@ -428,6 +448,8 @@ imaging = ["dep:craft-imaging", "color", "keymap"]
|
||||
|
||||
indd = ["dep:craft-indd"]
|
||||
|
||||
ipc = ["dep:craft-ipc"]
|
||||
|
||||
jmap = ["dep:craft-jmap"]
|
||||
|
||||
jpeg = ["dep:craft-jpeg"]
|
||||
@@ -478,6 +500,8 @@ media-mxf = ["media", "craft-media/mxf"]
|
||||
media-ogg = ["media", "craft-media/ogg"]
|
||||
media-serde = ["media", "craft-media/serde"]
|
||||
|
||||
media-source = ["dep:craft-media-source"]
|
||||
|
||||
menu = ["dep:craft-menu"]
|
||||
|
||||
mesh = ["dep:craft-mesh"]
|
||||
@@ -515,6 +539,8 @@ ora = ["dep:craft-ora"]
|
||||
|
||||
pages = ["dep:craft-pages"]
|
||||
|
||||
palette = ["dep:craft-palette"]
|
||||
|
||||
pcraft-bundle = ["dep:craft-pcraft-bundle"]
|
||||
|
||||
pcraft-format = ["dep:craft-pcraft-format"]
|
||||
@@ -564,6 +590,8 @@ prc = ["dep:craft-prc"]
|
||||
prepress = ["dep:craft-prepress"]
|
||||
prepress-flatten = ["prepress", "craft-prepress/flatten"]
|
||||
|
||||
press-print = ["dep:craft-press-print"]
|
||||
|
||||
print = ["dep:craft-print"]
|
||||
|
||||
proof = ["dep:craft-proof"]
|
||||
@@ -613,6 +641,8 @@ rtf = ["dep:craft-rtf"]
|
||||
rtf-doc = ["rtf", "craft-rtf/doc"]
|
||||
rtf-outlook = ["rtf", "craft-rtf/outlook"]
|
||||
|
||||
score = ["dep:craft-score"]
|
||||
|
||||
scraft-format = ["dep:craft-scraft-format"]
|
||||
|
||||
settings = ["dep:craft-settings"]
|
||||
@@ -633,6 +663,8 @@ smime = ["dep:craft-smime"]
|
||||
|
||||
sniff = ["dep:craft-sniff"]
|
||||
|
||||
ssip = ["dep:craft-ssip"]
|
||||
|
||||
stencilcraft-document = ["dep:craft-stencilcraft-document"]
|
||||
|
||||
stencilcraft-format = ["dep:craft-stencilcraft-format"]
|
||||
|
||||
@@ -12,6 +12,8 @@ pub use craft_aaf as aaf;
|
||||
pub use craft_aep as aep;
|
||||
#[cfg(feature = "affinity")]
|
||||
pub use craft_affinity as affinity;
|
||||
#[cfg(feature = "ai")]
|
||||
pub use craft_ai as ai;
|
||||
#[cfg(feature = "ai-core")]
|
||||
pub use craft_ai_core as ai_core;
|
||||
#[cfg(feature = "anicraft-format")]
|
||||
@@ -39,6 +41,8 @@ pub use craft_cms as cms;
|
||||
pub use craft_codepage as codepage;
|
||||
#[cfg(feature = "color")]
|
||||
pub use craft_color as color;
|
||||
#[cfg(feature = "control")]
|
||||
pub use craft_control as control;
|
||||
#[cfg(feature = "control-protocol")]
|
||||
pub use craft_control_protocol as control_protocol;
|
||||
#[cfg(feature = "crs")]
|
||||
@@ -49,6 +53,8 @@ pub use craft_crw as crw;
|
||||
pub use craft_csv as csv;
|
||||
#[cfg(feature = "dav")]
|
||||
pub use craft_dav as dav;
|
||||
#[cfg(feature = "dds")]
|
||||
pub use craft_dds as dds;
|
||||
#[cfg(feature = "deckcraft-format")]
|
||||
pub use craft_deckcraft_format as deckcraft_format;
|
||||
#[cfg(feature = "designcraft-format")]
|
||||
@@ -113,6 +119,8 @@ pub use craft_font_store as font_store;
|
||||
pub use craft_font_subset as font_subset;
|
||||
#[cfg(feature = "format")]
|
||||
pub use craft_format as format;
|
||||
#[cfg(feature = "frame")]
|
||||
pub use craft_frame as frame;
|
||||
#[cfg(feature = "gainmap")]
|
||||
pub use craft_gainmap as gainmap;
|
||||
#[cfg(feature = "gcode")]
|
||||
@@ -143,6 +151,8 @@ pub use craft_image_codecs as image_codecs;
|
||||
pub use craft_imaging as imaging;
|
||||
#[cfg(feature = "indd")]
|
||||
pub use craft_indd as indd;
|
||||
#[cfg(feature = "ipc")]
|
||||
pub use craft_ipc as ipc;
|
||||
#[cfg(feature = "jmap")]
|
||||
pub use craft_jmap as jmap;
|
||||
#[cfg(feature = "jpeg")]
|
||||
@@ -179,6 +189,8 @@ pub use craft_mcp as mcp;
|
||||
pub use craft_mcp_server as mcp_server;
|
||||
#[cfg(feature = "media")]
|
||||
pub use craft_media as media;
|
||||
#[cfg(feature = "media-source")]
|
||||
pub use craft_media_source as media_source;
|
||||
#[cfg(feature = "menu")]
|
||||
pub use craft_menu as menu;
|
||||
#[cfg(feature = "mesh")]
|
||||
@@ -205,6 +217,8 @@ pub use craft_openpgp as openpgp;
|
||||
pub use craft_ora as ora;
|
||||
#[cfg(feature = "pages")]
|
||||
pub use craft_pages as pages;
|
||||
#[cfg(feature = "palette")]
|
||||
pub use craft_palette as palette;
|
||||
#[cfg(feature = "pcraft-bundle")]
|
||||
pub use craft_pcraft_bundle as pcraft_bundle;
|
||||
#[cfg(feature = "pcraft-format")]
|
||||
@@ -231,6 +245,8 @@ pub use craft_pptx as pptx;
|
||||
pub use craft_prc as prc;
|
||||
#[cfg(feature = "prepress")]
|
||||
pub use craft_prepress as prepress;
|
||||
#[cfg(feature = "press-print")]
|
||||
pub use craft_press_print as press_print;
|
||||
#[cfg(feature = "print")]
|
||||
pub use craft_print as print;
|
||||
#[cfg(feature = "proof")]
|
||||
@@ -253,6 +269,8 @@ pub use craft_raw as raw;
|
||||
pub use craft_review as review;
|
||||
#[cfg(feature = "rtf")]
|
||||
pub use craft_rtf as rtf;
|
||||
#[cfg(feature = "score")]
|
||||
pub use craft_score as score;
|
||||
#[cfg(feature = "scraft-format")]
|
||||
pub use craft_scraft_format as scraft_format;
|
||||
#[cfg(feature = "settings")]
|
||||
@@ -271,6 +289,8 @@ pub use craft_sketch as sketch;
|
||||
pub use craft_smime as smime;
|
||||
#[cfg(feature = "sniff")]
|
||||
pub use craft_sniff as sniff;
|
||||
#[cfg(feature = "ssip")]
|
||||
pub use craft_ssip as ssip;
|
||||
#[cfg(feature = "stencilcraft-document")]
|
||||
pub use craft_stencilcraft_document as stencilcraft_document;
|
||||
#[cfg(feature = "stencilcraft-format")]
|
||||
|
||||
@@ -13,6 +13,7 @@ pub const NOT_IN_FACADE: &[(&str, &str)] = &[
|
||||
("plate-test", "a test harness: reference PDFs and PDF structure checks"),
|
||||
("render-parity", "test support: compares two renders of a photo against a stated tolerance"),
|
||||
("story-render", "a tool: renders a component's stories to PNG files offscreen"),
|
||||
("testkit", "test support for the apps' own tests: source guards, deadlines and hostile-input runs"),
|
||||
("ui-bench", "a tool: the release-mode benchmark of the UI library"),
|
||||
("ui-gallery", "a tool: the story gallery application"),
|
||||
("ui-macros", "a proc-macro crate that craft-ui re-exports"),
|
||||
@@ -25,6 +26,7 @@ pub const NOT_IN_FACADE: &[(&str, &str)] = &[
|
||||
("ui-test-kit", "test support shared by craft-ui's integration binaries"),
|
||||
("ui-widgets", "part of craft-ui, which the facade exposes as `ui`"),
|
||||
("win32", "the `unsafe` Win32 calls behind a safe API, used by craft-window"),
|
||||
("xtask", "a build tool: the step runner and reports of the apps' `cargo xtask` binaries"),
|
||||
];
|
||||
|
||||
/// Crates every app gets without a feature.
|
||||
|
||||
@@ -11,7 +11,7 @@ mapping a SYLK sheet onto a document are the caller's work.
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `decode_text(bytes)` | UTF-8 and UTF-16 byte order marks; without one, valid UTF-8 is kept; other bytes are Windows-1256 when they read as Arabic, else Windows-1252 |
|
||||
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
|
||||
| `sniff_delimiter(text, quote)` | comma, semicolon or tab: the one with the most consistent field count over the first 20 non-blank records in a 64 KiB sample; when no candidate finds coherent complete records with a delimiter in each, a candidate that finds no complete record counts the delimiters of the first record up to the end of the sample, or up to a quoted field still open there, and a quote inside a field that does not start with one makes that record incoherent; quote boundaries must be coherent, and ties prefer tab, comma, then semicolon |
|
||||
| `parse_records(text, delim, quote, max_fields, on_record)` | records of fields, one callback per record (return `false` to stop); CR, LF and CRLF line ends; quoted fields with doubled quotes and line breaks; a quote in the middle of a field is text |
|
||||
| `records(text, delim, quote, max_records, max_fields)` | the same, collected |
|
||||
| `push_field(out, field, delim)`, `field_len(field, delim)` | one field as written, quoted only when it holds the delimiter, a quote or a line break; its exact length, to enforce a size cap before writing |
|
||||
|
||||
+160
-66
@@ -34,8 +34,13 @@ pub use decode::{decode_text, decode_utf8};
|
||||
/// Quotes start only at a candidate field boundary; candidates with incoherent quoted fields
|
||||
/// are discarded when another coherent candidate separates the records. Ties prefer tab,
|
||||
/// then comma, then semicolon. Comma is the fallback when no candidate separates the text.
|
||||
/// Only the first 64 KiB are read. A record cut off at the end of that sample is left out,
|
||||
/// except when no candidate finds coherent complete records with a delimiter in each. Then a candidate
|
||||
/// that finds no complete record counts the delimiters read so far in the first record, and that
|
||||
/// record is incoherent when it has a quote inside a field that does not start with one. When one
|
||||
/// of those candidates ends the sample inside a quoted field, each of them counts only up to that
|
||||
/// field's opening quote.
|
||||
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
let mut candidates = [(b'\t', 0usize, 0usize, true), (b',', 0, 0, true), (b';', 0, 0, true)];
|
||||
const SAMPLE_BYTES: usize = 64 * 1024;
|
||||
let mut sample_end = text.len().min(SAMPLE_BYTES);
|
||||
while !text.is_char_boundary(sample_end) {
|
||||
@@ -43,78 +48,46 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
}
|
||||
let sample = &text[..sample_end];
|
||||
let truncated = sample_end < text.len();
|
||||
for (delimiter, min_count, total, coherent) in &mut candidates {
|
||||
let mut counts = Vec::new();
|
||||
let mut count = 0usize;
|
||||
let mut quoted = false;
|
||||
let mut after_quote = false;
|
||||
let mut field_start = true;
|
||||
let mut record_has_content = false;
|
||||
let mut record_coherent = true;
|
||||
let mut candidate_coherent = true;
|
||||
let mut reached_record_limit = false;
|
||||
let mut chars = sample.chars().peekable();
|
||||
while let Some(ch) = chars.next() {
|
||||
if quoted {
|
||||
if ch == quote {
|
||||
if chars.peek() == Some("e) {
|
||||
chars.next();
|
||||
record_has_content = true;
|
||||
} else {
|
||||
quoted = false;
|
||||
after_quote = true;
|
||||
}
|
||||
} else if !ch.is_whitespace() {
|
||||
record_has_content = true;
|
||||
}
|
||||
let mut scans = b"\t,;".map(|delimiter| scan_delimiter(sample, delimiter, quote));
|
||||
if truncated {
|
||||
if scans.iter().any(|scan| scan.coherent && scan.counts.iter().min().is_some_and(|&min| min > 0)) {
|
||||
return pick_delimiter(scans);
|
||||
}
|
||||
// Every quote in the scans is at a char boundary, so the cut is one too.
|
||||
let cut = scans.iter().filter(|scan| scan.counts.is_empty()).filter_map(|scan| scan.open_quote).min().unwrap_or(sample_end);
|
||||
for scan in &mut scans {
|
||||
if !scan.counts.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if ch == '\r' || ch == '\n' {
|
||||
if ch == '\r' && chars.peek() == Some(&'\n') {
|
||||
chars.next();
|
||||
}
|
||||
if record_has_content {
|
||||
counts.push(count);
|
||||
candidate_coherent &= record_coherent;
|
||||
if counts.len() >= 20 {
|
||||
reached_record_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
count = 0;
|
||||
quoted = false;
|
||||
after_quote = false;
|
||||
field_start = true;
|
||||
record_has_content = false;
|
||||
record_coherent = true;
|
||||
if cut < sample_end {
|
||||
let before_cut = scan_delimiter(&sample[..cut], scan.delimiter, quote);
|
||||
(scan.count, scan.record_has_content, scan.record_coherent, scan.record_bare_quote) =
|
||||
(before_cut.count, before_cut.record_has_content, before_cut.record_coherent, before_cut.record_bare_quote);
|
||||
}
|
||||
if scan.record_has_content {
|
||||
scan.coherent &= scan.record_coherent && !scan.record_bare_quote;
|
||||
scan.counts.push(scan.count);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for scan in &mut scans {
|
||||
if scan.reached_record_limit {
|
||||
continue;
|
||||
}
|
||||
if ch == quote && field_start {
|
||||
quoted = true;
|
||||
field_start = false;
|
||||
record_has_content = true;
|
||||
} else if ch == char::from(*delimiter) {
|
||||
count += 1;
|
||||
field_start = true;
|
||||
after_quote = false;
|
||||
} else {
|
||||
record_coherent &= !after_quote || ch.is_whitespace();
|
||||
record_has_content |= !ch.is_whitespace();
|
||||
field_start = false;
|
||||
if scan.open_quote.is_some() {
|
||||
scan.coherent = false;
|
||||
} else if scan.record_has_content {
|
||||
scan.coherent &= scan.record_coherent;
|
||||
scan.counts.push(scan.count);
|
||||
}
|
||||
}
|
||||
if !reached_record_limit && !truncated {
|
||||
if quoted {
|
||||
candidate_coherent = false;
|
||||
} else if record_has_content {
|
||||
candidate_coherent &= record_coherent;
|
||||
counts.push(count);
|
||||
}
|
||||
}
|
||||
*coherent &= candidate_coherent;
|
||||
*min_count = counts.iter().copied().min().unwrap_or(0);
|
||||
*total = counts.iter().sum();
|
||||
}
|
||||
pick_delimiter(scans)
|
||||
}
|
||||
|
||||
fn pick_delimiter(scans: [DelimiterScan; 3]) -> u8 {
|
||||
let candidates =
|
||||
scans.map(|scan| (scan.delimiter, scan.counts.iter().copied().min().unwrap_or(0), scan.counts.iter().sum::<usize>(), scan.coherent));
|
||||
let prefer_coherent = candidates.iter().any(|(_, min, _, coherent)| *min > 0 && *coherent);
|
||||
let mut best = (b',', 0usize, 0usize);
|
||||
for (delimiter, min, total, coherent) in candidates {
|
||||
@@ -125,6 +98,93 @@ pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
best.0
|
||||
}
|
||||
|
||||
/// One candidate delimiter's pass over the sample in [`sniff_delimiter`].
|
||||
struct DelimiterScan {
|
||||
delimiter: u8,
|
||||
/// Delimiter counts of the complete non-blank records, at most 20.
|
||||
counts: Vec<usize>,
|
||||
/// Whether every counted record had coherent quoted fields.
|
||||
coherent: bool,
|
||||
reached_record_limit: bool,
|
||||
/// The record still open at the end of the sample: its delimiter count so far, whether it
|
||||
/// has content, and whether its quoted fields so far are coherent.
|
||||
count: usize,
|
||||
record_has_content: bool,
|
||||
record_coherent: bool,
|
||||
/// Whether the open record has a quote inside a field that does not start with one.
|
||||
record_bare_quote: bool,
|
||||
/// The byte offset of the quote that opens a field still unterminated at the end.
|
||||
open_quote: Option<usize>,
|
||||
}
|
||||
|
||||
fn scan_delimiter(sample: &str, delimiter: u8, quote: char) -> DelimiterScan {
|
||||
let mut scan = DelimiterScan {
|
||||
delimiter,
|
||||
counts: Vec::new(),
|
||||
coherent: true,
|
||||
reached_record_limit: false,
|
||||
count: 0,
|
||||
record_has_content: false,
|
||||
record_coherent: true,
|
||||
record_bare_quote: false,
|
||||
open_quote: None,
|
||||
};
|
||||
let mut after_quote = false;
|
||||
let mut field_start = true;
|
||||
let mut chars = sample.char_indices().peekable();
|
||||
while let Some((at, ch)) = chars.next() {
|
||||
if scan.open_quote.is_some() {
|
||||
if ch == quote {
|
||||
if chars.peek().map(|&(_, next)| next) == Some(quote) {
|
||||
chars.next();
|
||||
scan.record_has_content = true;
|
||||
} else {
|
||||
scan.open_quote = None;
|
||||
after_quote = true;
|
||||
}
|
||||
} else if !ch.is_whitespace() {
|
||||
scan.record_has_content = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ch == '\r' || ch == '\n' {
|
||||
if ch == '\r' && chars.peek().map(|&(_, next)| next) == Some('\n') {
|
||||
chars.next();
|
||||
}
|
||||
if scan.record_has_content {
|
||||
scan.counts.push(scan.count);
|
||||
scan.coherent &= scan.record_coherent;
|
||||
if scan.counts.len() >= 20 {
|
||||
scan.reached_record_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
scan.count = 0;
|
||||
after_quote = false;
|
||||
field_start = true;
|
||||
scan.record_has_content = false;
|
||||
scan.record_coherent = true;
|
||||
scan.record_bare_quote = false;
|
||||
continue;
|
||||
}
|
||||
if ch == quote && field_start {
|
||||
scan.open_quote = Some(at);
|
||||
field_start = false;
|
||||
scan.record_has_content = true;
|
||||
} else if ch == char::from(delimiter) {
|
||||
scan.count += 1;
|
||||
field_start = true;
|
||||
after_quote = false;
|
||||
} else {
|
||||
scan.record_coherent &= !after_quote || ch.is_whitespace();
|
||||
scan.record_bare_quote |= ch == quote;
|
||||
scan.record_has_content |= !ch.is_whitespace();
|
||||
field_start = false;
|
||||
}
|
||||
}
|
||||
scan
|
||||
}
|
||||
|
||||
/// Splits delimited text into records of fields and hands each to `on_record`, which returns
|
||||
/// whether to go on. A record ends at CR, LF or CRLF; a blank line is a record with one empty
|
||||
/// field. A field that starts with `quote` runs to the closing quote (a doubled quote is one
|
||||
@@ -309,6 +369,40 @@ mod tests {
|
||||
assert_eq!(sniff_delimiter(&text, '"'), b'\t');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimiter_sniffing_counts_a_first_record_longer_than_the_sample() {
|
||||
let wide = |field: &str, delimiter: char| {
|
||||
let mut record = String::new();
|
||||
while record.len() <= 64 * 1024 {
|
||||
record.push_str(field);
|
||||
record.push(delimiter);
|
||||
}
|
||||
record
|
||||
};
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n1;2\n", wide("cell", ';')), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n1\t2\n", wide("cell", '\t')), '"'), b'\t');
|
||||
// Delimiters inside quoted fields of the long record are text.
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a,b\tc\"", ';')), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("{}end\n", wide("\"a;b\tc\"", ',')), '"'), b',');
|
||||
// A candidate that reads a quoted field as text inside an unquoted one is incoherent.
|
||||
assert_eq!(sniff_delimiter(&format!("a,\"{}\",{}", "x;y".repeat(20_000), "z".repeat(10_000)), '"'), b',');
|
||||
// The sample ends inside a quoted field: the delimiters before that field count.
|
||||
let text = format!("a;b;c;\"{}\"\n1;2;3;4\n", "long text ".repeat(8_000));
|
||||
assert_eq!(sniff_delimiter(&text, '"'), b';');
|
||||
// Candidates without a complete record do not count delimiters after that quote, also
|
||||
// when they read it as text.
|
||||
assert_eq!(sniff_delimiter(&format!("a,\"{}\"\n1,2\n", "x;y".repeat(30_000)), '"'), b',');
|
||||
assert_eq!(sniff_delimiter(&format!("a;b;\"{}\"\n1;2;3\n", "x,y\t".repeat(20_000)), '"'), b';');
|
||||
// A line break inside the open quoted field gives the other candidates a record without
|
||||
// a delimiter, which does not stop the first record from counting.
|
||||
assert_eq!(sniff_delimiter(&format!("a;b;\"hello\n{}\"\n", "x".repeat(70_000)), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("\"a\";b;\"hello,\n{}\"\n", "x".repeat(70_000)), '"'), b';');
|
||||
// When a candidate finds coherent complete records with a delimiter in each, a record cut
|
||||
// off at the end is left out.
|
||||
assert_eq!(sniff_delimiter(&format!("a;\"hello,\";b\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
|
||||
assert_eq!(sniff_delimiter(&format!("a,b,c;\"hello,\";d\n{}", "x;y;z\n".repeat(12_000)), '"'), b';');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_stops_collecting_fields_at_the_cap() {
|
||||
let (max, extra) = (16_384usize, 50);
|
||||
|
||||
+147
-3
@@ -219,12 +219,13 @@ fn position(arg: &str, max: u32) -> Option<u32> {
|
||||
arg.trim().parse::<u32>().ok().filter(|n| (1..=max).contains(n)).map(|n| n - 1)
|
||||
}
|
||||
|
||||
/// A `K` value: `"text"` (everything between the first and the last quote, as written), a
|
||||
/// boolean, a number, or other text (an error value when it starts with `#`).
|
||||
/// A `K` value: `"text"` (everything between the first and the last quote, with Excel's
|
||||
/// character escapes decoded), a boolean, a number, or other text (an error value when it starts
|
||||
/// with `#`).
|
||||
fn parse_value(arg: &str) -> Value {
|
||||
if let Some(text) = arg.strip_prefix('"') {
|
||||
let text = text.strip_suffix('"').unwrap_or(text);
|
||||
return Value::Text(text.to_string());
|
||||
return Value::Text(decode_escapes(text));
|
||||
}
|
||||
if arg.eq_ignore_ascii_case("TRUE") || arg.eq_ignore_ascii_case("FALSE") {
|
||||
return Value::Bool(arg.eq_ignore_ascii_case("TRUE"));
|
||||
@@ -238,6 +239,149 @@ fn parse_value(arg: &str) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// ISO 6937's diacritics (codes 0xC1–0xCF): each code, its combining mark, and the letters it
|
||||
/// composes with, as pairs of letter and precomposed letter (from Unicode's canonical composition).
|
||||
/// A diacritic on a SPACE is the spacing accent (`´`, `¨`, `ˇ`, ...), as in glibc's ISO_6937 map; the
|
||||
/// grave, circumflex and tilde have none, since `` ` ``, `^` and `~` are plain ASCII.
|
||||
const ISO6937_DIACRITICS: [(u8, char, &str); 13] = [
|
||||
(0xC1, '\u{0300}', "AÀaàEÈeèIÌiìNǸnǹOÒoòUÙuùWẀwẁYỲyỳ"), // grave
|
||||
(0xC2, '\u{0301}', " ´AÁaáCĆcćEÉeéGǴgǵIÍiíKḰkḱLĹlĺMḾmḿNŃnńOÓoóPṔpṕRŔrŕSŚsśUÚuúWẂwẃYÝyýZŹzź"), // acute
|
||||
(0xC3, '\u{0302}', "AÂaâCĈcĉEÊeêGĜgĝHĤhĥIÎiîJĴjĵOÔoôSŜsŝUÛuûWŴwŵYŶyŷZẐzẑ"), // circumflex
|
||||
(0xC4, '\u{0303}', "AÃaãEẼeẽIĨiĩNÑnñOÕoõUŨuũVṼvṽYỸyỹ"), // tilde
|
||||
(0xC5, '\u{0304}', " ¯AĀaāEĒeēGḠgḡIĪiīOŌoōUŪuūYȲyȳ"), // macron
|
||||
(0xC6, '\u{0306}', " ˘AĂaăEĔeĕGĞgğIĬiĭOŎoŏUŬuŭ"), // breve
|
||||
(0xC7, '\u{0307}', " ˙AȦaȧBḂbḃCĊcċDḊdḋEĖeėFḞfḟGĠgġHḢhḣIİMṀmṁNṄnṅOȮoȯPṖpṗRṘrṙSṠsṡTṪtṫWẆwẇXẊxẋYẎyẏZŻzż"), // dot above
|
||||
(0xC8, '\u{0308}', " ¨AÄaäEËeëHḦhḧIÏiïOÖoötẗUÜuüWẄwẅXẌxẍYŸyÿ"), // diaeresis
|
||||
(0xCA, '\u{030a}', " ˚AÅaåUŮuůwẘyẙ"), // ring
|
||||
(0xCB, '\u{0327}', " ¸CÇcçDḐdḑEȨeȩGĢgģHḨhḩKĶkķLĻlļNŅnņRŖrŗSŞsşTŢtţ"), // cedilla
|
||||
(0xCD, '\u{030b}', " ˝OŐoőUŰuű"), // double acute
|
||||
(0xCE, '\u{0328}', " ˛AĄaąEĘeęIĮiįOǪoǫUŲuų"), // ogonek
|
||||
(0xCF, '\u{030c}', " ˇAǍaǎCČcčDĎdďEĚeěGǦgǧHȞhȟIǏiǐjǰKǨkǩLĽlľNŇnňOǑoǒRŘrřSŠsšTŤtťUǓuǔZŽzž"), // caron
|
||||
];
|
||||
|
||||
/// ISO 6937's spacing characters above ASCII (codes 0xA1–0xFE that aren't diacritics).
|
||||
const ISO6937_SPACING: [(u8, char); 71] = [
|
||||
(0xA1, '¡'),
|
||||
(0xA2, '¢'),
|
||||
(0xA3, '£'),
|
||||
(0xA5, '¥'),
|
||||
(0xA7, '§'),
|
||||
(0xA8, '¤'),
|
||||
(0xA9, '‘'),
|
||||
(0xAA, '“'),
|
||||
(0xAB, '«'),
|
||||
(0xAC, '←'),
|
||||
(0xAD, '↑'),
|
||||
(0xAE, '→'),
|
||||
(0xAF, '↓'),
|
||||
(0xB0, '°'),
|
||||
(0xB1, '±'),
|
||||
(0xB2, '²'),
|
||||
(0xB3, '³'),
|
||||
(0xB4, '×'),
|
||||
(0xB5, 'µ'),
|
||||
(0xB6, '¶'),
|
||||
(0xB7, '·'),
|
||||
(0xB8, '÷'),
|
||||
(0xB9, '’'),
|
||||
(0xBA, '”'),
|
||||
(0xBB, '»'),
|
||||
(0xBC, '¼'),
|
||||
(0xBD, '½'),
|
||||
(0xBE, '¾'),
|
||||
(0xBF, '¿'),
|
||||
(0xD0, '―'),
|
||||
(0xD1, '¹'),
|
||||
(0xD2, '®'),
|
||||
(0xD3, '©'),
|
||||
(0xD4, '™'),
|
||||
(0xD5, '♪'),
|
||||
(0xD6, '¬'),
|
||||
(0xD7, '¦'),
|
||||
(0xDC, '⅛'),
|
||||
(0xDD, '⅜'),
|
||||
(0xDE, '⅝'),
|
||||
(0xDF, '⅞'),
|
||||
(0xE0, 'Ω'),
|
||||
(0xE1, 'Æ'),
|
||||
(0xE2, 'Đ'),
|
||||
(0xE3, 'ª'),
|
||||
(0xE4, 'Ħ'),
|
||||
(0xE6, 'IJ'),
|
||||
(0xE7, 'Ŀ'),
|
||||
(0xE8, 'Ł'),
|
||||
(0xE9, 'Ø'),
|
||||
(0xEA, 'Œ'),
|
||||
(0xEB, 'º'),
|
||||
(0xEC, 'Þ'),
|
||||
(0xED, 'Ŧ'),
|
||||
(0xEE, 'Ŋ'),
|
||||
(0xEF, 'ʼn'),
|
||||
(0xF0, 'ĸ'),
|
||||
(0xF1, 'æ'),
|
||||
(0xF2, 'đ'),
|
||||
(0xF3, 'ð'),
|
||||
(0xF4, 'ħ'),
|
||||
(0xF5, 'ı'),
|
||||
(0xF6, 'ij'),
|
||||
(0xF7, 'ŀ'),
|
||||
(0xF8, 'ł'),
|
||||
(0xF9, 'ø'),
|
||||
(0xFA, 'œ'),
|
||||
(0xFB, 'ß'),
|
||||
(0xFC, 'þ'),
|
||||
(0xFD, 'ŧ'),
|
||||
(0xFE, 'ŋ'),
|
||||
];
|
||||
|
||||
/// Text with Excel's SYLK character escapes decoded. Excel writes a character outside ASCII as
|
||||
/// `ESC N` and one byte, the character's ISO 6937 code less 0x80. A diacritic's code is followed
|
||||
/// by the letter it goes on (`ESC N H o` is `ö`, `ESC N {` is `ß`). A code ISO 6937 doesn't define,
|
||||
/// or an `ESC` that isn't followed by `N` and a code, is kept as written.
|
||||
fn decode_escapes(text: &str) -> String {
|
||||
if !text.contains('\u{1b}') {
|
||||
return text.to_string();
|
||||
}
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut chars = text.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
let mut ahead = chars.clone();
|
||||
let code = (c == '\u{1b}' && ahead.next() == Some('N'))
|
||||
.then(|| ahead.next())
|
||||
.flatten()
|
||||
.and_then(|b| u8::try_from(b).ok())
|
||||
.filter(|b| (0x20..0x80).contains(b));
|
||||
let Some(code) = code.map(|b| b + 0x80) else {
|
||||
out.push(c);
|
||||
continue;
|
||||
};
|
||||
if let Some(&(_, mark, pairs)) = ISO6937_DIACRITICS.iter().find(|(d, _, _)| *d == code) {
|
||||
// The letter it goes on: precomposed where Unicode has the pair, else letter and mark.
|
||||
// The base is an ordinary character: another escape (or any control character) after a
|
||||
// diacritic leaves this escape as written, and the next one is decoded on its own.
|
||||
let Some(base) = ahead.next().filter(|b| !b.is_control()) else {
|
||||
out.push(c);
|
||||
continue;
|
||||
};
|
||||
let letters: Vec<char> = pairs.chars().collect();
|
||||
match letters.chunks(2).find(|p| p.first() == Some(&base)).and_then(|p| p.get(1)) {
|
||||
Some(&composed) => out.push(composed),
|
||||
None => {
|
||||
out.push(base);
|
||||
out.push(mark);
|
||||
}
|
||||
}
|
||||
} else if let Some(&(_, ch)) = ISO6937_SPACING.iter().find(|(s, _)| *s == code) {
|
||||
out.push(ch);
|
||||
} else {
|
||||
out.push(c);
|
||||
continue;
|
||||
}
|
||||
chars = ahead;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One part of an R1C1 reference: `R`/`C` followed by an absolute number, a bracketed offset or
|
||||
/// nothing (the cell's own row or column).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
|
||||
@@ -115,3 +115,36 @@ fn survives_huge_relative_offsets_and_width_floods() {
|
||||
let _ = read(&b);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_excels_character_escapes_in_text() {
|
||||
// storytold/gridcraft#4321: Excel writes `Größe für Köln` with `ESC N` escapes (ISO 6937 codes
|
||||
// less 0x80: `H` is the diaeresis, put on the next letter; `{` is ß).
|
||||
let file = "ID;PWXL;N;E\r\nC;Y1;X1;K\"Gr\u{1b}NHo\u{1b}N{e f\u{1b}NHur K\u{1b}NHoln\"\r\nC;Y2;X1;K\"Plain ASCII 123\"\r\n\
|
||||
C;Y3;X1;K\"\u{1b}NBe \u{1b}NAa \u{1b}NCa \u{1b}NKc \u{1b}NOs \u{1b}NMo \u{1b}NJa \u{1b}NDn\"\r\n\
|
||||
C;Y4;X1;K\"\u{1b}NAa \u{1b}NAZ\"\r\nC;Y5;X1;K\"\u{1b}Nq \u{1b}NyStra\u{1b}N{e \u{1b}Na\u{1b}Ni\"\r\n\
|
||||
C;Y6;X1;K\"\u{1b}NHq \u{1b}N\u{1b}NHo end\u{1b}N\"\r\nC;Y7;X1;K\"\u{1b}N\u{7f} \u{1b}NL \u{1b}X\"\r\nC;Y8;X1;K\"\u{1b}NH\u{1b}NHo \u{1b}NB\tx\"\r\n\
|
||||
C;Y9;X1;K\"\u{1b}NB |\u{1b}NE |\u{1b}NF |\u{1b}NG |\u{1b}NH |\u{1b}NJ |\u{1b}NK |\u{1b}NM |\u{1b}NN |\u{1b}NO |\u{1b}NA |\u{1b}NC \"\r\nE\r\n";
|
||||
let sh = read(file.as_bytes()).unwrap();
|
||||
let value = |a: &str| sh.cells.get(&at(a)).map(|c| c.value.clone()).unwrap_or_default();
|
||||
let text = |s: &str| Value::Text(s.into());
|
||||
assert_eq!(value("A1"), text("Größe für Köln"));
|
||||
assert_eq!(value("A2"), text("Plain ASCII 123"), "plain ASCII is unchanged");
|
||||
// Acute, grave, circumflex, cedilla, caron, double acute, ring, tilde.
|
||||
assert_eq!(value("A3"), text("é à â ç š ő å ñ"));
|
||||
// A letter the mark has no precomposed form with keeps the combining mark.
|
||||
assert_eq!(value("A4"), text("à Z\u{300}"));
|
||||
// Spacing characters, one code each: `q` æ, `y` ø, `{` ß, `a` Æ, `i` Ø.
|
||||
assert_eq!(value("A5"), text("æ øStraße ÆØ"));
|
||||
// A diacritic on a letter with no precomposed form, and a stray `ESC N` before an escape.
|
||||
assert_eq!(value("A6"), text("q\u{308} \u{1b}Nö end\u{1b}N"));
|
||||
// Codes ISO 6937 doesn't define (0xFF, 0xCC) and an `ESC` without `N` stay as written.
|
||||
assert_eq!(value("A7"), text("\u{1b}N\u{7f} \u{1b}NL \u{1b}X"));
|
||||
// A diacritic followed by another escape (or a control character) is kept as written, and the
|
||||
// next escape still decodes.
|
||||
assert_eq!(value("A8"), text("\u{1b}NHö \u{1b}NB\tx"));
|
||||
// A diacritic on a SPACE is the spacing accent: acute, macron, breve, dot above, diaeresis, ring,
|
||||
// cedilla, double acute, ogonek, caron. The grave and circumflex have none (`` ` `` and `^` are
|
||||
// ASCII), so they stay a space with the combining mark.
|
||||
assert_eq!(value("A9"), text("´|¯|˘|˙|¨|˚|¸|˝|˛|ˇ| \u{300}| \u{302}"));
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "craft-dds"
|
||||
description = "DirectDraw Surface (DDS) textures: uncompressed and BC1-BC5 block compression, read and write, without dependencies"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Encode and decode rows of blocks (and mip levels) on several threads.
|
||||
parallel = ["dep:rayon"]
|
||||
|
||||
[dependencies]
|
||||
rayon = { workspace = true, optional = true }
|
||||
@@ -0,0 +1,64 @@
|
||||
# craft-dds
|
||||
|
||||
DirectDraw Surface (DDS) textures, after Microsoft's public DDS and block-compression
|
||||
documentation, with no dependencies. `craft-image-codecs` reads and writes DDS through this
|
||||
crate and re-exports its `DdsCompression` and `DdsOptions`; an app that only needs DDS (a texture
|
||||
exporter, say) can depend on it alone and leave out the other codecs and their licences.
|
||||
|
||||
```toml
|
||||
craft-dds = { git = "https://github.com/storytold/craft-libs", rev = "<commit>" }
|
||||
```
|
||||
|
||||
| | Read | Write |
|
||||
|---|---|---|
|
||||
| Uncompressed | 8, 16, 24 or 32 bits per pixel described by channel masks, RGB or luminance, with or without alpha; padded rows (`DDSD_PITCH`); 16 bits per channel: `R16_UNORM`, `R16G16_UNORM`, `R16G16B16A16_UNORM` (DXGI 56, 35, 11 and their typeless forms; legacy FourCC 36) | BGRA, BGRX, luminance, luminance with alpha; from 16-bit pixels `R16_UNORM`, `R16G16_UNORM`, `R16G16B16A16_UNORM` |
|
||||
| Block-compressed | BC1, BC2, BC3 (DXT1, DXT3, DXT5); unsigned BC4 (ATI1, BC4U, DXGI 79 and 80) and BC5 (ATI2, BC5U, DXGI 82 and 83) | BC1, BC2, BC3, BC4, BC5 |
|
||||
| Headers | legacy and DX10 | legacy; DX10 for BC4, BC5 and the 16-bit formats |
|
||||
| Mip levels | the top level only | optional full chain, box-filtered |
|
||||
|
||||
Cubemaps, texture arrays, volume textures, premultiplied-alpha textures (DXT2/DXT4, or DX10 alpha mode 2), signed (SNORM) BC4/BC5 and
|
||||
the other DXGI formats (BC6H, BC7, float, signed or integer 16-bit) are refused with `DdsError::Unsupported` rather than read
|
||||
in part.
|
||||
|
||||
Pixels are 8 bits per channel (`encode`, `Texture::decode`) or 16 bits per channel (`encode16`,
|
||||
`Texture::decode16`), interleaved and top row first, in one of four `Layout`s (`Gray`,
|
||||
`GrayA`, `Rgb`, `Rgba`). BC1 to BC3 are written from RGBA, BC4 from gray and BC5 from RGB
|
||||
(`DdsCompression::layout` names it). BC4 reads back as gray. BC5 stores red and green only, so it
|
||||
reads back as RGB with blue 0: for a tangent-space normal map, X and Y, with Z left to the
|
||||
renderer. ATI2 files are read as Microsoft's tools read them (red in the first block), not with
|
||||
the two blocks swapped as FFmpeg does.
|
||||
|
||||
From 16-bit pixels, uncompressed textures keep 16 bits per channel: gray as `R16_UNORM`, gray
|
||||
and alpha (or any two channels, such as a normal map's X and Y) as `R16G16_UNORM`, RGB (alpha
|
||||
65535) and RGBA as `R16G16B16A16_UNORM`, all with the DX10 header; they read back as gray, gray
|
||||
and alpha, and RGBA. BC4 and BC5 keep 8-bit endpoints, as the formats store them, but pick them
|
||||
and the indices against the 16-bit values, so a smooth height or normal map has less error than
|
||||
when its 8-bit rounding is compressed. BC1 to BC3 round the pixels to 8 bits first.
|
||||
`Texture::decode` rounds 16-bit formats to 8 bits; `Texture::decode16` returns them exactly,
|
||||
rescales channel-mask fields from their own width (a 16-bit luminance field exactly),
|
||||
interpolates BC4 and BC5 at 16 bits, and scales BC1 to BC3 by 257. `craft-image-codecs` still
|
||||
reads DDS at 8 bits.
|
||||
|
||||
```rust
|
||||
use craft_dds::{DdsCompression, DdsOptions, Layout, Texture};
|
||||
|
||||
let pixels = [10, 20, 30, 255].repeat(6 * 5);
|
||||
let file = craft_dds::encode(6, 5, Layout::Rgba, &pixels, DdsOptions { compression: DdsCompression::Bc1, mipmaps: true })?;
|
||||
// The header first: the size and layout are known before anything is allocated.
|
||||
let texture = Texture::parse(&file)?;
|
||||
assert_eq!((texture.width(), texture.height(), texture.layout()), (6, 5, Layout::Rgba));
|
||||
let decoded = texture.decode()?;
|
||||
```
|
||||
|
||||
`Texture::parse` requires a complete header and nonzero dimensions. `Texture::decode` bounds its output buffer to 1 GiB; use `decode_with_limit(max_bytes)` for an explicit per-buffer budget. Allocation is fallible. This is not a total process memory limit. The `craft-image-codecs` adapter forwards its own existing `Limits::max_alloc` instead of imposing the standalone default.
|
||||
|
||||
The block encoder is our own: RGB565 endpoints fitted along each block's principal axis and
|
||||
refined by least squares, indices picked against the palette the reader decodes; BC1 turns texels
|
||||
under half opacity into its transparent black, and BC3, BC4 and BC5 pick whichever mode of their
|
||||
value blocks fits. The block functions don't depend on the DDS header, so other containers of
|
||||
the same blocks can share them.
|
||||
|
||||
## Features
|
||||
|
||||
- `parallel`: encodes and decodes rows of blocks (and mip levels) on several threads with rayon.
|
||||
`craft-image-codecs` turns it on for native targets.
|
||||
@@ -0,0 +1,613 @@
|
||||
//! BC1 to BC5 (DXT1, DXT3, DXT5, ATI1, ATI2) blocks, after Microsoft's public block compression
|
||||
//! documentation ("Block Compression (Direct3D 10)" and "Texture Block Compression in Direct3D
|
||||
//! 11"): the block decoders and the block encoder the DDS reader and writer use. Nothing here
|
||||
//! knows about DDS headers, so other containers of the same blocks can use it.
|
||||
//!
|
||||
//! BC4 is one block of 16 single-channel values, the same block BC3 stores its alpha in; BC5 is
|
||||
//! two of them, red then green. [`value_block`] and [`decode_values`] serve all three;
|
||||
//! [`value_block16`] and [`decode_values16`] are the same blocks for 16-bit values (BC4 and BC5
|
||||
//! from 16-bit sources).
|
||||
//!
|
||||
//! The encoder fits a block's colours with two RGB565 endpoints along their principal axis,
|
||||
//! refines them by least squares, and scores every candidate against the exact palette the
|
||||
//! reader rebuilds, so the indices it picks are the best ones for what a decoder shows.
|
||||
|
||||
/// The 16 texels of a 4×4 block, row by row, as RGBA.
|
||||
pub(crate) type Block = [[u8; 4]; 16];
|
||||
|
||||
/// Bit `t` set for each texel `t` of a block that counts: edge blocks of a texture whose size is
|
||||
/// not a multiple of four have texels outside the image.
|
||||
pub(crate) type Mask = u16;
|
||||
|
||||
pub(crate) fn rgb565(c: u16) -> [u8; 3] {
|
||||
let scale = |v: u16, max: u16| ((u32::from(v) * 255 + u32::from(max) / 2) / u32::from(max)) as u8;
|
||||
[scale(c >> 11, 31), scale((c >> 5) & 63, 63), scale(c & 31, 31)]
|
||||
}
|
||||
|
||||
/// The colours a colour block's indices select. Four-colour mode (BC1 with `c0 > c1`, and every
|
||||
/// BC2 and BC3 block) adds two colours at thirds; three-colour mode adds the midpoint and
|
||||
/// transparent black.
|
||||
pub(crate) fn colour_palette(c0: u16, c1: u16, four: bool) -> [[u8; 4]; 4] {
|
||||
let (a, b) = (rgb565(c0), rgb565(c1));
|
||||
let mix = |wa: u32, wb: u32| {
|
||||
let d = wa + wb;
|
||||
let at = |i: usize| ((wa * u32::from(a[i]) + wb * u32::from(b[i]) + d / 2) / d) as u8;
|
||||
[at(0), at(1), at(2), 255]
|
||||
};
|
||||
if four {
|
||||
[[a[0], a[1], a[2], 255], [b[0], b[1], b[2], 255], mix(2, 1), mix(1, 2)]
|
||||
} else {
|
||||
[[a[0], a[1], a[2], 255], [b[0], b[1], b[2], 255], mix(1, 1), [0; 4]]
|
||||
}
|
||||
}
|
||||
|
||||
/// A value block's eight values: six interpolated at sevenths when `a0 > a1`, else four
|
||||
/// interpolated at fifths followed by 0 and 255.
|
||||
pub(crate) fn alpha_palette(a0: u8, a1: u8) -> [u8; 8] {
|
||||
let (a0, a1) = (u32::from(a0), u32::from(a1));
|
||||
let mut palette = [a0, a1, 0, 0, 0, 0, 0, 255];
|
||||
if a0 > a1 {
|
||||
for (k, v) in palette.iter_mut().enumerate().skip(2) {
|
||||
let k = k as u32;
|
||||
*v = ((8 - k) * a0 + (k - 1) * a1 + 3) / 7;
|
||||
}
|
||||
} else {
|
||||
for (k, v) in palette.iter_mut().enumerate().take(6).skip(2) {
|
||||
let k = k as u32;
|
||||
*v = ((6 - k) * a0 + (k - 1) * a1 + 2) / 5;
|
||||
}
|
||||
}
|
||||
palette.map(|v| v as u8)
|
||||
}
|
||||
|
||||
/// A colour block's 16 texels as RGBA. BC2 and BC3 colour blocks always have four colours
|
||||
/// (`one_bit_alpha` false); only BC1's `c0 <= c1` order selects three colours and transparent
|
||||
/// black. Missing bytes read as 0.
|
||||
pub(crate) fn decode_colour(block: &[u8], one_bit_alpha: bool) -> Block {
|
||||
let byte = |i: usize| block.get(i).copied().unwrap_or(0);
|
||||
let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)]));
|
||||
let palette = colour_palette(c0, c1, c0 > c1 || !one_bit_alpha);
|
||||
let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]);
|
||||
std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3])
|
||||
}
|
||||
|
||||
/// [`alpha_palette`] at 16 bits: the same eight values before they are rounded to 8 bits, as a
|
||||
/// decoder that interpolates at a higher precision shows them (endpoints scaled by 257, so 255
|
||||
/// is 65535).
|
||||
pub(crate) fn alpha_palette16(a0: u8, a1: u8) -> [u16; 8] {
|
||||
let (a0, a1) = (u32::from(a0) * 257, u32::from(a1) * 257);
|
||||
let mut palette = [a0, a1, 0, 0, 0, 0, 0, 65535];
|
||||
if a0 > a1 {
|
||||
for (k, v) in palette.iter_mut().enumerate().skip(2) {
|
||||
let k = k as u32;
|
||||
*v = ((8 - k) * a0 + (k - 1) * a1 + 3) / 7;
|
||||
}
|
||||
} else {
|
||||
for (k, v) in palette.iter_mut().enumerate().take(6).skip(2) {
|
||||
let k = k as u32;
|
||||
*v = ((6 - k) * a0 + (k - 1) * a1 + 2) / 5;
|
||||
}
|
||||
}
|
||||
palette.map(|v| v as u16)
|
||||
}
|
||||
|
||||
/// BC2's alpha: four explicit bits per texel.
|
||||
pub(crate) fn decode_bc2_alpha(block: &[u8]) -> [u8; 16] {
|
||||
let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0)));
|
||||
std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17)
|
||||
}
|
||||
|
||||
/// A value block (BC3's alpha, BC4, either half of BC5): two endpoints and a 3-bit index per
|
||||
/// texel into eight values, or into six plus 0 and 255 when `a0 <= a1`. Missing bytes read as 0.
|
||||
pub(crate) fn decode_values(block: &[u8]) -> [u8; 16] {
|
||||
let byte = |i: usize| block.get(i).copied().unwrap_or(0);
|
||||
let palette = alpha_palette(byte(0), byte(1));
|
||||
let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]);
|
||||
std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize])
|
||||
}
|
||||
|
||||
/// [`decode_values`] at 16 bits, from [`alpha_palette16`].
|
||||
pub(crate) fn decode_values16(block: &[u8]) -> [u16; 16] {
|
||||
let byte = |i: usize| block.get(i).copied().unwrap_or(0);
|
||||
let palette = alpha_palette16(byte(0), byte(1));
|
||||
let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]);
|
||||
std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize])
|
||||
}
|
||||
|
||||
/// A BC1 block. Texels under half opacity become BC1's transparent black, which needs the
|
||||
/// three-colour mode; an opaque block takes whichever mode fits its colours better.
|
||||
pub(crate) fn encode_bc1(block: &Block, inside: Mask) -> [u8; 8] {
|
||||
let transparent = select(block, inside, |t| t[3] < 128);
|
||||
colour_block(block, inside & !transparent, transparent, true)
|
||||
}
|
||||
|
||||
/// A BC2 block: four bits of alpha per texel, then a four-colour block.
|
||||
pub(crate) fn encode_bc2(block: &Block, inside: Mask) -> [u8; 16] {
|
||||
let bits = block.iter().enumerate().fold(0u64, |bits, (t, texel)| bits | ((u64::from(texel[3]) * 15 + 127) / 255) << (4 * t));
|
||||
join(bits.to_le_bytes(), colour_block(block, inside, 0, false))
|
||||
}
|
||||
|
||||
/// A BC3 block: an interpolated alpha block, then a four-colour block.
|
||||
pub(crate) fn encode_bc3(block: &Block, inside: Mask) -> [u8; 16] {
|
||||
let alphas = std::array::from_fn(|t| block[t][3]);
|
||||
join(value_block(&alphas, inside), colour_block(block, inside, 0, false))
|
||||
}
|
||||
|
||||
fn join(a: [u8; 8], b: [u8; 8]) -> [u8; 16] {
|
||||
std::array::from_fn(|i| if i < 8 { a.get(i) } else { b.get(i - 8) }.copied().unwrap_or(0))
|
||||
}
|
||||
|
||||
/// The texels of `inside` for which `f` holds.
|
||||
fn select(block: &Block, inside: Mask, f: impl Fn(&[u8; 4]) -> bool) -> Mask {
|
||||
block.iter().enumerate().filter(|(t, texel)| inside >> t & 1 != 0 && f(texel)).fold(0, |m, (t, _)| m | 1 << t)
|
||||
}
|
||||
|
||||
fn rgb_of(texel: &[u8; 4]) -> [f32; 3] {
|
||||
[f32::from(texel[0]), f32::from(texel[1]), f32::from(texel[2])]
|
||||
}
|
||||
|
||||
/// RGB565 of a colour, rounding each channel to the nearest level.
|
||||
fn to565(c: [f32; 3]) -> u16 {
|
||||
let q = |v: f32, max: f32| (v.clamp(0.0, 255.0) * max / 255.0).round() as u16;
|
||||
q(c[0], 31.0) << 11 | q(c[1], 63.0) << 5 | q(c[2], 31.0)
|
||||
}
|
||||
|
||||
fn distance(a: &[u8; 4], b: &[u8; 4]) -> u32 {
|
||||
let d = |i: usize| u32::from(a[i].abs_diff(b[i])).pow(2);
|
||||
d(0) + d(1) + d(2)
|
||||
}
|
||||
|
||||
/// The index of the first of `palette`'s entries nearest by `distance`, and that distance: the
|
||||
/// entry `Iterator::min_by_key` picks, found without the iterator (a texel equal to an entry
|
||||
/// stops the search, as no later entry can be nearer and an equal one doesn't win).
|
||||
fn nearest<T>(palette: &[T], distance: impl Fn(&T) -> u32) -> (usize, u32) {
|
||||
let mut best = (0, u32::MAX);
|
||||
for (i, p) in palette.iter().enumerate() {
|
||||
let d = distance(p);
|
||||
if d < best.1 {
|
||||
best = (i, d);
|
||||
if d == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if palette.is_empty() { (0, 0) } else { best }
|
||||
}
|
||||
|
||||
/// A colour block (endpoints, then 2-bit indices) for the `opaque` texels' colours, with the
|
||||
/// `transparent` ones on BC1's transparent index. `bc1` allows the three-colour mode; BC2 and
|
||||
/// BC3 always decode four colours.
|
||||
fn colour_block(block: &Block, opaque: Mask, transparent: Mask, bc1: bool) -> [u8; 8] {
|
||||
// The opaque texels' colours in texel order, on the stack (after storytold/photocraft#10669,
|
||||
// which took the per-block allocations out of PhotoCraft's copy of this encoder).
|
||||
let mut buffer = [[0f32; 3]; 16];
|
||||
let mut n = 0;
|
||||
for (t, texel) in block.iter().enumerate() {
|
||||
if opaque >> t & 1 != 0
|
||||
&& let Some(slot) = buffer.get_mut(n)
|
||||
{
|
||||
*slot = rgb_of(texel);
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
let colours = buffer.get(..n).unwrap_or_default();
|
||||
let Some((first, rest)) = colours.split_first() else {
|
||||
// Nothing visible: every texel on the transparent index (or index 0 of black).
|
||||
let indices = if transparent != 0 { u32::MAX } else { 0 };
|
||||
return pack(0, 0, indices);
|
||||
};
|
||||
// One colour has no axis: `principal_endpoints` returns that colour (their mean) for both
|
||||
// ends, so it is skipped. Everything after it runs as for any block.
|
||||
let (lo, hi) = if rest.iter().all(|c| c == first) { (*first, *first) } else { principal_endpoints(colours) };
|
||||
let mut candidate = (to565(lo), to565(hi));
|
||||
// The endpoint pairs fitted so far: at most one per round.
|
||||
let mut fitted = [candidate; 3];
|
||||
let mut count = 1;
|
||||
let modes: &[bool] = if transparent != 0 {
|
||||
&[false]
|
||||
} else if bc1 {
|
||||
&[true, false]
|
||||
} else {
|
||||
&[true]
|
||||
};
|
||||
let mut best: Option<(u32, [u8; 8])> = None;
|
||||
// Each refinement starts from the best block so far: its indices give the least-squares
|
||||
// endpoints, which become one more candidate.
|
||||
for round in 0..3 {
|
||||
let (e0, e1) = candidate;
|
||||
for &four in modes {
|
||||
let (bytes, error) = fit(block, opaque, transparent, e0, e1, four);
|
||||
if best.is_none_or(|(e, _)| error < e) {
|
||||
best = Some((error, bytes));
|
||||
}
|
||||
}
|
||||
let Some((error, bytes)) = best else { break };
|
||||
if error == 0 || round == 2 {
|
||||
break;
|
||||
}
|
||||
let Some(next) = least_squares(block, opaque, &bytes) else { break };
|
||||
// A pair fitted before (in either order: `fit` orders the endpoints itself) fits the same
|
||||
// again and can't beat the best block, which then gives the same pair again: the rounds
|
||||
// left would change nothing.
|
||||
let seen = fitted.get(..count).unwrap_or_default();
|
||||
if seen.iter().any(|&(a, b)| (a, b) == next || (b, a) == next) {
|
||||
break;
|
||||
}
|
||||
if let Some(slot) = fitted.get_mut(count) {
|
||||
*slot = next;
|
||||
count += 1;
|
||||
}
|
||||
candidate = next;
|
||||
}
|
||||
best.map_or([0; 8], |(_, bytes)| bytes)
|
||||
}
|
||||
|
||||
/// Orders endpoints `e0`, `e1` for the mode (`c0 > c1` for four colours, `c0 <= c1` for three)
|
||||
/// and picks each texel's nearest palette colour. Returns the block and its squared error.
|
||||
fn fit(block: &Block, opaque: Mask, transparent: Mask, e0: u16, e1: u16, four: bool) -> ([u8; 8], u32) {
|
||||
let (c0, c1) = if four == (e0 >= e1) { (e0, e1) } else { (e1, e0) };
|
||||
// Equal endpoints can't be in four-colour order; the palette is then one colour either way.
|
||||
let palette = colour_palette(c0, c1, four && c0 > c1);
|
||||
let choices = if four && c0 > c1 { 4 } else { 3 };
|
||||
let mut indices = 0u32;
|
||||
let mut error = 0u32;
|
||||
for (t, texel) in block.iter().enumerate() {
|
||||
let index = if transparent >> t & 1 != 0 {
|
||||
3
|
||||
} else if opaque >> t & 1 != 0 {
|
||||
let (i, d) = nearest(palette.get(..choices).unwrap_or_default(), |p| distance(texel, p));
|
||||
error += d;
|
||||
i as u32
|
||||
} else {
|
||||
0
|
||||
};
|
||||
indices |= index << (2 * t);
|
||||
}
|
||||
(pack(c0, c1, indices), error)
|
||||
}
|
||||
|
||||
fn pack(c0: u16, c1: u16, indices: u32) -> [u8; 8] {
|
||||
let (c0, c1, i) = (c0.to_le_bytes(), c1.to_le_bytes(), indices.to_le_bytes());
|
||||
[c0[0], c0[1], c1[0], c1[1], i[0], i[1], i[2], i[3]]
|
||||
}
|
||||
|
||||
/// The ends of the colours' spread along their principal axis (the covariance's dominant
|
||||
/// eigenvector, by power iteration), clamped to the RGB cube.
|
||||
fn principal_endpoints(colours: &[[f32; 3]]) -> ([f32; 3], [f32; 3]) {
|
||||
let n = colours.len().max(1) as f32;
|
||||
let mean = [0, 1, 2].map(|i| colours.iter().map(|c| c[i]).sum::<f32>() / n);
|
||||
let mut cov = [[0f32; 3]; 3];
|
||||
for c in colours {
|
||||
let d = [c[0] - mean[0], c[1] - mean[1], c[2] - mean[2]];
|
||||
for (i, row) in cov.iter_mut().enumerate() {
|
||||
for (j, v) in row.iter_mut().enumerate() {
|
||||
*v += d[i] * d[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
let times = |a: [f32; 3]| [0, 1, 2].map(|i| cov[i][0] * a[0] + cov[i][1] * a[1] + cov[i][2] * a[2]);
|
||||
let iterate = |start: usize| {
|
||||
let mut axis = [0f32; 3];
|
||||
axis[start] = 1.0;
|
||||
for _ in 0..8 {
|
||||
let next = times(axis);
|
||||
let len = next.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||
if !len.is_normal() {
|
||||
break;
|
||||
}
|
||||
let normalised = next.map(|v| v / len);
|
||||
// A step that returns the same axis, bit for bit, returns it on every later step too:
|
||||
// stopping there gives the very axis the remaining steps would. Kept only if timing
|
||||
// shows the comparison pays for itself.
|
||||
if normalised.map(f32::to_bits) == axis.map(f32::to_bits) {
|
||||
break;
|
||||
}
|
||||
axis = normalised;
|
||||
}
|
||||
axis
|
||||
};
|
||||
// From each channel's axis, keeping the one the colours spread along most: a channel that
|
||||
// varies on its own is already an eigenvector, so a single start there would never leave it
|
||||
// even when the others vary more together.
|
||||
let spread = |a: [f32; 3]| times(a).iter().zip(a).map(|(x, y)| x * y).sum::<f32>();
|
||||
let axis = [0, 1, 2].map(iterate).into_iter().max_by(|&a, &b| spread(a).total_cmp(&spread(b))).unwrap_or([1.0, 0.0, 0.0]);
|
||||
let along = |c: &[f32; 3]| (c[0] - mean[0]) * axis[0] + (c[1] - mean[1]) * axis[1] + (c[2] - mean[2]) * axis[2];
|
||||
let (lo, hi) = colours.iter().map(along).fold((f32::MAX, f32::MIN), |(lo, hi), t| (lo.min(t), hi.max(t)));
|
||||
let at = |t: f32| [0, 1, 2].map(|i| (mean[i] + t * axis[i]).clamp(0.0, 255.0));
|
||||
if lo > hi { (mean, mean) } else { (at(lo), at(hi)) }
|
||||
}
|
||||
|
||||
/// The endpoints that minimise the squared error for the indices `bytes` already chose: each
|
||||
/// texel's index fixes its weights between the endpoints, giving 2×2 normal equations per
|
||||
/// channel. `None` when the indices don't constrain both endpoints.
|
||||
fn least_squares(block: &Block, opaque: Mask, bytes: &[u8; 8]) -> Option<(u16, u16)> {
|
||||
let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]]));
|
||||
let indices = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
|
||||
let four = c0 > c1;
|
||||
// How much of c0 each index holds.
|
||||
let weight = |i: u32| match (four, i) {
|
||||
(_, 0) => Some(1.0f32),
|
||||
(_, 1) => Some(0.0),
|
||||
(true, 2) => Some(2.0 / 3.0),
|
||||
(true, _) => Some(1.0 / 3.0),
|
||||
(false, 2) => Some(0.5),
|
||||
(false, _) => None,
|
||||
};
|
||||
let (mut aa, mut ab, mut bb) = (0f32, 0f32, 0f32);
|
||||
let (mut ax, mut bx) = ([0f32; 3], [0f32; 3]);
|
||||
for (t, texel) in block.iter().enumerate() {
|
||||
if opaque >> t & 1 == 0 {
|
||||
continue;
|
||||
}
|
||||
let Some(a) = weight(indices >> (2 * t) & 3) else { continue };
|
||||
let b = 1.0 - a;
|
||||
let c = rgb_of(texel);
|
||||
aa += a * a;
|
||||
ab += a * b;
|
||||
bb += b * b;
|
||||
for ((x0, x1), v) in ax.iter_mut().zip(bx.iter_mut()).zip(c) {
|
||||
*x0 += a * v;
|
||||
*x1 += b * v;
|
||||
}
|
||||
}
|
||||
let det = aa * bb - ab * ab;
|
||||
if det.abs() < 1e-3 {
|
||||
return None;
|
||||
}
|
||||
let e0 = [0, 1, 2].map(|i| (ax[i] * bb - bx[i] * ab) / det);
|
||||
let e1 = [0, 1, 2].map(|i| (bx[i] * aa - ax[i] * ab) / det);
|
||||
Some((to565(e0), to565(e1)))
|
||||
}
|
||||
|
||||
/// A block of 16 single-channel values (BC3's alpha, BC4, either half of BC5): whichever of the two modes
|
||||
/// (eight values between the extremes, or six between the extremes other than 0 and 255, plus
|
||||
/// exact 0 and 255) fits better. Only the values of `inside` count.
|
||||
pub(crate) fn value_block(values: &[u8; 16], inside: Mask) -> [u8; 8] {
|
||||
// Both ranges in one pass: of all values, and of those other than 0 and 255.
|
||||
let mut all: Option<(u8, u8)> = None;
|
||||
let mut mid = all;
|
||||
let widen = |r: Option<(u8, u8)>, v: u8| Some(r.map_or((v, v), |(lo, hi)| (lo.min(v), hi.max(v))));
|
||||
for (t, &v) in values.iter().enumerate() {
|
||||
if inside >> t & 1 != 0 {
|
||||
all = widen(all, v);
|
||||
if !matches!(v, 0 | 255) {
|
||||
mid = widen(mid, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (lo, hi) = all.unwrap_or((0, 0));
|
||||
// Eight values need a0 > a1; equal ends fall to the six-value mode, which starts with a0 too.
|
||||
let eight = value_fit(values, inside, hi, lo);
|
||||
let (mid_lo, mid_hi) = mid.unwrap_or((0, 0));
|
||||
let six = value_fit(values, inside, mid_lo, mid_hi);
|
||||
if six.1 < eight.1 { six.0 } else { eight.0 }
|
||||
}
|
||||
|
||||
fn value_fit(values: &[u8; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) {
|
||||
let palette = alpha_palette(a0, a1);
|
||||
let mut bits = 0u64;
|
||||
let mut error = 0u32;
|
||||
for (t, &v) in values.iter().enumerate() {
|
||||
if inside >> t & 1 == 0 {
|
||||
continue;
|
||||
}
|
||||
let (i, d) = nearest(&palette, |&p| u32::from(p.abs_diff(v)).pow(2));
|
||||
error += d;
|
||||
bits |= (i as u64) << (3 * t);
|
||||
}
|
||||
let b = bits.to_le_bytes();
|
||||
([a0, a1, b[0], b[1], b[2], b[3], b[4], b[5]], error)
|
||||
}
|
||||
|
||||
/// [`value_block`] for 16-bit values: the endpoints are still 8 bits, but each end of each mode
|
||||
/// tries the 8-bit levels just below and above the 16-bit extreme, and every candidate is scored
|
||||
/// against the 16-bit palette ([`alpha_palette16`]). A block that spans less than one 8-bit level
|
||||
/// is flat from 8-bit values but gets the interpolated steps between two adjacent levels here.
|
||||
pub(crate) fn value_block16(values: &[u16; 16], inside: Mask) -> [u8; 8] {
|
||||
let mut all: Option<(u16, u16)> = None;
|
||||
let mut mid = all;
|
||||
let widen = |r: Option<(u16, u16)>, v: u16| Some(r.map_or((v, v), |(lo, hi)| (lo.min(v), hi.max(v))));
|
||||
for (t, &v) in values.iter().enumerate() {
|
||||
if inside >> t & 1 != 0 {
|
||||
all = widen(all, v);
|
||||
if !matches!(v, 0 | 65535) {
|
||||
mid = widen(mid, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (lo, hi) = all.unwrap_or((0, 0));
|
||||
let (mid_lo, mid_hi) = mid.unwrap_or((0, 0));
|
||||
let (down, up) = (|v: u16| (v / 257) as u8, |v: u16| v.div_ceil(257) as u8);
|
||||
// The eight-value mode's pairs first (a0 the high end), then the six-value mode's: on a tie
|
||||
// the eight-value mode wins, as in `value_block`. Each pair's palette follows its order, so a
|
||||
// pair that lands in the other mode is still scored as the decoder reads it.
|
||||
let candidates = [
|
||||
(up(hi), down(lo)),
|
||||
(up(hi), up(lo)),
|
||||
(down(hi), down(lo)),
|
||||
(down(hi), up(lo)),
|
||||
(down(mid_lo), down(mid_hi)),
|
||||
(down(mid_lo), up(mid_hi)),
|
||||
(up(mid_lo), down(mid_hi)),
|
||||
(up(mid_lo), up(mid_hi)),
|
||||
];
|
||||
let mut best: Option<([u8; 8], u64)> = None;
|
||||
for (a0, a1) in candidates {
|
||||
let fit = value_fit16(values, inside, a0, a1);
|
||||
if best.is_none_or(|(_, e)| fit.1 < e) {
|
||||
best = Some(fit);
|
||||
}
|
||||
}
|
||||
best.map_or([0; 8], |(bytes, _)| bytes)
|
||||
}
|
||||
|
||||
fn value_fit16(values: &[u16; 16], inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u64) {
|
||||
let palette = alpha_palette16(a0, a1);
|
||||
let mut bits = 0u64;
|
||||
let mut error = 0u64;
|
||||
for (t, &v) in values.iter().enumerate() {
|
||||
if inside >> t & 1 == 0 {
|
||||
continue;
|
||||
}
|
||||
let (i, d) = nearest(&palette, |&p| u32::from(p.abs_diff(v)).pow(2));
|
||||
error += u64::from(d);
|
||||
bits |= (i as u64) << (3 * t);
|
||||
}
|
||||
let b = bits.to_le_bytes();
|
||||
([a0, a1, b[0], b[1], b[2], b[3], b[4], b[5]], error)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn solid_blocks_land_on_the_nearest_565_colour() {
|
||||
for colour in [[0, 0, 0, 255], [255, 255, 255, 255], [12, 200, 77, 255]] {
|
||||
let block = [colour; 16];
|
||||
let out = decode_colour(&encode_bc1(&block, u16::MAX), true);
|
||||
for texel in out {
|
||||
assert!(texel.iter().zip(colour).all(|(a, b)| a.abs_diff(b) <= 4), "{texel:?} for {colour:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_colour_blocks_are_exact_when_the_colours_are_565() {
|
||||
let (a, b) = ([255, 0, 0, 255], [0, 0, 255, 255]);
|
||||
let block: Block = std::array::from_fn(|t| if t % 3 == 0 { a } else { b });
|
||||
for bc1 in [true, false] {
|
||||
let bytes = if bc1 { encode_bc1(&block, u16::MAX) } else { colour_block(&block, u16::MAX, 0, false) };
|
||||
assert_eq!(decode_colour(&bytes, bc1), block);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bc1_punches_through_texels_under_half_alpha() {
|
||||
let block: Block = std::array::from_fn(|t| if t < 8 { [200, 100, 50, 255] } else { [9, 9, 9, 127] });
|
||||
let bytes = encode_bc1(&block, u16::MAX);
|
||||
let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]]));
|
||||
assert!(c0 <= c1, "three-colour mode");
|
||||
let out = decode_colour(&bytes, true);
|
||||
assert!(out[..8].iter().all(|t| t[3] == 255) && out[8..].iter().all(|t| *t == [0; 4]), "{out:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn four_colour_blocks_keep_c0_above_c1() {
|
||||
// A ramp needs all four colours; BC2 and BC3 blocks must not rely on the order, but
|
||||
// older decoders read c0 <= c1 as three colours, so the encoder never writes that.
|
||||
let block: Block = std::array::from_fn(|t| [(t * 16) as u8, 0, 0, 255]);
|
||||
let bytes = colour_block(&block, u16::MAX, 0, false);
|
||||
assert!(u16::from_le_bytes([bytes[0], bytes[1]]) > u16::from_le_bytes([bytes[2], bytes[3]]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alpha_blocks_pick_the_mode_that_keeps_0_and_255() {
|
||||
// 0, 255 and a cluster in between: the six-value mode stores 0 and 255 exactly.
|
||||
let block: Block = std::array::from_fn(|t| [0, 0, 0, [0, 255, 100, 110][t % 4]]);
|
||||
let bytes = value_block(&block.map(|t| t[3]), u16::MAX);
|
||||
assert!(bytes[0] <= bytes[1], "six-value mode");
|
||||
let palette = alpha_palette(bytes[0], bytes[1]);
|
||||
let bits = u64::from_le_bytes([bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], 0, 0]);
|
||||
for (t, texel) in block.iter().enumerate() {
|
||||
let got = palette[(bits >> (3 * t)) as usize & 7];
|
||||
assert!(got.abs_diff(texel[3]) <= 2, "texel {t}: {got} for {}", texel[3]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_principal_axis_is_found_when_one_channel_varies_on_its_own() {
|
||||
// Red alternates on its own (variance 64); green and blue alternate together (variance 49
|
||||
// each, 98 along their shared axis), uncorrelated with red. Starting on red alone would
|
||||
// keep red's axis and flatten green and blue; the block must do better than its bounding box.
|
||||
let block: Block = std::array::from_fn(|t| {
|
||||
let gb = if (t / 2) % 2 == 0 { 121 } else { 135 };
|
||||
[if t % 2 == 0 { 120 } else { 136 }, gb, gb, 255]
|
||||
});
|
||||
let error = |bytes: &[u8; 8]| decode_colour(bytes, false).iter().zip(&block).map(|(a, b)| distance(a, b)).sum::<u32>();
|
||||
let (_, bounding_box) = fit(&block, u16::MAX, 0, to565([120.0, 121.0, 121.0]), to565([136.0, 135.0, 135.0]), true);
|
||||
let ours = error(&colour_block(&block, u16::MAX, 0, false));
|
||||
assert!(ours < bounding_box, "{ours} vs the bounding box's {bounding_box}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_blocks_decode_the_frozen_vectors() {
|
||||
// Eight-value mode (a0 = 200 > a1 = 50), every index 0.
|
||||
assert_eq!(decode_values(&[0xc8, 0x32, 0, 0, 0, 0, 0, 0]), [200; 16]);
|
||||
// Six-value mode (a0 = 10 <= a1 = 250), every index 1.
|
||||
assert_eq!(decode_values(&[0x0a, 0xfa, 0x49, 0x92, 0x24, 0x49, 0x92, 0x24]), [250; 16]);
|
||||
// Six-value mode's indices 6 and 7 are 0 and 255 whatever the endpoints.
|
||||
let bits: u64 = (0..16).fold(0, |b, t| b | (6 + (t % 2)) << (3 * t));
|
||||
let b = bits.to_le_bytes();
|
||||
let got = decode_values(&[10, 250, b[0], b[1], b[2], b[3], b[4], b[5]]);
|
||||
assert_eq!(got, std::array::from_fn(|t| if t % 2 == 0 { 0 } else { 255 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_blocks_round_trip_ramps_and_solids() {
|
||||
for values in [[200u8; 16], [0; 16], [255; 16], std::array::from_fn(|t| (t * 17) as u8), std::array::from_fn(|t| 100 + t as u8)] {
|
||||
let back = decode_values(&value_block(&values, u16::MAX));
|
||||
let worst = values.iter().zip(back).map(|(a, b)| a.abs_diff(b)).max().unwrap_or(0);
|
||||
// Within half a step of either mode: 255 / 7 apart in the eight-value mode, about
|
||||
// 221 / 5 in the six-value one.
|
||||
assert!(worst <= 23, "{values:?} -> {back:?}");
|
||||
}
|
||||
assert_eq!(decode_values(&value_block(&[200; 16], u16::MAX)), [200; 16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nearest_picks_what_min_by_key_picks() {
|
||||
// Ties (equal entries, entries equally far on both sides) and exact hits included.
|
||||
let mut seed = 0x2545_f491u32;
|
||||
for _ in 0..2000 {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
let palette = [seed as u8 & 0xf0, (seed >> 8) as u8 & 0xf0, (seed >> 16) as u8 & 0xf0, (seed >> 24) as u8 & 0xf0];
|
||||
let v = (seed >> 4) as u8 & 0xf8;
|
||||
let d = |p: &u8| u32::from(p.abs_diff(v)).pow(2);
|
||||
let want = palette.iter().enumerate().map(|(i, p)| (i, d(p))).min_by_key(|&(_, d)| d).unwrap();
|
||||
assert_eq!(nearest(&palette, d), want, "{palette:?} {v}");
|
||||
}
|
||||
assert_eq!(nearest::<u8>(&[], |_| 7), (0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_blocks_decode_the_frozen_vectors_at_16_bits() {
|
||||
// Eight-value mode (200 > 50): index 2 is (6 * 200 + 50) / 7 of 255 at 16 bits, which
|
||||
// the 8-bit palette rounds to 179.
|
||||
let bits: u64 = (0..16).fold(0, |b, t| b | 2 << (3 * t));
|
||||
let b = bits.to_le_bytes();
|
||||
let block = [200, 50, b[0], b[1], b[2], b[3], b[4], b[5]];
|
||||
assert_eq!(decode_values16(&block), [(((6 * 200 + 50) * 257 + 3) / 7_u32) as u16; 16]);
|
||||
assert_eq!(decode_values(&block), [179; 16]);
|
||||
// Six-value mode's indices 6 and 7 are 0 and 65535.
|
||||
let bits: u64 = (0..16).fold(0, |b, t| b | (6 + (t % 2)) << (3 * t));
|
||||
let b = bits.to_le_bytes();
|
||||
let got = decode_values16(&[10, 250, b[0], b[1], b[2], b[3], b[4], b[5]]);
|
||||
assert_eq!(got, std::array::from_fn(|t| if t % 2 == 0 { 0 } else { 65535 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn value_blocks_from_16_bits_step_between_two_8_bit_levels() {
|
||||
// A ramp across 0.8 of one 8-bit level: from 8 bits it is one or two flat values; from 16
|
||||
// bits the endpoints straddle it and the steps between them follow the ramp.
|
||||
let values: [u16; 16] = std::array::from_fn(|t| 25_700 + (t as u16) * 13);
|
||||
let error = |got: [u16; 16]| values.iter().zip(got).map(|(a, b)| u64::from(a.abs_diff(b)).pow(2)).sum::<u64>();
|
||||
let ours = error(decode_values16(&value_block16(&values, u16::MAX)));
|
||||
let from_8_bits = error(decode_values16(&value_block(&values.map(|v| ((u32::from(v) + 128) / 257) as u8), u16::MAX)));
|
||||
assert!(ours * 4 < from_8_bits, "{ours} vs {from_8_bits} from 8 bits");
|
||||
// Values outside the image don't count.
|
||||
let solid = value_block16(&std::array::from_fn(|t| if t == 0 { 1000 } else { 60_000 }), 1);
|
||||
assert!(decode_values16(&solid)[0].abs_diff(1000) < 129);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn texels_outside_the_image_are_ignored() {
|
||||
// Only the top-left texel is inside: its colour alone decides the endpoints.
|
||||
let mut block: Block = [[0, 255, 0, 255]; 16];
|
||||
block[0] = [255, 0, 255, 255];
|
||||
let out = decode_colour(&encode_bc1(&block, 1), true);
|
||||
assert_eq!(out[0], [255, 0, 255, 255]);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,46 @@
|
||||
use craft_dds::{DdsCompression, DdsOptions, Layout, Texture, encode};
|
||||
#[test]
|
||||
fn metadata_requires_a_complete_header() {
|
||||
for (compression, layout, channels, header) in [(DdsCompression::None, Layout::Gray, 1, 128), (DdsCompression::Bc4, Layout::Gray, 1, 148)] {
|
||||
let data = encode(4, 4, layout, &vec![7; 16 * channels], DdsOptions { compression, mipmaps: false }).unwrap();
|
||||
for n in 0..header {
|
||||
assert!(Texture::parse(&data[..n]).is_err(), "accepted {n}/{header} header bytes");
|
||||
}
|
||||
assert!(Texture::parse(&data[..header]).is_ok());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn empty_dimensions_are_not_textures() {
|
||||
for (w, h) in [(0, 0), (0, 4), (4, 0)] {
|
||||
assert!(encode(w, h, Layout::Gray, &[], DdsOptions::default()).is_err(), "encoded {w}x{h}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn decoded_output_respects_explicit_budget() {
|
||||
let data = encode(4, 4, Layout::Rgba, &[7; 64], DdsOptions { compression: DdsCompression::Bc1, mipmaps: false }).unwrap();
|
||||
let texture = Texture::parse(&data).unwrap();
|
||||
assert!(matches!(texture.decode_with_limit(63), Err(craft_dds::DdsError::LimitExceeded { required: 64, limit: 63 })));
|
||||
assert_eq!(texture.decode_with_limit(64).unwrap().len(), 64);
|
||||
assert_eq!(texture.decode().unwrap(), texture.decode_with_limit(u64::MAX).unwrap());
|
||||
}
|
||||
#[test]
|
||||
fn parse_rejects_empty_dimensions() {
|
||||
let valid = encode(1, 1, Layout::Gray, &[7], DdsOptions::default()).unwrap();
|
||||
for (w, h) in [(0u32, 0u32), (0, 4), (4, 0)] {
|
||||
let mut data = valid.clone();
|
||||
data[12..16].copy_from_slice(&h.to_le_bytes());
|
||||
data[16..20].copy_from_slice(&w.to_le_bytes());
|
||||
assert!(Texture::parse(&data).is_err());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn default_budget_rejects_oversized_header_before_payload_or_allocation() {
|
||||
let mut data = encode(1, 1, Layout::Gray, &[7], DdsOptions::default()).unwrap();
|
||||
data.truncate(128);
|
||||
data[12..16].copy_from_slice(&65536u32.to_le_bytes());
|
||||
data[16..20].copy_from_slice(&32768u32.to_le_bytes());
|
||||
data[20..24].copy_from_slice(&32768u32.to_le_bytes());
|
||||
let texture = Texture::parse(&data).unwrap();
|
||||
assert!(matches!(texture.decode(), Err(craft_dds::DdsError::LimitExceeded { limit: craft_dds::DEFAULT_MAX_DECODE_BYTES, .. })));
|
||||
assert!(matches!(texture.decode_with_limit(u64::MAX), Err(craft_dds::DdsError::Malformed(_))));
|
||||
}
|
||||
@@ -1,22 +1,33 @@
|
||||
# craft-doc-binary
|
||||
|
||||
Reader for the Word 97-2003 binary file format (`.doc`, [MS-DOC]) into the `craft-wordmodel`
|
||||
document model. It is WordCraft's reader, on `craft-cfb` for the compound file and
|
||||
`craft-codepage` for 8-bit text.
|
||||
Reader and writer for the Word 97-2003 binary file format (`.doc`, [MS-DOC]) and the
|
||||
`craft-wordmodel` document model. It is WordCraft's reader and writer, on `craft-cfb` for the
|
||||
compound file and `craft-codepage` for 8-bit text.
|
||||
|
||||
```rust
|
||||
// Not a compound file: an error, never a panic.
|
||||
assert!(craft_doc_binary::read(b"not a .doc").is_err());
|
||||
assert!(!craft_doc_binary::is_encrypted(b"not a .doc"));
|
||||
|
||||
// A document written as a .doc reads back.
|
||||
let bytes = craft_doc_binary::write(&craft_wordmodel::Document::from_text("Hello")).unwrap();
|
||||
let doc = craft_doc_binary::read(&bytes).unwrap();
|
||||
assert_eq!(doc.body[0].as_para().map(|p| p.text.as_str()), Some("Hello"));
|
||||
```
|
||||
|
||||
- `read(bytes)`, `read_with_password(bytes, Option<&str>)`, `is_encrypted(bytes)`.
|
||||
- `read(bytes)`, `read_with_password(bytes, Option<&str>)`, `is_encrypted(bytes)`, `write(&document)`.
|
||||
- Text in pieces (8-bit and UTF-16), paragraph and character formatting, styles, lists,
|
||||
tables, sections, headers and footers, notes, comments, fields, bookmarks, pictures and
|
||||
OfficeArt shapes, document properties.
|
||||
- XOR obfuscation, RC4 and RC4 CryptoAPI password protection.
|
||||
- Tracked insertions and deletions, with their author and time.
|
||||
|
||||
There is no writer.
|
||||
`write` produces a Word 97-2003 file (nFib 0x00C1, a version 3 compound file): text, character
|
||||
and paragraph formatting, styles, fonts, lists, tables, sections with headers and footers,
|
||||
footnotes and endnotes, fields and hyperlinks, bookmarks, inline pictures, tracked insertions
|
||||
and deletions, and the document properties. Floating pictures are written in line, nested tables
|
||||
become paragraphs of their cell, and shapes, charts, text boxes and comments are left out, each
|
||||
kind logged once per save. It never writes a password-protected file.
|
||||
|
||||
## Limits
|
||||
|
||||
|
||||
@@ -110,8 +110,8 @@ pub(crate) fn ico(i: u8) -> Option<Rgb> {
|
||||
}
|
||||
|
||||
/// The model border style a `brcType` stands for ([MS-DOC] §2.9.22 BrcType; 0x02, thick, is in
|
||||
/// the BRC of Microsoft's Word 97-2007 specification). Types the model doesn't carry (the gap
|
||||
/// variants, 3-D, inset/outset) draw as single lines, as the DOCX reader maps them.
|
||||
/// the BRC of Microsoft's Word 97-2007 specification); 0x0B-0x13 are the thin/thick compound
|
||||
/// lines. Types the model doesn't carry (3-D, inset/outset, art borders) draw as single lines.
|
||||
fn brc_style(t: u8) -> BorderStyle {
|
||||
match t {
|
||||
0x00 => BorderStyle::None,
|
||||
@@ -121,6 +121,15 @@ fn brc_style(t: u8) -> BorderStyle {
|
||||
0x07 | 0x16 => BorderStyle::Dashed,
|
||||
0x08 | 0x09 => BorderStyle::DotDash,
|
||||
0x0A => BorderStyle::Triple,
|
||||
0x0B => BorderStyle::ThinThickSmallGap,
|
||||
0x0C => BorderStyle::ThickThinSmallGap,
|
||||
0x0D => BorderStyle::ThinThickThinSmallGap,
|
||||
0x0E => BorderStyle::ThinThickMediumGap,
|
||||
0x0F => BorderStyle::ThickThinMediumGap,
|
||||
0x10 => BorderStyle::ThinThickThinMediumGap,
|
||||
0x11 => BorderStyle::ThinThickLargeGap,
|
||||
0x12 => BorderStyle::ThickThinLargeGap,
|
||||
0x13 => BorderStyle::ThinThickThinLargeGap,
|
||||
0x14 | 0x15 => BorderStyle::Wave,
|
||||
_ => BorderStyle::Single,
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
//! Reader for the Word 97-2003 binary file format (`.doc`, [MS-DOC]), WordCraft's.
|
||||
//! Reader and writer for the Word 97-2003 binary file format (`.doc`, [MS-DOC]), WordCraft's.
|
||||
//!
|
||||
//! [`read`] turns the bytes of a `.doc` file into a [`craft_wordmodel::Document`]. There is no
|
||||
//! writer: saving always goes through the other formats. The reader is written from the public
|
||||
//! specification ([MS-DOC], Word (.doc) Binary File Format, Microsoft Open Specifications) and,
|
||||
//! like the other Craft parsers, is lenient by design (unknown structures are skipped, bad numbers
|
||||
//! fall back to defaults) and bounded (stream sizes, piece counts and property counts are
|
||||
//! capped), so hostile input yields an error or a best-effort document, never a panic.
|
||||
//! [`read`] turns the bytes of a `.doc` file into a [`craft_wordmodel::Document`], and [`write`]
|
||||
//! turns a document into a Word 97-2003 `.doc` file (the `write` module lists what it keeps).
|
||||
//! Both follow the public specification ([MS-DOC], Word (.doc) Binary File Format, Microsoft Open
|
||||
//! Specifications). The reader, like the other Craft parsers, is lenient by design (unknown
|
||||
//! structures are skipped, bad numbers fall back to defaults) and bounded (stream sizes, piece
|
||||
//! counts and property counts are capped), so hostile input yields an error or a best-effort
|
||||
//! document, never a panic.
|
||||
//!
|
||||
//! Word 6.0/95 files, which that specification does not cover, are read by `word6` from their
|
||||
//! observed layout: their text, formatting and tables so far.
|
||||
@@ -28,11 +29,15 @@ mod metadata;
|
||||
mod notes;
|
||||
mod officeart;
|
||||
mod piece;
|
||||
mod rmark;
|
||||
mod sections;
|
||||
mod sprm;
|
||||
mod stsh;
|
||||
mod table;
|
||||
mod word6;
|
||||
mod write;
|
||||
|
||||
pub use write::write;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -303,6 +308,7 @@ fn read_limited(bytes: &[u8], password: Option<&str>, max_paragraphs: usize) ->
|
||||
// tight against aliasing — and never past the absolute ceiling.
|
||||
let budget = std::cell::Cell::new((word.len() as u64).saturating_mul(2).min(MAX_TOTAL_CHARS));
|
||||
let work = Work::new(MAX_WORK);
|
||||
let revs = rmark::Revisions::new(&table, &fib);
|
||||
let paragraphs = ParaBudget { left: std::cell::Cell::new(max_paragraphs), exceeded: std::cell::Cell::new(false) };
|
||||
|
||||
let ctx = WalkCtx {
|
||||
@@ -321,6 +327,7 @@ fn read_limited(bytes: &[u8], password: Option<&str>, max_paragraphs: usize) ->
|
||||
budget: &budget,
|
||||
paragraphs: ¶graphs,
|
||||
work: &work,
|
||||
revs: &revs,
|
||||
};
|
||||
|
||||
// Drawings: text box stories become parts first, then the floating shapes and pictures
|
||||
@@ -432,6 +439,7 @@ fn read_limited(bytes: &[u8], password: Option<&str>, max_paragraphs: usize) ->
|
||||
return Err(paragraph_limit(max_paragraphs));
|
||||
}
|
||||
doc.media.extend(pics.media);
|
||||
doc.revisions = revs.take();
|
||||
doc.body = table::assemble(paras);
|
||||
doc.ensure_nonempty();
|
||||
work.log();
|
||||
@@ -460,6 +468,7 @@ fn read_word6(comp: &craft_cfb::CompoundFile<'_>, word: &[u8], max_paragraphs: u
|
||||
let work = Work::new(MAX_WORK);
|
||||
let paragraphs = ParaBudget { left: std::cell::Cell::new(max_paragraphs), exceeded: std::cell::Cell::new(false) };
|
||||
let mut pics = media::Pictures::for_input(word.len());
|
||||
let revs = rmark::Revisions::none();
|
||||
let ctx = WalkCtx {
|
||||
word,
|
||||
data: &[],
|
||||
@@ -476,6 +485,7 @@ fn read_word6(comp: &craft_cfb::CompoundFile<'_>, word: &[u8], max_paragraphs: u
|
||||
budget: &budget,
|
||||
paragraphs: ¶graphs,
|
||||
work: &work,
|
||||
revs: &revs,
|
||||
};
|
||||
let paras = walk(&ctx, 0, fib.ccp_text, &mut pics);
|
||||
if paragraphs.exceeded.get() {
|
||||
@@ -574,6 +584,8 @@ struct WalkCtx<'a> {
|
||||
paragraphs: &'a ParaBudget,
|
||||
/// Repeated-work bytes left ([`MAX_WORK`]), and what they left out.
|
||||
work: &'a Work,
|
||||
/// Tracked insertions and deletions met so far.
|
||||
revs: &'a rmark::Revisions,
|
||||
}
|
||||
|
||||
/// The paragraph and cell marks still allowed in this document, and whether a walk ran into
|
||||
@@ -748,7 +760,10 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic
|
||||
pb.props.numbering = Some(NumRef { num, level: ilvl });
|
||||
}
|
||||
let fc = ctx.pieces.fc_of_cp(cp).unwrap_or(0);
|
||||
pb.mark = char_props(ctx.chpx_bins.chpx(ctx.word, fc), ctx.fonts, &CharProps::default());
|
||||
let chpx = ctx.chpx_bins.chpx(ctx.word, fc);
|
||||
pb.mark = char_props(chpx, ctx.fonts, &CharProps::default());
|
||||
// A tracked paragraph mark: an inserted or deleted paragraph break.
|
||||
ctx.revs.mark(&mut pb.mark, chpx, ctx.pieces.piece_prm(cp));
|
||||
let done = std::mem::take(&mut pb);
|
||||
out.push(ParaOut {
|
||||
para: Paragraph {
|
||||
@@ -769,7 +784,7 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic
|
||||
Some(f) => {
|
||||
f.depth = f.depth.saturating_add(1);
|
||||
}
|
||||
None => field = Some(FieldBuild { depth: 1, instr: String::new(), sep: None }),
|
||||
None => field = Some(FieldBuild { depth: 1, instr: String::new(), sep: None, anchor: anchor_props(ctx, cp) }),
|
||||
}
|
||||
cp = cp.saturating_add(1);
|
||||
continue;
|
||||
@@ -799,13 +814,16 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic
|
||||
'\u{1}' => {
|
||||
// Inline picture: the CHPX of the anchor carries sprmCPicLocation.
|
||||
let fc = ctx.pieces.fc_of_cp(cp).unwrap_or(0);
|
||||
let fc_pic = pic_location(ctx.chpx_bins.chpx(ctx.word, fc));
|
||||
let chpx = ctx.chpx_bins.chpx(ctx.word, fc);
|
||||
let fc_pic = pic_location(chpx);
|
||||
match fc_pic.and_then(|at| pics.get(ctx.data, at)) {
|
||||
Some(p) => {
|
||||
// Every anchor of a picture gets its own copy of the alt text; the
|
||||
// copies are charged to the work budget (#3709).
|
||||
let alt = if ctx.work.charge(p.alt.len() as u64, Cost::AltText) { p.alt.clone() } else { String::new() };
|
||||
pb.push('\u{FFFC}', CharProps::default());
|
||||
let mut props = CharProps::default();
|
||||
ctx.revs.mark(&mut props, chpx, ctx.pieces.piece_prm(cp));
|
||||
pb.push('\u{FFFC}', props);
|
||||
pb.objects.push(InlineObject::Image {
|
||||
media: p.key.clone(),
|
||||
w: p.w,
|
||||
@@ -829,7 +847,7 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic
|
||||
if !ctx.work.charge(object_cost(obj), Cost::FloatingObject) {
|
||||
continue;
|
||||
}
|
||||
pb.push('\u{FFFC}', CharProps::default());
|
||||
pb.push('\u{FFFC}', anchor_props(ctx, cp));
|
||||
pb.objects.push(obj.clone());
|
||||
}
|
||||
}
|
||||
@@ -837,7 +855,7 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic
|
||||
// Footnote or endnote reference; custom symbols share the same character.
|
||||
let at = ctx.notes.partition_point(|(c, _, _)| *c < cp);
|
||||
if let Some((_, kind, id)) = ctx.notes.get(at).filter(|(c, _, _)| *c == cp) {
|
||||
pb.push('\u{FFFC}', CharProps::default());
|
||||
pb.push('\u{FFFC}', anchor_props(ctx, cp));
|
||||
pb.objects.push(InlineObject::NoteRef { kind: *kind, id: *id, custom: String::new() });
|
||||
}
|
||||
}
|
||||
@@ -871,6 +889,8 @@ struct FieldBuild {
|
||||
depth: u32,
|
||||
instr: String,
|
||||
sep: Option<FieldResult>,
|
||||
/// The begin character's revision marks, for the field object's anchor.
|
||||
anchor: CharProps,
|
||||
}
|
||||
|
||||
/// A field's cached result: its text and, as runs, the formatting of that text. A run is added
|
||||
@@ -905,6 +925,15 @@ fn object_cost(obj: &InlineObject) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// The properties of an object anchor at `cp`: only its revision marks, so a tracked
|
||||
/// insertion or deletion of a picture, note reference or field survives.
|
||||
fn anchor_props(ctx: &WalkCtx<'_>, cp: u32) -> CharProps {
|
||||
let fc = ctx.pieces.fc_of_cp(cp).unwrap_or(0);
|
||||
let mut props = CharProps::default();
|
||||
ctx.revs.mark(&mut props, ctx.chpx_bins.chpx(ctx.word, fc), ctx.pieces.piece_prm(cp));
|
||||
props
|
||||
}
|
||||
|
||||
/// Emit a finished field: HYPERLINK keeps its result text as a linked run; every other
|
||||
/// field becomes one inline object with its cached result.
|
||||
///
|
||||
@@ -934,7 +963,7 @@ fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &Work) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
pb.push('\u{FFFC}', CharProps::default());
|
||||
pb.push('\u{FFFC}', f.anchor);
|
||||
pb.objects.push(InlineObject::Field { instr, result: result.text, locked: false, dirty: false, code: false, form: String::new() });
|
||||
}
|
||||
|
||||
@@ -1012,7 +1041,9 @@ fn push_formatted<'a>(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx<'a>, f
|
||||
}
|
||||
};
|
||||
let mut props = char_props(grpprl, ctx.fonts, base);
|
||||
apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base, ctx.work);
|
||||
let piece_prm = ctx.pieces.piece_prm(cp);
|
||||
apply_prm(&mut props, piece_prm, ctx.fonts, base, ctx.work);
|
||||
ctx.revs.mark(&mut props, grpprl, piece_prm);
|
||||
st.memo = Some(FmtMemo { chpx: grpprl, prm, style: style.map(str::to_owned), props });
|
||||
st.generation += 1;
|
||||
}
|
||||
@@ -1083,7 +1114,7 @@ fn prm_paragraph_sprms(prm: PrmRef<'_>, work: &Work) -> Vec<u8> {
|
||||
out.push(val);
|
||||
}
|
||||
}
|
||||
PrmRef::Grpprl(Some(g)) => {
|
||||
PrmRef::Grpprl { grpprl: Some(g), .. } => {
|
||||
if !work.charge(g.len() as u64, Cost::Formatting) {
|
||||
return out;
|
||||
}
|
||||
@@ -1092,7 +1123,7 @@ fn prm_paragraph_sprms(prm: PrmRef<'_>, work: &Work) -> Vec<u8> {
|
||||
out.extend_from_slice(prl.operand);
|
||||
}
|
||||
}
|
||||
PrmRef::None | PrmRef::Grpprl(None) => {}
|
||||
PrmRef::None | PrmRef::Grpprl { grpprl: None, .. } => {}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -1111,7 +1142,7 @@ fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &Ch
|
||||
}
|
||||
}
|
||||
}
|
||||
PrmRef::Grpprl(Some(g)) => {
|
||||
PrmRef::Grpprl { grpprl: Some(g), .. } => {
|
||||
if !work.charge(g.len() as u64, Cost::Formatting) {
|
||||
return;
|
||||
}
|
||||
@@ -1121,7 +1152,7 @@ fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &Ch
|
||||
}
|
||||
}
|
||||
}
|
||||
PrmRef::Grpprl(None) => {}
|
||||
PrmRef::Grpprl { grpprl: None, .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,15 +35,19 @@ struct RawLvl {
|
||||
xst: Vec<u16>,
|
||||
}
|
||||
|
||||
/// A bullet level's character, with the 0xF000 flag bits stripped and the common
|
||||
/// Symbol/Wingdings glyphs mapped to their Unicode equivalents so every font renders them.
|
||||
/// A bullet level's character: a symbol-font character (0xF000 to 0xF0FF, the font's own code
|
||||
/// in the low byte) with the common Symbol/Wingdings glyphs mapped to their Unicode equivalents
|
||||
/// so every font renders them; any other character is itself.
|
||||
fn bullet_char(c: u16) -> char {
|
||||
match c & 0x0FFF {
|
||||
if !(0xF000..=0xF0FF).contains(&c) {
|
||||
return char::from_u32(u32::from(c)).unwrap_or('\u{2022}');
|
||||
}
|
||||
match c & 0x00FF {
|
||||
0xB7 => '\u{2022}', // Symbol bullet
|
||||
0xA7 => '\u{25AA}', // Wingdings small square
|
||||
0x6F => '\u{25CB}', // Symbol open circle
|
||||
0x77 | 0x78 => '\u{25CF}', // Wingdings filled circles
|
||||
other => char::from_u32(other as u32).unwrap_or('\u{2022}'),
|
||||
other => char::from_u32(u32::from(other)).unwrap_or('\u{2022}'),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +55,7 @@ fn bullet_char(c: u16) -> char {
|
||||
/// 0x00 to 0x27 run in the order of ST_NumberFormat (decimal, upperRoman … 0x16 decimalZero, 0x17
|
||||
/// bullet … 0x25 chineseCounting, 0x26 chineseLegalSimplified, 0x27 chineseCountingThousand);
|
||||
/// later codes are drawn as decimal.
|
||||
fn num_format(nfc: u8) -> NumFormat {
|
||||
pub(crate) fn num_format(nfc: u8) -> NumFormat {
|
||||
match nfc {
|
||||
0xFF => NumFormat::None,
|
||||
0x00..=0x27 => NumFormat::ALL.get(nfc as usize).copied().unwrap_or(NumFormat::Decimal),
|
||||
|
||||
@@ -15,6 +15,12 @@ fn u32_at(bytes: &[u8], offset: usize) -> Option<u32> {
|
||||
Some(u32::from_le_bytes(bytes.get(offset..offset.checked_add(4)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
/// [`parse`], for the writer's tests.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn parse_for_tests(bytes: &[u8]) -> Option<CoreProps> {
|
||||
parse(bytes)
|
||||
}
|
||||
|
||||
fn parse(bytes: &[u8]) -> Option<CoreProps> {
|
||||
if bytes.get(..2)? != [0xfe, 0xff] || bytes.get(28..44)? != SUMMARY_FMTID || u32_at(bytes, 24)? < 1 {
|
||||
return None;
|
||||
|
||||
@@ -93,6 +93,11 @@ pub(crate) fn parse_bookmarks(table: &[u8], fib: &crate::fib::Fib) -> Vec<(u32,
|
||||
/// (1-byte length and Windows-1252 chars). `cbExtra` bytes after each string are skipped.
|
||||
/// Names are kept to 40 characters, but the reader always advances by the stored length.
|
||||
fn sttb_strings(table: &[u8], fc: u32, lcb: u32) -> Option<Vec<String>> {
|
||||
sttb_strings_max(table, fc, lcb, 40)
|
||||
}
|
||||
|
||||
/// [`sttb_strings`] keeping up to `max` characters of each string.
|
||||
pub(crate) fn sttb_strings_max(table: &[u8], fc: u32, lcb: u32, max: usize) -> Option<Vec<String>> {
|
||||
let b = table.get(fc as usize..(fc as usize).saturating_add(lcb as usize))?;
|
||||
let extended = b.get(0..2)? == [0xFF, 0xFF];
|
||||
let head = if extended { 2 } else { 0 };
|
||||
@@ -125,9 +130,10 @@ fn sttb_strings(table: &[u8], fc: u32, lcb: u32) -> Option<Vec<String>> {
|
||||
}
|
||||
at += cb_extra;
|
||||
let s: String = if extended {
|
||||
chars.as_chunks::<2>().0.iter().take(40).map(|c| char::from_u32(u16::from_le_bytes(*c) as u32).unwrap_or('\u{FFFD}')).collect()
|
||||
// UTF-16: a surrogate pair is one character; a lone surrogate becomes U+FFFD.
|
||||
char::decode_utf16(chars.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes(*c))).take(max).map(|c| c.unwrap_or('\u{FFFD}')).collect()
|
||||
} else {
|
||||
chars.iter().take(40).map(|&c| craft_codepage::cp1252(c)).collect()
|
||||
chars.iter().take(max).map(|&c| craft_codepage::cp1252(c)).collect()
|
||||
};
|
||||
out.push(s);
|
||||
}
|
||||
@@ -153,6 +159,20 @@ mod tests {
|
||||
b
|
||||
}
|
||||
|
||||
/// A character outside the BMP (a surrogate pair) stays one character; a lone surrogate
|
||||
/// is U+FFFD. The limit counts characters, not UTF-16 units.
|
||||
#[test]
|
||||
fn extended_strings_decode_surrogate_pairs() {
|
||||
let mut b = ext_sttb(&["Ada \u{1F600}"], &[]);
|
||||
let lone = [0x00u8, 0xD8];
|
||||
b[2] = 2;
|
||||
b.extend_from_slice(&2u16.to_le_bytes());
|
||||
b.extend_from_slice(&lone);
|
||||
b.extend_from_slice(&u16::from(b'x').to_le_bytes());
|
||||
let got = super::sttb_strings_max(&b, 0, b.len() as u32, 5).unwrap();
|
||||
assert_eq!(got, vec!["Ada \u{1F600}".to_string(), "\u{FFFD}x".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_bookmark_name_does_not_desync_table() {
|
||||
let long = "A".repeat(50);
|
||||
|
||||
@@ -228,7 +228,8 @@ impl PieceTable {
|
||||
}
|
||||
PrmRef::Sprm0 { isprm, val: (p.prm >> 8) as u8 }
|
||||
} else {
|
||||
PrmRef::Grpprl(self.prc_grpprls.get((p.prm >> 1) as usize).map(Vec::as_slice))
|
||||
let index = p.prm >> 1;
|
||||
PrmRef::Grpprl { index, grpprl: self.prc_grpprls.get(usize::from(index)).map(Vec::as_slice) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,6 +343,7 @@ impl PieceSpan<'_> {
|
||||
}
|
||||
|
||||
/// What a piece's `Prm` refers to.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum PrmRef<'a> {
|
||||
None,
|
||||
/// Prm0: a short sprm index into the isprm table, with a 1-byte operand.
|
||||
@@ -349,8 +351,11 @@ pub(crate) enum PrmRef<'a> {
|
||||
isprm: u8,
|
||||
val: u8,
|
||||
},
|
||||
/// Prm1: a grpprl from a Prc (absent if out of range).
|
||||
Grpprl(Option<&'a [u8]>),
|
||||
/// Prm1: the Prc at `index` and its grpprl (absent if out of range).
|
||||
Grpprl {
|
||||
index: u16,
|
||||
grpprl: Option<&'a [u8]>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Windows' code page number for symbol fonts (CP_SYMBOL).
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
//! Tracked insertions and deletions: the revision-mark sprms of a run's CHPX (`sprmCFRMarkIns`
|
||||
//! and `sprmCFRMarkDel`, each with its author index into `SttbfRMark` and its `DTTM`)
|
||||
//! ([MS-DOC] §2.6.1, §2.9.290), as model revisions.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use craft_wordmodel::props::CharProps;
|
||||
use craft_wordmodel::{Revision, RevisionKind};
|
||||
|
||||
use crate::fib::Fib;
|
||||
use crate::piece::PrmRef;
|
||||
use crate::sprm;
|
||||
|
||||
const C_RMARK_DEL: u16 = 0x0800;
|
||||
const C_RMARK_INS: u16 = 0x0801;
|
||||
const C_IBST_RMARK: u16 = 0x4804;
|
||||
const C_DTTM_RMARK: u16 = 0x6805;
|
||||
const C_IBST_RMARK_DEL: u16 = 0x4863;
|
||||
const C_DTTM_RMARK_DEL: u16 = 0x6864;
|
||||
/// `fcSttbfRMark`: the revision authors.
|
||||
const STTBF_RMARK: usize = 51;
|
||||
/// Most distinct revisions (author, time and kind) one document gets.
|
||||
const MAX_REVISIONS: usize = 100_000;
|
||||
/// Longest author name kept.
|
||||
const MAX_AUTHOR: usize = 255;
|
||||
|
||||
/// The document's revisions, made as the walk meets their marks: runs with the same author,
|
||||
/// time and kind share one.
|
||||
pub(crate) struct Revisions {
|
||||
authors: Vec<String>,
|
||||
list: RefCell<Vec<Revision>>,
|
||||
index: RefCell<HashMap<(bool, u16, u32), u32>>,
|
||||
/// The marks of each Prc a piece's Prm1 names, by its index (at most 32,768 of them): a Prc
|
||||
/// grpprl of up to 64 KB is scanned once, not at every run, paragraph mark and object anchor
|
||||
/// of its pieces.
|
||||
prcs: RefCell<HashMap<u16, Marks>>,
|
||||
}
|
||||
|
||||
impl Revisions {
|
||||
/// No revision authors: a file whose revision marks are not read (Word 6.0/95, whose
|
||||
/// revision sprms differ). Marks are still never set, as none of its sprms are Word 97's.
|
||||
pub(crate) fn none() -> Self {
|
||||
Self { authors: Vec::new(), list: RefCell::default(), index: RefCell::default(), prcs: RefCell::default() }
|
||||
}
|
||||
|
||||
pub(crate) fn new(table: &[u8], fib: &Fib) -> Self {
|
||||
let authors = fib
|
||||
.pair(STTBF_RMARK)
|
||||
.filter(|(_, lcb)| *lcb > 0)
|
||||
.and_then(|(fc, lcb)| crate::notes::sttb_strings_max(table, fc, lcb, MAX_AUTHOR))
|
||||
.unwrap_or_default();
|
||||
Revisions { authors, list: RefCell::new(Vec::new()), index: RefCell::new(HashMap::new()), prcs: RefCell::new(HashMap::new()) }
|
||||
}
|
||||
|
||||
/// Set `props.ins` / `props.del` from the revision marks of a run: those in its CHPX
|
||||
/// `grpprl`, then those of its piece's `prm`, which apply after the CHPX's.
|
||||
pub(crate) fn mark(&self, props: &mut CharProps, grpprl: &[u8], prm: PrmRef<'_>) {
|
||||
let mut m = Marks::of(grpprl);
|
||||
match prm {
|
||||
PrmRef::Sprm0 { isprm, val } => {
|
||||
if let Some(op) = sprm::from_prm0(isprm) {
|
||||
m.apply(op, &[val]);
|
||||
}
|
||||
}
|
||||
PrmRef::Grpprl { index, grpprl: Some(g) } => {
|
||||
let prc = *self.prcs.borrow_mut().entry(index).or_insert_with(|| Marks::of(g));
|
||||
m = m.then(prc);
|
||||
}
|
||||
PrmRef::Grpprl { grpprl: None, .. } | PrmRef::None => {}
|
||||
}
|
||||
if m.ins == Some(true) {
|
||||
props.ins = self.id(true, m.ibst_ins.unwrap_or(0), m.dttm_ins.unwrap_or(0));
|
||||
}
|
||||
if m.del == Some(true) {
|
||||
props.del = self.id(false, m.ibst_del.unwrap_or(0), m.dttm_del.unwrap_or(0));
|
||||
}
|
||||
}
|
||||
|
||||
/// The index of the revision of this kind, author and time, made on first use.
|
||||
fn id(&self, insert: bool, ibst: u16, dttm: u32) -> Option<u32> {
|
||||
let key = (insert, ibst, dttm);
|
||||
if let Some(&i) = self.index.borrow().get(&key) {
|
||||
return Some(i);
|
||||
}
|
||||
let mut list = self.list.borrow_mut();
|
||||
if list.len() >= MAX_REVISIONS {
|
||||
return None;
|
||||
}
|
||||
let author = self.authors.get(usize::from(ibst)).cloned().unwrap_or_else(|| "Unknown".to_string());
|
||||
let kind = if insert { RevisionKind::Insert } else { RevisionKind::Delete };
|
||||
list.push(Revision::new(kind, author, dttm_iso(dttm)));
|
||||
let i = u32::try_from(list.len() - 1).ok()?;
|
||||
self.index.borrow_mut().insert(key, i);
|
||||
Some(i)
|
||||
}
|
||||
|
||||
pub(crate) fn take(self) -> Vec<Revision> {
|
||||
self.list.into_inner()
|
||||
}
|
||||
}
|
||||
|
||||
/// The revision-mark sprms seen so far for one run, a later one replacing an earlier one; each
|
||||
/// is `None` until one is seen.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Marks {
|
||||
ins: Option<bool>,
|
||||
del: Option<bool>,
|
||||
ibst_ins: Option<u16>,
|
||||
dttm_ins: Option<u32>,
|
||||
ibst_del: Option<u16>,
|
||||
dttm_del: Option<u32>,
|
||||
}
|
||||
|
||||
impl Marks {
|
||||
/// The marks of one grpprl.
|
||||
fn of(grpprl: &[u8]) -> Self {
|
||||
let mut m = Marks::default();
|
||||
for prl in sprm::iter(grpprl) {
|
||||
m.apply(prl.op, prl.operand);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// These marks with `later`'s applied after them, as if its sprms came next.
|
||||
fn then(self, later: Marks) -> Self {
|
||||
Marks {
|
||||
ins: later.ins.or(self.ins),
|
||||
del: later.del.or(self.del),
|
||||
ibst_ins: later.ibst_ins.or(self.ibst_ins),
|
||||
dttm_ins: later.dttm_ins.or(self.dttm_ins),
|
||||
ibst_del: later.ibst_del.or(self.ibst_del),
|
||||
dttm_del: later.dttm_del.or(self.dttm_del),
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(&mut self, op: u16, operand: &[u8]) {
|
||||
let u16v = || operand.get(..2).map_or(0, |b| u16::from_le_bytes([b[0], b[1]]));
|
||||
let u32v = || operand.get(..4).map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
|
||||
match op {
|
||||
C_RMARK_INS => self.ins = Some(operand.first().is_some_and(|&v| v == 1 || v == 0x81)),
|
||||
C_RMARK_DEL => self.del = Some(operand.first().is_some_and(|&v| v == 1 || v == 0x81)),
|
||||
C_IBST_RMARK => self.ibst_ins = Some(u16v()),
|
||||
C_DTTM_RMARK => self.dttm_ins = Some(u32v()),
|
||||
C_IBST_RMARK_DEL => self.ibst_del = Some(u16v()),
|
||||
C_DTTM_RMARK_DEL => self.dttm_del = Some(u32v()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A `DTTM` ([MS-DOC] §2.9.75) as an ISO 8601 UTC time; empty for 0 or an impossible date.
|
||||
pub(crate) fn dttm_iso(d: u32) -> String {
|
||||
if d == 0 {
|
||||
return String::new();
|
||||
}
|
||||
let minute = d & 0x3F;
|
||||
let hour = (d >> 6) & 0x1F;
|
||||
let day = (d >> 11) & 0x1F;
|
||||
let month = (d >> 16) & 0xF;
|
||||
let year = 1900 + ((d >> 20) & 0x1FF) as i32;
|
||||
match chrono::NaiveDate::from_ymd_opt(year, month, day).and_then(|date| date.and_hms_opt(hour, minute, 0)) {
|
||||
Some(t) => t.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn revisions() -> Revisions {
|
||||
Revisions {
|
||||
authors: vec!["Ada".into(), "Grace".into()],
|
||||
list: RefCell::new(Vec::new()),
|
||||
index: RefCell::new(HashMap::new()),
|
||||
prcs: RefCell::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn marked(chpx: &[u8], prm: PrmRef<'_>) -> (CharProps, Vec<Revision>) {
|
||||
let r = revisions();
|
||||
let mut props = CharProps::default();
|
||||
r.mark(&mut props, chpx, prm);
|
||||
(props, r.take())
|
||||
}
|
||||
|
||||
const INS: [u8; 3] = [0x01, 0x08, 1];
|
||||
const NOT_INS: [u8; 3] = [0x01, 0x08, 0];
|
||||
/// sprmCFRMarkDel on, by the author at index 1.
|
||||
const DEL_BY_GRACE: [u8; 7] = [0x00, 0x08, 1, 0x63, 0x48, 1, 0];
|
||||
|
||||
#[test]
|
||||
fn a_piece_prm_marks_a_run() {
|
||||
// Prm1: a grpprl from the piece table's Prc.
|
||||
let (p, revs) = marked(&[], PrmRef::Grpprl { index: 0, grpprl: Some(&DEL_BY_GRACE) });
|
||||
assert_eq!(p.del, Some(0));
|
||||
assert_eq!((revs[0].kind, revs[0].author.as_str()), (RevisionKind::Delete, "Grace"));
|
||||
// Prm0: sprmCFRMarkIns by its isprm.
|
||||
let (p, revs) = marked(&[], PrmRef::Sprm0 { isprm: 0x42, val: 1 });
|
||||
assert_eq!(p.ins, Some(0));
|
||||
assert_eq!((revs[0].kind, revs[0].author.as_str()), (RevisionKind::Insert, "Ada"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_piece_prm_applies_after_the_chpx() {
|
||||
let (p, _) = marked(&INS, PrmRef::Grpprl { index: 0, grpprl: Some(&NOT_INS) });
|
||||
assert_eq!(p.ins, None, "the Prm turns the CHPX's insertion off");
|
||||
let (p, _) = marked(&NOT_INS, PrmRef::Sprm0 { isprm: 0x42, val: 1 });
|
||||
assert_eq!(p.ins, Some(0));
|
||||
let (p, _) = marked(&INS, PrmRef::None);
|
||||
assert_eq!(p.ins, Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prcs_marks_apply_after_the_chpx_one_by_one_and_are_scanned_once() {
|
||||
let r = revisions();
|
||||
// The CHPX inserts, by Grace; the Prc only sets the time, so the author stays Grace's.
|
||||
let chpx = [INS.as_slice(), &[0x04, 0x48, 1, 0]].concat();
|
||||
let prc = [0x05u8, 0x68, 0x21, 0x2B, 0xC3, 0x07];
|
||||
let mut p = CharProps::default();
|
||||
r.mark(&mut p, &chpx, PrmRef::Grpprl { index: 3, grpprl: Some(&prc) });
|
||||
assert_eq!(p.ins, Some(0));
|
||||
// The marks of Prc 3 are kept: the same index finds them without its grpprl's bytes.
|
||||
let mut again = CharProps::default();
|
||||
r.mark(&mut again, &INS, PrmRef::Grpprl { index: 3, grpprl: Some(&[]) });
|
||||
assert_eq!(again.ins, Some(1), "Ada's insertion at the Prc's time");
|
||||
let revs = r.take();
|
||||
assert_eq!((revs[0].author.as_str(), revs[0].date.as_str()), ("Grace", "2024-03-05T12:33:00Z"));
|
||||
assert_eq!((revs[1].author.as_str(), revs[1].date.as_str()), ("Ada", "2024-03-05T12:33:00Z"));
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,8 @@ pub(crate) fn from_prm0(isprm: u8) -> Option<u16> {
|
||||
0x09 => 0x2407, // sprmPFPageBreakBefore
|
||||
0x33 => 0x2435, // sprmPWidowControl (low byte of the operand is the value)
|
||||
0x78 => 0x2640, // sprmPOutLvl
|
||||
0x41 => 0x0800, // sprmCFRMarkDel
|
||||
0x42 => 0x0801, // sprmCFRMarkIns
|
||||
0x4D => 0x2A0C, // sprmCHighlight
|
||||
0x55 => 0x0835, // sprmCFBold
|
||||
0x56 => 0x0836, // sprmCFItalic
|
||||
|
||||
@@ -2822,7 +2822,12 @@ fn a_long_hyperlink_over_a_long_result_is_one_linked_run() {
|
||||
// what the copies charge to the work budget, which a busy machine can't skew.
|
||||
let mut result = super::FieldResult::default();
|
||||
(0..n).for_each(|_| result.push('y', craft_wordmodel::props::CharProps::default()));
|
||||
let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"http://{target}\""), sep: Some(result) };
|
||||
let f = super::FieldBuild {
|
||||
depth: 1,
|
||||
instr: format!("HYPERLINK \"http://{target}\""),
|
||||
sep: Some(result),
|
||||
anchor: craft_wordmodel::props::CharProps::default(),
|
||||
};
|
||||
let work = super::Work::new(u64::MAX);
|
||||
super::finish_field(&mut super::ParaBuild::default(), f, &work);
|
||||
assert_eq!(u64::MAX - work.left.get(), ("http://".len() + n) as u64, "one copy of the target");
|
||||
@@ -2836,7 +2841,12 @@ fn hyperlink_target_copies_are_charged_to_the_work_budget() {
|
||||
let bold = CharProps { bold: Some(true), ..Default::default() };
|
||||
let mut result = super::FieldResult::default();
|
||||
(0..1000).for_each(|i| result.push('a', if i % 2 == 0 { bold.clone() } else { CharProps::default() }));
|
||||
let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"{}\"", "u".repeat(100)), sep: Some(result) };
|
||||
let f = super::FieldBuild {
|
||||
depth: 1,
|
||||
instr: format!("HYPERLINK \"{}\"", "u".repeat(100)),
|
||||
sep: Some(result),
|
||||
anchor: craft_wordmodel::props::CharProps::default(),
|
||||
};
|
||||
let mut pb = super::ParaBuild::default();
|
||||
let budget = super::Work::new(10_000);
|
||||
super::finish_field(&mut pb, f, &budget);
|
||||
@@ -2901,15 +2911,46 @@ fn prm_grpprls_past_the_work_budget_are_skipped() {
|
||||
let work = super::Work::new(g.len() as u64 + 1);
|
||||
let base = CharProps::default();
|
||||
let mut first = CharProps::default();
|
||||
super::apply_prm(&mut first, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work);
|
||||
super::apply_prm(&mut first, crate::piece::PrmRef::Grpprl { index: 0, grpprl: Some(&g) }, &[], &base, &work);
|
||||
assert_eq!(first.bold, Some(true));
|
||||
let mut second = CharProps::default();
|
||||
super::apply_prm(&mut second, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work);
|
||||
super::apply_prm(&mut second, crate::piece::PrmRef::Grpprl { index: 0, grpprl: Some(&g) }, &[], &base, &work);
|
||||
assert_eq!(second.bold, None);
|
||||
assert_eq!(work.left.get(), 1);
|
||||
assert_eq!(work.dropped(), [super::Cost::Formatting]);
|
||||
}
|
||||
|
||||
/// A piece's Prm grpprl (a Prc of up to 64 KB) was scanned for revision marks again at every
|
||||
/// paragraph mark and object anchor, which the per-formatting-change memo (#3692) doesn't
|
||||
/// cover. Each Prc's marks are now found once.
|
||||
#[test]
|
||||
fn a_large_prc_is_scanned_for_revision_marks_once() {
|
||||
// 21,844 bold sprms, then sprmCFRMarkIns: the Prc's marks are only known at its end.
|
||||
let mut prls = vec![(0x0835u16, &[0x01u8][..]); 21_844];
|
||||
prls.push((0x0801, &[0x01]));
|
||||
let prc = grpprl(&prls);
|
||||
// A field and a paragraph mark per paragraph, all in the piece with that Prc.
|
||||
let text: &'static str = Box::leak("\u{13}\u{15}\r".repeat(100_000).into_boxed_str());
|
||||
let spec = RichSpec {
|
||||
text,
|
||||
styles: vec![(1, 0x0FFF, "Normal", Vec::new(), Vec::new())],
|
||||
fonts: vec!["Times New Roman"],
|
||||
chpx_runs: vec![(0, Vec::new())],
|
||||
papx_entries: vec![(0, 0, Vec::new())],
|
||||
};
|
||||
let bytes = rich_doc_with(&spec, &RichExtra { prcs: vec![prc], prm: 0x0001, ..Default::default() });
|
||||
let t = std::time::Instant::now();
|
||||
let doc = read(&bytes).expect("opens");
|
||||
// Well under a second; scanning the Prc at every mark and anchor took 17 s.
|
||||
assert!(t.elapsed().as_secs() < 10, "opened in {:?}", t.elapsed());
|
||||
assert_eq!(doc.body.len(), 100_000);
|
||||
assert_eq!(doc.revisions.len(), 1);
|
||||
for p in doc.body.iter().filter_map(|b| b.as_para()) {
|
||||
assert_eq!(p.mark.ins, Some(0), "the paragraph mark is a tracked insertion");
|
||||
assert_eq!(p.runs.first().and_then(|r| r.props.ins), Some(0), "so is the field");
|
||||
}
|
||||
}
|
||||
|
||||
/// A field's cached result keeps one `CharProps` per formatting run, not one per character
|
||||
/// (365 bytes each: 4 million result characters took 1.4 GB).
|
||||
#[test]
|
||||
@@ -2941,7 +2982,7 @@ fn field_results_keep_their_text_and_runs() {
|
||||
{
|
||||
result.push(c, p);
|
||||
}
|
||||
let f = super::FieldBuild { depth: 1, instr: "HYPERLINK \"http://x\"".into(), sep: Some(result) };
|
||||
let f = super::FieldBuild { depth: 1, instr: "HYPERLINK \"http://x\"".into(), sep: Some(result), anchor: CharProps::default() };
|
||||
let mut pb = super::ParaBuild::default();
|
||||
super::finish_field(&mut pb, f, &super::Work::new(u64::MAX));
|
||||
assert_eq!(pb.text, "a\u{e9}\u{20ac}bc");
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
//! Formatted disk pages ([MS-DOC] §2.9.43 ChpxFkp, §2.9.177 PapxFkp) and the bin tables that
|
||||
//! index them (`PlcBteChpx`, `PlcBtePapx`), plus the PLC and STTB containers the Table stream
|
||||
//! is made of.
|
||||
|
||||
/// Bytes in a formatted disk page.
|
||||
const PAGE: usize = 512;
|
||||
/// Most runs a `ChpxFkp` holds ([MS-DOC]: crun MUST be at most 0x65).
|
||||
const MAX_CHPX_RUNS: usize = 0x65;
|
||||
/// Most paragraphs a `PapxFkp` holds ([MS-DOC]: crun MUST be at most 0x1D).
|
||||
const MAX_PAPX_RUNS: usize = 0x1D;
|
||||
/// Size of a `BxPap`: a word offset and a 12-byte `PHE` (left zero: Word recomputes heights).
|
||||
const BX: usize = 13;
|
||||
|
||||
/// One run of text sharing its character formatting: `[fc_start, fc_end)` in the WordDocument
|
||||
/// stream, with the CHPX grpprl (empty = no direct formatting).
|
||||
pub(crate) struct ChpxRun<'a> {
|
||||
pub(crate) fc_start: u32,
|
||||
pub(crate) fc_end: u32,
|
||||
pub(crate) grpprl: &'a [u8],
|
||||
}
|
||||
|
||||
/// One paragraph: `[fc_start, fc_end)` (its mark included), its style and PAPX grpprl.
|
||||
pub(crate) struct PapxRun<'a> {
|
||||
pub(crate) fc_start: u32,
|
||||
pub(crate) fc_end: u32,
|
||||
pub(crate) istd: u16,
|
||||
pub(crate) grpprl: &'a [u8],
|
||||
}
|
||||
|
||||
/// Pages ready for the WordDocument stream and the bin table (`aFc` + `aPn`) that indexes them.
|
||||
pub(crate) struct Paged {
|
||||
pub(crate) pages: Vec<[u8; PAGE]>,
|
||||
/// `aFc`: the first FC of each page, then the last page's end.
|
||||
fcs: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Paged {
|
||||
/// The `PlcBte*` for these pages once they sit from page number `first_pn` on.
|
||||
pub(crate) fn plc_bte(&self, first_pn: u32) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(self.fcs.len() * 4 + self.pages.len() * 4);
|
||||
for fc in &self.fcs {
|
||||
v.extend_from_slice(&fc.to_le_bytes());
|
||||
}
|
||||
for i in 0..self.pages.len() {
|
||||
v.extend_from_slice(&first_pn.saturating_add(i as u32).to_le_bytes());
|
||||
}
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
/// A page being filled: the `rgfc` entries so far and the property bytes, which grow down from
|
||||
/// the end of the page (the last byte is `crun`).
|
||||
struct Page {
|
||||
buf: [u8; PAGE],
|
||||
fcs: Vec<u32>,
|
||||
/// Word offset (÷2) of each entry's properties (0 = none).
|
||||
offs: Vec<u8>,
|
||||
/// Lowest byte used by property data.
|
||||
low: usize,
|
||||
/// Property bytes already on the page and where they are, so equal ones are stored once.
|
||||
stored: Vec<(Vec<u8>, u8)>,
|
||||
}
|
||||
|
||||
impl Page {
|
||||
fn new() -> Page {
|
||||
Page { buf: [0; PAGE], fcs: Vec::new(), offs: Vec::new(), low: PAGE - 1, stored: Vec::new() }
|
||||
}
|
||||
|
||||
/// Bytes the header takes with `crun` entries of `entry` bytes each.
|
||||
fn header(crun: usize, entry: usize) -> usize {
|
||||
(crun + 1) * 4 + crun * entry
|
||||
}
|
||||
|
||||
/// Place `data` (already in its on-page form), returning its word offset, if it fits beside
|
||||
/// a header for `crun` entries.
|
||||
fn place(&mut self, data: &[u8], crun: usize, entry: usize) -> Option<u8> {
|
||||
if data.is_empty() {
|
||||
return (Page::header(crun, entry) <= self.low).then_some(0);
|
||||
}
|
||||
if let Some((_, off)) = self.stored.iter().find(|(d, _)| d == data) {
|
||||
return (Page::header(crun, entry) <= self.low).then_some(*off);
|
||||
}
|
||||
let at = self.low.checked_sub(data.len())? & !1;
|
||||
if at < Page::header(crun, entry) || at / 2 > u8::MAX as usize {
|
||||
return None;
|
||||
}
|
||||
self.buf.get_mut(at..at + data.len())?.copy_from_slice(data);
|
||||
self.low = at;
|
||||
let off = (at / 2) as u8;
|
||||
self.stored.push((data.to_vec(), off));
|
||||
Some(off)
|
||||
}
|
||||
|
||||
/// Write the header and `crun`; `entry` is the size of each `rgb`/`rgbx` element.
|
||||
fn finish(mut self, end_fc: u32, entry: usize) -> ([u8; PAGE], u32) {
|
||||
let crun = self.offs.len();
|
||||
let first = self.fcs.first().copied().unwrap_or(end_fc);
|
||||
self.fcs.push(end_fc);
|
||||
for (i, fc) in self.fcs.iter().enumerate() {
|
||||
if let Some(b) = self.buf.get_mut(i * 4..i * 4 + 4) {
|
||||
b.copy_from_slice(&fc.to_le_bytes());
|
||||
}
|
||||
}
|
||||
let rgb_at = (crun + 1) * 4;
|
||||
for (i, off) in self.offs.iter().enumerate() {
|
||||
if let Some(b) = self.buf.get_mut(rgb_at + i * entry) {
|
||||
*b = *off;
|
||||
}
|
||||
}
|
||||
self.buf[PAGE - 1] = crun as u8;
|
||||
(self.buf, first)
|
||||
}
|
||||
}
|
||||
|
||||
/// Page the character runs into `ChpxFkp`s: each run's `Chpx` is `cb` then the grpprl (at most
|
||||
/// 255 bytes; the caller truncates).
|
||||
pub(crate) fn chpx_pages(runs: &[ChpxRun<'_>]) -> Paged {
|
||||
let mut out = Paged { pages: Vec::new(), fcs: Vec::new() };
|
||||
let mut page = Page::new();
|
||||
for r in runs {
|
||||
let mut data = Vec::new();
|
||||
if !r.grpprl.is_empty() {
|
||||
data.push(r.grpprl.len().min(255) as u8);
|
||||
data.extend_from_slice(r.grpprl.get(..255).unwrap_or(r.grpprl));
|
||||
}
|
||||
let off = if page.offs.len() < MAX_CHPX_RUNS { page.place(&data, page.offs.len() + 1, 1) } else { None };
|
||||
let off = match off {
|
||||
Some(o) => o,
|
||||
None => {
|
||||
let (buf, first) = std::mem::replace(&mut page, Page::new()).finish(r.fc_start, 1);
|
||||
out.pages.push(buf);
|
||||
out.fcs.push(first);
|
||||
// A fresh page always has room for one Chpx of at most 256 bytes.
|
||||
page.place(&data, 1, 1).unwrap_or(0)
|
||||
}
|
||||
};
|
||||
page.fcs.push(r.fc_start);
|
||||
page.offs.push(off);
|
||||
}
|
||||
let end = runs.last().map_or(0, |r| r.fc_end);
|
||||
if !page.offs.is_empty() {
|
||||
let (buf, first) = page.finish(end, 1);
|
||||
out.pages.push(buf);
|
||||
out.fcs.push(first);
|
||||
}
|
||||
out.fcs.push(end);
|
||||
out
|
||||
}
|
||||
|
||||
/// A `PapxInFkp` ([MS-DOC] §2.9.175): the istd and grpprl with the size byte(s) in front: `cb`
|
||||
/// (2×cb−1 bytes follow) for an odd size, else 0 and `cb'` (2×cb' bytes follow).
|
||||
fn papx_in_fkp(istd: u16, grpprl: &[u8]) -> Vec<u8> {
|
||||
let mut body = istd.to_le_bytes().to_vec();
|
||||
body.extend_from_slice(grpprl);
|
||||
let mut v = Vec::with_capacity(body.len() + 2);
|
||||
if body.len() % 2 == 1 {
|
||||
v.push(body.len().div_ceil(2) as u8);
|
||||
} else {
|
||||
v.push(0);
|
||||
v.push((body.len() / 2) as u8);
|
||||
}
|
||||
v.extend_from_slice(&body);
|
||||
v
|
||||
}
|
||||
|
||||
/// Largest PAPX grpprl a page takes: one paragraph alone on a page must still fit.
|
||||
pub(crate) const MAX_PAPX_GRPPRL: usize = 480;
|
||||
|
||||
/// Page the paragraphs into `PapxFkp`s.
|
||||
pub(crate) fn papx_pages(paras: &[PapxRun<'_>]) -> Paged {
|
||||
let mut out = Paged { pages: Vec::new(), fcs: Vec::new() };
|
||||
let mut page = Page::new();
|
||||
for p in paras {
|
||||
let data = papx_in_fkp(p.istd, p.grpprl.get(..MAX_PAPX_GRPPRL).unwrap_or(p.grpprl));
|
||||
// Even an unformatted paragraph names its style, so every BX points at a PAPX.
|
||||
let off = if page.offs.len() < MAX_PAPX_RUNS { page.place(&data, page.offs.len() + 1, BX) } else { None };
|
||||
let off = match off {
|
||||
Some(o) => o,
|
||||
None => {
|
||||
let (buf, first) = std::mem::replace(&mut page, Page::new()).finish(p.fc_start, BX);
|
||||
out.pages.push(buf);
|
||||
out.fcs.push(first);
|
||||
page.place(&data, 1, BX).unwrap_or(0)
|
||||
}
|
||||
};
|
||||
page.fcs.push(p.fc_start);
|
||||
page.offs.push(off);
|
||||
}
|
||||
let end = paras.last().map_or(0, |p| p.fc_end);
|
||||
if !page.offs.is_empty() {
|
||||
let (buf, first) = page.finish(end, BX);
|
||||
out.pages.push(buf);
|
||||
out.fcs.push(first);
|
||||
}
|
||||
out.fcs.push(end);
|
||||
out
|
||||
}
|
||||
|
||||
/// A PLC ([MS-DOC] §2.2.2): `cps` (one more than the data elements), then the elements.
|
||||
pub(crate) fn plc(cps: &[u32], data: &[&[u8]]) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(cps.len() * 4 + data.iter().map(|d| d.len()).sum::<usize>());
|
||||
for cp in cps {
|
||||
v.extend_from_slice(&cp.to_le_bytes());
|
||||
}
|
||||
for d in data {
|
||||
v.extend_from_slice(d);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// An extended STTB ([MS-DOC] §2.2.4): `fExtend`, `cData`, `cbExtra`, then each string as a
|
||||
/// count of UTF-16 units and the units, followed by its `extra` bytes.
|
||||
pub(crate) fn sttb_ext(strings: &[(String, Vec<u8>)], cb_extra: u16) -> Vec<u8> {
|
||||
let mut v = vec![0xFF, 0xFF];
|
||||
v.extend_from_slice(&(strings.len().min(u16::MAX as usize) as u16).to_le_bytes());
|
||||
v.extend_from_slice(&cb_extra.to_le_bytes());
|
||||
for (s, extra) in strings.iter().take(u16::MAX as usize) {
|
||||
let units: Vec<u16> = s.encode_utf16().take(u16::MAX as usize).collect();
|
||||
v.extend_from_slice(&(units.len() as u16).to_le_bytes());
|
||||
for u in units {
|
||||
v.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
let mut e = extra.clone();
|
||||
e.resize(cb_extra as usize, 0);
|
||||
v.extend_from_slice(&e);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::fkp::Bins;
|
||||
|
||||
#[test]
|
||||
fn chpx_pages_read_back() {
|
||||
let bold = [0x35u8, 0x08, 1];
|
||||
let runs: Vec<ChpxRun> =
|
||||
(0..300u32).map(|i| ChpxRun { fc_start: 0x400 + i * 2, fc_end: 0x402 + i * 2, grpprl: if i % 2 == 0 { &bold } else { &[] } }).collect();
|
||||
let paged = chpx_pages(&runs);
|
||||
assert!(paged.pages.len() >= 3, "300 runs need several pages");
|
||||
let mut word = vec![0u8; 0x1000];
|
||||
for p in &paged.pages {
|
||||
word.extend_from_slice(p);
|
||||
}
|
||||
let bte = paged.plc_bte(8);
|
||||
let bins = Bins::parse(&bte, 0, bte.len() as u32).unwrap();
|
||||
for i in 0..300u32 {
|
||||
let g = bins.chpx(&word, 0x400 + i * 2);
|
||||
assert_eq!(g, if i % 2 == 0 { &bold[..] } else { &[][..] }, "run {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn papx_pages_read_back_odd_and_even_sizes() {
|
||||
let odd = [0x03u8, 0x24, 1]; // 2 + 3 = 5 bytes with the istd
|
||||
let even = [0x03u8, 0x24, 1, 0]; // padded to 6 by the istd
|
||||
let paras: Vec<PapxRun> = (0..70u32)
|
||||
.map(|i| PapxRun {
|
||||
fc_start: 0x400 + i * 10,
|
||||
fc_end: 0x40A + i * 10,
|
||||
istd: i as u16 % 3,
|
||||
grpprl: if i % 2 == 0 { &odd } else { &even[..3] },
|
||||
})
|
||||
.collect();
|
||||
let paged = papx_pages(¶s);
|
||||
assert!(paged.pages.len() >= 3);
|
||||
let mut word = vec![0u8; 0x1000];
|
||||
for p in &paged.pages {
|
||||
word.extend_from_slice(p);
|
||||
}
|
||||
let bte = paged.plc_bte(8);
|
||||
let bins = Bins::parse(&bte, 0, bte.len() as u32).unwrap();
|
||||
for i in 0..70u32 {
|
||||
let (istd, g) = bins.papx(&word, 0x400 + i * 10 + 9);
|
||||
assert_eq!(istd, i as u16 % 3);
|
||||
assert_eq!(g, &odd[..]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,753 @@
|
||||
//! Model properties → grpprls of sprms ([MS-DOC] §2.6): character (CHPX), paragraph (PAPX) and
|
||||
//! section (SEPX) formatting. The opcodes are the ones the reader decodes (`fmt.rs`,
|
||||
//! `sections.rs`, `table.rs`), so a saved file reads back with the same properties.
|
||||
|
||||
use craft_wordmodel::Rgb;
|
||||
use craft_wordmodel::props::{
|
||||
Align, Border, BorderStyle, Borders, CharProps, Highlight, LineSpacing, ParaProps, TabAlign, TabLeader, TextColor, Underline, VertAlign,
|
||||
};
|
||||
use craft_wordmodel::section::{NumFormat, PageVAlign, SectionProps, SectionStart};
|
||||
|
||||
/// A grpprl being built: Prls appended one by one.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
|
||||
pub(crate) struct Grpprl(pub(crate) Vec<u8>);
|
||||
|
||||
impl Grpprl {
|
||||
pub(crate) fn len(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
fn op(&mut self, op: u16) {
|
||||
self.0.extend_from_slice(&op.to_le_bytes());
|
||||
}
|
||||
|
||||
pub(crate) fn u8(&mut self, op: u16, v: u8) {
|
||||
self.op(op);
|
||||
self.0.push(v);
|
||||
}
|
||||
|
||||
pub(crate) fn u16(&mut self, op: u16, v: u16) {
|
||||
self.op(op);
|
||||
self.0.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
pub(crate) fn i16(&mut self, op: u16, v: i16) {
|
||||
self.u16(op, v as u16);
|
||||
}
|
||||
|
||||
pub(crate) fn u32(&mut self, op: u16, v: u32) {
|
||||
self.op(op);
|
||||
self.0.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
|
||||
/// A Prl whose operand is written as given (its size implied by the opcode's spra, or a
|
||||
/// leading size byte the caller includes).
|
||||
pub(crate) fn raw(&mut self, op: u16, operand: &[u8]) {
|
||||
self.op(op);
|
||||
self.0.extend_from_slice(operand);
|
||||
}
|
||||
|
||||
/// A variable-length Prl (spra 6): a size byte, then the operand.
|
||||
pub(crate) fn var(&mut self, op: u16, operand: &[u8]) {
|
||||
self.op(op);
|
||||
self.0.push(u8::try_from(operand.len()).unwrap_or(u8::MAX));
|
||||
self.0.extend_from_slice(operand.get(..255).unwrap_or(operand));
|
||||
}
|
||||
|
||||
pub(crate) fn extend(&mut self, other: &Grpprl) {
|
||||
self.0.extend_from_slice(&other.0);
|
||||
}
|
||||
|
||||
/// Keep whole Prls up to `max` bytes: a grpprl that doesn't fit where it is stored (a CHPX
|
||||
/// holds 255 bytes) loses its last properties rather than the file its shape.
|
||||
pub(crate) fn truncate_to(&mut self, max: usize) {
|
||||
if self.0.len() <= max {
|
||||
return;
|
||||
}
|
||||
let mut keep = 0usize;
|
||||
for prl in crate::sprm::iter(&self.0) {
|
||||
let next = keep + 2 + prl.operand.len();
|
||||
if next > max {
|
||||
break;
|
||||
}
|
||||
keep = next;
|
||||
}
|
||||
self.0.truncate(keep);
|
||||
}
|
||||
}
|
||||
|
||||
// Character sprms ([MS-DOC] §2.6.1).
|
||||
pub(crate) const C_RMARK_DEL: u16 = 0x0800;
|
||||
pub(crate) const C_RMARK_INS: u16 = 0x0801;
|
||||
pub(crate) const C_IBST_RMARK: u16 = 0x4804;
|
||||
pub(crate) const C_DTTM_RMARK: u16 = 0x6805;
|
||||
pub(crate) const C_IBST_RMARK_DEL: u16 = 0x4863;
|
||||
pub(crate) const C_DTTM_RMARK_DEL: u16 = 0x6864;
|
||||
pub(crate) const C_ISTD: u16 = 0x4A30;
|
||||
const C_BOLD: u16 = 0x0835;
|
||||
const C_ITALIC: u16 = 0x0836;
|
||||
const C_STRIKE: u16 = 0x0837;
|
||||
const C_OUTLINE: u16 = 0x0838;
|
||||
const C_SHADOW: u16 = 0x0839;
|
||||
const C_SMALL_CAPS: u16 = 0x083A;
|
||||
const C_CAPS: u16 = 0x083B;
|
||||
const C_VANISH: u16 = 0x083C;
|
||||
const C_IMPRINT: u16 = 0x0854;
|
||||
pub(crate) const C_SPEC: u16 = 0x0855;
|
||||
const C_EMBOSS: u16 = 0x0858;
|
||||
const C_RTL: u16 = 0x085A;
|
||||
const C_BOLD_BI: u16 = 0x085C;
|
||||
const C_ITALIC_BI: u16 = 0x085D;
|
||||
const C_NO_PROOF: u16 = 0x0875;
|
||||
const C_COMPLEX: u16 = 0x0882;
|
||||
const C_HIGHLIGHT: u16 = 0x2A0C;
|
||||
const C_KUL: u16 = 0x2A3E;
|
||||
const C_ICO: u16 = 0x2A42;
|
||||
const C_ISS: u16 = 0x2A48;
|
||||
const C_DSTRIKE: u16 = 0x2A53;
|
||||
const C_HPS: u16 = 0x4A43;
|
||||
const C_HPS_POS: u16 = 0x4845;
|
||||
const C_HPS_KERN: u16 = 0x484B;
|
||||
const C_CHAR_SCALE: u16 = 0x4852;
|
||||
const C_LID_BI: u16 = 0x485F;
|
||||
const C_LID0_80: u16 = 0x486D;
|
||||
const C_LID1_80: u16 = 0x486E;
|
||||
const C_LID0: u16 = 0x4873;
|
||||
const C_LID1: u16 = 0x4874;
|
||||
const C_FTC0: u16 = 0x4A4F;
|
||||
const C_FTC1: u16 = 0x4A50;
|
||||
const C_FTC2: u16 = 0x4A51;
|
||||
const C_FTC_BI: u16 = 0x4A5E;
|
||||
const C_HPS_BI: u16 = 0x4A61;
|
||||
pub(crate) const C_PIC_LOCATION: u16 = 0x6A03;
|
||||
const C_CV: u16 = 0x6870;
|
||||
const C_CV_UL: u16 = 0x6877;
|
||||
const C_DXA_SPACE: u16 = 0x8840;
|
||||
const C_SHD: u16 = 0xCA71;
|
||||
|
||||
/// What [`chpx`] needs besides the properties: the font table, the style table, and the
|
||||
/// tracked-change authors.
|
||||
pub(crate) trait CharCtx {
|
||||
/// Index of `name` in the font table (added on first use).
|
||||
fn font(&mut self, name: &str) -> u16;
|
||||
/// The istd of a character style id; `None` when the stylesheet doesn't have it.
|
||||
fn char_style(&self, id: &str) -> Option<u16>;
|
||||
/// The author index and `DTTM` of revision `rev` (`Document::revisions`).
|
||||
fn revision(&mut self, rev: u32) -> Option<(u16, u32)>;
|
||||
}
|
||||
|
||||
/// The CHPX grpprl for `p`: only the properties it sets, in the order Word applies them (the
|
||||
/// style first, then toggles, then values). Inherited (`None`) properties are left out.
|
||||
pub(crate) fn chpx(p: &CharProps, ctx: &mut dyn CharCtx) -> Grpprl {
|
||||
let mut g = Grpprl::default();
|
||||
if let Some(istd) = p.style.as_deref().and_then(|s| ctx.char_style(s)) {
|
||||
g.u16(C_ISTD, istd);
|
||||
}
|
||||
// Tracked changes: the mark, the author and the time ([MS-DOC] sprmCFRMarkIns and friends).
|
||||
if let Some((ibst, dttm)) = p.ins.and_then(|r| ctx.revision(r)) {
|
||||
g.u8(C_RMARK_INS, 1);
|
||||
g.u16(C_IBST_RMARK, ibst);
|
||||
g.u32(C_DTTM_RMARK, dttm);
|
||||
}
|
||||
if let Some((ibst, dttm)) = p.del.and_then(|r| ctx.revision(r)) {
|
||||
g.u8(C_RMARK_DEL, 1);
|
||||
g.u16(C_IBST_RMARK_DEL, ibst);
|
||||
g.u32(C_DTTM_RMARK_DEL, dttm);
|
||||
}
|
||||
for (op, v) in [
|
||||
(C_BOLD, p.bold),
|
||||
(C_ITALIC, p.italic),
|
||||
(C_STRIKE, p.strike),
|
||||
(C_OUTLINE, p.outline),
|
||||
(C_SHADOW, p.shadow),
|
||||
(C_SMALL_CAPS, p.small_caps),
|
||||
(C_CAPS, p.caps),
|
||||
(C_VANISH, p.hidden),
|
||||
(C_IMPRINT, p.engrave),
|
||||
(C_EMBOSS, p.emboss),
|
||||
(C_RTL, p.rtl),
|
||||
(C_BOLD_BI, p.bold_cs),
|
||||
(C_ITALIC_BI, p.italic_cs),
|
||||
(C_NO_PROOF, p.no_proof),
|
||||
(C_COMPLEX, p.cs),
|
||||
] {
|
||||
if let Some(on) = v {
|
||||
g.u8(op, u8::from(on));
|
||||
}
|
||||
}
|
||||
if let Some(d) = p.double_strike {
|
||||
g.u8(C_DSTRIKE, u8::from(d));
|
||||
}
|
||||
if let Some(u) = p.underline {
|
||||
g.u8(C_KUL, kul(u));
|
||||
}
|
||||
if let Some(c) = p.underline_color {
|
||||
g.u32(C_CV_UL, colorref(c));
|
||||
}
|
||||
match p.color {
|
||||
Some(TextColor::Auto) => g.u8(C_ICO, 0),
|
||||
Some(TextColor::Rgb(c)) => {
|
||||
// The palette colour for Word 97, then the exact colour for every later version.
|
||||
g.u8(C_ICO, nearest_ico(c));
|
||||
g.u32(C_CV, colorref(c));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if let Some(h) = p.highlight {
|
||||
g.u8(C_HIGHLIGHT, highlight_ico(h));
|
||||
}
|
||||
if let Some(s) = p.size {
|
||||
g.u16(C_HPS, half_points(s));
|
||||
}
|
||||
if let Some(s) = p.size_cs {
|
||||
g.u16(C_HPS_BI, half_points(s));
|
||||
}
|
||||
if let Some(v) = p.vert_align {
|
||||
g.u8(
|
||||
C_ISS,
|
||||
match v {
|
||||
VertAlign::Baseline => 0,
|
||||
VertAlign::Superscript => 1,
|
||||
VertAlign::Subscript => 2,
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(pos) = p.position {
|
||||
g.i16(C_HPS_POS, (pos * 2.0).round().clamp(-3168.0, 3168.0) as i16);
|
||||
}
|
||||
if let Some(sp) = p.spacing {
|
||||
g.i16(C_DXA_SPACE, twips(sp));
|
||||
}
|
||||
if let Some(sc) = p.scale {
|
||||
g.u16(C_CHAR_SCALE, sc.round().clamp(1.0, 600.0) as u16);
|
||||
}
|
||||
if let Some(k) = p.kern {
|
||||
g.u16(C_HPS_KERN, half_points(k.max(0.0)));
|
||||
}
|
||||
if let Some(f) = p.font.as_deref() {
|
||||
let i = ctx.font(f);
|
||||
g.u16(C_FTC0, i);
|
||||
// Text outside ASCII but not East Asian takes the third font; Word sets both together.
|
||||
g.u16(C_FTC2, i);
|
||||
}
|
||||
if let Some(f) = p.font_ea.as_deref() {
|
||||
let i = ctx.font(f);
|
||||
g.u16(C_FTC1, i);
|
||||
}
|
||||
if let Some(f) = p.font_cs.as_deref() {
|
||||
let i = ctx.font(f);
|
||||
g.u16(C_FTC_BI, i);
|
||||
}
|
||||
if let Some(lid) = p.lang.as_deref().and_then(lcid) {
|
||||
g.u16(C_LID0_80, lid);
|
||||
g.u16(C_LID0, lid);
|
||||
}
|
||||
if let Some(lid) = p.lang_ea.as_deref().and_then(lcid) {
|
||||
g.u16(C_LID1_80, lid);
|
||||
g.u16(C_LID1, lid);
|
||||
}
|
||||
if let Some(lid) = p.lang_bidi.as_deref().and_then(lcid) {
|
||||
g.u16(C_LID_BI, lid);
|
||||
}
|
||||
if let Some(c) = p.shading {
|
||||
g.var(C_SHD, &shd(Some(c)));
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
/// Points → half-points, as the size sprms store them.
|
||||
fn half_points(pt: f32) -> u16 {
|
||||
(pt * 2.0).round().clamp(2.0, 3276.0) as u16
|
||||
}
|
||||
|
||||
/// Points → twips, clamped to what an `XAS`/`YAS` holds.
|
||||
pub(crate) fn twips(pt: f32) -> i16 {
|
||||
if !pt.is_finite() {
|
||||
return 0;
|
||||
}
|
||||
(pt * 20.0).round().clamp(-31_680.0, 31_680.0) as i16
|
||||
}
|
||||
|
||||
/// A `COLORREF` (0x00BBGGRR, stored R, G, B, 0).
|
||||
pub(crate) fn colorref(c: Rgb) -> u32 {
|
||||
u32::from_le_bytes([c.0, c.1, c.2, 0])
|
||||
}
|
||||
|
||||
/// The automatic colour (`fAuto` in the high byte).
|
||||
pub(crate) const CV_AUTO: u32 = 0xFF00_0000;
|
||||
|
||||
/// A 10-byte `Shd` ([MS-DOC] §2.9.248): automatic foreground, `fill` as background, clear
|
||||
/// pattern; no fill is an all-automatic one.
|
||||
pub(crate) fn shd(fill: Option<Rgb>) -> [u8; 10] {
|
||||
let mut b = [0u8; 10];
|
||||
b[0..4].copy_from_slice(&CV_AUTO.to_le_bytes());
|
||||
let back = fill.map_or(CV_AUTO, colorref);
|
||||
b[4..8].copy_from_slice(&back.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
/// Word's 16-colour palette ([MS-DOC] §2.9.135 Ico), index 1..=16.
|
||||
const ICO: [Rgb; 16] = [
|
||||
Rgb(0, 0, 0),
|
||||
Rgb(0, 0, 255),
|
||||
Rgb(0, 255, 255),
|
||||
Rgb(0, 255, 0),
|
||||
Rgb(255, 0, 255),
|
||||
Rgb(255, 0, 0),
|
||||
Rgb(255, 255, 0),
|
||||
Rgb(255, 255, 255),
|
||||
Rgb(0, 0, 128),
|
||||
Rgb(0, 128, 128),
|
||||
Rgb(0, 128, 0),
|
||||
Rgb(128, 0, 128),
|
||||
Rgb(128, 0, 0),
|
||||
Rgb(128, 128, 0),
|
||||
Rgb(128, 128, 128),
|
||||
Rgb(192, 192, 192),
|
||||
];
|
||||
|
||||
/// The palette entry nearest `c` (by squared distance), 1-based.
|
||||
pub(crate) fn nearest_ico(c: Rgb) -> u8 {
|
||||
let d = |p: &Rgb| {
|
||||
let dr = i32::from(p.0) - i32::from(c.0);
|
||||
let dg = i32::from(p.1) - i32::from(c.1);
|
||||
let db = i32::from(p.2) - i32::from(c.2);
|
||||
dr * dr + dg * dg + db * db
|
||||
};
|
||||
ICO.iter().enumerate().min_by_key(|(_, p)| d(p)).map_or(1, |(i, _)| i as u8 + 1)
|
||||
}
|
||||
|
||||
/// A highlight colour as the `Ico` `sprmCHighlight` takes.
|
||||
fn highlight_ico(h: Highlight) -> u8 {
|
||||
match h {
|
||||
Highlight::None => 0,
|
||||
Highlight::Black => 1,
|
||||
Highlight::Blue => 2,
|
||||
Highlight::Turquoise => 3,
|
||||
Highlight::BrightGreen => 4,
|
||||
Highlight::Pink => 5,
|
||||
Highlight::Red => 6,
|
||||
Highlight::Yellow => 7,
|
||||
Highlight::White => 8,
|
||||
Highlight::DarkBlue => 9,
|
||||
Highlight::Teal => 10,
|
||||
Highlight::Green => 11,
|
||||
Highlight::Violet => 12,
|
||||
Highlight::DarkRed => 13,
|
||||
Highlight::DarkYellow => 14,
|
||||
Highlight::Gray50 => 15,
|
||||
Highlight::Gray25 => 16,
|
||||
}
|
||||
}
|
||||
|
||||
/// An underline as a `Kul` ([MS-DOC] §2.9.133).
|
||||
fn kul(u: Underline) -> u8 {
|
||||
match u {
|
||||
Underline::None => 0x00,
|
||||
Underline::Single => 0x01,
|
||||
Underline::Words => 0x02,
|
||||
Underline::Double => 0x03,
|
||||
Underline::Dotted => 0x04,
|
||||
Underline::Thick => 0x06,
|
||||
Underline::Dash => 0x07,
|
||||
Underline::DotDash => 0x09,
|
||||
Underline::DotDotDash => 0x0A,
|
||||
Underline::Wave => 0x0B,
|
||||
Underline::DoubleWave => 0x2B,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Windows language id (LCID) of a BCP 47 tag, for the common languages; `None` for a tag
|
||||
/// not in the table (the text keeps the style's language).
|
||||
pub(crate) fn lcid(tag: &str) -> Option<u16> {
|
||||
let t = tag.trim().to_ascii_lowercase().replace('_', "-");
|
||||
Some(match t.as_str() {
|
||||
"ar-sa" | "ar" => 0x0401,
|
||||
"bg-bg" | "bg" => 0x0402,
|
||||
"ca-es" | "ca" => 0x0403,
|
||||
"zh-tw" => 0x0404,
|
||||
"cs-cz" | "cs" => 0x0405,
|
||||
"da-dk" | "da" => 0x0406,
|
||||
"de-de" | "de" => 0x0407,
|
||||
"el-gr" | "el" => 0x0408,
|
||||
"en-us" | "en" => 0x0409,
|
||||
"es-es" | "es" => 0x0C0A,
|
||||
"fi-fi" | "fi" => 0x040B,
|
||||
"fr-fr" | "fr" => 0x040C,
|
||||
"he-il" | "he" => 0x040D,
|
||||
"hu-hu" | "hu" => 0x040E,
|
||||
"is-is" | "is" => 0x040F,
|
||||
"it-it" | "it" => 0x0410,
|
||||
"ja-jp" | "ja" => 0x0411,
|
||||
"ko-kr" | "ko" => 0x0412,
|
||||
"nl-nl" | "nl" => 0x0413,
|
||||
"nb-no" | "nb" | "no" => 0x0414,
|
||||
"pl-pl" | "pl" => 0x0415,
|
||||
"pt-br" => 0x0416,
|
||||
"ro-ro" | "ro" => 0x0418,
|
||||
"ru-ru" | "ru" => 0x0419,
|
||||
"hr-hr" | "hr" => 0x041A,
|
||||
"sk-sk" | "sk" => 0x041B,
|
||||
"sv-se" | "sv" => 0x041D,
|
||||
"th-th" | "th" => 0x041E,
|
||||
"tr-tr" | "tr" => 0x041F,
|
||||
"id-id" | "id" => 0x0421,
|
||||
"uk-ua" | "uk" => 0x0422,
|
||||
"sl-si" | "sl" => 0x0424,
|
||||
"et-ee" | "et" => 0x0425,
|
||||
"lv-lv" | "lv" => 0x0426,
|
||||
"lt-lt" | "lt" => 0x0427,
|
||||
"fa-ir" | "fa" => 0x0429,
|
||||
"vi-vn" | "vi" => 0x042A,
|
||||
"hi-in" | "hi" => 0x0439,
|
||||
"ne-np" | "ne" => 0x0461,
|
||||
"zh-cn" | "zh" => 0x0804,
|
||||
"de-ch" => 0x0807,
|
||||
"en-gb" => 0x0809,
|
||||
"es-mx" => 0x080A,
|
||||
"pt-pt" | "pt" => 0x0816,
|
||||
"sr-latn-rs" => 0x241A,
|
||||
"sr-cyrl-rs" | "sr" => 0x281A,
|
||||
"en-au" => 0x0C09,
|
||||
"en-ca" => 0x1009,
|
||||
"fr-ca" => 0x0C0C,
|
||||
"es-uy" => 0x380A,
|
||||
"es-419" => 0x580A,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
// Paragraph sprms ([MS-DOC] §2.6.2).
|
||||
const P_JC80: u16 = 0x2403;
|
||||
const P_JC: u16 = 0x2461;
|
||||
const P_KEEP: u16 = 0x2405;
|
||||
const P_KEEP_FOLLOW: u16 = 0x2406;
|
||||
const P_PAGE_BREAK_BEFORE: u16 = 0x2407;
|
||||
const P_NO_LINE_NUMB: u16 = 0x240C;
|
||||
const P_IN_TABLE: u16 = 0x2416;
|
||||
const P_TTP: u16 = 0x2417;
|
||||
const P_NO_AUTO_HYPH: u16 = 0x242A;
|
||||
const P_WIDOW: u16 = 0x2431;
|
||||
const P_BIDI: u16 = 0x2441;
|
||||
const P_OUTLVL: u16 = 0x2640;
|
||||
const P_CONTEXTUAL: u16 = 0x246D;
|
||||
const P_ILVL: u16 = 0x260A;
|
||||
const P_ILFO: u16 = 0x460B;
|
||||
const P_DXC_RIGHT: u16 = 0x4455;
|
||||
const P_DXC_LEFT: u16 = 0x4456;
|
||||
const P_DXC_FIRST: u16 = 0x4457;
|
||||
const P_DYA_LINE: u16 = 0x6412;
|
||||
const P_ITAP: u16 = 0x6649;
|
||||
const P_DXA_RIGHT80: u16 = 0x840E;
|
||||
const P_DXA_LEFT80: u16 = 0x840F;
|
||||
const P_DXA_FIRST80: u16 = 0x8411;
|
||||
const P_DXA_RIGHT: u16 = 0x845D;
|
||||
const P_DXA_LEFT: u16 = 0x845E;
|
||||
const P_DXA_FIRST: u16 = 0x8460;
|
||||
const P_DYA_BEFORE: u16 = 0xA413;
|
||||
const P_DYA_AFTER: u16 = 0xA414;
|
||||
const P_CHG_TABS_PAPX: u16 = 0xC60D;
|
||||
const P_SHD: u16 = 0xC64D;
|
||||
const P_BRC: [u16; 5] = [0xC64E, 0xC64F, 0xC650, 0xC651, 0xC652];
|
||||
const P_BRC80: [u16; 5] = [0x6424, 0x6425, 0x6426, 0x6427, 0x6428];
|
||||
|
||||
/// What [`papx`] needs besides the properties: the list (LFO) a model num id became.
|
||||
pub(crate) trait ParaCtx {
|
||||
/// The 1-based LFO index of model num `id` (0 = no list).
|
||||
fn lfo(&self, num: u32) -> u16;
|
||||
}
|
||||
|
||||
/// The PAPX grpprl for `p` (its style is the PAPX's istd, not a sprm).
|
||||
pub(crate) fn papx(p: &ParaProps, ctx: &dyn ParaCtx) -> Grpprl {
|
||||
let mut g = Grpprl::default();
|
||||
if let Some(a) = p.align {
|
||||
let jc = match a {
|
||||
Align::Left => 0,
|
||||
Align::Center => 1,
|
||||
Align::Right => 2,
|
||||
Align::Justify => 3,
|
||||
Align::Distribute => 4,
|
||||
};
|
||||
g.u8(P_JC80, jc);
|
||||
g.u8(P_JC, jc);
|
||||
}
|
||||
for (v, op80, op) in
|
||||
[(p.indent_left, P_DXA_LEFT80, P_DXA_LEFT), (p.indent_right, P_DXA_RIGHT80, P_DXA_RIGHT), (p.indent_first, P_DXA_FIRST80, P_DXA_FIRST)]
|
||||
{
|
||||
if let Some(v) = v {
|
||||
g.i16(op80, twips(v));
|
||||
g.i16(op, twips(v));
|
||||
}
|
||||
}
|
||||
for (v, op) in [(p.indent_left_chars, P_DXC_LEFT), (p.indent_right_chars, P_DXC_RIGHT), (p.indent_first_chars, P_DXC_FIRST)] {
|
||||
if let Some(v) = v.filter(|v| v.is_finite()) {
|
||||
g.i16(op, (v * 100.0).round().clamp(-31_680.0, 31_680.0) as i16);
|
||||
}
|
||||
}
|
||||
if let Some(v) = p.space_before {
|
||||
g.u16(P_DYA_BEFORE, twips(v.max(0.0)) as u16);
|
||||
}
|
||||
if let Some(v) = p.space_after {
|
||||
g.u16(P_DYA_AFTER, twips(v.max(0.0)) as u16);
|
||||
}
|
||||
if let Some(ls) = p.line_spacing {
|
||||
// LSPD: dyaLine, then fMultLinespace.
|
||||
let (line, mult): (i16, i16) = match ls {
|
||||
LineSpacing::Multiple(m) => ((m * 240.0).round().clamp(1.0, 31_680.0) as i16, 1),
|
||||
LineSpacing::AtLeast(pt) => (twips(pt.max(0.0)), 0),
|
||||
LineSpacing::Exactly(pt) => (twips(pt.max(0.05)).saturating_neg(), 0),
|
||||
};
|
||||
let mut op = line.to_le_bytes().to_vec();
|
||||
op.extend_from_slice(&mult.to_le_bytes());
|
||||
g.raw(P_DYA_LINE, &op);
|
||||
}
|
||||
for (op, v) in [
|
||||
(P_KEEP, p.keep_lines),
|
||||
(P_KEEP_FOLLOW, p.keep_next),
|
||||
(P_PAGE_BREAK_BEFORE, p.page_break_before),
|
||||
(P_WIDOW, p.widow_control),
|
||||
(P_CONTEXTUAL, p.contextual_spacing),
|
||||
(P_NO_AUTO_HYPH, p.suppress_hyphens),
|
||||
(P_NO_LINE_NUMB, p.suppress_line_numbers),
|
||||
(P_BIDI, p.bidi),
|
||||
] {
|
||||
if let Some(on) = v {
|
||||
g.u8(op, u8::from(on));
|
||||
}
|
||||
}
|
||||
if let Some(l) = p.outline_level {
|
||||
g.u8(P_OUTLVL, l.min(9));
|
||||
}
|
||||
if let Some(n) = p.numbering {
|
||||
g.u8(P_ILVL, n.level.min(8));
|
||||
g.u16(P_ILFO, if n.num == 0 { 0 } else { ctx.lfo(n.num) });
|
||||
}
|
||||
if let Some(c) = p.shading {
|
||||
g.var(P_SHD, &shd(Some(c)));
|
||||
}
|
||||
if let Some(b) = &p.borders {
|
||||
para_borders(&mut g, b);
|
||||
}
|
||||
if let Some(tabs) = &p.tabs {
|
||||
tab_stops(&mut g, tabs);
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
/// The paragraph's borders: the Word 97 form (palette colour) and the exact one, side by side.
|
||||
fn para_borders(g: &mut Grpprl, b: &Borders) {
|
||||
for (i, side) in [b.top, b.left, b.bottom, b.right, b.between].iter().enumerate() {
|
||||
let Some(side) = side else { continue };
|
||||
if let (Some(&op80), Some(&op)) = (P_BRC80.get(i), P_BRC.get(i)) {
|
||||
g.raw(op80, &brc80(side));
|
||||
g.var(op, &brc(side));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A border's `brcType` ([MS-DOC] §2.9.22).
|
||||
fn brc_type(s: BorderStyle) -> u8 {
|
||||
match s {
|
||||
BorderStyle::None => 0x00,
|
||||
BorderStyle::Single => 0x01,
|
||||
BorderStyle::Thick => 0x02,
|
||||
BorderStyle::Double => 0x03,
|
||||
BorderStyle::Dotted => 0x06,
|
||||
BorderStyle::Dashed => 0x07,
|
||||
BorderStyle::DotDash => 0x08,
|
||||
BorderStyle::Triple => 0x0A,
|
||||
BorderStyle::Wave => 0x14,
|
||||
BorderStyle::ThinThickSmallGap => 0x0B,
|
||||
BorderStyle::ThickThinSmallGap => 0x0C,
|
||||
BorderStyle::ThinThickThinSmallGap => 0x0D,
|
||||
BorderStyle::ThinThickMediumGap => 0x0E,
|
||||
BorderStyle::ThickThinMediumGap => 0x0F,
|
||||
BorderStyle::ThinThickThinMediumGap => 0x10,
|
||||
BorderStyle::ThinThickLargeGap => 0x11,
|
||||
BorderStyle::ThickThinLargeGap => 0x12,
|
||||
BorderStyle::ThinThickThinLargeGap => 0x13,
|
||||
}
|
||||
}
|
||||
|
||||
/// A `Brc80` ([MS-DOC] §2.9.18): width in eighths of a point, type, palette colour, space.
|
||||
pub(crate) fn brc80(b: &Border) -> [u8; 4] {
|
||||
if b.style == BorderStyle::None {
|
||||
return [0; 4];
|
||||
}
|
||||
let dpt = (b.width * 8.0).round().clamp(2.0, 255.0) as u8;
|
||||
let ico = b.color.map_or(0, nearest_ico);
|
||||
let space = b.space.round().clamp(0.0, 31.0) as u8;
|
||||
[dpt, brc_type(b.style), ico, space]
|
||||
}
|
||||
|
||||
/// A `Brc` ([MS-DOC] §2.9.16): `COLORREF`, width in eighths of a point, type, space and flags.
|
||||
pub(crate) fn brc(b: &Border) -> [u8; 8] {
|
||||
if b.style == BorderStyle::None {
|
||||
return [0; 8];
|
||||
}
|
||||
let cv = b.color.map_or(0, colorref);
|
||||
let [c0, c1, c2, c3] = cv.to_le_bytes();
|
||||
let dpt = (b.width * 8.0).round().clamp(2.0, 255.0) as u8;
|
||||
let space = b.space.round().clamp(0.0, 31.0) as u8;
|
||||
[c0, c1, c2, c3, dpt, brc_type(b.style), space, 0]
|
||||
}
|
||||
|
||||
/// The paragraph's own tab stops as a `PChgTabsPapxOperand`: the cleared stops in `rgdxaDel`,
|
||||
/// the others (with their alignment and leader) in `rgdxaAdd`, each list ascending.
|
||||
fn tab_stops(g: &mut Grpprl, tabs: &[craft_wordmodel::props::TabStop]) {
|
||||
let mut del: Vec<i16> = tabs.iter().filter(|t| t.align == TabAlign::Clear).map(|t| twips(t.pos)).collect();
|
||||
let mut add: Vec<(i16, u8)> = tabs
|
||||
.iter()
|
||||
.filter(|t| t.align != TabAlign::Clear)
|
||||
.map(|t| {
|
||||
let jc = match t.align {
|
||||
TabAlign::Center => 1,
|
||||
TabAlign::Right => 2,
|
||||
TabAlign::Decimal => 3,
|
||||
TabAlign::Bar => 4,
|
||||
_ => 0,
|
||||
};
|
||||
let tlc = match t.leader {
|
||||
TabLeader::None => 0,
|
||||
TabLeader::Dot => 1,
|
||||
TabLeader::Hyphen => 2,
|
||||
TabLeader::Underscore => 3,
|
||||
TabLeader::MiddleDot => 5,
|
||||
};
|
||||
(twips(t.pos), jc | (tlc << 3))
|
||||
})
|
||||
.collect();
|
||||
del.sort_unstable();
|
||||
del.dedup();
|
||||
add.sort_by_key(|t| t.0);
|
||||
add.dedup_by_key(|t| t.0);
|
||||
// The operand's size byte holds 255: 64 stops each way is Word's own limit and fits.
|
||||
del.truncate(64);
|
||||
add.truncate(64);
|
||||
if del.is_empty() && add.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut op = vec![del.len() as u8];
|
||||
for d in &del {
|
||||
op.extend_from_slice(&d.to_le_bytes());
|
||||
}
|
||||
op.push(add.len() as u8);
|
||||
for (x, _) in &add {
|
||||
op.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
op.extend(add.iter().map(|(_, tbd)| *tbd));
|
||||
g.var(P_CHG_TABS_PAPX, &op);
|
||||
}
|
||||
|
||||
/// The table membership of a paragraph inside a table cell (`sprmPFInTable`, `sprmPItap`); the
|
||||
/// row-end mark adds `sprmPFTtp`.
|
||||
pub(crate) fn in_table(g: &mut Grpprl, row_end: bool) {
|
||||
g.u8(P_IN_TABLE, 1);
|
||||
if row_end {
|
||||
g.u8(P_TTP, 1);
|
||||
}
|
||||
g.u32(P_ITAP, 1);
|
||||
}
|
||||
|
||||
// Section sprms ([MS-DOC] §2.6.4).
|
||||
const S_BKC: u16 = 0x3009;
|
||||
const S_TITLE_PAGE: u16 = 0x300A;
|
||||
const S_CCOLUMNS: u16 = 0x500B;
|
||||
const S_DXA_COLUMNS: u16 = 0x900C;
|
||||
const S_NFC_PGN: u16 = 0x300E;
|
||||
const S_PGN_RESTART: u16 = 0x3011;
|
||||
const S_LNC: u16 = 0x3013;
|
||||
const S_NLNN_MOD: u16 = 0x5015;
|
||||
const S_DXA_LNN: u16 = 0x9016;
|
||||
const S_HDR_TOP: u16 = 0xB017;
|
||||
const S_HDR_BOTTOM: u16 = 0xB018;
|
||||
const S_VJC: u16 = 0x301A;
|
||||
const S_LNN_MIN: u16 = 0x501B;
|
||||
const S_PGN_START: u16 = 0x501C;
|
||||
const S_ORIENTATION: u16 = 0x301D;
|
||||
const S_XA_PAGE: u16 = 0xB01F;
|
||||
const S_YA_PAGE: u16 = 0xB020;
|
||||
const S_DXA_LEFT: u16 = 0xB021;
|
||||
const S_DXA_RIGHT: u16 = 0xB022;
|
||||
const S_DYA_TOP: u16 = 0x9023;
|
||||
const S_DYA_BOTTOM: u16 = 0x9024;
|
||||
const S_DZA_GUTTER: u16 = 0xB025;
|
||||
const S_EVENLY_SPACED: u16 = 0x3005;
|
||||
const S_BIDI: u16 = 0x3228;
|
||||
|
||||
/// The SEPX grpprl of a section: every page property, so the section reads the same whatever
|
||||
/// the reader's defaults.
|
||||
pub(crate) fn sepx(s: &SectionProps) -> Grpprl {
|
||||
let mut g = Grpprl::default();
|
||||
g.u8(
|
||||
S_BKC,
|
||||
match s.start {
|
||||
SectionStart::Continuous => 0,
|
||||
SectionStart::NextColumn => 1,
|
||||
SectionStart::NextPage => 2,
|
||||
SectionStart::EvenPage => 3,
|
||||
SectionStart::OddPage => 4,
|
||||
},
|
||||
);
|
||||
if s.title_page {
|
||||
g.u8(S_TITLE_PAGE, 1);
|
||||
}
|
||||
let cols = s.columns.count.clamp(1, 44);
|
||||
if cols > 1 {
|
||||
g.u16(S_CCOLUMNS, (cols - 1) as u16);
|
||||
g.u16(S_DXA_COLUMNS, twips(s.columns.space.max(0.0)) as u16);
|
||||
g.u8(S_EVENLY_SPACED, 1);
|
||||
}
|
||||
if let Some(start) = s.page_num_start {
|
||||
g.u8(S_PGN_RESTART, 1);
|
||||
g.u16(S_PGN_START, start.min(0x7FFF) as u16);
|
||||
}
|
||||
if s.page_num_format != NumFormat::Decimal {
|
||||
g.u8(S_NFC_PGN, nfc(s.page_num_format));
|
||||
}
|
||||
if let Some(ln) = &s.line_numbers {
|
||||
g.u8(
|
||||
S_LNC,
|
||||
match ln.restart {
|
||||
craft_wordmodel::section::LineNumberRestart::Page => 0,
|
||||
craft_wordmodel::section::LineNumberRestart::Section => 1,
|
||||
craft_wordmodel::section::LineNumberRestart::Continuous => 2,
|
||||
},
|
||||
);
|
||||
g.u16(S_NLNN_MOD, ln.count_by.clamp(1, 100) as u16);
|
||||
g.i16(S_DXA_LNN, twips(ln.distance.max(0.0)));
|
||||
g.u16(S_LNN_MIN, ln.start.saturating_sub(1).min(0x7FFF) as u16);
|
||||
}
|
||||
g.u8(S_ORIENTATION, if s.landscape { 2 } else { 1 });
|
||||
g.u16(S_XA_PAGE, twips(s.page_w.max(7.2)) as u16);
|
||||
g.u16(S_YA_PAGE, twips(s.page_h.max(7.2)) as u16);
|
||||
g.u16(S_DXA_LEFT, twips(s.margin_left.max(0.0)) as u16);
|
||||
g.u16(S_DXA_RIGHT, twips(s.margin_right.max(0.0)) as u16);
|
||||
g.i16(S_DYA_TOP, twips(s.margin_top));
|
||||
g.i16(S_DYA_BOTTOM, twips(s.margin_bottom));
|
||||
g.u16(S_DZA_GUTTER, twips(s.gutter.max(0.0)) as u16);
|
||||
g.u16(S_HDR_TOP, twips(s.header.max(0.0)) as u16);
|
||||
g.u16(S_HDR_BOTTOM, twips(s.footer.max(0.0)) as u16);
|
||||
let vjc = match s.valign {
|
||||
PageVAlign::Top => 0,
|
||||
PageVAlign::Center => 1,
|
||||
PageVAlign::Justify => 2,
|
||||
PageVAlign::Bottom => 3,
|
||||
};
|
||||
if vjc != 0 {
|
||||
g.u8(S_VJC, vjc);
|
||||
}
|
||||
if s.rtl {
|
||||
g.u8(S_BIDI, 1);
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
/// A number format as an `MSONFC` ([MS-OSHARED] §2.2.1.3): its place in the ST_NumberFormat
|
||||
/// order up to 0x27, decimal past it (the formats Word 97 never had), 0xFF for none.
|
||||
pub(crate) fn nfc(f: NumFormat) -> u8 {
|
||||
match f {
|
||||
NumFormat::None => 0xFF,
|
||||
f => NumFormat::ALL.iter().position(|x| *x == f).filter(|&i| i <= 0x27).map_or(0, |i| i as u8),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
//! Lists: `PlfLst` (one LSTF per abstract list, its nine LVLs after the array) and `PlfLfo`
|
||||
//! (one LFO per model num, with LFOLVL start and level overrides) ([MS-DOC] §2.9.147, §2.9.150).
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use craft_wordmodel::numbering::{Level, LevelSuffix, Numbering};
|
||||
use craft_wordmodel::props::{Align, ParaProps};
|
||||
use craft_wordmodel::section::NumFormat;
|
||||
|
||||
use super::grpprl::{self, CharCtx, ParaCtx};
|
||||
|
||||
/// Levels of a multi-level list.
|
||||
const LEVELS: usize = 9;
|
||||
/// Most lists written (the format counts LFOs from 1 to 0x07FE).
|
||||
const MAX_LISTS: usize = 0x07FE;
|
||||
|
||||
/// The written lists: `PlfLst` (with its LVLs appended) and how much of it `lcbPlfLst` covers,
|
||||
/// `PlfLfo`, and each model num id's LFO index.
|
||||
pub(crate) struct Lists {
|
||||
pub(crate) plf_lst: Vec<u8>,
|
||||
pub(crate) lcb_plf_lst: u32,
|
||||
pub(crate) plf_lfo: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Each model num's 1-based LFO index, in `Numbering::nums` order.
|
||||
pub(crate) fn lfo_indexes(n: &Numbering) -> HashMap<u32, u16> {
|
||||
n.nums.iter().take(MAX_LISTS).enumerate().map(|(i, num)| (num.id, (i + 1) as u16)).collect()
|
||||
}
|
||||
|
||||
/// The list id (`lsid`) of abstract list `i`: any unique value other than 0 and -1.
|
||||
fn lsid(i: usize) -> i32 {
|
||||
0x1000_0000 + i as i32
|
||||
}
|
||||
|
||||
pub(crate) fn write(n: &Numbering, ctx: &mut (impl CharCtx + ParaCtx)) -> Option<Lists> {
|
||||
let abstracts: Vec<_> = n.abstracts.iter().take(MAX_LISTS).collect();
|
||||
if abstracts.is_empty() || n.nums.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut lst = Vec::new();
|
||||
lst.extend_from_slice(&(abstracts.len() as i16).to_le_bytes());
|
||||
for (i, _) in abstracts.iter().enumerate() {
|
||||
// LSTF: lsid, tplc, rgistdPara[9] (none), flags (multi-level), grfhic.
|
||||
lst.extend_from_slice(&lsid(i).to_le_bytes());
|
||||
lst.extend_from_slice(&0u32.to_le_bytes());
|
||||
for _ in 0..LEVELS {
|
||||
lst.extend_from_slice(&0x0FFFu16.to_le_bytes());
|
||||
}
|
||||
lst.push(0);
|
||||
lst.push(0);
|
||||
}
|
||||
let lcb_plf_lst = lst.len() as u32;
|
||||
for a in &abstracts {
|
||||
for l in 0..LEVELS {
|
||||
let level = a.levels.get(l).cloned().unwrap_or_else(|| default_level(l));
|
||||
lst.extend_from_slice(&lvl(&level, ctx));
|
||||
}
|
||||
}
|
||||
|
||||
let nums: Vec<_> = n.nums.iter().take(MAX_LISTS).collect();
|
||||
let mut lfo = Vec::new();
|
||||
lfo.extend_from_slice(&(nums.len() as i32).to_le_bytes());
|
||||
let mut data = Vec::new();
|
||||
for num in &nums {
|
||||
let ai = abstracts.iter().position(|a| a.id == num.abstract_id).unwrap_or(0);
|
||||
// The overrides per level: a start value, a whole level, or both.
|
||||
let mut over: Vec<(u8, Option<u32>, Option<&Level>)> = Vec::new();
|
||||
for &(l, start) in &num.start_overrides {
|
||||
if usize::from(l) < LEVELS {
|
||||
over.push((l, Some(start), None));
|
||||
}
|
||||
}
|
||||
for (l, level) in &num.level_overrides {
|
||||
if usize::from(*l) >= LEVELS {
|
||||
continue;
|
||||
}
|
||||
match over.iter_mut().find(|o| o.0 == *l) {
|
||||
Some(o) => o.2 = Some(level),
|
||||
None => over.push((*l, None, Some(level))),
|
||||
}
|
||||
}
|
||||
over.sort_by_key(|o| o.0);
|
||||
// LFO: lsid, unused1, unused2, clfolvl, ibstFltAutoNum, grfhic, unused3.
|
||||
lfo.extend_from_slice(&lsid(ai).to_le_bytes());
|
||||
lfo.extend_from_slice(&[0u8; 8]);
|
||||
lfo.push(over.len() as u8);
|
||||
lfo.extend_from_slice(&[0u8; 3]);
|
||||
// LFOData: cp (unused, 0xFFFFFFFF), then the LFOLVLs.
|
||||
data.extend_from_slice(&u32::MAX.to_le_bytes());
|
||||
for (l, start, level) in over {
|
||||
let start_at = start.or(level.map(|lv| lv.start)).unwrap_or(1).min(0x7FFF);
|
||||
data.extend_from_slice(&(start_at as i32).to_le_bytes());
|
||||
let mut flags = l & 0x0F;
|
||||
if start.is_some() {
|
||||
flags |= 0x10; // fStartAt
|
||||
}
|
||||
if level.is_some() {
|
||||
flags |= 0x20; // fFormatting
|
||||
}
|
||||
data.push(flags);
|
||||
data.extend_from_slice(&[0u8; 3]);
|
||||
if let Some(level) = level {
|
||||
data.extend_from_slice(&lvl(level, ctx));
|
||||
}
|
||||
}
|
||||
}
|
||||
lfo.extend_from_slice(&data);
|
||||
Some(Lists { plf_lst: lst, lcb_plf_lst, plf_lfo: lfo })
|
||||
}
|
||||
|
||||
/// A level the model's list doesn't have: decimal, `%n.`, indented a quarter inch a level.
|
||||
fn default_level(l: usize) -> Level {
|
||||
Level { start: 1, format: NumFormat::Decimal, text: format!("%{}.", l + 1), indent: 18.0 * (l as f32 + 1.0), hanging: 18.0, ..Level::default() }
|
||||
}
|
||||
|
||||
/// One LVL: LVLF, grpprlPapx (the indents), grpprlChpx (the number's formatting), number text.
|
||||
fn lvl(level: &Level, ctx: &mut (impl CharCtx + ParaCtx)) -> Vec<u8> {
|
||||
// Number text: `%n` placeholders become the level index (n − 1), their 1-based positions
|
||||
// go in rgbxchNums.
|
||||
let mut xst: Vec<u16> = Vec::new();
|
||||
let mut nums: Vec<u8> = Vec::new();
|
||||
let mut chars = level.text.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '%'
|
||||
&& let Some(d) = chars.peek().and_then(|d| d.to_digit(10)).filter(|d| (1..=9).contains(d))
|
||||
{
|
||||
chars.next();
|
||||
xst.push((d - 1) as u16);
|
||||
if nums.len() < 9 {
|
||||
nums.push(xst.len() as u8);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let mut buf = [0u16; 2];
|
||||
xst.extend_from_slice(c.encode_utf16(&mut buf));
|
||||
}
|
||||
xst.truncate(64);
|
||||
nums.retain(|&p| usize::from(p) <= xst.len());
|
||||
|
||||
let mut papx = grpprl::papx(&ParaProps { indent_left: Some(level.indent), indent_first: Some(-level.hanging), ..Default::default() }, ctx);
|
||||
papx.truncate_to(255);
|
||||
let mut chpx = grpprl::chpx(&level.chr, ctx);
|
||||
chpx.truncate_to(255);
|
||||
|
||||
let mut v = Vec::with_capacity(28 + papx.len() + chpx.len() + 2 + xst.len() * 2);
|
||||
v.extend_from_slice(&(level.start.min(0x7FFF) as i32).to_le_bytes());
|
||||
v.push(grpprl::nfc(level.format));
|
||||
let jc: u8 = match level.align {
|
||||
Align::Center => 1,
|
||||
Align::Right => 2,
|
||||
_ => 0,
|
||||
};
|
||||
let mut flags = jc;
|
||||
if level.legal {
|
||||
flags |= 0x04;
|
||||
}
|
||||
if !level.restart {
|
||||
flags |= 0x08;
|
||||
}
|
||||
v.push(flags);
|
||||
let mut rgbxch = [0u8; 9];
|
||||
for (slot, p) in rgbxch.iter_mut().zip(&nums) {
|
||||
*slot = *p;
|
||||
}
|
||||
v.extend_from_slice(&rgbxch);
|
||||
v.push(match level.suffix {
|
||||
LevelSuffix::Tab => 0,
|
||||
LevelSuffix::Space => 1,
|
||||
LevelSuffix::Nothing => 2,
|
||||
});
|
||||
v.extend_from_slice(&0i32.to_le_bytes()); // dxaIndentSav
|
||||
v.extend_from_slice(&0u32.to_le_bytes()); // unused2
|
||||
v.push(chpx.len() as u8);
|
||||
v.push(papx.len() as u8);
|
||||
v.push(level.restart_after.map_or(0, |r| r.saturating_add(1)).min(LEVELS as u8));
|
||||
v.push(0); // grfhic
|
||||
v.extend_from_slice(&papx.0);
|
||||
v.extend_from_slice(&chpx.0);
|
||||
v.extend_from_slice(&(xst.len() as u16).to_le_bytes());
|
||||
for u in &xst {
|
||||
v.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
v
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
//! The `.doc` writer: a [`Document`] as a Word 97-2003 binary file ([MS-DOC]).
|
||||
//!
|
||||
//! The file is what Word 97 itself writes, minus the parts the model has nothing for: a compound file
|
||||
//! with the WordDocument stream (the FIB, the text as one Unicode piece, the formatting pages and
|
||||
//! the section properties), the 1Table stream (stylesheet, fonts, piece table, document
|
||||
//! properties, the PLCs locating sections, headers, notes, fields and bookmarks, the lists), a
|
||||
//! Data stream with the pictures, and the summary information.
|
||||
//!
|
||||
//! Kept: text, character and paragraph formatting, styles, lists, tables (one level; nested
|
||||
//! tables become paragraphs of their cell), sections with headers and footers, footnotes and
|
||||
//! endnotes, fields and hyperlinks, bookmarks, inline pictures, tracked insertions and deletions,
|
||||
//! and the document properties. Floating pictures are written in line; shapes, charts, text
|
||||
//! boxes and comments are left out, each kind logged once per save.
|
||||
|
||||
mod fkp;
|
||||
mod grpprl;
|
||||
mod lists;
|
||||
mod picture;
|
||||
mod story;
|
||||
mod styles;
|
||||
mod summary;
|
||||
mod tables;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use craft_wordmodel::Document;
|
||||
use craft_wordmodel::para::{NoteKind, Paragraph};
|
||||
use craft_wordmodel::section::SectionProps;
|
||||
|
||||
use crate::DocbinError;
|
||||
use crate::fib::pair;
|
||||
use grpprl::{CharCtx, Grpprl};
|
||||
use story::{Lookup, PARA, Text};
|
||||
|
||||
/// Where the text starts in the WordDocument stream: right after the FIB (900 bytes).
|
||||
const TEXT_FC: u32 = 0x400;
|
||||
/// Bytes in the FIB this writer produces: FibBase, csw + FibRgW97, cslw + FibRgLw97,
|
||||
/// cbRgFcLcb + FibRgFcLcb97 (0x5D pairs), cswNew.
|
||||
const FIB_LEN: usize = 0x9A + 0x5D * 8 + 2;
|
||||
/// Most characters written: the WordDocument stream's FCs are 32-bit and each character takes
|
||||
/// two bytes; stay far inside.
|
||||
const MAX_CP: usize = 256 << 20;
|
||||
/// Size of a Dop97.
|
||||
const DOP97_LEN: usize = 500;
|
||||
|
||||
/// Write `doc` as a Word 97-2003 document.
|
||||
pub fn write(doc: &Document) -> Result<Vec<u8>, DocbinError> {
|
||||
let plan = styles::plan(&doc.styles);
|
||||
let normal = plan.istd.get("Normal").copied().unwrap_or(0);
|
||||
let lfos = lists::lfo_indexes(&doc.numbering);
|
||||
let mut look = Lookup::new(doc, plan.istd.clone(), normal, lfos);
|
||||
// Font 0 is the document's default font, as in every Word file; the stylesheet header names
|
||||
// it as the standard font.
|
||||
let default_font = doc.styles.default_chr.font.clone().unwrap_or_else(|| "Times New Roman".to_string());
|
||||
let f_ascii = look.font(&default_font);
|
||||
let f_ea = doc.styles.default_chr.font_ea.as_deref().map_or(f_ascii, |f| look.font(f));
|
||||
let f_cs = doc.styles.default_chr.font_cs.as_deref().map_or(f_ascii, |f| look.font(f));
|
||||
let stsh = styles::stsh(&doc.styles, &plan, &mut look, [f_ascii, f_ea, f_cs]);
|
||||
let lists = lists::write(&doc.numbering, &mut look);
|
||||
|
||||
let mut t = Text::new(look);
|
||||
// The main document; it ends with a plain paragraph (a table or a section break can't end it).
|
||||
t.blocks(&doc.body, true);
|
||||
t.close_fields();
|
||||
let ends_plain = doc.body.last().and_then(|b| b.as_para()).is_some_and(|p| p.section.is_none());
|
||||
if !ends_plain || t.paras.last().is_none_or(|p| p.end_cp != t.cp()) {
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
}
|
||||
let ccp_text = t.cp();
|
||||
let mut sections: Vec<(u32, SectionProps)> = std::mem::take(&mut t.sections);
|
||||
sections.push((ccp_text, doc.last_section.clone()));
|
||||
|
||||
// Footnotes, then the header document, then endnotes, each a subdocument.
|
||||
let (ccp_ftn, ftn_txt) = notes(&mut t, doc, NoteKind::Footnote);
|
||||
let hdd_start = t.cp();
|
||||
let hdd_stories = headers(&mut t, doc, §ions);
|
||||
let ccp_hdd = t.cp() - hdd_start;
|
||||
let (ccp_edn, edn_txt) = notes(&mut t, doc, NoteKind::Endnote);
|
||||
if ccp_ftn + ccp_hdd + ccp_edn > 0 {
|
||||
// One more paragraph mark after the last subdocument ([MS-DOC] §2.8.35).
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
}
|
||||
let ccp_all = t.cp();
|
||||
if t.units.len() > MAX_CP {
|
||||
return Err(DocbinError::Limit(format!("the document has {} characters, more than a .doc holds", t.units.len())));
|
||||
}
|
||||
log_dropped(&t.dropped);
|
||||
|
||||
// ---- WordDocument stream: FIB, text, formatting pages, section properties.
|
||||
let mut word = vec![0u8; TEXT_FC as usize];
|
||||
for u in &t.units {
|
||||
word.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
let fc = |cp: u32| TEXT_FC + cp * 2;
|
||||
pad_to(&mut word, 512);
|
||||
let runs = t.char_runs();
|
||||
let chpx_runs: Vec<fkp::ChpxRun> = runs.iter().map(|&(a, b, g)| fkp::ChpxRun { fc_start: fc(a), fc_end: fc(b), grpprl: g }).collect();
|
||||
let chpx = fkp::chpx_pages(&chpx_runs);
|
||||
let chpx_pn = (word.len() / 512) as u32;
|
||||
for p in &chpx.pages {
|
||||
word.extend_from_slice(p);
|
||||
}
|
||||
let mut start = 0u32;
|
||||
let papx_runs: Vec<fkp::PapxRun> = t
|
||||
.paras
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let r = fkp::PapxRun { fc_start: fc(start), fc_end: fc(p.end_cp), istd: p.istd, grpprl: p.grpprl.as_slice() };
|
||||
start = p.end_cp;
|
||||
r
|
||||
})
|
||||
.collect();
|
||||
let papx = fkp::papx_pages(&papx_runs);
|
||||
let papx_pn = (word.len() / 512) as u32;
|
||||
for p in &papx.pages {
|
||||
word.extend_from_slice(p);
|
||||
}
|
||||
let mut sepx_at = Vec::with_capacity(sections.len());
|
||||
for (_, s) in §ions {
|
||||
pad_to(&mut word, 2);
|
||||
sepx_at.push(word.len() as u32);
|
||||
let g = grpprl::sepx(s);
|
||||
word.extend_from_slice(&(g.len() as u16).to_le_bytes());
|
||||
word.extend_from_slice(&g.0);
|
||||
}
|
||||
let cb_mac = word.len() as u32;
|
||||
|
||||
// ---- Table stream.
|
||||
let mut tbl = TableStream::default();
|
||||
tbl.put(pair::STSHF, &stsh);
|
||||
if let Some(&v) = tbl.pairs.get(&pair::STSHF) {
|
||||
tbl.pairs.insert(0, v); // fcStshfOrig: the same stylesheet
|
||||
}
|
||||
// Notes: references in the main text and the stories in their subdocuments.
|
||||
for (kind, ref_pair, txt_pair, ccp, txt) in [
|
||||
(NoteKind::Footnote, pair::PLCF_FND_REF, pair::PLCF_FND_TXT, ccp_ftn, &ftn_txt),
|
||||
(NoteKind::Endnote, pair::PLCF_END_REF, pair::PLCF_END_TXT, ccp_edn, &edn_txt),
|
||||
] {
|
||||
if ccp == 0 {
|
||||
continue;
|
||||
}
|
||||
let refs: Vec<&story::NoteRefAt> = t.notes.iter().filter(|n| n.kind == kind).collect();
|
||||
let mut cps: Vec<u32> = refs.iter().map(|n| n.cp).collect();
|
||||
cps.push(ccp_text);
|
||||
let idx: Vec<[u8; 2]> = refs.iter().enumerate().map(|(i, n)| if n.auto { (i as u16 + 1).to_le_bytes() } else { [0, 0] }).collect();
|
||||
tbl.put(ref_pair, &fkp::plc(&cps, &idx.iter().map(|x| &x[..]).collect::<Vec<_>>()));
|
||||
let mut tcps = txt.clone();
|
||||
tcps.push(ccp - 1);
|
||||
tcps.push(ccp);
|
||||
tbl.put(txt_pair, &fkp::plc(&tcps, &[]));
|
||||
}
|
||||
// Sections.
|
||||
let mut sed_cps = vec![0u32];
|
||||
sed_cps.extend(sections.iter().map(|(end, _)| *end));
|
||||
let seds: Vec<[u8; 12]> = sepx_at
|
||||
.iter()
|
||||
.map(|at| {
|
||||
let mut s = [0u8; 12];
|
||||
s[2..6].copy_from_slice(&at.to_le_bytes());
|
||||
s[8..12].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
s
|
||||
})
|
||||
.collect();
|
||||
tbl.put(pair::PLCF_SED, &fkp::plc(&sed_cps, &seds.iter().map(|x| &x[..]).collect::<Vec<_>>()));
|
||||
if ccp_hdd > 0 {
|
||||
let mut cps = hdd_stories;
|
||||
cps.push(ccp_hdd - 1);
|
||||
cps.push(ccp_hdd);
|
||||
tbl.put(pair::PLCF_HDD, &fkp::plc(&cps, &[]));
|
||||
}
|
||||
tbl.put(pair::PLCF_BTE_CHPX, &chpx.plc_bte(chpx_pn));
|
||||
tbl.put(pair::PLCF_BTE_PAPX, &papx.plc_bte(papx_pn));
|
||||
// Fields, one PLC per subdocument with CPs relative to it.
|
||||
let ftn_start = ccp_text;
|
||||
let edn_start = hdd_start + ccp_hdd;
|
||||
for (i, a, len) in [(16usize, 0, ccp_text), (18, ftn_start, ccp_ftn), (17, hdd_start, ccp_hdd), (48, edn_start, ccp_edn)] {
|
||||
let f: Vec<&story::FieldChar> = t.fields.iter().filter(|f| f.cp >= a && f.cp < a + len).collect();
|
||||
if f.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut cps: Vec<u32> = f.iter().map(|f| f.cp - a).collect();
|
||||
cps.push(len);
|
||||
tbl.put(i, &fkp::plc(&cps, &f.iter().map(|f| &f.fld[..]).collect::<Vec<_>>()));
|
||||
}
|
||||
// Bookmarks in the main text (with an end): names, starts with the index of their end, ends.
|
||||
let mut marks: Vec<(String, u32, u32)> =
|
||||
t.bookmarks.iter().filter_map(|(n, a, b)| b.filter(|&b| b <= ccp_text && *a <= b).map(|b| (n.clone(), *a, b))).take(0x3FFF).collect();
|
||||
marks.sort_by_key(|m| m.1);
|
||||
if !marks.is_empty() {
|
||||
let mut ends: Vec<(u32, usize)> = marks.iter().enumerate().map(|(i, m)| (m.2, i)).collect();
|
||||
ends.sort();
|
||||
let mut ibkl = vec![0u16; marks.len()];
|
||||
for (pos, (_, i)) in ends.iter().enumerate() {
|
||||
if let Some(slot) = ibkl.get_mut(*i) {
|
||||
*slot = pos as u16;
|
||||
}
|
||||
}
|
||||
let names: Vec<(String, Vec<u8>)> = marks.iter().map(|m| (m.0.chars().take(40).collect(), Vec::new())).collect();
|
||||
tbl.put(pair::STTBF_BKMK, &fkp::sttb_ext(&names, 0));
|
||||
let mut cps: Vec<u32> = marks.iter().map(|m| m.1).collect();
|
||||
cps.push(ccp_text + 1);
|
||||
let fbkf: Vec<[u8; 4]> = ibkl
|
||||
.iter()
|
||||
.map(|i| {
|
||||
let mut b = [0u8; 4];
|
||||
b[0..2].copy_from_slice(&i.to_le_bytes());
|
||||
b
|
||||
})
|
||||
.collect();
|
||||
tbl.put(pair::PLCF_BKF, &fkp::plc(&cps, &fbkf.iter().map(|x| &x[..]).collect::<Vec<_>>()));
|
||||
let mut cps: Vec<u32> = ends.iter().map(|e| e.0).collect();
|
||||
cps.push(ccp_text + 1);
|
||||
tbl.put(pair::PLCF_BKL, &fkp::plc(&cps, &[]));
|
||||
}
|
||||
tbl.put(31, &dop(doc));
|
||||
// SttbfAssoc MUST be there: 18 strings (template, title, author…), all empty here.
|
||||
tbl.put(32, &fkp::sttb_ext(&vec![(String::new(), Vec::new()); 18], 0));
|
||||
// The piece table: one Unicode piece holding every character.
|
||||
let mut plc = fkp::plc(&[0, ccp_all], &[]);
|
||||
plc.extend_from_slice(&0u16.to_le_bytes());
|
||||
plc.extend_from_slice(&TEXT_FC.to_le_bytes());
|
||||
plc.extend_from_slice(&0u16.to_le_bytes());
|
||||
let mut clx = vec![0x02];
|
||||
clx.extend_from_slice(&(plc.len() as u32).to_le_bytes());
|
||||
clx.extend_from_slice(&plc);
|
||||
tbl.put(pair::CLX, &clx);
|
||||
if t.look.authors.len() > 1 {
|
||||
let authors: Vec<(String, Vec<u8>)> = t.look.authors.iter().map(|a| (a.clone(), Vec::new())).collect();
|
||||
tbl.put(51, &fkp::sttb_ext(&authors, 0));
|
||||
}
|
||||
if let Some(l) = &lists {
|
||||
pad_to(&mut tbl.bytes, 2);
|
||||
tbl.pairs.insert(pair::PLF_LST, (tbl.bytes.len() as u32, l.lcb_plf_lst));
|
||||
tbl.bytes.extend_from_slice(&l.plf_lst);
|
||||
tbl.put(pair::PLF_LFO, &l.plf_lfo);
|
||||
}
|
||||
// The font table last: every font the text and styles name is in it now.
|
||||
tbl.put(pair::STTBF_FFN, &styles::font_table(&t.look.fonts));
|
||||
|
||||
// ---- FIB.
|
||||
let has_pic = !t.pictures.data.is_empty();
|
||||
let fib = fib(cb_mac, [ccp_text, ccp_ftn, ccp_hdd, ccp_edn], &tbl.pairs, has_pic);
|
||||
if let Some(head) = word.get_mut(..FIB_LEN) {
|
||||
head.copy_from_slice(&fib);
|
||||
}
|
||||
|
||||
// ---- The compound file.
|
||||
let mut streams: Vec<(&str, Vec<u8>)> = vec![("WordDocument", word), ("1Table", tbl.bytes)];
|
||||
if has_pic {
|
||||
streams.push(("Data", std::mem::take(&mut t.pictures.data)));
|
||||
}
|
||||
streams.push(("\u{5}SummaryInformation", summary::stream(&doc.core)));
|
||||
compound(&streams)
|
||||
}
|
||||
|
||||
/// The Table stream being built, and the FIB pair (fc, lcb) of each structure in it.
|
||||
#[derive(Default)]
|
||||
struct TableStream {
|
||||
bytes: Vec<u8>,
|
||||
pairs: HashMap<usize, (u32, u32)>,
|
||||
}
|
||||
|
||||
impl TableStream {
|
||||
/// Append a structure (word-aligned) and record it as FIB pair `i`; nothing for an empty one.
|
||||
fn put(&mut self, i: usize, bytes: &[u8]) {
|
||||
if bytes.is_empty() {
|
||||
return;
|
||||
}
|
||||
pad_to(&mut self.bytes, 2);
|
||||
self.pairs.insert(i, (self.bytes.len() as u32, bytes.len() as u32));
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the stories of the notes of `kind` the main text referred to, in reference order:
|
||||
/// the subdocument's length and each story's start CP (relative to it).
|
||||
fn notes(t: &mut Text<'_>, doc: &Document, kind: NoteKind) -> (u32, Vec<u32>) {
|
||||
let start = t.cp();
|
||||
let parts: Vec<u32> = t.notes.iter().filter(|n| n.kind == kind).map(|n| n.part).collect();
|
||||
if parts.is_empty() {
|
||||
return (0, Vec::new());
|
||||
}
|
||||
let mut starts = Vec::with_capacity(parts.len());
|
||||
for id in parts {
|
||||
starts.push(t.cp() - start);
|
||||
let blocks = doc.parts.get(&id).map(|p| p.blocks.as_slice()).unwrap_or_default();
|
||||
t.note_story(kind, id, blocks);
|
||||
t.close_fields();
|
||||
}
|
||||
// The guard paragraph mark after the last story.
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
(t.cp() - start, starts)
|
||||
}
|
||||
|
||||
/// Write the header document: the six (empty) note separator stories, then six stories per
|
||||
/// section (even header, header, even footer, footer, first-page header, first-page footer),
|
||||
/// each ended by a guard paragraph mark. The story start CPs (relative); nothing is written
|
||||
/// when no section has a header or footer.
|
||||
fn headers(t: &mut Text<'_>, doc: &Document, sections: &[(u32, SectionProps)]) -> Vec<u32> {
|
||||
let any = sections.iter().any(|(_, s)| {
|
||||
[s.headers.default, s.headers.first, s.headers.even, s.footers.default, s.footers.first, s.footers.even].iter().any(Option::is_some)
|
||||
});
|
||||
if !any {
|
||||
return Vec::new();
|
||||
}
|
||||
let start = t.cp();
|
||||
let mut cps = vec![0u32; 6];
|
||||
for (_, s) in sections {
|
||||
for id in [s.headers.even, s.headers.default, s.footers.even, s.footers.default, s.headers.first, s.footers.first] {
|
||||
cps.push(t.cp() - start);
|
||||
let Some(part) = id.and_then(|id| doc.parts.get(&id)) else { continue };
|
||||
if part.blocks.is_empty() {
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
} else {
|
||||
t.blocks(&part.blocks, false);
|
||||
}
|
||||
t.close_fields();
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
}
|
||||
}
|
||||
// The final mark the last CP of Plcfhdd ends at ([MS-DOC] §2.8.22).
|
||||
t.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
cps
|
||||
}
|
||||
|
||||
/// Log, once per save, each kind of content the file leaves out.
|
||||
fn log_dropped(d: &story::Dropped) {
|
||||
for (n, what) in [
|
||||
(d.shapes, "shapes, charts and text boxes are left out"),
|
||||
(d.comments, "comments are left out"),
|
||||
(d.nested_tables, "tables inside table cells become paragraphs of their cell"),
|
||||
(d.floating_pictures, "floating pictures are placed in line with the text"),
|
||||
(d.pictures, "pictures in formats a .doc can't hold are left out"),
|
||||
(d.equations, "equations are written as their linear text"),
|
||||
] {
|
||||
if n > 0 {
|
||||
log::warn!(".doc save: {n} × {what}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pad_to(v: &mut Vec<u8>, align: usize) {
|
||||
while !v.len().is_multiple_of(align) {
|
||||
v.push(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// The FIB for nFib 0x00C1 ([MS-DOC] §2.5): `ccp` is text, footnotes, headers, endnotes.
|
||||
fn fib(cb_mac: u32, ccp: [u32; 4], pairs: &HashMap<usize, (u32, u32)>, has_pic: bool) -> Vec<u8> {
|
||||
let mut v = vec![0u8; FIB_LEN];
|
||||
let put16 = |v: &mut Vec<u8>, at: usize, x: u16| {
|
||||
if let Some(b) = v.get_mut(at..at + 2) {
|
||||
b.copy_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
};
|
||||
put16(&mut v, 0x00, 0xA5EC); // wIdent
|
||||
put16(&mut v, 0x02, 0x00C1); // nFib
|
||||
put16(&mut v, 0x06, 0x0409); // lid
|
||||
// fHasPic, fWhichTblStm (1Table), fExtChar.
|
||||
put16(&mut v, 0x0A, 0x0200 | 0x1000 | if has_pic { 0x0008 } else { 0 });
|
||||
put16(&mut v, 0x0C, 0x00BF); // nFibBack
|
||||
put16(&mut v, 0x20, 0x000E); // csw
|
||||
put16(&mut v, 0x22 + 13 * 2, 0x0409); // FibRgW97.lidFE
|
||||
put16(&mut v, 0x3E, 0x0016); // cslw
|
||||
let put32 = |v: &mut Vec<u8>, at: usize, x: u32| {
|
||||
if let Some(b) = v.get_mut(at..at + 4) {
|
||||
b.copy_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
};
|
||||
put32(&mut v, 0x40, cb_mac);
|
||||
put32(&mut v, 0x40 + 3 * 4, ccp[0]);
|
||||
put32(&mut v, 0x40 + 4 * 4, ccp[1]);
|
||||
put32(&mut v, 0x40 + 5 * 4, ccp[2]);
|
||||
put32(&mut v, 0x40 + 8 * 4, ccp[3]);
|
||||
put16(&mut v, 0x98, 0x005D); // cbRgFcLcb
|
||||
for (&i, &(fc, lcb)) in pairs {
|
||||
if i < 0x5D {
|
||||
put32(&mut v, 0x9A + i * 8, fc);
|
||||
put32(&mut v, 0x9A + i * 8 + 4, lcb);
|
||||
}
|
||||
}
|
||||
// cswNew = 0: the last two bytes stay zero.
|
||||
v
|
||||
}
|
||||
|
||||
/// A Dop97 ([MS-DOC] §2.7.4): footnotes at the bottom of the page, endnotes at the end of the
|
||||
/// document, both numbered from 1; the default tab stop, hyphenation zone and the settings the
|
||||
/// document carries; 100 % zoom.
|
||||
fn dop(doc: &Document) -> Vec<u8> {
|
||||
let s = &doc.settings;
|
||||
let mut v = vec![0u8; DOP97_LEN];
|
||||
// fFacingPages (different odd and even headers), fpc = 1 (bottom of page).
|
||||
v[0] = u8::from(s.even_odd_headers) | (1 << 5);
|
||||
v[2..4].copy_from_slice(&(1u16 << 2).to_le_bytes()); // rncFtn 0, nFtn 1
|
||||
let mut flags: u32 = 0;
|
||||
if s.auto_hyphenation {
|
||||
flags |= 1 << 12; // fAutoHyphen
|
||||
}
|
||||
if s.hyphenate_caps {
|
||||
flags |= 1 << 11; // fHyphCapitals
|
||||
}
|
||||
if s.track_changes {
|
||||
flags |= 1 << 15; // fRevMarking
|
||||
}
|
||||
if s.mirror_margins {
|
||||
flags |= 1 << 21; // fMirrorMargins
|
||||
}
|
||||
v[4..8].copy_from_slice(&flags.to_le_bytes());
|
||||
let tab = if s.default_tab > 0.0 { grpprl::twips(s.default_tab) } else { 720 };
|
||||
v[10..12].copy_from_slice(&tab.to_le_bytes()); // dxaTab
|
||||
let hot = if s.hyphenation_zone > 0.0 { grpprl::twips(s.hyphenation_zone) } else { 360 };
|
||||
v[14..16].copy_from_slice(&hot.to_le_bytes()); // dxaHotZ
|
||||
v[20..24].copy_from_slice(&dttm(&doc.core.created).to_le_bytes());
|
||||
v[24..28].copy_from_slice(&dttm(&doc.core.modified).to_le_bytes());
|
||||
v[32..34].copy_from_slice(&(doc.core.revision.clamp(1, u16::MAX as u32) as u16).to_le_bytes());
|
||||
v[52..54].copy_from_slice(&(1u16 << 2).to_le_bytes()); // rncEdn 0, nEdn 1
|
||||
v[54..56].copy_from_slice(&3u16.to_le_bytes()); // epc: end of document
|
||||
v[82..84].copy_from_slice(&((100u16 << 3) | (1 << 12)).to_le_bytes()); // pctWwdSaved, zkSaved
|
||||
v
|
||||
}
|
||||
|
||||
/// A `DTTM` ([MS-DOC] §2.9.75) from an ISO 8601 date; 0 when there is none.
|
||||
pub(crate) fn dttm(iso: &str) -> u32 {
|
||||
use chrono::{Datelike, Timelike};
|
||||
let Some(t) = parse_time(iso) else { return 0 };
|
||||
let year = t.year();
|
||||
if !(1900..=2411).contains(&year) {
|
||||
return 0;
|
||||
}
|
||||
(t.minute() & 0x3F)
|
||||
| ((t.hour() & 0x1F) << 6)
|
||||
| ((t.day() & 0x1F) << 11)
|
||||
| ((t.month() & 0xF) << 16)
|
||||
| ((((year - 1900) as u32) & 0x1FF) << 20)
|
||||
| ((t.weekday().num_days_from_sunday() & 7) << 29)
|
||||
}
|
||||
|
||||
/// An ISO 8601 date or date-time (UTC when it doesn't say).
|
||||
pub(crate) fn parse_time(iso: &str) -> Option<chrono::NaiveDateTime> {
|
||||
let s = iso.trim();
|
||||
if s.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if let Ok(t) = chrono::DateTime::parse_from_rfc3339(s) {
|
||||
return Some(t.naive_utc());
|
||||
}
|
||||
for f in ["%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S"] {
|
||||
if let Ok(t) = chrono::NaiveDateTime::parse_from_str(s.trim_end_matches('Z'), f) {
|
||||
return Some(t);
|
||||
}
|
||||
}
|
||||
chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").ok().and_then(|d| d.and_hms_opt(0, 0, 0))
|
||||
}
|
||||
|
||||
/// The compound file holding `streams` at its root. Version 3 (512-byte sectors), the version
|
||||
/// Word 97-2003 writes; readers of that era expect it.
|
||||
fn compound(streams: &[(&str, Vec<u8>)]) -> Result<Vec<u8>, DocbinError> {
|
||||
let refs: Vec<(&str, &[u8])> = streams.iter().map(|(name, data)| (*name, data.as_slice())).collect();
|
||||
craft_cfb::write_streams(craft_cfb::Version::V3, &refs).map_err(|e| DocbinError::Container(e.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,211 @@
|
||||
//! Inline pictures in the Data stream: a `PICF` header ([MS-DOC] §2.9.193) followed by an
|
||||
//! `OfficeArtInlineSpContainer` ([MS-ODRAW]): the picture-frame shape, its properties (the
|
||||
//! picture's place in the BLIP store, crop, alt text) and the `OfficeArtFBSE` with the BLIP
|
||||
//! embedded in it. The character 0x01 in the text points here through `sprmCPicLocation`.
|
||||
|
||||
use sha1::Digest;
|
||||
|
||||
/// `PICF.cbHeader`.
|
||||
const CB_HEADER: u16 = 0x44;
|
||||
/// `MM_SHAPE`: the picture is the OfficeArt shape that follows the header.
|
||||
const MM_SHAPE: u16 = 0x0064;
|
||||
/// Largest picture written (bytes of its BLIP): Word's own limit for one picture is far above
|
||||
/// what the Data stream's 32-bit offsets allow many of.
|
||||
const MAX_PICTURE: usize = 64 << 20;
|
||||
/// All pictures together: `sprmCPicLocation` offsets are 32-bit, keep well inside them.
|
||||
const MAX_DATA: usize = 1 << 30;
|
||||
|
||||
/// OfficeArt record types and BLIP instances ([MS-ODRAW] §2.2).
|
||||
const SP_CONTAINER: u16 = 0xF004;
|
||||
const FSP: u16 = 0xF00A;
|
||||
const FOPT: u16 = 0xF00B;
|
||||
const FBSE: u16 = 0xF007;
|
||||
const BLIP_EMF: (u16, u16) = (0xF01A, 0x3D4);
|
||||
const BLIP_WMF: (u16, u16) = (0xF01B, 0x216);
|
||||
const BLIP_JPEG: (u16, u16) = (0xF01D, 0x46A);
|
||||
const BLIP_PNG: (u16, u16) = (0xF01E, 0x6E0);
|
||||
/// `msosptPictureFrame`.
|
||||
const SPT_PICTURE_FRAME: u16 = 75;
|
||||
|
||||
/// The Data stream being built.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Pictures {
|
||||
pub(crate) data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A picture in a form the BLIP store holds, with its `btWin32` type.
|
||||
enum Blip {
|
||||
Png(Vec<u8>),
|
||||
Jpeg(Vec<u8>),
|
||||
/// EMF (true) or WMF metafile bytes.
|
||||
Metafile(Vec<u8>, bool),
|
||||
}
|
||||
|
||||
impl Blip {
|
||||
/// Sniff `bytes`: PNG and JPEG go in as they are, metafiles compressed, other raster
|
||||
/// formats (GIF, BMP, TIFF, WebP) re-encoded as PNG.
|
||||
fn of(bytes: &[u8]) -> Option<Blip> {
|
||||
if bytes.starts_with(b"\x89PNG") {
|
||||
return Some(Blip::Png(bytes.to_vec()));
|
||||
}
|
||||
if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||
return Some(Blip::Jpeg(bytes.to_vec()));
|
||||
}
|
||||
// EMF: an EMR_HEADER record (type 1) with " EMF" at offset 40.
|
||||
if bytes.get(0..4) == Some(&[1, 0, 0, 0]) && bytes.get(40..44) == Some(b" EMF") {
|
||||
return Some(Blip::Metafile(bytes.to_vec(), true));
|
||||
}
|
||||
// WMF: the placeable header's key, or a plain header (type 1 or 2, header size 9).
|
||||
if bytes.starts_with(&[0xD7, 0xCD, 0xC6, 0x9A]) || (matches!(bytes.get(0..4), Some([1 | 2, 0, 9, 0]))) {
|
||||
// The BLIP holds the metafile without the placeable header.
|
||||
let body = if bytes.starts_with(&[0xD7, 0xCD, 0xC6, 0x9A]) { bytes.get(22..)? } else { bytes };
|
||||
return Some(Blip::Metafile(body.to_vec(), false));
|
||||
}
|
||||
let img = image::load_from_memory(bytes).ok()?;
|
||||
let mut png = Vec::new();
|
||||
img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).ok()?;
|
||||
Some(Blip::Png(png))
|
||||
}
|
||||
|
||||
/// `btWin32` ([MS-ODRAW] §2.4.1 MSOBLIPTYPE).
|
||||
fn bt(&self) -> u8 {
|
||||
match self {
|
||||
Blip::Png(_) => 6,
|
||||
Blip::Jpeg(_) => 5,
|
||||
Blip::Metafile(_, true) => 2,
|
||||
Blip::Metafile(_, false) => 3,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `OfficeArtBlip*` record.
|
||||
fn record(&self, w_emu: i32, h_emu: i32) -> Vec<u8> {
|
||||
let (raw, (typ, inst)) = match self {
|
||||
Blip::Png(b) => (b, BLIP_PNG),
|
||||
Blip::Jpeg(b) => (b, BLIP_JPEG),
|
||||
Blip::Metafile(b, emf) => (b, if *emf { BLIP_EMF } else { BLIP_WMF }),
|
||||
};
|
||||
let mut body = uid(raw).to_vec();
|
||||
match self {
|
||||
Blip::Png(b) | Blip::Jpeg(b) => {
|
||||
body.push(0xFF); // tag
|
||||
body.extend_from_slice(b);
|
||||
}
|
||||
Blip::Metafile(b, _) => {
|
||||
let packed = miniz_oxide::deflate::compress_to_vec_zlib(b, 6);
|
||||
// OfficeArtMetafileHeader: cbSize, rcBounds, ptSize (EMUs), cbSave, compression
|
||||
// (0 = deflate), filter (0xFE = none).
|
||||
body.extend_from_slice(&(b.len() as u32).to_le_bytes());
|
||||
for v in [0i32, 0, w_emu / 12_700, h_emu / 12_700] {
|
||||
body.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
body.extend_from_slice(&w_emu.to_le_bytes());
|
||||
body.extend_from_slice(&h_emu.to_le_bytes());
|
||||
body.extend_from_slice(&(packed.len() as u32).to_le_bytes());
|
||||
body.push(0);
|
||||
body.push(0xFE);
|
||||
body.extend_from_slice(&packed);
|
||||
}
|
||||
}
|
||||
rec(0, inst, typ, &body)
|
||||
}
|
||||
}
|
||||
|
||||
/// A 16-byte id for a BLIP (the format wants an MD4; any digest identifies it).
|
||||
fn uid(bytes: &[u8]) -> [u8; 16] {
|
||||
let d = sha1::Sha1::digest(bytes);
|
||||
let mut u = [0u8; 16];
|
||||
u.copy_from_slice(d.get(..16).unwrap_or(&[0; 16]));
|
||||
u
|
||||
}
|
||||
|
||||
/// An OfficeArt record: header (version, instance, type, length) then the body.
|
||||
fn rec(ver: u16, inst: u16, typ: u16, body: &[u8]) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(8 + body.len());
|
||||
v.extend_from_slice(&((ver & 0xF) | (inst << 4)).to_le_bytes());
|
||||
v.extend_from_slice(&typ.to_le_bytes());
|
||||
v.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
v.extend_from_slice(body);
|
||||
v
|
||||
}
|
||||
|
||||
/// A 16.16 fixed-point fraction.
|
||||
fn fixed(f: f32) -> u32 {
|
||||
((f.clamp(0.0, 0.99) * 65_536.0).round() as i32) as u32
|
||||
}
|
||||
|
||||
impl Pictures {
|
||||
/// Add a picture shown `w` × `h` points with `crop` (left, top, right, bottom fractions):
|
||||
/// the offset of its `PICF` in the Data stream, `None` when the picture can't be stored.
|
||||
pub(crate) fn add(&mut self, bytes: &[u8], w: f32, h: f32, crop: [f32; 4], alt: &str) -> Option<u32> {
|
||||
if bytes.len() > MAX_PICTURE {
|
||||
return None;
|
||||
}
|
||||
let blip = Blip::of(bytes)?;
|
||||
let [cl, ct, cr, cb] = crop.map(|c| if c.is_finite() { c.clamp(0.0, 0.99) } else { 0.0 });
|
||||
// The goal size is the uncropped picture's: the crop takes its share off it.
|
||||
let gw = (w.max(1.0) / (1.0 - cl - cr).max(0.01)).min(1584.0);
|
||||
let gh = (h.max(1.0) / (1.0 - ct - cb).max(0.01)).min(1584.0);
|
||||
let emu = |pt: f32| (pt * 12_700.0).round() as i32;
|
||||
|
||||
let blip_rec = blip.record(emu(gw), emu(gh));
|
||||
let mut fbse = vec![blip.bt(), blip.bt()];
|
||||
fbse.extend_from_slice(&uid(bytes));
|
||||
fbse.extend_from_slice(&0x00FFu16.to_le_bytes()); // tag
|
||||
fbse.extend_from_slice(&(blip_rec.len() as u32).to_le_bytes());
|
||||
fbse.extend_from_slice(&1u32.to_le_bytes()); // cRef
|
||||
fbse.extend_from_slice(&0u32.to_le_bytes()); // foDelay: embedded
|
||||
fbse.extend_from_slice(&[0, 0, 0, 0]); // unused1, cbName, unused2, unused3
|
||||
fbse.extend_from_slice(&blip_rec);
|
||||
let fbse = rec(2, u16::from(blip.bt()), FBSE, &fbse);
|
||||
|
||||
// The shape: an FSP (spid, fHaveAnchor | fHaveSpt) and its properties.
|
||||
let mut fsp = 1025u32.to_le_bytes().to_vec();
|
||||
fsp.extend_from_slice(&0x0A00u32.to_le_bytes());
|
||||
let fsp = rec(2, SPT_PICTURE_FRAME, FSP, &fsp);
|
||||
let mut props: Vec<(u16, u32)> = Vec::new();
|
||||
for (pid, f) in [(0x0100u16, ct), (0x0101, cb), (0x0102, cl), (0x0103, cr)] {
|
||||
if f > 0.0 {
|
||||
props.push((pid, fixed(f)));
|
||||
}
|
||||
}
|
||||
props.push((0x4104, 1)); // pib, fBid: the first (only) store entry
|
||||
let alt_units: Vec<u16> = alt.encode_utf16().take(4095).chain(std::iter::once(0)).collect();
|
||||
let has_alt = alt_units.len() > 1;
|
||||
if has_alt {
|
||||
props.push((0x8381, (alt_units.len() * 2) as u32)); // wzDescription, complex
|
||||
}
|
||||
let mut fopt = Vec::new();
|
||||
for (pid, v) in &props {
|
||||
fopt.extend_from_slice(&pid.to_le_bytes());
|
||||
fopt.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
if has_alt {
|
||||
for u in &alt_units {
|
||||
fopt.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
}
|
||||
let fopt = rec(3, props.len() as u16, FOPT, &fopt);
|
||||
let mut sp = fsp;
|
||||
sp.extend_from_slice(&fopt);
|
||||
let sp = rec(0xF, 0, SP_CONTAINER, &sp);
|
||||
|
||||
let lcb = usize::from(CB_HEADER) + sp.len() + fbse.len();
|
||||
let at = self.data.len();
|
||||
if at.checked_add(lcb)? > MAX_DATA {
|
||||
return None;
|
||||
}
|
||||
let mut picf = vec![0u8; usize::from(CB_HEADER)];
|
||||
picf[0..4].copy_from_slice(&(lcb as u32).to_le_bytes());
|
||||
picf[4..6].copy_from_slice(&CB_HEADER.to_le_bytes());
|
||||
picf[6..8].copy_from_slice(&MM_SHAPE.to_le_bytes());
|
||||
// PICMID: dxaGoal, dyaGoal, mx, my (1000 = 100 %).
|
||||
picf[28..30].copy_from_slice(&super::grpprl::twips(gw).to_le_bytes());
|
||||
picf[30..32].copy_from_slice(&super::grpprl::twips(gh).to_le_bytes());
|
||||
picf[32..34].copy_from_slice(&1000u16.to_le_bytes());
|
||||
picf[34..36].copy_from_slice(&1000u16.to_le_bytes());
|
||||
self.data.extend_from_slice(&picf);
|
||||
self.data.extend_from_slice(&sp);
|
||||
self.data.extend_from_slice(&fbse);
|
||||
u32::try_from(at).ok()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,735 @@
|
||||
//! The text of every story, as Word's character stream: paragraphs and table cells end with
|
||||
//! their marks, fields become begin/separator/end characters, pictures and note references
|
||||
//! special characters, each character carrying its CHPX and each paragraph its PAPX
|
||||
//! ([MS-DOC] §2.4.1 Retrieving Text, §2.4.3 Table Text).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use craft_wordmodel::para::{InlineObject, NoteKind, Paragraph};
|
||||
use craft_wordmodel::props::{CharProps, ParaProps};
|
||||
use craft_wordmodel::section::SectionProps;
|
||||
use craft_wordmodel::table::Table;
|
||||
use craft_wordmodel::{Block, Document};
|
||||
|
||||
use super::grpprl::{self, CharCtx, Grpprl, ParaCtx};
|
||||
use super::picture::Pictures;
|
||||
use super::tables;
|
||||
|
||||
/// Paragraph mark, cell or row end, section mark and the other characters Word gives meaning.
|
||||
pub(crate) const PARA: u16 = 0x0D;
|
||||
pub(crate) const CELL: u16 = 0x07;
|
||||
pub(crate) const SECTION: u16 = 0x0C;
|
||||
const FIELD_BEGIN: u16 = 0x13;
|
||||
const FIELD_SEP: u16 = 0x14;
|
||||
const FIELD_END: u16 = 0x15;
|
||||
const NOTE_REF: u16 = 0x02;
|
||||
const PICTURE: u16 = 0x01;
|
||||
|
||||
/// Deepest field nesting written; deeper fields keep their result text only.
|
||||
const MAX_FIELD_DEPTH: usize = 20;
|
||||
|
||||
/// One paragraph's mark: the CP after it, the paragraph's style and PAPX grpprl.
|
||||
pub(crate) struct ParaEnd {
|
||||
pub(crate) end_cp: u32,
|
||||
pub(crate) istd: u16,
|
||||
pub(crate) grpprl: Arc<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// One field character ([MS-DOC] §2.9.84 Fld): its CP (absolute) and its two bytes.
|
||||
pub(crate) struct FieldChar {
|
||||
pub(crate) cp: u32,
|
||||
pub(crate) fld: [u8; 2],
|
||||
}
|
||||
|
||||
/// A note reference in the main text: its CP, kind, story (part id) and whether it is numbered
|
||||
/// automatically.
|
||||
pub(crate) struct NoteRefAt {
|
||||
pub(crate) cp: u32,
|
||||
pub(crate) kind: NoteKind,
|
||||
pub(crate) part: u32,
|
||||
pub(crate) auto: bool,
|
||||
}
|
||||
|
||||
/// What the writer needs to look up while building the text.
|
||||
pub(crate) struct Lookup<'d> {
|
||||
pub(crate) doc: &'d Document,
|
||||
/// Style id → istd.
|
||||
pub(crate) styles: HashMap<String, u16>,
|
||||
/// The istd of `Normal`.
|
||||
pub(crate) normal: u16,
|
||||
/// Model num id → 1-based LFO index.
|
||||
pub(crate) lfos: HashMap<u32, u16>,
|
||||
/// Font names in table order and their indexes.
|
||||
pub(crate) fonts: Vec<String>,
|
||||
font_index: HashMap<String, u16>,
|
||||
/// Revision authors (`SttbfRMark`, index 0 = "Unknown") and their indexes.
|
||||
pub(crate) authors: Vec<String>,
|
||||
}
|
||||
|
||||
impl<'d> Lookup<'d> {
|
||||
pub(crate) fn new(doc: &'d Document, styles: HashMap<String, u16>, normal: u16, lfos: HashMap<u32, u16>) -> Self {
|
||||
Lookup { doc, styles, normal, lfos, fonts: Vec::new(), font_index: HashMap::new(), authors: vec!["Unknown".to_string()] }
|
||||
}
|
||||
}
|
||||
|
||||
impl CharCtx for Lookup<'_> {
|
||||
fn font(&mut self, name: &str) -> u16 {
|
||||
let name = name.trim();
|
||||
if let Some(&i) = self.font_index.get(name) {
|
||||
return i;
|
||||
}
|
||||
// The font table keeps names Word can store (it counts them in 16-bit units).
|
||||
let i = u16::try_from(self.fonts.len()).unwrap_or(0).min(0x7FFF);
|
||||
if self.fonts.len() < 0x7FFF {
|
||||
self.fonts.push(name.chars().take(31).collect());
|
||||
self.font_index.insert(name.to_string(), i);
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
fn char_style(&self, id: &str) -> Option<u16> {
|
||||
let st = self.doc.styles.get(id)?;
|
||||
(st.kind == craft_wordmodel::styles::StyleKind::Character).then(|| self.styles.get(id).copied()).flatten()
|
||||
}
|
||||
|
||||
fn revision(&mut self, rev: u32) -> Option<(u16, u32)> {
|
||||
let r = self.doc.revisions.get(rev as usize)?;
|
||||
let author = if r.author.trim().is_empty() { "Unknown" } else { r.author.trim() };
|
||||
let ibst = match self.authors.iter().position(|a| a == author) {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
self.authors.push(author.to_string());
|
||||
self.authors.len() - 1
|
||||
}
|
||||
};
|
||||
Some((u16::try_from(ibst).unwrap_or(0), super::dttm(&r.date)))
|
||||
}
|
||||
}
|
||||
|
||||
impl ParaCtx for Lookup<'_> {
|
||||
fn lfo(&self, num: u32) -> u16 {
|
||||
self.lfos.get(&num).copied().unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Grpprls are shared: a run or paragraph names one by its index.
|
||||
#[derive(Default)]
|
||||
struct Interned {
|
||||
all: Vec<Arc<Vec<u8>>>,
|
||||
index: HashMap<Arc<Vec<u8>>, usize>,
|
||||
}
|
||||
|
||||
impl Interned {
|
||||
fn get(&mut self, g: Vec<u8>) -> usize {
|
||||
if let Some(&i) = self.index.get(&g) {
|
||||
return i;
|
||||
}
|
||||
let g = Arc::new(g);
|
||||
self.all.push(g.clone());
|
||||
self.index.insert(g, self.all.len() - 1);
|
||||
self.all.len() - 1
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts of content the binary format (as written here) leaves out, logged once per save.
|
||||
#[derive(Default, Debug)]
|
||||
pub(crate) struct Dropped {
|
||||
pub(crate) shapes: usize,
|
||||
pub(crate) comments: usize,
|
||||
pub(crate) nested_tables: usize,
|
||||
pub(crate) floating_pictures: usize,
|
||||
pub(crate) pictures: usize,
|
||||
pub(crate) equations: usize,
|
||||
}
|
||||
|
||||
/// The character stream of every story, with what each character and paragraph carries.
|
||||
pub(crate) struct Text<'d> {
|
||||
pub(crate) look: Lookup<'d>,
|
||||
pub(crate) units: Vec<u16>,
|
||||
/// Run starts: (CP, interned CHPX), each run up to the next start.
|
||||
runs: Vec<(u32, usize)>,
|
||||
chpx: Interned,
|
||||
/// CHPX → the same with `sprmCFSpec`, for field characters.
|
||||
spec: HashMap<usize, usize>,
|
||||
pub(crate) paras: Vec<ParaEnd>,
|
||||
/// Field characters of the subdocument being written, then of each finished one.
|
||||
pub(crate) fields: Vec<FieldChar>,
|
||||
/// Bookmarks: name, start CP, end CP (absolute), in the order they start.
|
||||
pub(crate) bookmarks: Vec<(String, u32, Option<u32>)>,
|
||||
pub(crate) notes: Vec<NoteRefAt>,
|
||||
/// Section ends in the main text: the CP after each section mark, and its properties.
|
||||
pub(crate) sections: Vec<(u32, SectionProps)>,
|
||||
pub(crate) pictures: Pictures,
|
||||
pub(crate) dropped: Dropped,
|
||||
/// Fields begun with a `FieldStart` and not yet ended (their nesting depth counts here).
|
||||
open_fields: usize,
|
||||
/// The note story being written: its own reference mark is its leading 0x02.
|
||||
note: Option<(NoteKind, u32)>,
|
||||
/// Inside a table cell: section marks can't end a cell's paragraph.
|
||||
in_cell: bool,
|
||||
}
|
||||
|
||||
impl<'d> Text<'d> {
|
||||
pub(crate) fn new(look: Lookup<'d>) -> Self {
|
||||
Text {
|
||||
look,
|
||||
units: Vec::new(),
|
||||
runs: Vec::new(),
|
||||
chpx: Interned::default(),
|
||||
spec: HashMap::new(),
|
||||
paras: Vec::new(),
|
||||
fields: Vec::new(),
|
||||
bookmarks: Vec::new(),
|
||||
notes: Vec::new(),
|
||||
sections: Vec::new(),
|
||||
pictures: Pictures::default(),
|
||||
dropped: Dropped::default(),
|
||||
open_fields: 0,
|
||||
note: None,
|
||||
in_cell: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The CP the next character gets.
|
||||
pub(crate) fn cp(&self) -> u32 {
|
||||
u32::try_from(self.units.len()).unwrap_or(u32::MAX)
|
||||
}
|
||||
|
||||
/// The character runs: (start CP, end CP, CHPX grpprl).
|
||||
pub(crate) fn char_runs(&self) -> Vec<(u32, u32, &[u8])> {
|
||||
let end = self.cp();
|
||||
self.runs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &(cp, g))| {
|
||||
let next = self.runs.get(i + 1).map_or(end, |r| r.0);
|
||||
(cp, next, self.chpx.all.get(g).map_or(&[][..], |v| v.as_slice()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn emit(&mut self, unit: u16, chpx: usize) {
|
||||
let cp = self.cp();
|
||||
if self.runs.last().is_none_or(|&(_, g)| g != chpx) {
|
||||
self.runs.push((cp, chpx));
|
||||
}
|
||||
self.units.push(unit);
|
||||
}
|
||||
|
||||
/// The interned CHPX of `props`, plus `extra` sprms.
|
||||
fn chpx_of(&mut self, props: &CharProps, extra: Option<&Grpprl>) -> usize {
|
||||
let mut g = grpprl::chpx(props, &mut self.look);
|
||||
if let Some(e) = extra {
|
||||
g.extend(e);
|
||||
}
|
||||
g.truncate_to(255);
|
||||
self.chpx.get(g.0)
|
||||
}
|
||||
|
||||
fn text(&mut self, s: &str, chpx: usize) {
|
||||
for c in s.chars() {
|
||||
self.char(c, chpx);
|
||||
}
|
||||
}
|
||||
|
||||
/// One character of text, mapped to what Word stores for it.
|
||||
fn char(&mut self, c: char, chpx: usize) {
|
||||
let unit = match c {
|
||||
'\n' => 0x0B,
|
||||
'\t' => 0x09,
|
||||
'\u{C}' => 0x0C,
|
||||
'\u{E}' => 0x0E,
|
||||
'\u{2011}' => 0x1E,
|
||||
'\u{AD}' => 0x1F,
|
||||
'\r' | '\u{FFFC}' => return,
|
||||
c if (c as u32) < 0x20 => return,
|
||||
c => {
|
||||
let mut buf = [0u16; 2];
|
||||
for u in c.encode_utf16(&mut buf).iter() {
|
||||
self.emit(*u, chpx);
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.emit(unit, chpx);
|
||||
}
|
||||
|
||||
/// A field character: the text's formatting with `sprmCFSpec` on, as every field character
|
||||
/// MUST have ([MS-DOC] §2.8.25).
|
||||
fn field_char(&mut self, ch: u16, second: u8, chpx: usize) {
|
||||
let spec = match self.spec.get(&chpx) {
|
||||
Some(&s) => s,
|
||||
None => {
|
||||
let mut g = Grpprl(self.chpx.all.get(chpx).map_or_else(Vec::new, |g| g.as_ref().clone()));
|
||||
g.truncate_to(252);
|
||||
g.u8(grpprl::C_SPEC, 1);
|
||||
let s = self.chpx.get(g.0);
|
||||
self.spec.insert(chpx, s);
|
||||
s
|
||||
}
|
||||
};
|
||||
self.fields.push(FieldChar { cp: self.cp(), fld: [ch as u8, second] });
|
||||
self.emit(ch, spec);
|
||||
}
|
||||
|
||||
/// A whole field: begin, its code, separator and result, end.
|
||||
fn field(&mut self, instr: &str, result: Option<&str>, locked: bool, dirty: bool, chpx: usize) {
|
||||
if self.open_fields >= MAX_FIELD_DEPTH {
|
||||
if let Some(r) = result {
|
||||
self.text(r, chpx);
|
||||
}
|
||||
return;
|
||||
}
|
||||
self.field_char(FIELD_BEGIN, flt(instr), chpx);
|
||||
self.text(&format!(" {} ", instr.trim()), chpx);
|
||||
if let Some(r) = result {
|
||||
self.field_char(FIELD_SEP, 0, chpx);
|
||||
self.text(r, chpx);
|
||||
}
|
||||
self.field_char(FIELD_END, end_flags(result.is_some(), locked, dirty, self.open_fields > 0), chpx);
|
||||
}
|
||||
|
||||
/// Write `blocks` as a story's paragraphs and tables.
|
||||
pub(crate) fn blocks(&mut self, blocks: &[Arc<Block>], top: bool) {
|
||||
for b in blocks {
|
||||
match &**b {
|
||||
Block::Para(p) => self.paragraph(p, top, None),
|
||||
Block::Table(t) => self.table(t),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a note's story, its reference mark first.
|
||||
pub(crate) fn note_story(&mut self, kind: NoteKind, part: u32, blocks: &[Arc<Block>]) {
|
||||
let own_mark = blocks
|
||||
.first()
|
||||
.and_then(|b| b.as_para())
|
||||
.is_some_and(|p| p.objects.iter().any(|o| matches!(o, InlineObject::NoteRef { kind: k, id, .. } if *k == kind && *id == part)));
|
||||
self.note = Some((kind, part));
|
||||
if !own_mark {
|
||||
let style = self.note_ref_style(kind);
|
||||
let chpx = self.special(&style);
|
||||
self.emit(NOTE_REF, chpx);
|
||||
}
|
||||
if blocks.is_empty() {
|
||||
self.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
}
|
||||
self.blocks(blocks, false);
|
||||
self.note = None;
|
||||
}
|
||||
|
||||
/// A character style for note references ("footnote reference"), when the document has it.
|
||||
fn note_ref_style(&self, kind: NoteKind) -> CharProps {
|
||||
let id = match kind {
|
||||
NoteKind::Footnote => "FootnoteReference",
|
||||
NoteKind::Endnote => "EndnoteReference",
|
||||
};
|
||||
let style = self.look.styles.contains_key(id).then(|| id.to_string());
|
||||
CharProps { style, ..Default::default() }
|
||||
}
|
||||
|
||||
/// The CHPX of a special character (a note reference, a picture): `sprmCFSpec` on.
|
||||
fn special(&mut self, props: &CharProps) -> usize {
|
||||
let mut extra = Grpprl::default();
|
||||
extra.u8(grpprl::C_SPEC, 1);
|
||||
self.chpx_of(props, Some(&extra))
|
||||
}
|
||||
|
||||
/// One paragraph: its text, then its mark (`mark` overrides the paragraph mark, for cells;
|
||||
/// a paragraph ending a main-text section takes a section mark).
|
||||
pub(crate) fn paragraph(&mut self, p: &Paragraph, top: bool, cell: Option<(u16, &Grpprl)>) {
|
||||
let mut objects = p.objects.iter();
|
||||
let mut link: Option<(String, usize)> = None;
|
||||
let mut at = 0usize;
|
||||
for run in &p.runs {
|
||||
let end = at.saturating_add(run.len).min(p.text.len());
|
||||
let Some(s) = p.text.get(at..end) else { break };
|
||||
at = end;
|
||||
if s.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// A run's link becomes a HYPERLINK field around the runs that share it.
|
||||
let mut props = run.props.clone();
|
||||
let target = props.link.take();
|
||||
if link.as_ref().map(|l| &l.0) != target.as_ref() {
|
||||
if let Some((_, chpx)) = link.take() {
|
||||
self.field_char(FIELD_END, end_flags(true, false, false, self.open_fields > 0), chpx);
|
||||
}
|
||||
if let Some(t) = &target {
|
||||
let chpx = self.chpx_of(&props, None);
|
||||
self.field_char(FIELD_BEGIN, FLT_HYPERLINK, chpx);
|
||||
self.text(&format!(" {} ", hyperlink_instr(t)), chpx);
|
||||
self.field_char(FIELD_SEP, 0, chpx);
|
||||
link = Some((t.clone(), chpx));
|
||||
}
|
||||
}
|
||||
let chpx = self.chpx_of(&props, None);
|
||||
for c in s.chars() {
|
||||
if c == '\u{FFFC}' {
|
||||
if let Some(o) = objects.next() {
|
||||
self.object(o, &props, chpx);
|
||||
}
|
||||
} else {
|
||||
self.char(c, chpx);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((_, chpx)) = link.take() {
|
||||
self.field_char(FIELD_END, end_flags(true, false, false, self.open_fields > 0), chpx);
|
||||
}
|
||||
match cell {
|
||||
Some((mark, extra)) => self.mark(p, mark, extra),
|
||||
None => {
|
||||
let section = top && !self.in_cell && p.section.is_some();
|
||||
self.mark(p, if section { SECTION } else { PARA }, &Grpprl::default());
|
||||
if section && let Some(s) = &p.section {
|
||||
self.sections.push((self.cp(), (**s).clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The paragraph mark: its character formatting is the paragraph's mark formatting; the
|
||||
/// PAPX carries the style and paragraph properties (plus `extra`, for table cells).
|
||||
pub(crate) fn mark(&mut self, p: &Paragraph, mark: u16, extra: &Grpprl) {
|
||||
let chpx = self.chpx_of(&p.mark, None);
|
||||
self.emit(mark, chpx);
|
||||
let (istd, g) = self.papx_of(&p.props, extra);
|
||||
self.paras.push(ParaEnd { end_cp: self.cp(), istd, grpprl: g });
|
||||
}
|
||||
|
||||
pub(crate) fn papx_of(&mut self, props: &ParaProps, extra: &Grpprl) -> (u16, Arc<Vec<u8>>) {
|
||||
let istd = props.style.as_deref().and_then(|s| self.look.styles.get(s).copied()).unwrap_or(self.look.normal);
|
||||
let mut g = grpprl::papx(props, &self.look);
|
||||
g.extend(extra);
|
||||
g.truncate_to(super::fkp::MAX_PAPX_GRPPRL);
|
||||
(istd, Arc::new(g.0))
|
||||
}
|
||||
|
||||
fn object(&mut self, o: &InlineObject, props: &CharProps, chpx: usize) {
|
||||
match o {
|
||||
InlineObject::Field { instr, result, locked, dirty, .. } => self.field(instr, Some(result), *locked, *dirty, chpx),
|
||||
InlineObject::FieldStart { instr, locked, dirty } => {
|
||||
if self.open_fields >= MAX_FIELD_DEPTH {
|
||||
self.open_fields += 1;
|
||||
return;
|
||||
}
|
||||
self.field_char(FIELD_BEGIN, flt(instr), chpx);
|
||||
self.text(&format!(" {} ", instr.trim()), chpx);
|
||||
self.field_char(FIELD_SEP, 0, chpx);
|
||||
// The end's flags are decided now: they don't depend on the result.
|
||||
self.open_fields += 1;
|
||||
let _ = (locked, dirty);
|
||||
}
|
||||
InlineObject::FieldEnd => {
|
||||
if self.open_fields == 0 {
|
||||
return;
|
||||
}
|
||||
self.open_fields -= 1;
|
||||
if self.open_fields < MAX_FIELD_DEPTH {
|
||||
self.field_char(FIELD_END, end_flags(true, false, false, self.open_fields > 0), chpx);
|
||||
}
|
||||
}
|
||||
InlineObject::NoteRef { kind, id, custom } => {
|
||||
if self.note == Some((*kind, *id)) {
|
||||
let c = self.special(props);
|
||||
self.emit(NOTE_REF, c);
|
||||
} else if self.note.is_none() {
|
||||
let cp = self.cp();
|
||||
let c = self.special(props);
|
||||
self.emit(NOTE_REF, c);
|
||||
self.notes.push(NoteRefAt { cp, kind: *kind, part: *id, auto: custom.is_empty() });
|
||||
}
|
||||
}
|
||||
InlineObject::BookmarkStart { name } => {
|
||||
if !name.is_empty() && self.bookmarks.iter().all(|(n, _, _)| n != name) {
|
||||
self.bookmarks.push((name.clone(), self.cp(), None));
|
||||
}
|
||||
}
|
||||
InlineObject::BookmarkEnd { name } => {
|
||||
let cp = self.cp();
|
||||
if let Some(b) = self.bookmarks.iter_mut().find(|(n, _, e)| n == name && e.is_none()) {
|
||||
b.2 = Some(cp);
|
||||
}
|
||||
}
|
||||
InlineObject::Image { media, w, h, crop, float, alt, .. } => {
|
||||
if float.wrap != craft_wordmodel::para::Wrap::Inline {
|
||||
self.dropped.floating_pictures += 1;
|
||||
}
|
||||
let Some(bytes) = self.look.doc.media.get(media).cloned() else {
|
||||
self.dropped.pictures += 1;
|
||||
return;
|
||||
};
|
||||
match self.pictures.add(&bytes, *w, *h, *crop, alt) {
|
||||
Some(fc) => {
|
||||
let mut extra = Grpprl::default();
|
||||
extra.u8(grpprl::C_SPEC, 1);
|
||||
extra.u32(grpprl::C_PIC_LOCATION, fc);
|
||||
let c = self.chpx_of(props, Some(&extra));
|
||||
self.emit(PICTURE, c);
|
||||
}
|
||||
None => self.dropped.pictures += 1,
|
||||
}
|
||||
}
|
||||
InlineObject::Equation { linear, .. } => {
|
||||
self.dropped.equations += 1;
|
||||
self.text(linear, chpx);
|
||||
}
|
||||
InlineObject::Ruby { base, .. } => self.text(base, chpx),
|
||||
InlineObject::PositionalTab { .. } => self.emit(0x09, chpx),
|
||||
InlineObject::Opaque { text, .. } => self.text(text, chpx),
|
||||
InlineObject::CommentStart { .. } => self.dropped.comments += 1,
|
||||
InlineObject::Shape { .. } | InlineObject::Group { .. } | InlineObject::Graphic { .. } => self.dropped.shapes += 1,
|
||||
InlineObject::CommentEnd { .. }
|
||||
| InlineObject::ControlStart { .. }
|
||||
| InlineObject::ControlEnd
|
||||
| InlineObject::PermStart { .. }
|
||||
| InlineObject::PermEnd { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// End fields left open at the end of a story, so every begin has its end; the ends get a
|
||||
/// paragraph of their own, as the story must end with a mark.
|
||||
pub(crate) fn close_fields(&mut self) {
|
||||
let mut wrote = false;
|
||||
while self.open_fields > 0 {
|
||||
self.open_fields -= 1;
|
||||
if self.open_fields < MAX_FIELD_DEPTH {
|
||||
let chpx = self.chpx.get(Vec::new());
|
||||
self.field_char(FIELD_END, end_flags(true, false, false, self.open_fields > 0), chpx);
|
||||
wrote = true;
|
||||
}
|
||||
}
|
||||
if wrote {
|
||||
self.mark(&Paragraph::new(), PARA, &Grpprl::default());
|
||||
}
|
||||
}
|
||||
|
||||
/// A table: each cell's paragraphs end with a cell mark, each row with a row-end mark that
|
||||
/// carries the row's table properties ([MS-DOC] §2.4.3).
|
||||
fn table(&mut self, t: &Table) {
|
||||
let was = self.in_cell;
|
||||
self.in_cell = true;
|
||||
let rows = t.rows.len();
|
||||
let style_borders = t.props.style.as_deref().and_then(|s| self.look.doc.styles.get(s)).and_then(|s| s.table.as_ref()).and_then(|p| p.borders);
|
||||
for (ri, row) in t.rows.iter().enumerate() {
|
||||
// A row has at most 63 cells: the text of any past them goes in the last one.
|
||||
let last = row.cells.len().min(tables::MAX_CELLS).saturating_sub(1);
|
||||
let mut in_table = Grpprl::default();
|
||||
grpprl::in_table(&mut in_table, false);
|
||||
for (ci, cell) in row.cells.iter().enumerate() {
|
||||
let mut paras: Vec<&Paragraph> = Vec::new();
|
||||
let mut owned: Vec<Paragraph> = Vec::new();
|
||||
flatten(&cell.blocks, &mut paras, &mut owned, &mut self.dropped.nested_tables);
|
||||
let all: Vec<&Paragraph> = if owned.is_empty() { paras } else { owned.iter().collect() };
|
||||
// Whether this cell's last paragraph ends the (possibly merged) cell.
|
||||
let ends_cell = ci < last || ci + 1 == row.cells.len();
|
||||
if all.is_empty() {
|
||||
let mark = if ends_cell { CELL } else { PARA };
|
||||
self.paragraph(&Paragraph::new(), false, Some((mark, &in_table)));
|
||||
continue;
|
||||
}
|
||||
for (i, p) in all.iter().enumerate() {
|
||||
let mark = if ends_cell && i + 1 == all.len() { CELL } else { PARA };
|
||||
self.paragraph(p, false, Some((mark, &in_table)));
|
||||
}
|
||||
}
|
||||
let tapx = tables::row_end(t, ri, rows, style_borders);
|
||||
let mut extra = Grpprl::default();
|
||||
grpprl::in_table(&mut extra, true);
|
||||
extra.extend(&tapx);
|
||||
self.mark(&Paragraph::new(), CELL, &extra);
|
||||
}
|
||||
self.in_cell = was;
|
||||
}
|
||||
}
|
||||
|
||||
/// A cell's paragraphs, nested tables flattened into their own paragraphs (the writer keeps one
|
||||
/// level of tables). When there is nothing to flatten `paras` borrows; else `owned` holds all.
|
||||
fn flatten<'a>(blocks: &'a [Arc<Block>], paras: &mut Vec<&'a Paragraph>, owned: &mut Vec<Paragraph>, nested: &mut usize) {
|
||||
if blocks.iter().all(|b| b.as_para().is_some()) {
|
||||
paras.extend(blocks.iter().filter_map(|b| b.as_para()));
|
||||
return;
|
||||
}
|
||||
*nested += 1;
|
||||
fn walk(blocks: &[Arc<Block>], out: &mut Vec<Paragraph>, depth: usize) {
|
||||
for b in blocks {
|
||||
match &**b {
|
||||
Block::Para(p) => out.push(p.clone()),
|
||||
Block::Table(t) if depth < 16 => {
|
||||
for row in &t.rows {
|
||||
for c in &row.cells {
|
||||
walk(&c.blocks, out, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Block::Table(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(blocks, owned, 0);
|
||||
}
|
||||
|
||||
/// `flt` of a HYPERLINK field.
|
||||
const FLT_HYPERLINK: u8 = 88;
|
||||
|
||||
/// The field type (`flt`, [MS-DOC] §2.9.87) of a field code, from its first word; 0x01 (an
|
||||
/// unknown keyword) for anything not in the table, which Word reparses from the code anyway.
|
||||
fn flt(instr: &str) -> u8 {
|
||||
let word = instr.split_whitespace().next().unwrap_or("").trim_start_matches('=').to_ascii_uppercase();
|
||||
if instr.trim_start().starts_with('=') {
|
||||
return 34;
|
||||
}
|
||||
match word.as_str() {
|
||||
"REF" => 3,
|
||||
"FTNREF" => 5,
|
||||
"SET" => 6,
|
||||
"IF" => 7,
|
||||
"INDEX" => 8,
|
||||
"TC" => 9,
|
||||
"STYLEREF" => 10,
|
||||
"SEQ" => 12,
|
||||
"TOC" => 13,
|
||||
"INFO" => 14,
|
||||
"TITLE" => 15,
|
||||
"SUBJECT" => 16,
|
||||
"AUTHOR" => 17,
|
||||
"KEYWORDS" => 18,
|
||||
"COMMENTS" => 19,
|
||||
"LASTSAVEDBY" => 20,
|
||||
"CREATEDATE" => 21,
|
||||
"SAVEDATE" => 22,
|
||||
"PRINTDATE" => 23,
|
||||
"REVNUM" => 24,
|
||||
"EDITTIME" => 25,
|
||||
"NUMPAGES" => 26,
|
||||
"NUMWORDS" => 27,
|
||||
"NUMCHARS" => 28,
|
||||
"FILENAME" => 29,
|
||||
"TEMPLATE" => 30,
|
||||
"DATE" => 31,
|
||||
"TIME" => 32,
|
||||
"PAGE" => 33,
|
||||
"QUOTE" => 35,
|
||||
"INCLUDE" => 36,
|
||||
"PAGEREF" => 37,
|
||||
"ASK" => 38,
|
||||
"FILLIN" => 39,
|
||||
"DATA" => 40,
|
||||
"NEXT" => 41,
|
||||
"NEXTIF" => 42,
|
||||
"SKIPIF" => 43,
|
||||
"MERGEREC" => 44,
|
||||
"DDE" => 45,
|
||||
"DDEAUTO" => 46,
|
||||
"GLOSSARY" => 47,
|
||||
"PRINT" => 48,
|
||||
"EQ" => 49,
|
||||
"GOTOBUTTON" => 50,
|
||||
"MACROBUTTON" => 51,
|
||||
"AUTONUMOUT" => 52,
|
||||
"AUTONUMLGL" => 53,
|
||||
"AUTONUM" => 54,
|
||||
"IMPORT" => 55,
|
||||
"LINK" => 56,
|
||||
"SYMBOL" => 57,
|
||||
"EMBED" => 58,
|
||||
"MERGEFIELD" => 59,
|
||||
"USERNAME" => 60,
|
||||
"USERINITIALS" => 61,
|
||||
"USERADDRESS" => 62,
|
||||
"BARCODE" => 63,
|
||||
"DOCVARIABLE" => 64,
|
||||
"SECTION" => 65,
|
||||
"SECTIONPAGES" => 66,
|
||||
"INCLUDEPICTURE" => 67,
|
||||
"INCLUDETEXT" => 68,
|
||||
"FILESIZE" => 69,
|
||||
"FORMTEXT" => 70,
|
||||
"FORMCHECKBOX" => 71,
|
||||
"NOTEREF" => 72,
|
||||
"TOA" => 73,
|
||||
"TA" => 74,
|
||||
"MERGESEQ" => 75,
|
||||
"AUTOTEXT" => 79,
|
||||
"COMPARE" => 80,
|
||||
"ADDIN" => 81,
|
||||
"FORMDROPDOWN" => 83,
|
||||
"ADVANCE" => 84,
|
||||
"DOCPROPERTY" => 85,
|
||||
"CONTROL" => 87,
|
||||
"HYPERLINK" => 88,
|
||||
"AUTOTEXTLIST" => 89,
|
||||
"LISTNUM" => 90,
|
||||
"HTMLCONTROL" => 91,
|
||||
"BIDIOUTLINE" => 92,
|
||||
"ADDRESSBLOCK" => 93,
|
||||
"GREETINGLINE" => 94,
|
||||
"SHAPE" => 95,
|
||||
"CITATION" => 96,
|
||||
"BIBLIOGRAPHY" => 97,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// The second byte of a field end ([MS-DOC] §2.9.84 grffldEnd): `fHasSep` when there is a
|
||||
/// result, `fLocked`, `fResultsDirty` and `fNested`.
|
||||
fn end_flags(has_sep: bool, locked: bool, dirty: bool, nested: bool) -> u8 {
|
||||
let mut f = 0u8;
|
||||
if dirty {
|
||||
f |= 0x04;
|
||||
}
|
||||
if locked {
|
||||
f |= 0x10;
|
||||
}
|
||||
if nested {
|
||||
f |= 0x40;
|
||||
}
|
||||
if has_sep {
|
||||
f |= 0x80;
|
||||
}
|
||||
f
|
||||
}
|
||||
|
||||
/// The field code of a link: `HYPERLINK "url"`, `HYPERLINK \l "bookmark"` for an internal
|
||||
/// one, or both for a link into another document's bookmark. A quote or backslash in an
|
||||
/// argument is escaped with a backslash, as field codes do.
|
||||
fn hyperlink_instr(target: &str) -> String {
|
||||
let q = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
match target.split_once('#') {
|
||||
Some(("", frag)) => format!("HYPERLINK \\l \"{}\"", q(frag)),
|
||||
Some((url, frag)) if !frag.is_empty() && !url.contains("://") => format!("HYPERLINK \"{}\" \\l \"{}\"", q(url), q(frag)),
|
||||
_ => format!("HYPERLINK \"{}\"", q(target)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn field_types() {
|
||||
assert_eq!(flt("PAGE"), 33);
|
||||
assert_eq!(flt(" hyperlink \"x\""), 88);
|
||||
assert_eq!(flt("= 2 + 2"), 34);
|
||||
assert_eq!(flt("NOSUCHFIELD"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hyperlink_codes() {
|
||||
assert_eq!(hyperlink_instr("https://example.org/a#b"), "HYPERLINK \"https://example.org/a#b\"");
|
||||
assert_eq!(hyperlink_instr("#Intro"), "HYPERLINK \\l \"Intro\"");
|
||||
assert_eq!(hyperlink_instr("other.doc#Part"), "HYPERLINK \"other.doc\" \\l \"Part\"");
|
||||
assert_eq!(craft_wordmodel::link::hyperlink_target(&hyperlink_instr("#Intro")).as_deref(), Some("#Intro"));
|
||||
assert_eq!(
|
||||
craft_wordmodel::link::hyperlink_target(&hyperlink_instr("https://example.org/?q=\"x\"")).as_deref(),
|
||||
Some("https://example.org/?q=\"x\"")
|
||||
);
|
||||
assert_eq!(
|
||||
craft_wordmodel::link::hyperlink_target(&hyperlink_instr(r"file:///C:\docs\report.doc")).as_deref(),
|
||||
Some(r"file:///C:\docs\report.doc")
|
||||
);
|
||||
assert_eq!(craft_wordmodel::link::hyperlink_target(&hyperlink_instr("#a\"b")).as_deref(), Some("#a\"b"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
//! The stylesheet (`STSH`, [MS-DOC] §2.9.271) and the font table (`SttbfFfn`, §2.9.286).
|
||||
//!
|
||||
//! Istds 0–14 are fixed: 0 Normal, 1–9 heading 1–9, 10 Default Paragraph Font, 11 Normal
|
||||
//! Table, 12 No List, 13–14 unused. Every other style follows from istd 15 in stylesheet
|
||||
//! order. A style Word has built in keeps its `sti` and the name Word gives it, so Word treats
|
||||
//! it as its own; the rest are user styles.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use craft_wordmodel::props::CharProps;
|
||||
use craft_wordmodel::styles::{Style, StyleKind, StyleSheet};
|
||||
|
||||
use super::grpprl::{self, CharCtx, Grpprl, ParaCtx};
|
||||
|
||||
/// Istds reserved for built-in styles.
|
||||
const FIXED: u16 = 15;
|
||||
/// `sti` of a user-defined style.
|
||||
const STI_USER: u16 = 0x0FFE;
|
||||
/// `istdBase` / `istdNext` meaning "none".
|
||||
const ISTD_NIL: u16 = 0x0FFF;
|
||||
/// Most styles written (the format allows 0x0FFE).
|
||||
const MAX_STYLES: usize = 4000;
|
||||
|
||||
/// Built-in styles: the name Word stores (lower-case where Word's is), and the `sti`.
|
||||
const BUILTIN: &[(&str, u16)] = &[
|
||||
("Normal", 0),
|
||||
("heading 1", 1),
|
||||
("heading 2", 2),
|
||||
("heading 3", 3),
|
||||
("heading 4", 4),
|
||||
("heading 5", 5),
|
||||
("heading 6", 6),
|
||||
("heading 7", 7),
|
||||
("heading 8", 8),
|
||||
("heading 9", 9),
|
||||
("toc 1", 19),
|
||||
("toc 2", 20),
|
||||
("toc 3", 21),
|
||||
("toc 4", 22),
|
||||
("toc 5", 23),
|
||||
("toc 6", 24),
|
||||
("toc 7", 25),
|
||||
("toc 8", 26),
|
||||
("toc 9", 27),
|
||||
("Normal Indent", 28),
|
||||
("footnote text", 29),
|
||||
("annotation text", 30),
|
||||
("header", 31),
|
||||
("footer", 32),
|
||||
("index heading", 33),
|
||||
("caption", 34),
|
||||
("table of figures", 35),
|
||||
("footnote reference", 38),
|
||||
("annotation reference", 39),
|
||||
("line number", 40),
|
||||
("page number", 41),
|
||||
("endnote reference", 42),
|
||||
("endnote text", 43),
|
||||
("List", 47),
|
||||
("List Bullet", 48),
|
||||
("List Number", 49),
|
||||
("Title", 62),
|
||||
("Signature", 64),
|
||||
("Default Paragraph Font", 65),
|
||||
("Body Text", 66),
|
||||
("Body Text Indent", 67),
|
||||
("List Continue", 68),
|
||||
("Subtitle", 74),
|
||||
("Salutation", 75),
|
||||
("Date", 76),
|
||||
("Hyperlink", 85),
|
||||
("FollowedHyperlink", 86),
|
||||
("Strong", 87),
|
||||
("Emphasis", 88),
|
||||
("Plain Text", 91),
|
||||
("Normal (Web)", 94),
|
||||
("Normal Table", 105),
|
||||
("No List", 107),
|
||||
("Balloon Text", 153),
|
||||
("Table Grid", 154),
|
||||
("Placeholder Text", 156),
|
||||
("No Spacing", 157),
|
||||
("List Paragraph", 179),
|
||||
("Quote", 180),
|
||||
("Intense Quote", 181),
|
||||
("Subtle Emphasis", 260),
|
||||
("Intense Emphasis", 261),
|
||||
("Subtle Reference", 262),
|
||||
("Intense Reference", 263),
|
||||
("Book Title", 264),
|
||||
("Bibliography", 265),
|
||||
("TOC Heading", 266),
|
||||
];
|
||||
|
||||
/// The built-in style a model style is, by its name or id (`Heading 1`, `Heading1` and
|
||||
/// `heading 1` are the same), with the name Word stores.
|
||||
fn builtin(st: &Style) -> Option<(&'static str, u16)> {
|
||||
let norm = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::<String>().to_ascii_lowercase();
|
||||
let (name, id) = (norm(&st.name), norm(&st.id));
|
||||
// The docx reader's ids for a few styles differ from their names.
|
||||
let alias = |s: &str| match s {
|
||||
"tablenormal" => "normaltable".to_string(),
|
||||
"commenttext" => "annotationtext".to_string(),
|
||||
"commentreference" => "annotationreference".to_string(),
|
||||
other => other.to_string(),
|
||||
};
|
||||
BUILTIN
|
||||
.iter()
|
||||
.find(|(n, _)| {
|
||||
let n = norm(n);
|
||||
n == alias(&name) || n == alias(&id)
|
||||
})
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// The stylesheet as written: each model style's istd.
|
||||
pub(crate) struct Plan {
|
||||
/// Model style id → istd.
|
||||
pub(crate) istd: HashMap<String, u16>,
|
||||
/// Istd → model style (index into the stylesheet), `None` for an empty slot.
|
||||
slots: Vec<Option<usize>>,
|
||||
/// Istd → the name and sti it is written with.
|
||||
names: Vec<(String, u16)>,
|
||||
}
|
||||
|
||||
/// Assign istds: the fixed built-ins to their slots, everything else after them.
|
||||
pub(crate) fn plan(sheet: &StyleSheet) -> Plan {
|
||||
let mut slots: Vec<Option<usize>> = vec![None; FIXED as usize];
|
||||
let mut names: Vec<(String, u16)> = vec![(String::new(), 0); FIXED as usize];
|
||||
let mut istd = HashMap::new();
|
||||
let mut taken = std::collections::HashSet::new();
|
||||
for (i, st) in sheet.styles.iter().enumerate().take(MAX_STYLES) {
|
||||
if st.id.is_empty() || istd.contains_key(&st.id) {
|
||||
continue;
|
||||
}
|
||||
let bi = builtin(st);
|
||||
// The fixed slot a built-in style takes, when its kind is the one Word gives it.
|
||||
let fixed = bi.and_then(|(_, sti)| match (sti, st.kind) {
|
||||
(0..=9, StyleKind::Paragraph) => Some(sti),
|
||||
(65, StyleKind::Character) => Some(10),
|
||||
(105, StyleKind::Table) => Some(11),
|
||||
(107, StyleKind::Numbering) => Some(12),
|
||||
_ => None,
|
||||
});
|
||||
let slot = match fixed {
|
||||
Some(s) if !taken.contains(&s) => s,
|
||||
_ => {
|
||||
slots.push(None);
|
||||
names.push((String::new(), 0));
|
||||
(slots.len() - 1) as u16
|
||||
}
|
||||
};
|
||||
taken.insert(slot);
|
||||
if let Some(s) = slots.get_mut(slot as usize) {
|
||||
*s = Some(i);
|
||||
}
|
||||
// A built-in name is Word's own spelling; its sti only where the kind matches.
|
||||
let (name, sti) = match bi {
|
||||
Some((n, sti)) if builtin_kind(sti) == st.kind => (n.to_string(), sti),
|
||||
_ => (st.name.clone(), STI_USER),
|
||||
};
|
||||
if let Some(n) = names.get_mut(slot as usize) {
|
||||
*n = (name, sti);
|
||||
}
|
||||
istd.insert(st.id.clone(), slot);
|
||||
}
|
||||
// Normal and Default Paragraph Font are always there.
|
||||
if slots.first().is_some_and(Option::is_none)
|
||||
&& let Some(n) = names.get_mut(0)
|
||||
{
|
||||
*n = ("Normal".into(), 0);
|
||||
}
|
||||
for (i, name, sti) in [(10usize, "Default Paragraph Font", 65u16), (11, "Normal Table", 105), (12, "No List", 107)] {
|
||||
if slots.get(i).is_some_and(Option::is_none)
|
||||
&& let Some(n) = names.get_mut(i)
|
||||
{
|
||||
*n = (name.into(), sti);
|
||||
}
|
||||
}
|
||||
Plan { istd, slots, names }
|
||||
}
|
||||
|
||||
/// The kind of style Word's built-in `sti` is.
|
||||
fn builtin_kind(sti: u16) -> StyleKind {
|
||||
match sti {
|
||||
38 | 39 | 40 | 41 | 42 | 65 | 85 | 86 | 87 | 88 | 156 | 260..=264 => StyleKind::Character,
|
||||
105 | 154 => StyleKind::Table,
|
||||
107 => StyleKind::Numbering,
|
||||
_ => StyleKind::Paragraph,
|
||||
}
|
||||
}
|
||||
|
||||
/// The STSH bytes. `default_font` is the font-table index of the document's default font
|
||||
/// (the standard font of the stylesheet header).
|
||||
pub(crate) fn stsh(sheet: &StyleSheet, plan: &Plan, ctx: &mut (impl CharCtx + ParaCtx), default_fonts: [u16; 3]) -> Vec<u8> {
|
||||
// LPStshi: cbStshi, then Stshi: Stshif (18 bytes), ftcBi, and the latent style data (an
|
||||
// empty LSD for every sti below stiMaxWhenSaved).
|
||||
let cstd = plan.slots.len().min(0x0FFE) as u16;
|
||||
let sti_max = plan.names.iter().map(|(_, sti)| *sti).filter(|&s| s != STI_USER).max().map_or(0x5B, |m| (m + 1).max(0x5B));
|
||||
let cb_stshi = 18 + 2 + 2 + 4 * usize::from(sti_max);
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&(cb_stshi as u16).to_le_bytes());
|
||||
v.extend_from_slice(&cstd.to_le_bytes());
|
||||
v.extend_from_slice(&0x000Au16.to_le_bytes()); // cbSTDBaseInFile
|
||||
v.extend_from_slice(&1u16.to_le_bytes()); // fStdStylenamesWritten
|
||||
v.extend_from_slice(&sti_max.to_le_bytes()); // stiMaxWhenSaved
|
||||
v.extend_from_slice(&FIXED.to_le_bytes()); // istdMaxFixedWhenSaved
|
||||
v.extend_from_slice(&0u16.to_le_bytes()); // nVerBuiltInNamesWhenSaved
|
||||
for f in default_fonts {
|
||||
v.extend_from_slice(&f.to_le_bytes()); // rgftcStandardChpStsh
|
||||
}
|
||||
v.extend_from_slice(&default_fonts[2].to_le_bytes()); // ftcBi
|
||||
v.extend_from_slice(&4u16.to_le_bytes()); // StshiLsd.cbLSD
|
||||
v.extend(std::iter::repeat_n(0u8, 4 * usize::from(sti_max)));
|
||||
for (istd, slot) in plan.slots.iter().enumerate().take(cstd as usize) {
|
||||
let (name, sti) = plan.names.get(istd).cloned().unwrap_or_default();
|
||||
let std = match slot.and_then(|i| sheet.styles.get(i)) {
|
||||
Some(st) => std_bytes(st, &name, sti, sheet, plan, ctx, istd == 0),
|
||||
None if !name.is_empty() => empty_builtin(&name, sti, istd as u16),
|
||||
None => Vec::new(),
|
||||
};
|
||||
v.extend_from_slice(&(std.len() as u16).to_le_bytes());
|
||||
v.extend_from_slice(&std);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// A built-in style the document doesn't define (Normal, Default Paragraph Font).
|
||||
fn empty_builtin(name: &str, sti: u16, istd: u16) -> Vec<u8> {
|
||||
let own = istd.to_le_bytes().to_vec();
|
||||
let (stk, upx): (u16, Vec<Vec<u8>>) = match sti {
|
||||
65 => (2, vec![Vec::new()]),
|
||||
105 => (3, vec![Vec::new(), own, Vec::new()]),
|
||||
107 => (4, vec![own]),
|
||||
_ => (1, vec![own, Vec::new()]),
|
||||
};
|
||||
let next = if stk == 1 { istd } else { ISTD_NIL };
|
||||
assemble(sti, stk, ISTD_NIL, next, upx.len() as u16, 0, name, &upx)
|
||||
}
|
||||
|
||||
/// One STD: `StdfBase`, the name, then the UPXs for its kind.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn std_bytes(st: &Style, name: &str, sti: u16, sheet: &StyleSheet, plan: &Plan, ctx: &mut (impl CharCtx + ParaCtx), normal: bool) -> Vec<u8> {
|
||||
let base = st.based_on.as_deref().and_then(|b| plan.istd.get(b).copied()).unwrap_or(ISTD_NIL);
|
||||
let istd_self = plan.istd.get(&st.id).copied().unwrap_or(0);
|
||||
let next = st.next.as_deref().and_then(|b| plan.istd.get(b).copied()).unwrap_or(istd_self);
|
||||
// Document defaults have nowhere else to go in this format: Normal (and any other root
|
||||
// paragraph or character style) starts from them.
|
||||
let root = base == ISTD_NIL;
|
||||
let mut chr = st.chr.clone();
|
||||
let mut para = st.para.clone();
|
||||
if root && matches!(st.kind, StyleKind::Paragraph) {
|
||||
chr = overlay(&sheet.default_chr, &chr);
|
||||
para = craft_wordmodel::props::ParaProps { style: None, ..overlay_para(&sheet.default_para, ¶) };
|
||||
}
|
||||
if normal {
|
||||
chr.style = None;
|
||||
}
|
||||
para.style = None;
|
||||
para.numbering = para.numbering.filter(|n| n.num != 0);
|
||||
chr.style = None;
|
||||
let mut grfstd: u16 = 0;
|
||||
if st.hidden {
|
||||
grfstd |= 0x0002 | 0x0100; // fHidden, fSemiHidden
|
||||
}
|
||||
if st.locked {
|
||||
grfstd |= 0x0200;
|
||||
}
|
||||
if st.quick {
|
||||
grfstd |= 0x1000; // fQFormat
|
||||
}
|
||||
let (stk, cupx, upx): (u16, u16, Vec<Vec<u8>>) = match st.kind {
|
||||
StyleKind::Paragraph => {
|
||||
let mut papx = istd_self.to_le_bytes().to_vec();
|
||||
let mut g = grpprl::papx(¶, ctx);
|
||||
g.truncate_to(super::fkp::MAX_PAPX_GRPPRL);
|
||||
papx.extend_from_slice(&g.0);
|
||||
let mut c = grpprl::chpx(&chr, ctx);
|
||||
c.truncate_to(255);
|
||||
(1, 2, vec![papx, c.0])
|
||||
}
|
||||
StyleKind::Character => {
|
||||
let mut c = grpprl::chpx(&chr, ctx);
|
||||
c.truncate_to(255);
|
||||
(2, 1, vec![c.0])
|
||||
}
|
||||
// Table styles: their paragraph and character formatting; the conditional formats stay
|
||||
// with the model (tables carry their own borders and shading when written).
|
||||
StyleKind::Table => {
|
||||
let mut papx = istd_self.to_le_bytes().to_vec();
|
||||
papx.extend_from_slice(&grpprl::papx(¶, ctx).0);
|
||||
(3, 3, vec![Vec::new(), papx, grpprl::chpx(&chr, ctx).0])
|
||||
}
|
||||
StyleKind::Numbering => {
|
||||
let mut papx = istd_self.to_le_bytes().to_vec();
|
||||
papx.extend_from_slice(&Grpprl::default().0);
|
||||
(4, 1, vec![papx])
|
||||
}
|
||||
};
|
||||
assemble(sti, stk, base, next, cupx, grfstd, name, &upx)
|
||||
}
|
||||
|
||||
/// `StdfBase` + `xstzName` + `GrLPUpxSw` (each UPX padded to an even size).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn assemble(sti: u16, stk: u16, base: u16, next: u16, cupx: u16, grfstd: u16, name: &str, upx: &[Vec<u8>]) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&(sti & 0x0FFF).to_le_bytes());
|
||||
v.extend_from_slice(&((stk & 0xF) | ((base & 0x0FFF) << 4)).to_le_bytes());
|
||||
v.extend_from_slice(&((cupx & 0xF) | ((next & 0x0FFF) << 4)).to_le_bytes());
|
||||
let bch_at = v.len();
|
||||
v.extend_from_slice(&0u16.to_le_bytes()); // bchUpe, filled below
|
||||
v.extend_from_slice(&grfstd.to_le_bytes());
|
||||
let units: Vec<u16> = name.encode_utf16().take(253).collect();
|
||||
v.extend_from_slice(&(units.len() as u16).to_le_bytes());
|
||||
for u in &units {
|
||||
v.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
v.extend_from_slice(&0u16.to_le_bytes());
|
||||
for u in upx {
|
||||
v.extend_from_slice(&(u.len() as u16).to_le_bytes());
|
||||
v.extend_from_slice(u);
|
||||
if u.len() % 2 == 1 {
|
||||
v.push(0);
|
||||
}
|
||||
}
|
||||
let len = v.len() as u16;
|
||||
if let Some(b) = v.get_mut(bch_at..bch_at + 2) {
|
||||
b.copy_from_slice(&len.to_le_bytes());
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// `over`'s character properties over `under`'s.
|
||||
fn overlay(under: &CharProps, over: &CharProps) -> CharProps {
|
||||
let mut out = under.clone();
|
||||
out.overlay(over);
|
||||
out
|
||||
}
|
||||
|
||||
fn overlay_para(under: &craft_wordmodel::props::ParaProps, over: &craft_wordmodel::props::ParaProps) -> craft_wordmodel::props::ParaProps {
|
||||
let mut out = under.clone();
|
||||
out.overlay(over);
|
||||
out
|
||||
}
|
||||
|
||||
/// The `SttbfFfn`: a non-extended STTB whose entries are `FFN`s (a byte count, then the FFN).
|
||||
pub(crate) fn font_table(names: &[String]) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&(names.len().min(0x7FFF) as u16).to_le_bytes());
|
||||
v.extend_from_slice(&0u16.to_le_bytes()); // cbExtra
|
||||
for name in names.iter().take(0x7FFF) {
|
||||
let ffn = ffn(name);
|
||||
v.push(ffn.len() as u8);
|
||||
v.extend_from_slice(&ffn);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// One `FFN` ([MS-DOC] §2.9.82): pitch/TrueType/family bits, `wWeight`, `chs`,
|
||||
/// `ixchSzAlt`, `panose`, `fs`, then the name, NUL-terminated UTF-16.
|
||||
fn ffn(name: &str) -> Vec<u8> {
|
||||
let units: Vec<u16> = name.encode_utf16().take(31).collect();
|
||||
let mut v = Vec::with_capacity(40 + units.len() * 2 + 2);
|
||||
let symbol = matches!(name.to_ascii_lowercase().as_str(), "symbol" | "wingdings" | "wingdings 2" | "wingdings 3" | "webdings");
|
||||
let family: u8 = if symbol {
|
||||
5 // FF_DECORATIVE
|
||||
} else if name.to_ascii_lowercase().contains("mono") || name.eq_ignore_ascii_case("courier new") {
|
||||
3 // FF_MODERN
|
||||
} else if name.to_ascii_lowercase().contains("sans")
|
||||
|| ["arial", "calibri", "aptos", "verdana", "tahoma", "segoe ui"].contains(&name.to_ascii_lowercase().as_str())
|
||||
{
|
||||
2 // FF_SWISS
|
||||
} else {
|
||||
1 // FF_ROMAN
|
||||
};
|
||||
// prq (2 = variable pitch), fTrueType, ff.
|
||||
v.push(2 | 0x04 | (family << 4));
|
||||
v.extend_from_slice(&400u16.to_le_bytes()); // wWeight
|
||||
v.push(if symbol { 2 } else { 0 }); // chs: SYMBOL_CHARSET or ANSI_CHARSET
|
||||
v.push(0); // ixchSzAlt: no alternate name
|
||||
v.extend_from_slice(&[0u8; 10]); // panose
|
||||
v.extend_from_slice(&[0u8; 24]); // fs
|
||||
for u in &units {
|
||||
v.extend_from_slice(&u.to_le_bytes());
|
||||
}
|
||||
v.extend_from_slice(&0u16.to_le_bytes());
|
||||
v
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn font_table_reads_back() {
|
||||
let names = vec!["Times New Roman".to_string(), "Symbol".to_string(), "Noto Sans Ünï".to_string()];
|
||||
let t = font_table(&names);
|
||||
assert_eq!(crate::stsh::fonts(&t, 0, t.len() as u32), names);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//! The `\u{5}SummaryInformation` stream ([MS-OLEPS] §2.21): title, subject, author, keywords,
|
||||
//! comments, last author, revision and the creation and save times.
|
||||
|
||||
use craft_wordmodel::CoreProps;
|
||||
|
||||
/// `FMTID_SummaryInformation` as stored (little-endian GUID fields).
|
||||
const SUMMARY_FMTID: [u8; 16] = [0xe0, 0x85, 0x9f, 0xf2, 0xf9, 0x4f, 0x68, 0x10, 0xab, 0x91, 0x08, 0x00, 0x2b, 0x27, 0xb3, 0xd9];
|
||||
const VT_I2: u32 = 2;
|
||||
const VT_LPSTR: u32 = 30;
|
||||
const VT_FILETIME: u32 = 64;
|
||||
/// Longest string property written (characters).
|
||||
const MAX_CHARS: usize = 32 << 10;
|
||||
|
||||
/// The text of `s` in code page 1252, when every character has a byte there.
|
||||
fn cp1252(s: &str) -> Option<Vec<u8>> {
|
||||
s.chars().map(|c| (0u8..=255).find(|&b| craft_codepage::cp1252(b) == c && (b >= 0x20 || c == '\t' || c == '\n' || c == '\r'))).collect()
|
||||
}
|
||||
|
||||
/// The stream's bytes. Strings are Windows-1252 when they all fit it, else UTF-16 (code page
|
||||
/// 1200).
|
||||
pub(crate) fn stream(core: &CoreProps) -> Vec<u8> {
|
||||
let strings: Vec<(u32, String)> =
|
||||
[(2u32, &core.title), (3, &core.subject), (4, &core.creator), (5, &core.keywords), (6, &core.description), (8, &core.last_modified_by)]
|
||||
.into_iter()
|
||||
.filter(|(_, s)| !s.is_empty())
|
||||
.map(|(id, s)| (id, s.chars().take(MAX_CHARS).collect()))
|
||||
.chain((core.revision > 0).then(|| (9u32, core.revision.to_string())))
|
||||
.collect();
|
||||
let ansi = strings.iter().all(|(_, s)| cp1252(s).is_some());
|
||||
let codepage: u16 = if ansi { 1252 } else { 1200 };
|
||||
|
||||
let mut props: Vec<(u32, Vec<u8>)> = Vec::new();
|
||||
let mut cp = VT_I2.to_le_bytes().to_vec();
|
||||
cp.extend_from_slice(&codepage.to_le_bytes());
|
||||
cp.extend_from_slice(&[0, 0]);
|
||||
props.push((1, cp));
|
||||
for (id, s) in &strings {
|
||||
let mut bytes = if ansi { cp1252(s).unwrap_or_default() } else { s.encode_utf16().flat_map(u16::to_le_bytes).collect() };
|
||||
bytes.extend_from_slice(if ansi { &[0][..] } else { &[0, 0][..] });
|
||||
let mut v = VT_LPSTR.to_le_bytes().to_vec();
|
||||
v.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
|
||||
v.extend_from_slice(&bytes);
|
||||
while !v.len().is_multiple_of(4) {
|
||||
v.push(0);
|
||||
}
|
||||
props.push((*id, v));
|
||||
}
|
||||
for (id, iso) in [(12u32, &core.created), (13, &core.modified)] {
|
||||
if let Some(ft) = filetime(iso) {
|
||||
let mut v = VT_FILETIME.to_le_bytes().to_vec();
|
||||
v.extend_from_slice(&(ft as u32).to_le_bytes());
|
||||
v.extend_from_slice(&((ft >> 32) as u32).to_le_bytes());
|
||||
props.push((id, v));
|
||||
}
|
||||
}
|
||||
|
||||
// PropertySet: size, count, (id, offset) pairs, then the values.
|
||||
let table = 8 + props.len() * 8;
|
||||
let mut set = Vec::new();
|
||||
let mut values = Vec::new();
|
||||
let mut index = Vec::new();
|
||||
for (id, v) in &props {
|
||||
index.extend_from_slice(&id.to_le_bytes());
|
||||
index.extend_from_slice(&((table + values.len()) as u32).to_le_bytes());
|
||||
values.extend_from_slice(v);
|
||||
}
|
||||
set.extend_from_slice(&((table + values.len()) as u32).to_le_bytes());
|
||||
set.extend_from_slice(&(props.len() as u32).to_le_bytes());
|
||||
set.extend_from_slice(&index);
|
||||
set.extend_from_slice(&values);
|
||||
|
||||
// PropertySetStream header: byte order, version, system id, CLSID, one set, its FMTID and
|
||||
// offset.
|
||||
let mut out = vec![0xFE, 0xFF, 0, 0];
|
||||
out.extend_from_slice(&0x0002_0006u32.to_le_bytes());
|
||||
out.extend_from_slice(&[0u8; 16]);
|
||||
out.extend_from_slice(&1u32.to_le_bytes());
|
||||
out.extend_from_slice(&SUMMARY_FMTID);
|
||||
out.extend_from_slice(&48u32.to_le_bytes());
|
||||
out.extend_from_slice(&set);
|
||||
out
|
||||
}
|
||||
|
||||
/// A FILETIME (100 ns since 1601) from an ISO 8601 date.
|
||||
fn filetime(iso: &str) -> Option<u64> {
|
||||
let t = super::parse_time(iso)?.and_utc().timestamp();
|
||||
let ft = (t.checked_add(11_644_473_600)?).checked_mul(10_000_000)?;
|
||||
u64::try_from(ft).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn summary_reads_back() {
|
||||
let core = craft_wordmodel::CoreProps {
|
||||
title: "Annual Report".into(),
|
||||
creator: "Zoë Writer".into(),
|
||||
keywords: "a, b".into(),
|
||||
revision: 3,
|
||||
created: "2024-05-06T07:08:09Z".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let read = crate::metadata::parse_for_tests(&super::stream(&core)).unwrap();
|
||||
assert_eq!(read.title, "Annual Report");
|
||||
assert_eq!(read.creator, "Zoë Writer");
|
||||
assert_eq!(read.keywords, "a, b");
|
||||
assert_eq!(read.revision, 3);
|
||||
assert_eq!(read.created, "2024-05-06T07:08:09Z");
|
||||
let unicode = craft_wordmodel::CoreProps { title: "Отчёт 报告".into(), ..Default::default() };
|
||||
assert_eq!(crate::metadata::parse_for_tests(&super::stream(&unicode)).unwrap().title, "Отчёт 报告");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
//! A table row's properties, carried by its row-end mark ([MS-DOC] §2.4.3, §2.6.3): the cell
|
||||
//! edges and per-cell borders (`sprmTDefTable`), the table's borders, alignment and indent, row
|
||||
//! height and flags, and the cells' shading, widths, vertical merges and alignment.
|
||||
|
||||
use craft_wordmodel::props::{Align, Border, Borders, HeightRule, VAlign, VMerge};
|
||||
use craft_wordmodel::table::Table;
|
||||
|
||||
use super::grpprl::{Grpprl, brc, brc80, shd, twips};
|
||||
|
||||
const T_JC90: u16 = 0x5400;
|
||||
const T_JC: u16 = 0x548A;
|
||||
const T_DXA_LEFT: u16 = 0x9601;
|
||||
const T_DXA_GAP_HALF: u16 = 0x9602;
|
||||
const T_CANT_SPLIT90: u16 = 0x3403;
|
||||
const T_TABLE_HEADER: u16 = 0x3404;
|
||||
const T_CANT_SPLIT: u16 = 0x3466;
|
||||
const T_TABLE_BORDERS80: u16 = 0xD605;
|
||||
const T_ROW_HEIGHT: u16 = 0x9407;
|
||||
const T_DEF_TABLE: u16 = 0xD608;
|
||||
const T_DEF_TABLE_SHD: [u16; 3] = [0xD612, 0xD616, 0xD60C];
|
||||
const T_TABLE_BORDERS: u16 = 0xD613;
|
||||
const T_TABLE_WIDTH: u16 = 0xF614;
|
||||
const T_VERT_MERGE: u16 = 0xD62B;
|
||||
const T_VERT_ALIGN: u16 = 0xD62C;
|
||||
const T_CELL_PADDING_DEFAULT: u16 = 0xD634;
|
||||
const T_CELL_WIDTH: u16 = 0xD635;
|
||||
|
||||
/// Most cells a row holds (`itcMac` MUST be at most 63).
|
||||
pub(crate) const MAX_CELLS: usize = 63;
|
||||
/// Word's default space left of a cell's text (5.4 pt), when the table doesn't say.
|
||||
const DEFAULT_GAP: f32 = 5.4;
|
||||
|
||||
/// The TAPX sprms of row `ri` of `t` (of `rows`), for its row-end mark; `style_borders` are
|
||||
/// the borders of the table's style, for a table without its own.
|
||||
pub(crate) fn row_end(t: &Table, ri: usize, rows: usize, style_borders: Option<Borders>) -> Grpprl {
|
||||
let mut g = Grpprl::default();
|
||||
let Some(row) = t.rows.get(ri) else { return g };
|
||||
let cells = &row.cells.get(..row.cells.len().min(MAX_CELLS)).unwrap_or(&[]);
|
||||
let n = cells.len();
|
||||
let tp = &t.props;
|
||||
|
||||
// Alignment, indent and the cell gap.
|
||||
let jc: u16 = match tp.align {
|
||||
Some(Align::Center) => 1,
|
||||
Some(Align::Right) => 2,
|
||||
_ => 0,
|
||||
};
|
||||
if jc != 0 {
|
||||
g.u16(T_JC90, jc);
|
||||
g.u16(T_JC, jc);
|
||||
}
|
||||
let gap = tp.cell_margins.map_or(DEFAULT_GAP, |m| m[1]).clamp(0.0, 100.0);
|
||||
g.i16(T_DXA_GAP_HALF, twips(gap));
|
||||
let indent = tp.indent.unwrap_or(0.0);
|
||||
g.i16(T_DXA_LEFT, twips(indent + gap));
|
||||
if let Some(m) = tp.cell_margins {
|
||||
// CSSAOperand: itcFirst, itcLim, grfbrc (top 1, left 2, bottom 4, right 8), ftsWidth
|
||||
// (3 = twips), wWidth.
|
||||
for (side, v) in [(1u8, m[0]), (2, m[1]), (4, m[2]), (8, m[3])] {
|
||||
let w = twips(v.max(0.0)) as u16;
|
||||
let mut op = vec![0, n.max(1) as u8, side, 3];
|
||||
op.extend_from_slice(&w.to_le_bytes());
|
||||
g.var(T_CELL_PADDING_DEFAULT, &op);
|
||||
}
|
||||
}
|
||||
if let Some(pct) = tp.width_pct {
|
||||
// ftsWidth 2: fiftieths of a percent.
|
||||
let mut op = vec![2u8];
|
||||
op.extend_from_slice(&((pct.clamp(0.0, 600.0) * 50.0).round() as u16).to_le_bytes());
|
||||
g.raw(T_TABLE_WIDTH, &op);
|
||||
} else if let Some(w) = tp.width {
|
||||
let mut op = vec![3u8];
|
||||
op.extend_from_slice(&(twips(w.max(0.0)) as u16).to_le_bytes());
|
||||
g.raw(T_TABLE_WIDTH, &op);
|
||||
}
|
||||
|
||||
// Row height and flags.
|
||||
if let Some(h) = row.props.height {
|
||||
let tw = twips(h.max(0.0));
|
||||
g.i16(T_ROW_HEIGHT, if row.props.height_rule == HeightRule::Exact { tw.saturating_neg() } else { tw });
|
||||
}
|
||||
if row.props.header {
|
||||
g.u8(T_TABLE_HEADER, 1);
|
||||
}
|
||||
if row.props.cant_split {
|
||||
g.u8(T_CANT_SPLIT90, 1);
|
||||
g.u8(T_CANT_SPLIT, 1);
|
||||
}
|
||||
|
||||
// The table's borders: its own, else its style's.
|
||||
let borders = tp.borders.or(style_borders).unwrap_or_default();
|
||||
let six = [borders.top, borders.left, borders.bottom, borders.right, borders.between, borders.inside_v];
|
||||
let mut b80 = Vec::with_capacity(24);
|
||||
let mut b = Vec::with_capacity(48);
|
||||
for side in &six {
|
||||
b80.extend_from_slice(&side.as_ref().map_or([0; 4], brc80));
|
||||
b.extend_from_slice(&side.as_ref().map_or([0; 8], brc));
|
||||
}
|
||||
g.var(T_TABLE_BORDERS80, &b80);
|
||||
g.var(T_TABLE_BORDERS, &b);
|
||||
|
||||
// Cell edges: each cell spans its grid columns (from the row's `grid_before`), else its own
|
||||
// preferred width; the edges start at the table's indent.
|
||||
let widths = cell_widths(t, ri);
|
||||
let mut edges: Vec<i16> = Vec::with_capacity(n + 1);
|
||||
let mut x = twips(indent);
|
||||
edges.push(x);
|
||||
for w in &widths {
|
||||
x = x.saturating_add(twips(*w).max(1));
|
||||
edges.push(x);
|
||||
}
|
||||
// TDefTableOperand: cb, itcMac, rgdxaCenter, rgTc80.
|
||||
let mut body = vec![n as u8];
|
||||
for e in &edges {
|
||||
body.extend_from_slice(&e.to_le_bytes());
|
||||
}
|
||||
for (ci, cell) in cells.iter().enumerate() {
|
||||
let mut grf: u16 = 0;
|
||||
match cell.props.vmerge {
|
||||
VMerge::Restart => grf |= 0x20 | 0x40,
|
||||
VMerge::Continue => grf |= 0x20,
|
||||
VMerge::None => {}
|
||||
}
|
||||
grf |= valign_bits(cell.props.valign) << 7;
|
||||
grf |= 3 << 9; // ftsWidth: twips
|
||||
if cell.props.no_wrap {
|
||||
grf |= 1 << 13;
|
||||
}
|
||||
body.extend_from_slice(&grf.to_le_bytes());
|
||||
body.extend_from_slice(&(twips(widths.get(ci).copied().unwrap_or(0.0)) as u16).to_le_bytes());
|
||||
let cb = cell_borders(&borders, cell.props.borders.as_ref(), ri, rows, ci, n);
|
||||
for side in [cb.top, cb.left, cb.bottom, cb.right] {
|
||||
body.extend_from_slice(&side.as_ref().map_or([0; 4], brc80));
|
||||
}
|
||||
}
|
||||
let cb = u16::try_from(body.len() + 1).unwrap_or(u16::MAX);
|
||||
let mut op = cb.to_le_bytes().to_vec();
|
||||
op.extend_from_slice(&body);
|
||||
g.raw(T_DEF_TABLE, &op);
|
||||
|
||||
// Cell shading, in three arrays of up to 22 cells each.
|
||||
if cells.iter().any(|c| c.props.shading.is_some()) || tp.shading.is_some() {
|
||||
for (k, op) in T_DEF_TABLE_SHD.iter().enumerate() {
|
||||
let chunk = cells.get(k * 22..((k + 1) * 22).min(n)).unwrap_or(&[]);
|
||||
if chunk.is_empty() {
|
||||
break;
|
||||
}
|
||||
let mut v = Vec::with_capacity(chunk.len() * 10);
|
||||
for c in chunk {
|
||||
v.extend_from_slice(&shd(c.props.shading.or(tp.shading)));
|
||||
}
|
||||
g.var(*op, &v);
|
||||
}
|
||||
}
|
||||
// Per-cell preferred widths, vertical merges and alignment, for the readers that take them
|
||||
// from sprms rather than from the TC80s.
|
||||
for (ci, cell) in cells.iter().enumerate() {
|
||||
let i = ci as u8;
|
||||
let mut op = vec![i, i + 1];
|
||||
if let Some(pct) = cell.props.width_pct {
|
||||
op.push(2);
|
||||
op.extend_from_slice(&((pct.clamp(0.0, 100.0) * 50.0).round() as u16).to_le_bytes());
|
||||
} else {
|
||||
op.push(3);
|
||||
op.extend_from_slice(&(twips(widths.get(ci).copied().unwrap_or(0.0)) as u16).to_le_bytes());
|
||||
}
|
||||
g.var(T_CELL_WIDTH, &op);
|
||||
match cell.props.vmerge {
|
||||
VMerge::Restart => g.var(T_VERT_MERGE, &[i, 3]),
|
||||
VMerge::Continue => g.var(T_VERT_MERGE, &[i, 1]),
|
||||
VMerge::None => {}
|
||||
}
|
||||
if let Some(v) = cell.props.valign
|
||||
&& v != VAlign::Top
|
||||
{
|
||||
g.var(T_VERT_ALIGN, &[i, i + 1, valign_bits(Some(v)) as u8]);
|
||||
}
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
fn valign_bits(v: Option<VAlign>) -> u16 {
|
||||
match v {
|
||||
Some(VAlign::Center) => 1,
|
||||
Some(VAlign::Bottom) => 2,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Each cell's width in points: the grid columns it spans, else its preferred width, else an
|
||||
/// equal share of a 6-inch row.
|
||||
fn cell_widths(t: &Table, ri: usize) -> Vec<f32> {
|
||||
let Some(row) = t.rows.get(ri) else { return Vec::new() };
|
||||
let n = row.cells.len().min(MAX_CELLS);
|
||||
let mut col = row.props.grid_before as usize;
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for cell in row.cells.iter().take(n) {
|
||||
let span = (cell.props.span.max(1) as usize).min(MAX_CELLS);
|
||||
let from_grid: f32 = t.grid.get(col..col.saturating_add(span).min(t.grid.len())).map_or(0.0, |g| g.iter().sum());
|
||||
col = col.saturating_add(span);
|
||||
let w = if from_grid > 0.0 && from_grid.is_finite() {
|
||||
from_grid
|
||||
} else {
|
||||
cell.props.width.filter(|w| *w > 0.0 && w.is_finite()).unwrap_or(432.0 / n.max(1) as f32)
|
||||
};
|
||||
out.push(w.clamp(1.0, 1584.0));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The borders cell `ci` of `n` in row `ri` of `rows` shows: the table's outer borders on the
|
||||
/// table's edges, its inside ones between cells, the cell's own over both.
|
||||
fn cell_borders(table: &Borders, own: Option<&Borders>, ri: usize, rows: usize, ci: usize, n: usize) -> Borders {
|
||||
let pick = |own: Option<Border>, edge: bool, outer: Option<Border>, inner: Option<Border>| own.or(if edge { outer } else { inner });
|
||||
let own = own.copied().unwrap_or_default();
|
||||
Borders {
|
||||
top: pick(own.top, ri == 0, table.top, table.between),
|
||||
bottom: pick(own.bottom, ri + 1 == rows, table.bottom, table.between),
|
||||
left: pick(own.left, ci == 0, table.left, table.inside_v),
|
||||
right: pick(own.right, ci + 1 == n, table.right, table.inside_v),
|
||||
between: None,
|
||||
inside_v: None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,707 @@
|
||||
//! Round trips: documents written as `.doc` and read back with the reader, which follows the
|
||||
//! same specification independently of the writer.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use craft_wordmodel::para::{InlineObject, NoteKind, Paragraph};
|
||||
use craft_wordmodel::props::*;
|
||||
use craft_wordmodel::section::{Columns, SectionProps, SectionStart};
|
||||
use craft_wordmodel::table::{Cell, Row, Table};
|
||||
use craft_wordmodel::{Block, Document, PartKind, para_block};
|
||||
|
||||
use crate::{read, write};
|
||||
|
||||
fn doc_of(paras: Vec<Paragraph>) -> Document {
|
||||
let mut d = Document::new();
|
||||
d.body = paras.into_iter().map(para_block).collect();
|
||||
d
|
||||
}
|
||||
|
||||
fn round(d: &Document) -> Document {
|
||||
let bytes = write(d).expect("write");
|
||||
read(&bytes).expect("read back")
|
||||
}
|
||||
|
||||
fn texts(d: &Document) -> Vec<String> {
|
||||
d.body.iter().filter_map(|b| b.as_para().map(|p| p.plain_text())).collect()
|
||||
}
|
||||
|
||||
fn para(text: &str) -> Paragraph {
|
||||
Paragraph::with_text(text, CharProps::default())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_text_round_trips() {
|
||||
let d = doc_of(vec![
|
||||
para("Hello, world."),
|
||||
para(""),
|
||||
para("Ünïcødé — “quotes” 😀 𝄞 中文"),
|
||||
para("tab\there, line\nbreak, nb\u{2011}hyphen, soft\u{AD}hyphen"),
|
||||
]);
|
||||
let r = round(&d);
|
||||
assert_eq!(texts(&r), texts(&d));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_document_is_one_paragraph() {
|
||||
let r = round(&Document::new());
|
||||
assert_eq!(texts(&r), vec![String::new()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn character_formatting_round_trips() {
|
||||
let mut p = para("bold italic under red big font high sup");
|
||||
let fmt = |p: &mut Paragraph, a: usize, b: usize, f: &dyn Fn(&mut CharProps)| p.format(a, b, f).unwrap();
|
||||
fmt(&mut p, 0, 4, &|c| c.bold = Some(true));
|
||||
fmt(&mut p, 5, 11, &|c| c.italic = Some(true));
|
||||
fmt(&mut p, 12, 17, &|c| c.underline = Some(Underline::Double));
|
||||
fmt(&mut p, 18, 21, &|c| c.color = Some(TextColor::Rgb(craft_wordmodel::Rgb(0xC0, 0x10, 0x20))));
|
||||
fmt(&mut p, 22, 25, &|c| c.size = Some(18.5));
|
||||
fmt(&mut p, 26, 30, &|c| c.font = Some("Georgia".into()));
|
||||
fmt(&mut p, 31, 35, &|c| c.highlight = Some(Highlight::Yellow));
|
||||
fmt(&mut p, 36, 39, &|c| {
|
||||
c.strike = Some(true);
|
||||
c.small_caps = Some(true);
|
||||
});
|
||||
let r = round(&doc_of(vec![p.clone()]));
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
assert_eq!(q.plain_text(), p.plain_text());
|
||||
let at = |i: usize| q.props_at(i + 1).clone();
|
||||
assert_eq!(at(0).bold, Some(true));
|
||||
assert_eq!(at(5).italic, Some(true));
|
||||
assert_eq!(at(12).underline, Some(Underline::Double));
|
||||
assert_eq!(at(18).color, Some(TextColor::Rgb(craft_wordmodel::Rgb(0xC0, 0x10, 0x20))));
|
||||
assert_eq!(at(22).size, Some(18.5));
|
||||
assert_eq!(at(26).font.as_deref(), Some("Georgia"));
|
||||
assert_eq!(at(36).strike, Some(true));
|
||||
assert_eq!(at(36).small_caps, Some(true));
|
||||
assert_ne!(at(5).bold, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paragraph_formatting_round_trips() {
|
||||
let mut p = para("Centred and indented");
|
||||
p.props.align = Some(Align::Center);
|
||||
p.props.indent_left = Some(36.0);
|
||||
p.props.indent_right = Some(18.0);
|
||||
p.props.indent_first = Some(-9.0);
|
||||
p.props.space_before = Some(12.0);
|
||||
p.props.space_after = Some(6.0);
|
||||
p.props.line_spacing = Some(LineSpacing::Multiple(1.5));
|
||||
p.props.keep_next = Some(true);
|
||||
p.props.page_break_before = Some(true);
|
||||
p.props.tabs = Some(vec![TabStop { pos: 72.0, align: TabAlign::Right, leader: TabLeader::Dot }]);
|
||||
p.props.borders = Some(Borders {
|
||||
bottom: Some(Border { style: BorderStyle::Double, width: 1.5, color: Some(craft_wordmodel::Rgb(0, 0, 255)), space: 1.0 }),
|
||||
..Default::default()
|
||||
});
|
||||
let mut exact = para("Exactly 14 pt");
|
||||
exact.props.line_spacing = Some(LineSpacing::Exactly(14.0));
|
||||
let r = round(&doc_of(vec![p.clone(), exact]));
|
||||
let q = &r.body[0].as_para().unwrap().props;
|
||||
assert_eq!(q.align, Some(Align::Center));
|
||||
assert_eq!(q.indent_left, Some(36.0));
|
||||
assert_eq!(q.indent_right, Some(18.0));
|
||||
assert_eq!(q.indent_first, Some(-9.0));
|
||||
assert_eq!(q.space_before, Some(12.0));
|
||||
assert_eq!(q.space_after, Some(6.0));
|
||||
assert_eq!(q.line_spacing, Some(LineSpacing::Multiple(1.5)));
|
||||
assert_eq!(q.keep_next, Some(true));
|
||||
assert_eq!(q.page_break_before, Some(true));
|
||||
assert_eq!(q.tabs, p.props.tabs);
|
||||
assert_eq!(q.borders.and_then(|b| b.bottom), p.props.borders.and_then(|b| b.bottom));
|
||||
assert_eq!(r.body[1].as_para().unwrap().props.line_spacing, Some(LineSpacing::Exactly(14.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn styles_round_trip() {
|
||||
let mut d = Document::new();
|
||||
let h1 = d.styles.styles.iter().find(|s| s.name == "Heading 1").map(|s| s.id.clone()).expect("a Heading 1 style");
|
||||
Arc::make_mut(&mut d.styles).styles.push(craft_wordmodel::styles::Style {
|
||||
id: "MyQuote".into(),
|
||||
name: "My Quote".into(),
|
||||
kind: craft_wordmodel::styles::StyleKind::Paragraph,
|
||||
based_on: Some("Normal".into()),
|
||||
chr: CharProps { italic: Some(true), ..Default::default() },
|
||||
para: ParaProps { indent_left: Some(24.0), ..Default::default() },
|
||||
..Default::default()
|
||||
});
|
||||
d.body = vec![para_block(para("Title text").styled(&h1)), para_block(para("Quoted").styled("MyQuote")), para_block(para("Body"))];
|
||||
let r = round(&d);
|
||||
assert_eq!(texts(&r), vec!["Title text", "Quoted", "Body"]);
|
||||
let style_of = |i: usize| r.body[i].as_para().unwrap().props.style.clone().and_then(|id| r.styles.get(&id).map(|s| s.name.clone()));
|
||||
assert_eq!(style_of(0).as_deref(), Some("Heading 1"));
|
||||
assert_eq!(style_of(1).as_deref(), Some("My Quote"));
|
||||
let q = r.styles.styles.iter().find(|s| s.name == "My Quote").unwrap();
|
||||
assert_eq!(q.chr.italic, Some(true));
|
||||
assert_eq!(q.para.indent_left, Some(24.0));
|
||||
// The heading keeps its outline level, the stylesheet its default font.
|
||||
let h = r.styles.styles.iter().find(|s| s.name == "Heading 1").unwrap();
|
||||
assert_eq!(h.para.outline_level, Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn many_paragraphs_page_their_formatting() {
|
||||
let paras: Vec<Paragraph> = (0..2000)
|
||||
.map(|i| {
|
||||
let mut p = para(&format!("Paragraph {i} with some text"));
|
||||
if i % 3 == 0 {
|
||||
p.format(0, 9, &|c| c.bold = Some(true)).unwrap();
|
||||
}
|
||||
if i % 5 == 0 {
|
||||
p.props.indent_left = Some(i as f32 % 100.0);
|
||||
}
|
||||
p
|
||||
})
|
||||
.collect();
|
||||
let d = doc_of(paras);
|
||||
let r = round(&d);
|
||||
assert_eq!(texts(&r), texts(&d));
|
||||
for i in [0usize, 3, 999, 1998] {
|
||||
let q = r.body[i].as_para().unwrap();
|
||||
assert_eq!(q.props_at(1).bold, if i % 3 == 0 { Some(true) } else { q.props_at(1).bold.filter(|_| false) }, "paragraph {i}");
|
||||
if i % 5 == 0 {
|
||||
assert_eq!(q.props.indent_left, Some(i as f32 % 100.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cell(text: &str) -> Cell {
|
||||
Cell { props: CellProps { span: 1, ..Default::default() }, blocks: vec![para_block(para(text))], controls: Default::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tables_round_trip() {
|
||||
let mut a = cell("A1");
|
||||
a.props.shading = Some(craft_wordmodel::Rgb(0xEE, 0xDD, 0x00));
|
||||
a.props.vmerge = VMerge::Restart;
|
||||
let mut b = cell("");
|
||||
b.props.vmerge = VMerge::Continue;
|
||||
let mut multi = cell("B1 first");
|
||||
multi.blocks.push(para_block(para("B1 second")));
|
||||
let t = Table {
|
||||
props: TableProps {
|
||||
borders: Some(Borders { top: Some(Border::single(0.5)), bottom: Some(Border::single(0.5)), ..Default::default() }),
|
||||
..Default::default()
|
||||
},
|
||||
grid: vec![100.0, 150.0, 80.0],
|
||||
rows: vec![
|
||||
Row { props: RowProps { header: true, ..Default::default() }, cells: vec![a, multi, cell("C1")], controls: Default::default() },
|
||||
Row {
|
||||
props: RowProps { height: Some(30.0), height_rule: HeightRule::Exact, ..Default::default() },
|
||||
cells: vec![b, cell("B2"), cell("C2")],
|
||||
controls: Default::default(),
|
||||
},
|
||||
],
|
||||
controls: Default::default(),
|
||||
};
|
||||
let mut d = Document::new();
|
||||
d.body = vec![para_block(para("Before")), Arc::new(Block::Table(t)), para_block(para("After"))];
|
||||
let r = round(&d);
|
||||
assert_eq!(r.body.len(), 3, "{:?}", r.body.iter().map(|b| b.as_para().map(|p| p.plain_text())).collect::<Vec<_>>());
|
||||
let t = r.body[1].as_table().expect("a table");
|
||||
assert_eq!(t.rows.len(), 2);
|
||||
assert_eq!(t.grid, vec![100.0, 150.0, 80.0]);
|
||||
let cell_text = |r: usize, c: usize| t.rows[r].cells[c].blocks.iter().filter_map(|b| b.as_para().map(|p| p.plain_text())).collect::<Vec<_>>();
|
||||
assert_eq!(cell_text(0, 0), vec!["A1"]);
|
||||
assert_eq!(cell_text(0, 1), vec!["B1 first", "B1 second"]);
|
||||
assert_eq!(cell_text(1, 2), vec!["C2"]);
|
||||
assert_eq!(t.rows[0].cells[0].props.shading, Some(craft_wordmodel::Rgb(0xEE, 0xDD, 0x00)));
|
||||
assert_eq!(t.rows[0].cells[0].props.vmerge, VMerge::Restart);
|
||||
assert_eq!(t.rows[1].cells[0].props.vmerge, VMerge::Continue);
|
||||
assert!(t.rows[0].props.header);
|
||||
assert_eq!(t.rows[1].props.height, Some(30.0));
|
||||
assert_eq!(t.rows[1].props.height_rule, HeightRule::Exact);
|
||||
assert_eq!(texts(&r), vec!["Before", "After"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sections_headers_and_footers_round_trip() {
|
||||
let mut d = Document::new();
|
||||
let h1 = d.add_part(PartKind::Header, vec![para_block(para("First header"))]);
|
||||
let f1 = d.add_part(PartKind::Footer, vec![para_block(para("First footer"))]);
|
||||
let h2 = d.add_part(PartKind::Header, vec![para_block(para("Landscape header"))]);
|
||||
let mut first = para("Portrait section");
|
||||
let mut s1 = SectionProps::default();
|
||||
s1.headers.default = Some(h1);
|
||||
s1.footers.default = Some(f1);
|
||||
s1.margin_left = 54.0;
|
||||
first.section = Some(Box::new(s1));
|
||||
let mut last =
|
||||
SectionProps { landscape: true, start: SectionStart::NextPage, columns: Columns { count: 2, ..Default::default() }, ..Default::default() };
|
||||
(last.page_w, last.page_h) = (792.0, 612.0);
|
||||
last.headers.default = Some(h2);
|
||||
d.body = vec![para_block(first), para_block(para("Landscape section"))];
|
||||
d.last_section = last;
|
||||
let r = round(&d);
|
||||
assert_eq!(texts(&r), vec!["Portrait section", "Landscape section"]);
|
||||
let s = r.body[0].as_para().unwrap().section.as_ref().expect("the first section ends here");
|
||||
assert_eq!(s.margin_left, 54.0);
|
||||
assert!(!s.landscape);
|
||||
let part_text =
|
||||
|id: Option<u32>| id.and_then(|i| r.parts.get(&i)).and_then(|p| p.blocks.first()).and_then(|b| b.as_para()).map(|p| p.plain_text());
|
||||
assert_eq!(part_text(s.headers.default).as_deref(), Some("First header"));
|
||||
assert_eq!(part_text(s.footers.default).as_deref(), Some("First footer"));
|
||||
assert!(r.last_section.landscape);
|
||||
assert_eq!((r.last_section.page_w, r.last_section.page_h), (792.0, 612.0));
|
||||
assert_eq!(r.last_section.columns.count, 2);
|
||||
assert_eq!(part_text(r.last_section.headers.default).as_deref(), Some("Landscape header"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notes_round_trip() {
|
||||
let mut d = Document::new();
|
||||
let f = d.add_part(PartKind::Footnote, vec![para_block(para("The footnote."))]);
|
||||
let e = d.add_part(PartKind::Endnote, vec![para_block(para("The endnote."))]);
|
||||
let mut p = para("Text.");
|
||||
p.insert_object(5, InlineObject::NoteRef { kind: NoteKind::Footnote, id: f, custom: String::new() }, &CharProps::default()).unwrap();
|
||||
let end = p.text.len();
|
||||
p.insert_object(end, InlineObject::NoteRef { kind: NoteKind::Endnote, id: e, custom: String::new() }, &CharProps::default()).unwrap();
|
||||
d.body = vec![para_block(p), para_block(para("After."))];
|
||||
let r = round(&d);
|
||||
assert_eq!(texts(&r)[1], "After.");
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
let notes: Vec<(NoteKind, String)> = q
|
||||
.objects
|
||||
.iter()
|
||||
.filter_map(|o| match o {
|
||||
InlineObject::NoteRef { kind, id, .. } => {
|
||||
Some((*kind, r.parts.get(id).and_then(|p| p.blocks.first()).and_then(|b| b.as_para()).map(|p| p.plain_text()).unwrap_or_default()))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(notes, vec![(NoteKind::Footnote, "The footnote.".to_string()), (NoteKind::Endnote, "The endnote.".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fields_links_and_bookmarks_round_trip() {
|
||||
let mut p = para("Page , see Example and here.");
|
||||
p.insert_object(
|
||||
5,
|
||||
InlineObject::Field { instr: "PAGE".into(), result: "7".into(), locked: false, dirty: false, code: false, form: String::new() },
|
||||
&CharProps::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let at = p.text.find("Example").unwrap();
|
||||
p.format(at, at + 7, &|c| c.link = Some("https://example.org/a?b=1".into())).unwrap();
|
||||
let here = p.text.find("here").unwrap();
|
||||
p.insert_object(here + 4, InlineObject::BookmarkEnd { name: "Spot".into() }, &CharProps::default()).unwrap();
|
||||
p.insert_object(here, InlineObject::BookmarkStart { name: "Spot".into() }, &CharProps::default()).unwrap();
|
||||
let r = round(&doc_of(vec![p]));
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
assert!(q.objects.iter().any(|o| matches!(o, InlineObject::Field { instr, result, .. } if instr == "PAGE" && result == "7")), "{:?}", q.objects);
|
||||
let link_at = q.text.find("Example").unwrap();
|
||||
assert_eq!(q.props_at(link_at + 1).link.as_deref(), Some("https://example.org/a?b=1"));
|
||||
assert!(q.objects.iter().any(|o| matches!(o, InlineObject::BookmarkStart { name } if name == "Spot")));
|
||||
assert!(q.objects.iter().any(|o| matches!(o, InlineObject::BookmarkEnd { name } if name == "Spot")));
|
||||
}
|
||||
|
||||
/// A 2×1 PNG.
|
||||
fn png() -> Vec<u8> {
|
||||
let img = image::RgbaImage::from_pixel(2, 1, image::Rgba([10, 200, 30, 255]));
|
||||
let mut out = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(img).write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_pictures_round_trip() {
|
||||
let mut d = Document::new();
|
||||
let bytes = png();
|
||||
d.media.insert("pic.png".into(), Arc::new(bytes.clone()));
|
||||
let mut p = para("Picture: .");
|
||||
p.insert_object(
|
||||
9,
|
||||
InlineObject::Image {
|
||||
media: "pic.png".into(),
|
||||
w: 144.0,
|
||||
h: 72.0,
|
||||
alt: "A green bar".into(),
|
||||
float: Default::default(),
|
||||
crop: [0.0; 4],
|
||||
ole: None,
|
||||
fx: Default::default(),
|
||||
},
|
||||
&CharProps::default(),
|
||||
)
|
||||
.unwrap();
|
||||
d.body = vec![para_block(p)];
|
||||
let r = round(&d);
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
let Some(InlineObject::Image { media, w, h, alt, .. }) = q.objects.first() else { panic!("no picture: {:?}", q.objects) };
|
||||
assert!((w - 144.0).abs() < 0.1 && (h - 72.0).abs() < 0.1, "{w}×{h}");
|
||||
assert_eq!(alt, "A green bar");
|
||||
assert_eq!(r.media.get(media).map(|m| m.as_slice()), Some(bytes.as_slice()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_properties_round_trip() {
|
||||
let mut d = Document::new();
|
||||
d.core.title = "Quarterly".into();
|
||||
d.core.creator = "A. Writer".into();
|
||||
let r = round(&d);
|
||||
assert_eq!(r.core.title, "Quarterly");
|
||||
assert_eq!(r.core.creator, "A. Writer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lists_round_trip() {
|
||||
use craft_wordmodel::numbering::{AbstractNum, Level, Num};
|
||||
use craft_wordmodel::section::NumFormat;
|
||||
let mut d = Document::new();
|
||||
let levels = vec![
|
||||
Level { start: 1, format: NumFormat::Decimal, text: "%1.".into(), indent: 36.0, hanging: 18.0, ..Level::default() },
|
||||
Level { start: 1, format: NumFormat::LowerLetter, text: "%1.%2)".into(), indent: 72.0, hanging: 18.0, ..Level::default() },
|
||||
Level { start: 1, format: NumFormat::Bullet, text: "\u{2022}".into(), indent: 108.0, hanging: 18.0, ..Level::default() },
|
||||
];
|
||||
let numbering = Arc::make_mut(&mut d.numbering);
|
||||
numbering.abstracts = vec![AbstractNum { id: 7, levels, ..Default::default() }];
|
||||
numbering.nums = vec![Num { id: 3, abstract_id: 7, start_overrides: vec![(0, 4)], ..Default::default() }];
|
||||
let item = |text: &str, level: u8| {
|
||||
let mut p = para(text);
|
||||
p.props.numbering = Some(NumRef { num: 3, level });
|
||||
para_block(p)
|
||||
};
|
||||
d.body = vec![item("One", 0), item("One a", 1), item("Dot", 2)];
|
||||
let r = round(&d);
|
||||
let num = |i: usize| r.body[i].as_para().unwrap().props.numbering.expect("numbered");
|
||||
assert_eq!((num(0).level, num(1).level, num(2).level), (0, 1, 2));
|
||||
let n = r.numbering.nums.iter().find(|n| n.id == num(0).num).expect("the list");
|
||||
assert_eq!(n.start_overrides, vec![(0, 4)]);
|
||||
let a = r.numbering.abstracts.iter().find(|a| a.id == n.abstract_id).expect("its levels");
|
||||
assert_eq!(a.levels[0].text, "%1.");
|
||||
assert_eq!(a.levels[1].text, "%1.%2)");
|
||||
assert_eq!(a.levels[1].format, NumFormat::LowerLetter);
|
||||
assert_eq!(a.levels[2].text, "\u{2022}");
|
||||
assert_eq!(a.levels[2].format, NumFormat::Bullet);
|
||||
assert_eq!((a.levels[1].indent, a.levels[1].hanging), (72.0, 18.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_changes_round_trip() {
|
||||
use craft_wordmodel::{Revision, RevisionKind};
|
||||
let mut d = Document::new();
|
||||
d.revisions = vec![
|
||||
Revision::new(RevisionKind::Insert, "Ada", "2025-03-04T05:06:00Z"),
|
||||
Revision::new(RevisionKind::Delete, "Grace", "2025-03-05T10:20:00Z"),
|
||||
];
|
||||
let mut p = para("kept added removed");
|
||||
p.format(5, 10, &|c| c.ins = Some(0)).unwrap();
|
||||
p.format(11, 18, &|c| c.del = Some(1)).unwrap();
|
||||
d.body = vec![para_block(p)];
|
||||
let r = round(&d);
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
assert_eq!(q.plain_text(), "kept added removed");
|
||||
let ins = q.props_at(6).ins.and_then(|i| r.revisions.get(i as usize)).expect("an insertion");
|
||||
assert_eq!((ins.kind, ins.author.as_str(), ins.date.as_str()), (RevisionKind::Insert, "Ada", "2025-03-04T05:06:00Z"));
|
||||
let del = q.props_at(12).del.and_then(|i| r.revisions.get(i as usize)).expect("a deletion");
|
||||
assert_eq!((del.kind, del.author.as_str(), del.date.as_str()), (RevisionKind::Delete, "Grace", "2025-03-05T10:20:00Z"));
|
||||
assert_eq!(q.props_at(1).ins, None);
|
||||
assert_eq!(q.mark.ins, None);
|
||||
}
|
||||
|
||||
/// An inserted or deleted paragraph break (the paragraph mark's revision) round-trips too.
|
||||
#[test]
|
||||
fn tracked_paragraph_marks_round_trip() {
|
||||
use craft_wordmodel::{Revision, RevisionKind};
|
||||
let mut d = Document::new();
|
||||
d.revisions = vec![
|
||||
Revision::new(RevisionKind::Insert, "Ada", "2025-03-04T05:06:00Z"),
|
||||
Revision::new(RevisionKind::Delete, "Grace", "2025-03-05T10:20:00Z"),
|
||||
];
|
||||
let (mut inserted, mut deleted) = (para("inserted break"), para("deleted break"));
|
||||
inserted.mark.ins = Some(0);
|
||||
deleted.mark.del = Some(1);
|
||||
d.body = vec![para_block(inserted), para_block(deleted), para_block(para("last"))];
|
||||
let r = round(&d);
|
||||
let mark = |i: usize| r.body[i].as_para().unwrap().mark.clone();
|
||||
let ins = mark(0).ins.and_then(|i| r.revisions.get(i as usize)).expect("an inserted mark");
|
||||
assert_eq!((ins.kind, ins.author.as_str()), (RevisionKind::Insert, "Ada"));
|
||||
let del = mark(1).del.and_then(|i| r.revisions.get(i as usize)).expect("a deleted mark");
|
||||
assert_eq!((del.kind, del.author.as_str()), (RevisionKind::Delete, "Grace"));
|
||||
assert_eq!((mark(2).ins, mark(2).del), (None, None));
|
||||
assert_eq!(r.body[0].as_para().unwrap().props_at(1).ins, None, "the text itself isn't marked");
|
||||
}
|
||||
|
||||
/// A tracked insertion or deletion of a picture, a note reference or a field round-trips: the
|
||||
/// reader takes each anchor's revision marks from its CHPX.
|
||||
#[test]
|
||||
fn tracked_object_anchors_round_trip() {
|
||||
use craft_wordmodel::{Revision, RevisionKind};
|
||||
let mut d = Document::new();
|
||||
d.revisions = vec![
|
||||
Revision::new(RevisionKind::Insert, "Ada", "2025-03-04T05:06:00Z"),
|
||||
Revision::new(RevisionKind::Delete, "Grace", "2025-03-05T10:20:00Z"),
|
||||
];
|
||||
d.media.insert("pic.png".into(), Arc::new(png()));
|
||||
let f = d.add_part(PartKind::Footnote, vec![para_block(para("The footnote."))]);
|
||||
let inserted = CharProps { ins: Some(0), ..Default::default() };
|
||||
let deleted = CharProps { del: Some(1), ..Default::default() };
|
||||
let mut p = para("a b c d");
|
||||
let picture = InlineObject::Image {
|
||||
media: "pic.png".into(),
|
||||
w: 20.0,
|
||||
h: 10.0,
|
||||
alt: String::new(),
|
||||
float: Default::default(),
|
||||
crop: [0.0; 4],
|
||||
ole: None,
|
||||
fx: Default::default(),
|
||||
};
|
||||
let page = InlineObject::Field { instr: "PAGE".into(), result: "1".into(), locked: false, dirty: false, code: false, form: String::new() };
|
||||
let note = InlineObject::NoteRef { kind: NoteKind::Footnote, id: f, custom: String::new() };
|
||||
// Inserted back to front so the offsets stay put: picture after "a", note after "b",
|
||||
// field after "c" and an untracked field after "d".
|
||||
p.insert_object(7, page.clone(), &CharProps::default()).unwrap();
|
||||
p.insert_object(5, page, &inserted).unwrap();
|
||||
p.insert_object(3, note, &deleted).unwrap();
|
||||
p.insert_object(1, picture, &inserted).unwrap();
|
||||
d.body = vec![para_block(p)];
|
||||
let r = round(&d);
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
let anchors: Vec<(&str, Option<RevisionKind>, Option<RevisionKind>)> = q
|
||||
.text
|
||||
.match_indices('\u{FFFC}')
|
||||
.zip(&q.objects)
|
||||
.map(|((at, _), o)| {
|
||||
let props = q.props_of_char(at);
|
||||
let kind = |rev: Option<u32>| rev.and_then(|i| r.revisions.get(i as usize)).map(|rev| rev.kind);
|
||||
let name = match o {
|
||||
InlineObject::Image { .. } => "picture",
|
||||
InlineObject::NoteRef { .. } => "note",
|
||||
InlineObject::Field { .. } => "field",
|
||||
_ => "other",
|
||||
};
|
||||
(name, kind(props.ins), kind(props.del))
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
anchors,
|
||||
vec![
|
||||
("picture", Some(RevisionKind::Insert), None),
|
||||
("note", None, Some(RevisionKind::Delete)),
|
||||
("field", Some(RevisionKind::Insert), None),
|
||||
("field", None, None),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "the reader pairs bookmark ends through FBKF.ibkl once craft-libs#1817 is in"]
|
||||
fn overlapping_bookmarks_keep_their_ends() {
|
||||
// Outer starts first and ends last; inner ends before it: the ends sort apart from the starts.
|
||||
let mut p = para("abcdef");
|
||||
p.insert_object(6, InlineObject::BookmarkEnd { name: "Outer".into() }, &CharProps::default()).unwrap();
|
||||
p.insert_object(4, InlineObject::BookmarkEnd { name: "Inner".into() }, &CharProps::default()).unwrap();
|
||||
p.insert_object(2, InlineObject::BookmarkStart { name: "Inner".into() }, &CharProps::default()).unwrap();
|
||||
p.insert_object(0, InlineObject::BookmarkStart { name: "Outer".into() }, &CharProps::default()).unwrap();
|
||||
let r = round(&doc_of(vec![p.clone()]));
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
let marks = |p: &Paragraph| -> Vec<String> {
|
||||
p.objects
|
||||
.iter()
|
||||
.filter_map(|o| match o {
|
||||
InlineObject::BookmarkStart { name } => Some(format!("+{name}")),
|
||||
InlineObject::BookmarkEnd { name } => Some(format!("-{name}")),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
assert_eq!(marks(q), marks(&p));
|
||||
assert_eq!(q.text, p.text);
|
||||
}
|
||||
|
||||
/// What the writer writes for a character's vertical position: superscript and subscript as
|
||||
/// `sprmCIss`, a raised or lowered position as `sprmCHpsPos`, character spacing as
|
||||
/// `sprmCDxaSpace`. The reader reads them once craft-libs#1799 is in.
|
||||
#[test]
|
||||
#[ignore = "the reader reads sprmCIss, sprmCHpsPos and sprmCDxaSpace once craft-libs#1799 is in"]
|
||||
fn position_and_spacing_round_trip() {
|
||||
let mut p = para("sup sub up sp");
|
||||
p.format(0, 3, &|c| c.vert_align = Some(VertAlign::Superscript)).unwrap();
|
||||
p.format(4, 7, &|c| c.vert_align = Some(VertAlign::Subscript)).unwrap();
|
||||
p.format(8, 10, &|c| c.position = Some(3.0)).unwrap();
|
||||
p.format(11, 13, &|c| c.spacing = Some(1.5)).unwrap();
|
||||
let r = round(&doc_of(vec![p]));
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
assert_eq!(q.props_at(1).vert_align, Some(VertAlign::Superscript));
|
||||
assert_eq!(q.props_at(5).vert_align, Some(VertAlign::Subscript));
|
||||
assert_eq!(q.props_at(9).position, Some(3.0));
|
||||
assert_ne!(q.props_at(9).vert_align, Some(VertAlign::Superscript));
|
||||
assert_eq!(q.props_at(12).spacing, Some(1.5));
|
||||
}
|
||||
|
||||
/// Double strikethrough, emboss and the double-wave underline. The reader reads them once
|
||||
/// craft-libs#1969 is fixed.
|
||||
#[test]
|
||||
#[ignore = "the reader reads sprmCFDStrike, sprmCFEmboss and Kul 0x2B once craft-libs#1969 is fixed"]
|
||||
fn character_effects_round_trip() {
|
||||
let mut p = para("dbl emb");
|
||||
p.format(0, 3, &|c| c.double_strike = Some(true)).unwrap();
|
||||
p.format(4, 7, &|c| {
|
||||
c.emboss = Some(true);
|
||||
c.underline = Some(Underline::DoubleWave);
|
||||
})
|
||||
.unwrap();
|
||||
let r = round(&doc_of(vec![p]));
|
||||
let q = r.body[0].as_para().unwrap();
|
||||
assert_eq!(q.props_at(1).double_strike, Some(true));
|
||||
assert_eq!(q.props_at(5).emboss, Some(true));
|
||||
assert_eq!(q.props_at(5).underline, Some(Underline::DoubleWave));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "the reader reads sprmCHighlight as an Ico once craft-libs#1854 is in"]
|
||||
fn every_highlight_colour_round_trips() {
|
||||
for h in Highlight::ALL {
|
||||
let mut p = para("x");
|
||||
p.format(0, 1, &|c| c.highlight = Some(h)).unwrap();
|
||||
let r = round(&doc_of(vec![p]));
|
||||
assert_eq!(r.body[0].as_para().unwrap().props_at(0).highlight, Some(h), "{h:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A section's column count survives a save.
|
||||
#[test]
|
||||
fn section_columns_round_trip() {
|
||||
let mut d = Document::new();
|
||||
d.last_section.columns = Columns { count: 3, space: 18.0, ..Default::default() };
|
||||
d.body = vec![para_block(para("Three columns"))];
|
||||
let r = round(&d);
|
||||
assert_eq!(r.last_section.columns.count, 3);
|
||||
}
|
||||
|
||||
/// The page-number start and format, the column spacing, and the paragraph's shading, widow
|
||||
/// control and contextual spacing, as the writer writes them. Each needs a reader change that is
|
||||
/// still open: page numbers craft-libs#1354, column spacing and widow control craft-libs#1799,
|
||||
/// shading craft-libs#1808, contextual spacing craft-libs#1969.
|
||||
#[test]
|
||||
#[ignore = "needs the reader changes in craft-libs#1354, #1799 and #1808, and craft-libs#1969"]
|
||||
fn page_numbering_and_paragraph_shading_round_trip() {
|
||||
let mut d = Document::new();
|
||||
d.last_section.page_num_start = Some(5);
|
||||
d.last_section.page_num_format = craft_wordmodel::section::NumFormat::LowerRoman;
|
||||
d.last_section.columns = Columns { count: 3, space: 18.0, ..Default::default() };
|
||||
let mut p = para("Shaded");
|
||||
p.props.shading = Some(craft_wordmodel::Rgb(0xDD, 0xEE, 0xFF));
|
||||
p.props.widow_control = Some(false);
|
||||
p.props.contextual_spacing = Some(true);
|
||||
d.body = vec![para_block(p)];
|
||||
let r = round(&d);
|
||||
assert_eq!(r.last_section.page_num_start, Some(5));
|
||||
assert_eq!(r.last_section.page_num_format, craft_wordmodel::section::NumFormat::LowerRoman);
|
||||
assert_eq!((r.last_section.columns.count, r.last_section.columns.space), (3, 18.0));
|
||||
let q = &r.body[0].as_para().unwrap().props;
|
||||
assert_eq!(q.shading, Some(craft_wordmodel::Rgb(0xDD, 0xEE, 0xFF)));
|
||||
assert_eq!(q.widow_control, Some(false));
|
||||
assert_eq!(q.contextual_spacing, Some(true));
|
||||
}
|
||||
|
||||
fn table_of(grid: Vec<f32>, rows: Vec<Vec<Cell>>) -> Table {
|
||||
Table {
|
||||
props: TableProps::default(),
|
||||
grid,
|
||||
rows: rows.into_iter().map(|cells| Row { props: RowProps::default(), cells, controls: Default::default() }).collect(),
|
||||
controls: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_tables_become_paragraphs_of_their_cell() {
|
||||
let inner = table_of(vec![50.0, 50.0], vec![vec![cell("i1"), cell("i2")]]);
|
||||
let mut outer_cell = cell("before");
|
||||
outer_cell.blocks.push(Arc::new(Block::Table(inner)));
|
||||
let mut d = Document::new();
|
||||
d.body = vec![Arc::new(Block::Table(table_of(vec![200.0], vec![vec![outer_cell]]))), para_block(para("end"))];
|
||||
let r = round(&d);
|
||||
let t = r.body[0].as_table().expect("the outer table");
|
||||
let texts: Vec<String> = t.rows[0].cells[0].blocks.iter().filter_map(|b| b.as_para().map(|p| p.plain_text())).collect();
|
||||
assert_eq!(texts, vec!["before", "i1", "i2"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_table_ending_the_document_gets_a_paragraph_after_it() {
|
||||
let mut d = Document::new();
|
||||
d.body = vec![Arc::new(Block::Table(table_of(vec![100.0], vec![vec![cell("only")]])))];
|
||||
let r = round(&d);
|
||||
assert!(r.body[0].as_table().is_some());
|
||||
assert_eq!(texts(&r), vec![String::new()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rows_wider_than_the_format_keep_their_first_cells() {
|
||||
let cells: Vec<Cell> = (0..80).map(|i| cell(&format!("c{i}"))).collect();
|
||||
let mut d = Document::new();
|
||||
d.body = vec![Arc::new(Block::Table(table_of(vec![10.0; 80], vec![cells]))), para_block(para("after"))];
|
||||
// Never a panic; the first 63 cells (the format's most) keep their text.
|
||||
let r = round(&d);
|
||||
let all: Vec<String> = r
|
||||
.body
|
||||
.iter()
|
||||
.flat_map(|b| match &**b {
|
||||
Block::Para(p) => vec![p.plain_text()],
|
||||
Block::Table(t) => {
|
||||
t.rows.iter().flat_map(|r| r.cells.iter()).flat_map(|c| c.blocks.iter()).filter_map(|b| b.as_para().map(|p| p.plain_text())).collect()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
for i in [0, 30, 62, 79] {
|
||||
assert!(all.iter().any(|t| t == &format!("c{i}")), "c{i} missing: {all:?}");
|
||||
}
|
||||
assert_eq!(texts(&r).last().map(String::as_str), Some("after"));
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
#![proptest_config(proptest::prelude::ProptestConfig::with_cases(64))]
|
||||
/// Any text, with any run boundaries and formatting, writes without panicking and reads back
|
||||
/// as the same text.
|
||||
#[test]
|
||||
fn any_text_round_trips(lines in proptest::collection::vec("\\PC{0,40}", 1..8), bold in proptest::collection::vec(proptest::prelude::any::<bool>(), 8)) {
|
||||
let paras: Vec<Paragraph> = lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| {
|
||||
let clean: String = l.chars().filter(|c| !c.is_control() && *c != '\u{FFFC}').collect();
|
||||
let mut p = para(&clean);
|
||||
if bold.get(i).copied().unwrap_or(false) && !clean.is_empty() {
|
||||
let end = clean.char_indices().nth(clean.chars().count() / 2).map_or(clean.len(), |(i, _)| i);
|
||||
p.format(0, end, &|c| c.bold = Some(true)).unwrap();
|
||||
}
|
||||
p
|
||||
})
|
||||
.collect();
|
||||
let d = doc_of(paras);
|
||||
let r = round(&d);
|
||||
proptest::prop_assert_eq!(texts(&r), texts(&d));
|
||||
}
|
||||
}
|
||||
|
||||
/// The thin/thick compound border lines (`brcType` 0x0B-0x13) keep their style through a save,
|
||||
/// rather than coming back as single lines.
|
||||
#[test]
|
||||
fn compound_border_lines_round_trip() {
|
||||
let styles = [
|
||||
BorderStyle::ThinThickSmallGap,
|
||||
BorderStyle::ThickThinSmallGap,
|
||||
BorderStyle::ThinThickThinSmallGap,
|
||||
BorderStyle::ThinThickMediumGap,
|
||||
BorderStyle::ThickThinMediumGap,
|
||||
BorderStyle::ThinThickThinMediumGap,
|
||||
BorderStyle::ThinThickLargeGap,
|
||||
BorderStyle::ThickThinLargeGap,
|
||||
BorderStyle::ThinThickThinLargeGap,
|
||||
];
|
||||
let mut d = Document::new();
|
||||
d.body = styles
|
||||
.iter()
|
||||
.map(|&style| {
|
||||
let mut p = para("Bordered");
|
||||
p.props.borders = Some(Borders { bottom: Some(Border { style, width: 3.0, color: None, space: 1.0 }), ..Default::default() });
|
||||
para_block(p)
|
||||
})
|
||||
.collect();
|
||||
let r = round(&d);
|
||||
let got: Vec<_> = r.body.iter().filter_map(|b| b.as_para()).filter_map(|p| p.props.borders.as_ref()?.bottom.map(|b| b.style)).collect();
|
||||
assert_eq!(got, styles);
|
||||
}
|
||||
@@ -20,6 +20,19 @@ mod custom;
|
||||
mod flat;
|
||||
mod fonts;
|
||||
mod package;
|
||||
|
||||
static APPLICATION_NAME: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
|
||||
/// Names the application written to `docProps/app.xml` (`<Application>`), for an app that
|
||||
/// embeds this writer under its own name; "WordCraft" until set. Only the first call counts.
|
||||
pub fn set_application_name(name: &str) {
|
||||
let _ = APPLICATION_NAME.set(name.to_owned());
|
||||
}
|
||||
|
||||
/// The application `docProps/app.xml` names (see [`set_application_name`]).
|
||||
pub fn application_name() -> &'static str {
|
||||
APPLICATION_NAME.get().map_or("WordCraft", String::as_str)
|
||||
}
|
||||
mod read;
|
||||
mod smart_art;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -12,6 +12,9 @@ pub const MAX_PART: u64 = 256 * 1024 * 1024;
|
||||
pub const MAX_TOTAL: u64 = 512 * 1024 * 1024;
|
||||
/// Most zip entries we look at.
|
||||
pub const MAX_ENTRIES: usize = 20_000;
|
||||
// ZIP names have at most u16::MAX raw bytes. CP437/lossy UTF-8 decoding can
|
||||
// expand each byte to three UTF-8 bytes. Bound from_parts lookup copies too.
|
||||
const MAX_SCAN_NAME: usize = 3 * u16::MAX as usize;
|
||||
|
||||
/// Relationship type suffixes (after `.../relationships/`).
|
||||
pub mod rt {
|
||||
@@ -233,6 +236,11 @@ impl Package {
|
||||
self.opc.size()
|
||||
}
|
||||
|
||||
/// The entries of the zip that could not be read and were left out, by name.
|
||||
pub fn skipped(&self) -> &[String] {
|
||||
self.opc.skipped()
|
||||
}
|
||||
|
||||
/// A package from parts already in memory (a Flat OPC file, a converted document), by part
|
||||
/// name without the leading `/`.
|
||||
pub fn from_parts(files: BTreeMap<String, Vec<u8>>) -> Package {
|
||||
@@ -265,6 +273,61 @@ impl Package {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read only XML attributes, once per canonical package part. This never invokes
|
||||
/// an altChunk importer or fetches additional data. Non-XML extensions are valid OPC
|
||||
/// targets too. Errors end a lenient part exactly as the normal XML reader does;
|
||||
/// the subsequent strict main-document parse remains authoritative.
|
||||
pub(crate) fn file_orders(&self) -> Result<craft_wordmodel::para::FileOrders, DocxError> {
|
||||
use craft_wordmodel::para::FileOrderScan;
|
||||
let mut bytes = 0u64;
|
||||
for (i, name) in self.opc.names().enumerate() {
|
||||
if name.len() > MAX_SCAN_NAME {
|
||||
return Err(DocxError::Limit("stacking-order source part name is too long".into()));
|
||||
}
|
||||
if i >= MAX_ENTRIES {
|
||||
return Err(DocxError::Limit("too many stacking-order source parts".into()));
|
||||
}
|
||||
bytes = bytes
|
||||
.checked_add(self.opc.get(name).map_or(0, |b| b.len() as u64))
|
||||
.ok_or_else(|| DocxError::Limit("stacking-order source size overflow".into()))?;
|
||||
if bytes > FileOrderScan::MAX_SOURCE_BYTES {
|
||||
return Err(DocxError::Limit("stacking-order source exceeds 2 GiB".into()));
|
||||
}
|
||||
}
|
||||
let mut scan = FileOrderScan::new(bytes).map_err(DocxError::Limit)?;
|
||||
let limits = craft_xml::Limits {
|
||||
max_input: 2 << 30,
|
||||
max_depth: xml::MAX_DEPTH + 1,
|
||||
max_nodes: 2 * xml::MAX_ELEMENTS,
|
||||
max_elements: xml::MAX_ELEMENTS,
|
||||
max_attrs: xml::MAX_ATTRS,
|
||||
max_attrs_total: 2 * xml::MAX_ELEMENTS,
|
||||
};
|
||||
let opts = craft_xml::Options::default().with_limits(limits);
|
||||
for name in self.opc.names() {
|
||||
let Some(bytes) = self.opc.get(name) else { continue };
|
||||
let text = craft_xml::to_utf8(bytes);
|
||||
// Attribute names cannot contain entity references. This only skips parts
|
||||
// that cannot possibly supply an order, including ordinary binary media.
|
||||
if !text.windows(b"relativeHeight".len()).any(|w| w == b"relativeHeight") {
|
||||
continue;
|
||||
}
|
||||
let mut events = opts.events(&text);
|
||||
while let Ok(Some(ev)) = events.next() {
|
||||
if let craft_xml::Event::Start { attrs, .. } = ev {
|
||||
for (name, value) in attrs {
|
||||
if name.rsplit(':').next() == Some("relativeHeight")
|
||||
&& let Ok(raw) = value.trim().parse::<u32>()
|
||||
{
|
||||
scan.observe(raw);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
scan.finish().map_err(DocxError::Limit)
|
||||
}
|
||||
|
||||
/// Relationships of `part` (from `dir/_rels/name.rels`). Internal targets name the part as
|
||||
/// stored, so a part reached under several spellings is kept, cached and written once.
|
||||
pub fn rels(&self, part: &str) -> Rels {
|
||||
@@ -386,3 +449,39 @@ mod tests {
|
||||
assert!(junk.parts.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod file_order_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn file_order_scan_bounds_decoded_part_names_before_lookup() {
|
||||
let at_limit = "n".repeat(MAX_SCAN_NAME);
|
||||
assert!(Package::from_parts(BTreeMap::from([(at_limit, Vec::new())])).file_orders().is_ok());
|
||||
let too_long = "n".repeat(MAX_SCAN_NAME + 1);
|
||||
assert!(Package::from_parts(BTreeMap::from([(too_long, Vec::new())])).file_orders().is_err());
|
||||
}
|
||||
|
||||
fn scan(xml: &str) -> craft_wordmodel::para::FileOrders {
|
||||
Package::from_parts(BTreeMap::from([("odd.part".into(), xml.as_bytes().to_vec())])).file_orders().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_order_scan_matches_lenient_attributes_entities_and_parser_limits() {
|
||||
let malformed = r#"<r><x alien:relativeHeight="4294967295"></wrong><x relativeHeight="4294967294"/></r>"#;
|
||||
let tree = xml::parse(malformed.as_bytes()).unwrap();
|
||||
assert_eq!(tree.find("?:x").or_else(|| tree.find("x")).unwrap().attr("relativeHeight"), Some("4294967295"));
|
||||
let map = scan(malformed);
|
||||
assert_eq!((map.model_order(Some(u32::MAX - 1)), map.model_order(Some(u32::MAX))), (u32::MAX - 1, u32::MAX));
|
||||
let custom = r#"<!DOCTYPE r [<!ENTITY rank "4294967295">]><r relativeHeight="&rank;"/>"#;
|
||||
assert_ne!(xml::parse(custom.as_bytes()).unwrap().attr("relativeHeight"), Some("4294967295"));
|
||||
assert_eq!(scan(custom).model_order(Some(u32::MAX - 1)), u32::MAX);
|
||||
let too_many =
|
||||
format!(r#"<r {} relativeHeight="4294967295"/>"#, (0..xml::MAX_ATTRS).map(|i| format!(r#"a{i}="x""#)).collect::<Vec<_>>().join(" "));
|
||||
assert!(xml::parse(too_many.as_bytes()).is_err());
|
||||
assert_eq!(scan(&too_many).model_order(Some(u32::MAX - 1)), u32::MAX);
|
||||
let too_deep = format!(r#"{}<r relativeHeight="4294967295"/>{}"#, "<r>".repeat(xml::MAX_DEPTH + 1), "</r>".repeat(xml::MAX_DEPTH + 1));
|
||||
assert!(xml::parse(too_deep.as_bytes()).is_err());
|
||||
assert_eq!(scan(&too_deep).model_order(Some(u32::MAX - 1)), u32::MAX);
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,7 @@ const MAX_TEMPLATE_PATH: usize = 32 * 1024;
|
||||
|
||||
pub(crate) struct Reader<'p> {
|
||||
pkg: &'p Package,
|
||||
file_orders: craft_wordmodel::para::FileOrders,
|
||||
pub doc: Document,
|
||||
pub pc: PropCtx,
|
||||
revs: HashMap<(u8, String, String, Option<String>), u32>,
|
||||
@@ -167,7 +168,9 @@ fn read_opened_in(pkg: &Package, chunks: ChunkState) -> Result<(Document, usize,
|
||||
let rels = pkg.rels(&main);
|
||||
let mut doc = Document::new();
|
||||
doc.body.clear();
|
||||
let file_orders = pkg.file_orders()?;
|
||||
let mut r = Reader {
|
||||
file_orders,
|
||||
pkg,
|
||||
doc,
|
||||
pc: PropCtx::default(),
|
||||
@@ -370,6 +373,7 @@ fn read_opened_in(pkg: &Package, chunks: ChunkState) -> Result<(Document, usize,
|
||||
}
|
||||
let (left, bytes, graphic) = (r.chunks.left, r.chunks.bytes, r.chunks.graphic);
|
||||
let mut doc = r.doc;
|
||||
doc.skipped_parts.extend(pkg.skipped().iter().cloned());
|
||||
doc.ensure_nonempty();
|
||||
Ok((doc, left, bytes, graphic))
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ use parse::{axis, category_text, labels, numbers};
|
||||
use parse::{read_series, text_size, title_of};
|
||||
#[cfg(test)]
|
||||
use plot::{Scale, narrow, nice_scale};
|
||||
use plot::{backdrop, cartesian};
|
||||
use plot::{backdrop, cartesian, label};
|
||||
use polar::{pie, radar};
|
||||
|
||||
/// Most series read from one chart.
|
||||
@@ -129,7 +129,14 @@ fn placeholder(out: &mut Vec<GraphicItem>, space: &El, theme: &[Rgb], w: f32, h:
|
||||
}
|
||||
let Some(ty) = plot.els().find(|e| e.name.ends_with("Chart")) else { return };
|
||||
backdrop(out, space, theme, w, h);
|
||||
out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill: None, stroke: Some(AXIS), stroke_width: 0.75 });
|
||||
out.push(GraphicItem::Shape {
|
||||
rect: [0.0, 0.0, w, h],
|
||||
kind: ShapeKind::Rectangle,
|
||||
fill: None,
|
||||
stroke: Some(AXIS),
|
||||
stroke_width: 0.75,
|
||||
alpha: 1.0,
|
||||
});
|
||||
let mut area = Rect { x: PAD, y: PAD, w: (w - 2.0 * PAD).max(1.0), h: (h - 2.0 * PAD).max(1.0) };
|
||||
if let Some(t) = title_of(chart, &read_series(ty, false, theme)) {
|
||||
area = title_block(out, &t, area);
|
||||
@@ -141,6 +148,58 @@ fn placeholder(out: &mut Vec<GraphicItem>, space: &El, theme: &[Rgb], w: f32, h:
|
||||
text(out, line, &clip_text(&msg, area.w, size), size, TextAlign::Center, false);
|
||||
}
|
||||
|
||||
/// Items for an Office 2016 chart (`cx:chartSpace`: waterfall, funnel, treemap…), sized `w` x `h`
|
||||
/// points: none of these types is drawn yet, so this is the same placeholder an unsupported
|
||||
/// classic chart gets (frame, title, the type it is), at the size the chart takes in Word.
|
||||
/// `space` is `None` when the part can't be read: the frame and "Chart" are still drawn.
|
||||
pub(crate) fn chartex_items(space: Option<&El>, w: f32, h: f32) -> Vec<GraphicItem> {
|
||||
let mut out = Vec::new();
|
||||
if !(w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0) {
|
||||
return out;
|
||||
}
|
||||
out.push(GraphicItem::Shape {
|
||||
rect: [0.0, 0.0, w, h],
|
||||
kind: ShapeKind::Rectangle,
|
||||
fill: None,
|
||||
stroke: Some(AXIS),
|
||||
stroke_width: 0.75,
|
||||
alpha: 1.0,
|
||||
});
|
||||
let mut area = Rect { x: PAD, y: PAD, w: (w - 2.0 * PAD).max(1.0), h: (h - 2.0 * PAD).max(1.0) };
|
||||
let chart = space.and_then(|s| s.child("cx:chart"));
|
||||
let title = chart.and_then(|c| c.child("cx:title")).and_then(|t| {
|
||||
// Rich text keeps its runs together ("Sales " + "funnel"); a text-data title is one value.
|
||||
let s = t.find("cx:rich").or_else(|| t.find("cx:v")).map(|e| e.deep_text()).unwrap_or_default();
|
||||
let s = label(s.trim());
|
||||
(!s.is_empty()).then_some(s)
|
||||
});
|
||||
if let Some(t) = title {
|
||||
area = title_block(&mut out, &t, area);
|
||||
}
|
||||
let kind = chart.and_then(|c| c.find("cx:series")).and_then(|s| s.attr("layoutId")).unwrap_or("");
|
||||
let size = FONT;
|
||||
let lh = size * LINE;
|
||||
let msg = format!("{} (not supported yet)", chartex_name(kind));
|
||||
let line = Rect { x: area.x, y: area.y + ((area.h - lh) / 2.0).max(0.0), w: area.w, h: lh };
|
||||
text(&mut out, line, &clip_text(&msg, area.w, size), size, TextAlign::Center, false);
|
||||
out
|
||||
}
|
||||
|
||||
/// What a placeholder calls an Office 2016 chart type (`cx:series` `layoutId`).
|
||||
fn chartex_name(layout: &str) -> &'static str {
|
||||
match layout {
|
||||
"waterfall" => "Waterfall chart",
|
||||
"funnel" => "Funnel chart",
|
||||
"treemap" => "Treemap chart",
|
||||
"sunburst" => "Sunburst chart",
|
||||
"boxWhisker" => "Box and whisker chart",
|
||||
"clusteredColumn" => "Histogram chart",
|
||||
"paretoLine" => "Pareto chart",
|
||||
"regionMap" => "Map chart",
|
||||
_ => "Chart",
|
||||
}
|
||||
}
|
||||
|
||||
/// What a placeholder calls a chart type element WordCraft doesn't draw.
|
||||
fn unsupported_name(name: &str) -> &'static str {
|
||||
match name {
|
||||
|
||||
@@ -69,7 +69,14 @@ pub(super) fn stroke(out: &mut Vec<GraphicItem>, segs: Vec<PathSeg>, color: Rgb,
|
||||
}
|
||||
|
||||
fn swatch(out: &mut Vec<GraphicItem>, x: f32, y: f32, color: Rgb) {
|
||||
out.push(GraphicItem::Shape { rect: [x, y, SWATCH, SWATCH], kind: ShapeKind::Rectangle, fill: Some(color), stroke: None, stroke_width: 0.0 });
|
||||
out.push(GraphicItem::Shape {
|
||||
rect: [x, y, SWATCH, SWATCH],
|
||||
kind: ShapeKind::Rectangle,
|
||||
fill: Some(color),
|
||||
stroke: None,
|
||||
stroke_width: 0.0,
|
||||
alpha: 1.0,
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn seg(x0: f32, y0: f32, x1: f32, y1: f32) -> Vec<PathSeg> {
|
||||
|
||||
@@ -24,7 +24,7 @@ pub(super) fn backdrop(out: &mut Vec<GraphicItem>, space: &El, theme: &[Rgb], w:
|
||||
let stroke = ln.and_then(|l| fill_colour(l, theme));
|
||||
let stroke_width = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).unwrap_or(0.75).clamp(0.0, 20.0);
|
||||
if fill.is_some() || stroke.is_some() {
|
||||
out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill, stroke, stroke_width });
|
||||
out.push(GraphicItem::Shape { rect: [0.0, 0.0, w, h], kind: ShapeKind::Rectangle, fill, stroke, stroke_width, alpha: 1.0 });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +212,8 @@ fn title_left(c: &Chart, cat_on: bool, val_on: bool, area_w: f32) -> f32 {
|
||||
fn flat_title_lines(c: &Chart, area_w: f32) -> Option<Vec<String>> {
|
||||
let left = if c.horizontal { &c.cat } else { &c.val };
|
||||
let r = left.title_rot?.rem_euclid(360.0);
|
||||
if !(r < 1.0 || r > 359.0) {
|
||||
// Only near-zero rotations fit the horizontal strip; non-finite rotations do not.
|
||||
if !r.is_finite() || (1.0..=359.0).contains(&r) {
|
||||
return None;
|
||||
}
|
||||
let width = if area_w.is_finite() { (area_w * 0.18).max(1.0) } else { 1.0 };
|
||||
@@ -544,7 +545,7 @@ fn markers(out: &mut Vec<GraphicItem>, pts: impl Iterator<Item = (f32, f32)>, co
|
||||
let size = s.marker_size.filter(|v| v.is_finite()).unwrap_or(MARKER).clamp(2.0, 72.0);
|
||||
let h = size / 2.0;
|
||||
let shape = |out: &mut Vec<GraphicItem>, rect: [f32; 4], kind: ShapeKind| {
|
||||
out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0 });
|
||||
out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0, alpha: 1.0 });
|
||||
};
|
||||
let mut strokes = Vec::new();
|
||||
for (x, y) in pts {
|
||||
|
||||
@@ -96,7 +96,7 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
|
||||
for (i, p) in ring.iter().enumerate() {
|
||||
if c.radar_marks && s.values.get(i).copied().flatten().is_some() {
|
||||
let rect = [p.0 as f32 - MARKER / 2.0, p.1 as f32 - MARKER / 2.0, MARKER, MARKER];
|
||||
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0 });
|
||||
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0, alpha: 1.0 });
|
||||
}
|
||||
if let Some(text) = label_of(s, i) {
|
||||
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false, pos: LabelPos::Above, off: 0.0 });
|
||||
|
||||
@@ -745,6 +745,11 @@ fn value_axis_title_turns_counter_clockwise_like_word() {
|
||||
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="5400000" vert="horz"/>"#))), Some(90.0));
|
||||
// A horizontal title (`rot="0"`) stays horizontal: the plot makes room for it (see below).
|
||||
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="0" vert="horz"/>"#))), Some(0.0));
|
||||
// The horizontal tolerance excludes both boundary angles and keeps equivalent full turns.
|
||||
for (rot, expected) in [(30000, 0.0), (-30000, 0.0), (60000, -90.0), (-60000, -90.0), (10800000, -90.0), (21600000, 0.0)] {
|
||||
let body = format!(r#"<a:bodyPr rot="{rot}" vert="horz"/>"#);
|
||||
assert_eq!(angle(&line_with("", &title(&body))), Some(expected), "rotation {rot}");
|
||||
}
|
||||
// Hostile values stay finite.
|
||||
let odd = angle(&line_with("", &title(r#"<a:bodyPr rot="99999999999999999999"/>"#)));
|
||||
assert!(odd.is_some_and(f32::is_finite));
|
||||
@@ -899,3 +904,27 @@ fn source_linked_series_labels_do_not_inherit_the_chart_format() {
|
||||
let inherited = run(r#"<c:dLbls><c:showVal val="1"/></c:dLbls>"#);
|
||||
assert!(inherited.contains(&"320%".to_string()), "{inherited:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn office_2016_chart_draws_a_placeholder_of_its_size() {
|
||||
let cx = |series: &str, title: &str| {
|
||||
let x = format!(
|
||||
r#"<cx:chartSpace xmlns:cx="http://schemas.microsoft.com/office/drawing/2014/chartex" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><cx:chart>{title}<cx:plotArea><cx:plotAreaRegion>{series}</cx:plotAreaRegion></cx:plotArea></cx:chart></cx:chartSpace>"#
|
||||
);
|
||||
crate::xml::parse(x.as_bytes()).unwrap()
|
||||
};
|
||||
let title = r#"<cx:title pos="t"><cx:tx><cx:txData><cx:v>Quarterly steps</cx:v></cx:txData></cx:tx></cx:title>"#;
|
||||
let rich = r#"<cx:title><cx:tx><cx:rich><a:p><a:r><a:t>Sales </a:t></a:r><a:r><a:t>funnel</a:t></a:r></a:p></cx:rich></cx:tx></cx:title>"#;
|
||||
let waterfall = chartex_items(Some(&cx(r#"<cx:series layoutId="waterfall"/>"#, title)), 432.0, 252.0);
|
||||
assert_eq!(texts(&waterfall), ["Quarterly steps", "Waterfall chart (not supported yet)"]);
|
||||
let framed = waterfall.iter().any(|it| matches!(it, GraphicItem::Shape { rect, stroke: Some(_), .. } if *rect == [0.0, 0.0, 432.0, 252.0]));
|
||||
assert!(framed, "the frame has the chart's size: {waterfall:?}");
|
||||
let funnel = chartex_items(Some(&cx(r#"<cx:series layoutId="funnel"/>"#, rich)), 300.0, 200.0);
|
||||
assert_eq!(texts(&funnel), ["Sales funnel", "Funnel chart (not supported yet)"]);
|
||||
let other = chartex_items(Some(&cx(r#"<cx:series layoutId="newKind"/>"#, "")), 300.0, 200.0);
|
||||
assert_eq!(texts(&other), ["Chart (not supported yet)"]);
|
||||
// An unreadable part still reserves and frames the chart's place.
|
||||
let missing = chartex_items(None, 300.0, 200.0);
|
||||
assert_eq!(texts(&missing), ["Chart (not supported yet)"]);
|
||||
assert!(chartex_items(None, 0.0, 200.0).is_empty() && chartex_items(None, f32::NAN, 200.0).is_empty());
|
||||
}
|
||||
Loaded 100 of 581 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user