diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8182802..b80f4ed 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -66,6 +66,15 @@ jobs: with: submodules: recursive + # Cache the extracted Odin install so only the first run downloads the + # ~60 MB tarball; keyed on the pinned version (cache paths live outside + # the checkout, so no sharing between jobs/refs). + - name: Cache Odin + uses: actions/cache@v4 + with: + path: /tmp/odin + key: odin-${{ env.ODIN_VERSION }}-${{ runner.arch }} + - name: Install Odin ${{ env.ODIN_VERSION }} run: scripts/install_odin.sh diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index f3e347e..fb2877f 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -34,6 +34,12 @@ jobs: with: submodules: recursive + - name: Cache Odin + uses: actions/cache@v4 + with: + path: /tmp/odin + key: odin-${{ env.ODIN_VERSION }}-${{ runner.arch }} + - name: Install Odin ${{ env.ODIN_VERSION }} run: scripts/install_odin.sh @@ -76,6 +82,12 @@ jobs: with: submodules: recursive + - name: Cache Odin + uses: actions/cache@v4 + with: + path: /tmp/odin + key: odin-${{ env.ODIN_VERSION }}-${{ runner.arch }} + - name: Install Odin ${{ env.ODIN_VERSION }} run: scripts/install_odin.sh @@ -139,6 +151,12 @@ jobs: with: submodules: recursive + - name: Cache Odin + uses: actions/cache@v4 + with: + path: /tmp/odin + key: odin-${{ env.ODIN_VERSION }}-${{ runner.arch }} + - name: Install Odin ${{ env.ODIN_VERSION }} run: scripts/install_odin.sh diff --git a/.gitignore b/.gitignore index 0753ab7..d075350 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ resources/ai/sessions infra/Pulumi.*.yaml.backup infra/desi-explorer-infra api/target/ +.env +.env.* +*.env diff --git a/AGENTS.md b/AGENTS.md index 1e11b77..144596b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ The root `Makefile` is a lean delegator: base commands (`run`, `build`, `test`, - Odin code lives in `gui/src/`; external deps go in `gui/lib/` and are wired via `-collection:lib=lib/local` (or a git submodule imported by relative path). - Everything is plain `make` — no Taskfile — so CI (Gitea Actions) can call `make` directly. - Keep the renderer (gui/), API (api/), and infra (infra/) logically separated; each owns its own Makefile, and the root Makefile is the only place that ties them together. +- **Diagrams in this repo's documentation are Mermaid flowcharts.** Gitea renders ` ```mermaid ` fenced blocks natively. Prefer a Mermaid flowchart over ASCII art / box-drawing diagrams; if a diagram genuinely can't be expressed as a flowchart, fall back to a plain text-based markdown diagram (e.g. a code block or table) rather than hand-rawn ASCII boxes. ## Gotchas - Odin version is pinned in `.gitea/workflows/*.yml` (`ODIN_VERSION`) and defaults in `scripts/install_odin.sh`; bump both together when tracking a new release. diff --git a/Makefile b/Makefile index f4c2f0b..064264a 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,14 @@ GUI := gui API := api INFRA := infra +# Local dev/test assets live under resources/dev. These are the defaults for +# the run/*-web targets; override any of them on the command line, e.g. +# make run GUI_ENV_FILE=/path/to/gui.env API_DESI_DATA=/path/to/data.json +RESOURCE_DIR := $(CURDIR)/resources/dev +GUI_ENV_FILE ?= $(RESOURCE_DIR)/gui.env.example +API_ENV_FILE ?= $(RESOURCE_DIR)/api.env.example +API_DESI_DATA ?= $(RESOURCE_DIR)/desi_subset.json + .PHONY: help setup run run-web build build-debug build-web test clean fmt \ renovate-validate @@ -32,9 +40,9 @@ setup: ## Setup all sub-projects (submodules, gui deps, api deps, infra deps) ## ---- Renderer (Odin) ----------------------------------------------------- run: ## Run the native app (gui/) - @$(MAKE) -C $(API) run & api_pid=$$!; \ + @API_ENV_FILE="$(API_ENV_FILE)" API_DESI_DATA="$(API_DESI_DATA)" $(MAKE) -C $(API) run & api_pid=$$!; \ trap 'kill $$api_pid 2>/dev/null' INT TERM EXIT; \ - $(MAKE) -C $(GUI) run; \ + GUI_ENV_FILE="$(GUI_ENV_FILE)" $(MAKE) -C $(GUI) run; \ kill $$api_pid 2>/dev/null build: ## Release build (gui/ + api/) @@ -49,9 +57,9 @@ build-web: ## WebAssembly build -> build/web (gui/, needs emscripten) @$(MAKE) -C $(GUI) build-web run-web: ## Start WASM build + API server for web dev - @$(MAKE) -C $(API) run & api_pid=$$!; \ + @API_ENV_FILE="$(API_ENV_FILE)" API_DESI_DATA="$(API_DESI_DATA)" $(MAKE) -C $(API) run & api_pid=$$!; \ trap 'kill $$api_pid 2>/dev/null' INT TERM EXIT; \ - $(MAKE) -C $(GUI) build-web; \ + GUI_ENV_FILE="$(GUI_ENV_FILE)" $(MAKE) -C $(GUI) build-web; \ kill $$api_pid 2>/dev/null ## ---- Aggregates ---------------------------------------------------------- diff --git a/README.md b/README.md index fe21849..af603a9 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,22 @@ make clean # remove build artifacts from all projects make fmt # format all projects ``` +`make run` / `make run-web` configure the API and renderer from the example +assets under `resources/dev/` (see the dotenv lib in `gui/lib/local/dotenv`). +Override any of them on the command line: + +```sh +make run \ + GUI_ENV_FILE=/path/to/gui.env \ + API_ENV_FILE=/path/to/api.env \ + API_DESI_DATA=/path/to/desi_data.json +``` + +The defaults point at `resources/dev/gui.env.example` (renderer's `API_URL`), +`resources/dev/api.env.example` (API `API_BIND_ADDR`), and +`resources/dev/desi_subset.json` (a small JSON catalog subset served by the +API's `/api/v1/catalogs` and `/api/v1/objects` endpoints). + Project-specific targets live in their own `Makefile` and are reached with `make -C `: diff --git a/api/Cargo.lock b/api/Cargo.lock index 3c2a245..c3b08b5 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -99,6 +99,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "dotenvy", "serde", "serde_json", "tokio", @@ -108,6 +109,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "errno" version = "0.3.14" diff --git a/api/Cargo.toml b/api/Cargo.toml index 7dfe638..aff779a 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -5,17 +5,18 @@ edition = "2021" description = "Backend API for DESI Explorer — serves DESI survey catalog data" [dependencies] +anyhow = "1" axum = "0.8" -tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } +dotenvy = "0.15" serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } +tower-http = { version = "0.7", features = ["cors", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tower-http = { version = "0.7", features = ["cors", "trace"] } -anyhow = "1" [dev-dependencies] tower = { version = "0.5", features = ["util"] } -serde_json = "1" [profile.release] lto = true diff --git a/api/Makefile b/api/Makefile index 36d94a7..3de9787 100644 --- a/api/Makefile +++ b/api/Makefile @@ -4,6 +4,20 @@ # `api-*` convenience targets). CARGO ?= cargo +ROOT := .. + +# Normalize API_ENV_FILE / API_DESI_DATA (given relative to the repo root) to +# absolute paths so the API process can open them regardless of its working +# directory. "override" is required because they are usually passed as +# command-line/env vars, which would otherwise override any assignment here. +define normalize_path +ifdef $1 +ifneq ($(abspath $($1)),$($1)) +override $1 := $(abspath $(ROOT)/$($1)) +endif +endif +endef +$(foreach v,API_ENV_FILE API_DESI_DATA,$(eval $(call normalize_path,$v))) .PHONY: help setup run build test check fmt clean diff --git a/api/src/config.rs b/api/src/config.rs index e168014..85141ef 100644 --- a/api/src/config.rs +++ b/api/src/config.rs @@ -1,5 +1,10 @@ +use std::path::PathBuf; + pub struct Config { pub bind_addr: String, + /// Path to a DESI data file (JSON) to serve; `None` falls back to the + /// built-in placeholder catalogs. + pub desi_data: Option, } impl Config { @@ -7,6 +12,14 @@ impl Config { let bind_addr = std::env::var("API_BIND_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); - Ok(Self { bind_addr }) + let desi_data = std::env::var("API_DESI_DATA") + .ok() + .filter(|s| !s.is_empty()) + .map(PathBuf::from); + + Ok(Self { + bind_addr, + desi_data, + }) } } diff --git a/api/src/lib.rs b/api/src/lib.rs index b77fb25..2530105 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -1,3 +1,4 @@ pub mod config; pub mod models; pub mod routes; +pub mod store; diff --git a/api/src/main.rs b/api/src/main.rs index fef0a0c..20261c2 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -1,4 +1,7 @@ -use desi_explorer_api::{config, routes}; +use std::path::Path; +use std::sync::Arc; + +use desi_explorer_api::{config, routes, store}; use tracing_subscriber::EnvFilter; @@ -11,8 +14,22 @@ async fn main() -> anyhow::Result<()> { ) .init(); + load_env_file()?; + let config = config::Config::from_env()?; - let app = routes::app(); + + let catalog_store = match &config.desi_data { + Some(path) => { + tracing::info!(path = %path.display(), "loading DESI data"); + store::CatalogStore::load(Path::new(path))? + } + None => { + tracing::warn!("API_DESI_DATA not set, serving placeholder catalogs"); + store::CatalogStore::placeholder() + } + }; + + let app = routes::app_with_state(Arc::new(catalog_store)); let listener = tokio::net::TcpListener::bind(&config.bind_addr).await?; tracing::info!("DESI Explorer API listening on {}", config.bind_addr); @@ -24,6 +41,27 @@ async fn main() -> anyhow::Result<()> { Ok(()) } +/// Loads the env file named by `API_ENV_FILE` (if set) into the process +/// environment. Existing env vars are not overridden, so values passed +/// directly on the command line or by the Makefile take precedence. +fn load_env_file() -> anyhow::Result<()> { + let path = std::env::var("API_ENV_FILE").unwrap_or_default(); + if path.is_empty() { + return Ok(()); + } + + match dotenvy::from_path(&path) { + Ok(_) => tracing::info!(%path, "loaded env file"), + Err(err) => { + return Err(anyhow::anyhow!( + "failed to load API_ENV_FILE {path:?}: {err}" + )) + } + } + + Ok(()) +} + async fn shutdown_signal() { let _ = tokio::signal::ctrl_c().await; tracing::info!("shutting down"); diff --git a/api/src/models.rs b/api/src/models.rs index 0090bcd..acdd6a6 100644 --- a/api/src/models.rs +++ b/api/src/models.rs @@ -1,17 +1,17 @@ -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// Catalog metadata for a DESI data release/survey. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct Catalog { pub name: String, pub release: String, - pub description: &'static str, + pub description: String, pub object_count: Option, } /// A single catalog object (galaxy / quasar / star) with its survey /// coordinates. `ra` and `dec` are in degrees; `redshift` is dimensionless. -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct CatalogObject { pub id: String, pub catalog: String, diff --git a/api/src/routes/catalogs.rs b/api/src/routes/catalogs.rs index 5e5d220..c6fbc7d 100644 --- a/api/src/routes/catalogs.rs +++ b/api/src/routes/catalogs.rs @@ -1,34 +1,16 @@ +use std::sync::Arc; + use axum::{ - extract::Query, - http::StatusCode, - response::{IntoResponse, Response}, + extract::{Query, State}, Json, }; use serde::Deserialize; -use std::sync::LazyLock; use crate::models::{Catalog, CatalogObject}; +use crate::store::CatalogStore; -/// Placeholder catalogs until real DESI EDR/DR1 ingestion lands. -static CATALOGS: LazyLock> = LazyLock::new(|| { - vec![ - Catalog { - name: "edr".to_string(), - release: "EDR".to_string(), - description: "DESI Early Data Release", - object_count: None, - }, - Catalog { - name: "dr1".to_string(), - release: "DR1".to_string(), - description: "DESI Data Release 1", - object_count: None, - }, - ] -}); - -pub async fn list_catalogs() -> Json> { - Json(CATALOGS.clone()) +pub async fn list_catalogs(State(state): State>) -> Json> { + Json(state.catalogs.clone()) } #[derive(Debug, Deserialize)] @@ -38,17 +20,29 @@ pub struct ObjectQuery { limit: Option, } -/// Placeholder object query. Real implementation will page through the -/// centralized DESI catalog store rather than return an empty result set. -pub async fn list_objects(Query(query): Query) -> Response { +pub async fn list_objects( + State(state): State>, + Query(query): Query, +) -> Json> { let limit = query.limit.unwrap_or(100).min(10_000); tracing::debug!( %limit, catalog = query.catalog.as_deref().unwrap_or("all"), - "querying catalog objects (placeholder)" + objects = state.objects.len(), + "querying catalog objects" ); - let objects: Vec = Vec::new(); - (StatusCode::OK, Json(objects)).into_response() + let objects: Vec = match &query.catalog { + Some(catalog) => state + .objects + .iter() + .filter(|o| &o.catalog == catalog) + .take(limit) + .cloned() + .collect(), + None => state.objects.iter().take(limit).cloned().collect(), + }; + + Json(objects) } diff --git a/api/src/routes/mod.rs b/api/src/routes/mod.rs index 2a7119f..6ec05d1 100644 --- a/api/src/routes/mod.rs +++ b/api/src/routes/mod.rs @@ -1,13 +1,23 @@ pub mod catalogs; pub mod health; +use std::sync::Arc; + use axum::{routing::get, Router}; -/// Builds the application router. Kept separate from `main` so tests can -/// construct it without binding a socket. +use crate::store::CatalogStore; + +/// Builds the application router with a static placeholder store. Kept +/// separate from `main` so tests can construct it without binding a socket. pub fn app() -> Router { + app_with_state(Arc::new(CatalogStore::placeholder())) +} + +/// Builds the application router serving the given catalog store. +pub fn app_with_state(state: Arc) -> Router { Router::new() .route("/health", get(health::health)) .route("/api/v1/catalogs", get(catalogs::list_catalogs)) .route("/api/v1/objects", get(catalogs::list_objects)) + .with_state(state) } diff --git a/api/src/store.rs b/api/src/store.rs new file mode 100644 index 0000000..9220dcb --- /dev/null +++ b/api/src/store.rs @@ -0,0 +1,57 @@ +use std::path::Path; + +use serde::Deserialize; + +use crate::models::{Catalog, CatalogObject}; + +/// In-memory catalog store, shared (via `Arc`) across routes. Populated either +/// from a DESI data file loaded at startup or from `placeholder`. +#[derive(Debug, Clone, Default)] +pub struct CatalogStore { + pub catalogs: Vec, + pub objects: Vec, +} + +/// JSON layout of the DESI data file referenced by `API_DESI_DATA`. +#[derive(Debug, Deserialize)] +pub struct DataFile { + pub catalogs: Vec, + #[serde(default)] + pub objects: Vec, +} + +impl CatalogStore { + /// Static fallback catalogs used when no `API_DESI_DATA` file is given + /// (and by the `routes::app()` test helper). + pub fn placeholder() -> Self { + Self { + catalogs: vec![ + Catalog { + name: "edr".to_string(), + release: "EDR".to_string(), + description: "DESI Early Data Release".to_string(), + object_count: None, + }, + Catalog { + name: "dr1".to_string(), + release: "DR1".to_string(), + description: "DESI Data Release 1".to_string(), + object_count: None, + }, + ], + objects: Vec::new(), + } + } + + /// Loads catalogs and objects from a JSON data file. Errors on unreadable + /// files or malformed JSON so the caller can fail loudly instead of + /// silently serving empty data. + pub fn load(path: &Path) -> anyhow::Result { + let text = std::fs::read_to_string(path)?; + let file: DataFile = serde_json::from_str(&text)?; + Ok(Self { + catalogs: file.catalogs, + objects: file.objects, + }) + } +} diff --git a/api/tests/objects.rs b/api/tests/objects.rs new file mode 100644 index 0000000..d0a682d --- /dev/null +++ b/api/tests/objects.rs @@ -0,0 +1,93 @@ +use axum::body::{to_bytes, Body}; +use axum::http::{Request, StatusCode}; +use std::sync::Arc; +use tower::ServiceExt; + +use desi_explorer_api::models::CatalogObject; +use desi_explorer_api::routes; +use desi_explorer_api::store::CatalogStore; + +fn sample_store() -> CatalogStore { + CatalogStore { + catalogs: Vec::new(), + objects: vec![ + CatalogObject { + id: "o1".to_string(), + catalog: "edr".to_string(), + object_type: "GALAXY".to_string(), + ra: 1.5, + dec: 2.5, + redshift: 0.8, + }, + CatalogObject { + id: "o2".to_string(), + catalog: "dr1".to_string(), + object_type: "STAR".to_string(), + ra: 3.5, + dec: 4.5, + redshift: 0.0, + }, + ], + } +} + +#[tokio::test] +async fn objects_returns_all_when_no_filter() { + let app = routes::app_with_state(Arc::new(sample_store())); + + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/objects") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let objects: Vec = serde_json::from_slice(&body).unwrap(); + assert_eq!(objects.len(), 2); +} + +#[tokio::test] +async fn objects_filters_by_catalog() { + let app = routes::app_with_state(Arc::new(sample_store())); + + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/objects?catalog=edr") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let objects: Vec = serde_json::from_slice(&body).unwrap(); + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].id, "o1"); +} + +#[tokio::test] +async fn objects_respects_limit() { + let app = routes::app_with_state(Arc::new(sample_store())); + + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/objects?limit=1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let objects: Vec = serde_json::from_slice(&body).unwrap(); + assert_eq!(objects.len(), 1); +} diff --git a/api/tests/store.rs b/api/tests/store.rs new file mode 100644 index 0000000..4954aaa --- /dev/null +++ b/api/tests/store.rs @@ -0,0 +1,54 @@ +use desi_explorer_api::store::CatalogStore; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn load_parses_catalogs_and_objects() { + let json = r#"{ + "catalogs": [ + {"name":"edr","release":"EDR","description":"test","object_count":2} + ], + "objects": [ + {"id":"o1","catalog":"edr","object_type":"GALAXY","ra":1.5,"dec":2.5,"redshift":0.8} + ] + }"#; + + let dir = std::env::temp_dir().join(format!( + "desi_explorer_store_{}_{}", + std::process::id(), + line!() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("data.json"); + std::fs::write(&path, json).unwrap(); + + let store = CatalogStore::load(&path).unwrap(); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(store.catalogs.len(), 1); + assert_eq!(store.catalogs[0].name, "edr"); + assert_eq!(store.catalogs[0].object_count, Some(2)); + assert_eq!(store.objects.len(), 1); + assert_eq!(store.objects[0].id, "o1"); + assert_eq!(store.objects[0].ra, 1.5); + } + + #[test] + fn load_rejects_malformed_json() { + let dir = std::env::temp_dir().join(format!( + "desi_explorer_store_{}_{}", + std::process::id(), + line!() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("data.json"); + std::fs::write(&path, "not json").unwrap(); + + let result = CatalogStore::load(&path); + let _ = std::fs::remove_dir_all(&dir); + + assert!(result.is_err()); + } +} diff --git a/gui/Makefile b/gui/Makefile index 84951f2..a0c71a9 100644 --- a/gui/Makefile +++ b/gui/Makefile @@ -8,13 +8,28 @@ # output dirs live at the repo root and are referenced through `$(ROOT)`. ODIN ?= odin +GDB ?= gdb ROOT := .. BIN := $(ROOT)/bin BINARY := $(BIN)/desi_explorer ODIN_FLAGS := -collection:lib=lib/local WASM_DEFINE := RAYLIB_WASM_LIB=env.o -.PHONY: help setup add-dep run build build-debug build-web test clean fmt +# Normalize GUI_ENV_FILE (given relative to the repo root) to an absolute path +# so the Odin process can open it regardless of its working directory. +# "override" is required because GUI_ENV_FILE is usually set on the command +# line (or passed as an env var to this sub-make), which would otherwise +# override any assignment made here. +ifdef GUI_ENV_FILE +ifneq ($(abspath $(GUI_ENV_FILE)),$(GUI_ENV_FILE)) +override GUI_ENV_FILE := $(abspath $(ROOT)/$(GUI_ENV_FILE)) +endif +endif +# "override" on a command-line variable silently drops it from the recipe +# environment; re-export it so the app can find its env file (run + gdb). +export GUI_ENV_FILE + +.PHONY: help setup add-dep run build build-debug gdb build-web test clean fmt help: ## List available targets @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ @@ -41,12 +56,22 @@ build-debug: ## Debug build -> bin/desi_explorer @mkdir -p lib/local $(BIN) $(ODIN) build src $(ODIN_FLAGS) -o:none -debug -out:$(BINARY) +gdb: build-debug ## Run the native app under gdb (type 'run', then 'bt' on a crash) + $(GDB) -q --args $(BINARY) $(ARGS) + build-web: ## WebAssembly build -> build/web (needs emscripten) @scripts/build_web.sh test: ## Run Odin unit tests @mkdir -p lib/local - $(ODIN) test src $(ODIN_FLAGS) + @if ls test/*.odin >/dev/null 2>&1; then \ + echo "== gui/test =="; \ + $(ODIN) test test $(ODIN_FLAGS); \ + fi + @for dir in $$(find lib/local -name '*_test.odin' -exec dirname {} \; | sort -u); do \ + echo "== $$dir =="; \ + $(ODIN) test "$$dir" $(ODIN_FLAGS); \ + done clean: ## Remove build artifacts rm -rf $(BIN) build diff --git a/gui/lib/local/dotenv/src/dotenv.odin b/gui/lib/local/dotenv/src/dotenv.odin new file mode 100644 index 0000000..524bc3c --- /dev/null +++ b/gui/lib/local/dotenv/src/dotenv.odin @@ -0,0 +1,167 @@ +package dotenv + +import "base:runtime" +import "core:os" +import "core:reflect" +import "core:strconv" +import "core:strings" + +// parse parses dotenv-format source (KEY=VALUE lines) into a map allocated +// with allocator. Blank lines, lines starting with '#', and lines without a +// '=' are skipped. Keys and values are trimmed; values may be wrapped in +// double quotes. Real process environment variables take precedence over the +// file. The returned map owns its keys/values; release it with destroy. +@(private) +parse :: proc(src: string, allocator := context.allocator) -> map[string]string { + result := make(map[string]string, allocator) + + it := src + for line in strings.split_lines_iterator(&it) { + tr := strings.trim_space(line) + if len(tr) == 0 || strings.has_prefix(tr, "#") { + continue + } + + eq := strings.index_byte(tr, '=') + if eq < 0 { + continue + } + + key := strings.trim_space(tr[:eq]) + if key == "" { + continue + } + + value := strings.trim_space(tr[eq + 1:]) + if len(value) >= 2 && value[0] == '"' && value[len(value) - 1] == '"' { + value = value[1:len(value) - 1] + } + + // real process env vars win over the file + if override, found := os.lookup_env(key, allocator); found { + result[strings.clone(key, allocator)] = override + continue + } + + // clone so the map outlives the source buffer (e.g. a freed file read) + result[strings.clone(key, allocator)] = strings.clone(value, allocator) + } + + return result +} + +// parse_file reads a dotenv file from disk and parses it into a map. It +// returns (nil, false) when the file cannot be read (e.g. it does not exist). +parse_file :: proc(filename: string, allocator := context.allocator) -> (map[string]string, bool) { + data, err := os.read_entire_file(filename, allocator) + if err != nil { + return nil, false + } + defer delete(data, allocator) + + return parse(string(data), allocator), true +} + +// destroy frees the cloned keys/values and the map itself. Use it to release +// a map returned by parse/parse_file (plain delete does not free the strings). +// Any allocator passed to parse/parse_file must be passed here too. +destroy :: proc(env: map[string]string, allocator := context.allocator) { + for key, value in env { + delete(key, allocator) + delete(value, allocator) + } + delete(env) +} + +// env_key returns the env key that should bind to a struct field. It prefers +// an explicit `env:"NAME"` tag; when the tag is absent or empty it falls +// back to the field's name. Matching against the parsed map is +// case-insensitive, so API_URL maps onto api_url (or an `env:"API_URL"` tag). +@(private) +env_key_for_field :: proc(field: reflect.Struct_Field) -> string { + if tag_key, ok := reflect.struct_tag_lookup(field.tag, "env"); ok && tag_key != "" { + return tag_key + } + return field.name +} + +// decode populates dest's fields from env, matching each field by name +// (or by an `env:"NAME"` struct tag). Values are converted to the field's +// type: string is cloned as-is into allocator, integers are parsed with +// strconv.parse_int (decimal/hex/negative), booleans with strconv.parse_bool, +// and floats with strconv.parse_f64. Keys missing from env leave the field +// at its zero value. It returns false if a present value cannot be converted +// to the field's type. +decode :: proc(env: map[string]string, dest: ^$T, allocator := context.allocator) -> bool { + ti := reflect.type_info_base(type_info_of(T)) + fields, ok := ti.variant.(runtime.Type_Info_Struct) + if !ok { + return false + } + + value: string + field_ptr := rawptr(dest) + st: reflect.Struct_Field + for _, i in fields.names[:fields.field_count] { + st = reflect.struct_field_at(T, i) + name := env_key_for_field(st) + value = "" + found := false + for key, v in env { + if key == name || strings.equal_fold(key, name) { + value, found = v, true + break + } + } + if !found { + continue + } + + field_ptr = rawptr(uintptr(dest) + fields.offsets[i]) + field_ti := reflect.type_info_base(fields.types[i]) + #partial switch variant in field_ti.variant { + case runtime.Type_Info_String: + (^string)(field_ptr)^ = strings.clone(value, allocator) + case runtime.Type_Info_Integer: + parsed, err := strconv.parse_int(value) + if !err { + return false + } + switch field_ti.size { + case 1: + (^i8)(field_ptr)^ = cast(i8)parsed + case 2: + (^i16)(field_ptr)^ = cast(i16)parsed + case 4: + (^i32)(field_ptr)^ = cast(i32)parsed + case 8: + (^i64)(field_ptr)^ = cast(i64)parsed + case: + return false + } + case runtime.Type_Info_Boolean: + parsed, err := strconv.parse_bool(value) + if !err { + return false + } + (^bool)(field_ptr)^ = parsed + case runtime.Type_Info_Float: + parsed, err := strconv.parse_f64(value) + if !err { + return false + } + switch field_ti.size { + case 4: + (^f32)(field_ptr)^ = cast(f32)parsed + case 8: + (^f64)(field_ptr)^ = parsed + case: + return false + } + case: + // unsupported field type (slices, pointers, ...) is left untouched + } + } + + return true +} diff --git a/gui/lib/local/dotenv/test/dotenv_test.odin b/gui/lib/local/dotenv/test/dotenv_test.odin new file mode 100644 index 0000000..b14fc73 --- /dev/null +++ b/gui/lib/local/dotenv/test/dotenv_test.odin @@ -0,0 +1,245 @@ +package dotenv_tests + +import "core:os" +import "core:strings" +import "core:testing" +import dotenv "lib:dotenv/src" + +Test_Config :: struct { + api_url: string, + debug: bool, + port: int, + ratio: f64, +} + +Tagged_Config :: struct { + api_url: string `env:"API_URL"`, + port: int `env:"PORT"`, + debug: bool `env:"DEBUG"`, +} + +Tagged_Empty :: struct { + api_url: string `env:""`, +} + +// load_env writes src to a unique temp file and parses it via parse_file. +// The returned map owns its strings; callers must destroy it. +load_env :: proc(t: ^testing.T, src: string) -> map[string]string { + dir, err := os.make_directory_temp("", "dotenv_test_*", context.allocator) + testing.expect(t, err == nil, "expected temp dir to be created") + defer os.remove_all(dir) + defer delete(dir) + + path := strings.concatenate({dir, "/.env"}) + defer delete(path) + + testing.expect( + t, + os.write_entire_file(path, src) == nil, + "expected file write to succeed", + ) + + env, ok := dotenv.parse_file(path) + testing.expect(t, ok, "expected parse_file to succeed") + return env +} + +@(test) +test_parse_basic :: proc(t: ^testing.T) { + env := load_env(t, "API_URL=http://127.0.0.1:8080\nDEBUG=true\nPORT=8080\n") + defer dotenv.destroy(env) + + testing.expect(t, env["API_URL"] == "http://127.0.0.1:8080") + testing.expect(t, env["DEBUG"] == "true") + testing.expect(t, env["PORT"] == "8080") +} + +@(test) +test_parse_ignores_comments_and_blank_lines :: proc(t: ^testing.T) { + env := load_env(t, "# leading comment\n\n \nFOO=bar \nBAZ = qux \n") + defer dotenv.destroy(env) + + testing.expect(t, env["FOO"] == "bar", "value should be trimmed") + testing.expect( + t, + env["BAZ"] == "qux", + "key and value should be trimmed around '='", + ) + testing.expect( + t, + "API_URL" not_in env, + "comment-only lines should not be parsed", + ) +} + +@(test) +test_parse_quoted_values :: proc(t: ^testing.T) { + env := load_env(t, "GREETING=\"hello world\"\nEMPTY=\"\"\n") + defer dotenv.destroy(env) + + testing.expect( + t, + env["GREETING"] == "hello world", + "quoted value with inner space", + ) + testing.expect(t, env["EMPTY"] == "", "double-quoted empty value") +} + +@(test) +test_parse_skips_lines_without_equals :: proc(t: ^testing.T) { + env := load_env(t, "not-an-assignment\nOK=yep\n") + defer dotenv.destroy(env) + + testing.expect(t, env["OK"] == "yep") + testing.expect( + t, + "not-an-assignment" not_in env, + "line without '=' should be skipped", + ) +} + +@(test) +test_parse_missing_file :: proc(t: ^testing.T) { + env, ok := dotenv.parse_file("/nonexistent/dotenv_test_does_not_exist.env") + testing.expect(t, !ok, "missing file should report failure") + testing.expect(t, env == nil, "missing file should return nil map") +} + +@(test) +test_real_env_overrides_file :: proc(t: ^testing.T) { + testing.expect(t, os.set_env("DESI_EXPLORER_TEST_FOO", "from_env") == nil) + defer os.unset_env("DESI_EXPLORER_TEST_FOO") + + env := load_env(t, "DESI_EXPLORER_TEST_FOO=from_file\n") + defer dotenv.destroy(env) + + testing.expect( + t, + env["DESI_EXPLORER_TEST_FOO"] == "from_env", + "real env var should win over file", + ) +} + +@(test) +test_decode_maps_fields_case_insensitively :: proc(t: ^testing.T) { + env := load_env( + t, + "API_URL=http://127.0.0.1:8080\nDEBUG=true\nPORT=8080\nRATIO=0.5\n", + ) + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect(t, dotenv.decode(env, &cfg)) + defer delete(cfg.api_url) + + testing.expect( + t, + cfg.api_url == "http://127.0.0.1:8080", + "API_URL maps onto api_url", + ) + testing.expect(t, cfg.debug == true, "DEBUG=true should decode to true") + testing.expect(t, cfg.port == 8080, "PORT=8080 should decode to int 8080") + testing.expect(t, cfg.ratio == 0.5, "RATIO=0.5 should decode to f64 0.5") +} + +@(test) +test_decode_matches_exact_and_lowercase_keys :: proc(t: ^testing.T) { + env := load_env(t, "api_url=http://exact\nPort=9090\n") + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect(t, dotenv.decode(env, &cfg)) + defer delete(cfg.api_url) + + testing.expect( + t, + cfg.api_url == "http://exact", + "exact-case key should match", + ) + testing.expect(t, cfg.port == 9090, "mixed-case key should match field") +} + +@(test) +test_decode_missing_keys_leave_zero_values :: proc(t: ^testing.T) { + env := load_env(t, "UNRELATED=value\n") + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect(t, dotenv.decode(env, &cfg)) + + testing.expect(t, cfg.api_url == "") + testing.expect(t, !cfg.debug) + testing.expect(t, cfg.port == 0) + testing.expect(t, cfg.ratio == 0) +} + +@(test) +test_decode_unparsable_int_fails :: proc(t: ^testing.T) { + env := load_env(t, "PORT=oops\n") + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect( + t, + !dotenv.decode(env, &cfg), + "unparsable int should make decode fail", + ) +} + +@(test) +test_decode_unparsable_bool_fails :: proc(t: ^testing.T) { + env := load_env(t, "DEBUG=maybe\nAPI_URL=http://127.0.0.1:8080\n") + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect( + t, + !dotenv.decode(env, &cfg), + "unparsable bool should make decode fail", + ) + defer delete(cfg.api_url) +} + +@(test) +test_decode_hex_and_negative_ints :: proc(t: ^testing.T) { + env := load_env(t, "PORT=0x1F\n") + defer dotenv.destroy(env) + + cfg := Test_Config{} + testing.expect(t, dotenv.decode(env, &cfg)) + testing.expect(t, cfg.port == 31, "hex int should decode") +} + +@(test) +test_decode_honors_env_tags :: proc(t: ^testing.T) { + env := load_env(t, "API_URL=http://127.0.0.1:8080\nPORT=9090\nDEBUG=true\n") + defer dotenv.destroy(env) + + cfg := Tagged_Config{} + testing.expect(t, dotenv.decode(env, &cfg)) + defer delete(cfg.api_url) + + testing.expect( + t, + cfg.api_url == "http://127.0.0.1:8080", + "env tag should bind API_URL", + ) + testing.expect(t, cfg.port == 9090, "env tag should bind PORT") + testing.expect(t, cfg.debug == true, "env tag should bind DEBUG") +} + +@(test) +test_decode_empty_env_tag_falls_back_to_field_name :: proc(t: ^testing.T) { + env := load_env(t, "api_url=http://fallback\n") + defer dotenv.destroy(env) + + cfg := Tagged_Empty{} + testing.expect(t, dotenv.decode(env, &cfg)) + defer delete(cfg.api_url) + + testing.expect( + t, + cfg.api_url == "http://fallback", + "empty env tag should use field name", + ) +} diff --git a/gui/src/config.odin b/gui/src/config.odin new file mode 100644 index 0000000..1a2355b --- /dev/null +++ b/gui/src/config.odin @@ -0,0 +1,44 @@ +package main + +import "core:log" +import "core:os" +import dotenv "lib:dotenv/src" + +Config :: struct { + api_url: string `env:"API_URL"`, +} + +get_config :: proc(env_file: ^string = nil) -> (^Config, ^Error) { + path := ".env" + if env_file != nil && env_file^ != "" { + path = env_file^ + } else if from_env, ok := os.lookup_env("GUI_ENV_FILE", context.temp_allocator); ok { + path = from_env + } + + log.debugf("getting config from file: %s", path) + + env, _ := dotenv.parse_file(path, context.temp_allocator) + defer dotenv.destroy(env, context.temp_allocator) + + c := new(Config) + + if env == nil { + c.api_url = os.get_env("API_URL", context.temp_allocator) + } else if !dotenv.decode(env, c) { + return nil, new_clone(Error{.Config, "failed to decode .env into Config"}) + } + + if err := validate_config(c); err != nil { + return nil, err + } + + return c, nil +} + +validate_config :: proc(c: ^Config) -> (err: ^Error) { + if c.api_url == "" { + err = new_clone(Error{.Config, "'API_URL' is required"}) + } + return err +} diff --git a/gui/src/data.odin b/gui/src/data.odin new file mode 100644 index 0000000..e4da099 --- /dev/null +++ b/gui/src/data.odin @@ -0,0 +1,48 @@ +package main + +import "core:net" +import "vendor:curl" + +Catalog :: struct { + name: string, + release: string, + description: ^string, + object_count: ^u64, +} + +CatalogObject :: struct { + id: string, + catalog: string, + object_type: string, + ra: f64, + dec: f64, + redshift: f64, +} + +APIError :: struct { + code: int, + message: string, +} + +get_catalogs :: proc(url: string) -> ([dynamic]Catalog, ^APIError) { + ucurl := curl.url() + + + defer curl.url_cleanup(ucurl) + return nil, nil +} + +get_catalog :: proc(url: string, name: string) -> (^Catalog, ^APIError) { + return nil, nil +} + +get_catalog_objects :: proc( + url: string, + catalog_name: string, +) -> ( + [dynamic]CatalogObject, + ^APIError, +) { + return nil, nil +} + diff --git a/gui/src/error.odin b/gui/src/error.odin new file mode 100644 index 0000000..daeb84f --- /dev/null +++ b/gui/src/error.odin @@ -0,0 +1,20 @@ +package main + +import "core:fmt" +import "core:strings" +ErrorType :: enum { + Config, + API, +} + +Error :: struct { + type: ErrorType, + message: string, +} + +format_error :: proc(err: ^Error) -> string { + sb := strings.builder_make(context.temp_allocator) + + return fmt.sbprintf(&sb, "[%s] => %s", err.type, err.message) +} + diff --git a/gui/src/main.odin b/gui/src/main.odin index 67285fb..b971bac 100644 --- a/gui/src/main.odin +++ b/gui/src/main.odin @@ -1,7 +1,9 @@ package main +import "base:runtime" import "core:math" import "core:math/rand" +import "core:os" import rl "vendor:raylib" WIDTH :: 1280 @@ -24,6 +26,16 @@ universe: [dynamic]Galaxy rng: rand.Default_Random_State main :: proc() { + + c: ^Config + err: ^Error + + s := os.get_env("GUI_ENV_FILE", context.temp_allocator) + + if c, err = get_config(&s); err != nil { + panic(format_error(err)) + } + rng = rand.create(0xDE51_0000) context.random_generator = rand.default_random_generator(&rng) @@ -103,11 +115,5 @@ draw :: proc() { } rl.DrawFPS(10, 10) - rl.DrawText( - "DESI Explorer — drag to rotate, scroll to zoom", - 10, - 34, - 18, - rl.RAYWHITE, - ) + rl.DrawText("DESI Explorer — drag to rotate, scroll to zoom", 10, 34, 18, rl.RAYWHITE) } diff --git a/resources/ai/research/data-streaming/01-architecture.md b/resources/ai/research/data-streaming/01-architecture.md new file mode 100644 index 0000000..3504594 --- /dev/null +++ b/resources/ai/research/data-streaming/01-architecture.md @@ -0,0 +1,159 @@ +# 01 — Current Architecture and Where FlatBuffers Fits + +## What exists today + +### API (Rust + axum) + +- **Entrypoint**: `api/src/main.rs` — binds `API_BIND_ADDR` (default `0.0.0.0:8080`), + wires `tracing`, serves the axum app with graceful shutdown. +- **Routes** (`api/src/routes/`): + - `GET /health` → static `{"status":"ok"}` + - `GET /api/v1/catalogs` → hardcoded list of 2 catalogs (`edr`, `dr1`) + - `GET /api/v1/objects?catalog=&limit=` → **stubbed**, returns `[]` +- **Models** (`api/src/models.rs`): + +```rust +pub struct Catalog { + pub name: String, + pub release: String, + pub description: &'static str, + pub object_count: Option, +} + +pub struct CatalogObject { + pub id: String, + pub catalog: String, + pub object_type: String, // galaxy / quasar / star + pub ra: f64, // degrees + pub dec: f64, // degrees + pub redshift: f64, // dimensionless +} +``` + +- **Serialization**: serde `Serialize` derives + `axum::Json`. `serde_json` is a + dev-dependency only (used by integration tests). +- **CORS**: `tower-http` is compiled with `cors` + `trace` features, but no CORS + layer is currently added to `routes::app()`. This matters for the WASM GUI. + +### GUI (Odin + raylib) + +- **Entrypoint**: `gui/src/main.odin` — `update()` → `draw()` loop, orbital + `Camera3D`, 4,000 procedurally generated points via `make_universe()`. +- **Placeholder data**: `Galaxy { position: rl.Vector3, color: rl.Color }` + generated in a 500-unit-radius sphere. Colors are a distance-based redshift + stand-in. +- **Data layer**: `gui/src/data.odin` defines hand-written Odin structs that + mirror the Rust API JSON contract (including pointer types `^string`, `^u64` + to mirror Rust `Option`), plus stubbed procedures: + +```odin +get_catalogs :: proc(url: string) -> ([dynamic]Catalog, ^APIError) // nil, nil +get_catalog :: proc(url: string, name: string) -> (^Catalog, ^APIError) // nil, nil +get_catalog_objects :: proc(url: string, catalog_name: string) // nil, nil +``` + +- **No HTTP client exists** on the Odin side — no `core:net`, no curl bindings, + nothing wired into the render loop. The structs are a *planned* interface. +- **Web build**: `gui/www/` — WASM shell (Emscripten) loading `index.wasm` + + `odin.js`, no API URL wiring yet. + +## Data flow gap + +```mermaid +flowchart LR + A["DESI catalog store"] + B["Rust / axum API
serde JSON models"] + C["Odin + raylib GUI
hand-mirrored structs
stubs, never used"] + A -. "future ingestion" .-> B + B --x|"nothing today"| C +``` + +There is **no live data flow**. The API currently returns JSON placeholders; the +GUI renders procedural points. The intended shape (per code comments): +- The /api/v1/objects endpoint "will page through the centralized DESI catalog store." +- Real survey data replaces the procedural cloud "once ingestion lands." + +## Where FlatBuffers fits + +There are three natural insertion points: + +### 1. Wire format for the object-stream endpoint (primary fit) + +Replace the JSON array response of `GET /api/v1/objects` (or add an +`application/flatbuffer` variant / new `/api/v1/stream` endpoint) with a framed +stream of FlatBuffer messages: + +- **Rust side**: build FlatBuffers with `FlatBufferBuilder` per batch (e.g. + 10k-50k objects per batch), send as `application/octet-stream`. +- **Odin side**: receive the byte blob, verify once, and access `ra`/`dec`/ + `redshift` **directly from the network buffer** — no per-field copy. This is + the killer use case for the render loop, where every point is projected and + colored per frame. + +### 2. Catalog metadata (secondary fit) + +`GET /api/v1/catalogs` returns tiny JSON today. The FlatBuffers win here is +marginal (5-10 records), but using the *same* serialization everywhere keeps the +codebase uniform — one schema, one reader path. Recommend serving both a JSON +response (for curl/debugging) and a FlatBuffers variant (for the client). + +### 3. Bidirectional client → server channel (future fit) + +If the GUI ever sends selection/filter requests (pan region, redshift range, +object type filters), a WebSocket carrying FlatBuffers both ways gives +consistent framing. The `get_catalog_objects()` stub signature suggests the GUI +is expected to query **on demand** as the camera moves — a WebSocket request/ +response protocol with a cheap binary payload would fit this well. + +## Integration points (concrete anchors) + +| Layer | Today | FlatBuffers insertion point | +|---|---|---| +| Rust models | `Catalog`, `CatalogObject` (serde) | Generated `catalog_generated.rs` alongside or replacing | +| Rust routes | `routes/catalogs.rs` | New route (e.g. `/ws/catalogs`) or response-format negotiation | +| Rust Cargo | `serde`, `axum::Json` | `flatbuffers` crate + `flatbuffers-build` for `.fbs → .rs` | +| Odin structs | `data.odin` mirrors JSON | Generated/FFI FlatBuffers reader; or hand-rolled reader | +| Odin HTTP | none | `core:net` HTTP, WebSocket, or curl FFI | +| Build | `make build`, `make test` per project | Schema shared & compiled by both sides | + +## High-level target architecture + +```mermaid +flowchart TB + S["catalog.fbs
single source of truth
checked into repo"] + R["flatc --rust"] + C["flatcc --c
or hand-rolled Odin reader"] + API["api/ · Rust"] + GUI["gui/ · Odin + raylib"] + S --> R --> API + S --> C --> GUI + API <-->|"HTTP / WebSocket
framed FlatBuffer binary stream"| GUI +``` + +- One schema file. Two generators. Byte-for-byte identical wire format. +- A shared conformance test suite (see `08-testing-strategies.md`) ensures both + sides stay in sync. + +## Key architectural decisions to make later + +1. **Transport**: HTTP chunked/range requests vs. WebSocket vs. both + (see `05-streaming-protocols.md`). +2. **Framing**: size-prefixed FlatBuffers (`size_prefixed_root_*`) vs. a custom + length prefix (e.g. flatstream's 4-byte LE length). Needed because a TCP/ + WebSocket stream is byte-ambiguous — you must know where one message ends and + the next begins. +3. **Batching**: per-object tables vs. packed tables of arrays (columnar-style). + For millions of DESI objects, a `table { ra: [double]; dec: [double]; ... }` + layout is dramatically smaller and faster than 1 table per object. +4. **Schema ownership**: options include a top-level `schema/` directory shared + by both `api/` and `gui/`, checked-in generated code, or generated at build + time. Checked-in generated code is the simplest CI-safe option for both + `cargo build` and `odin build` isolation. + +## What to read next + +- `02-flatbuffers-overview.md` — the format's internals and guarantees +- `03-rust-integration.md` — how the Rust side consumes `.fbs` +- `04-odin-client-integration.md` — how the Odin side consumes the same format +- `05-streaming-protocols.md` — transport and framing choices +- `06-schema-design.md` — proposed DESI schema \ No newline at end of file diff --git a/resources/ai/research/data-streaming/02-flatbuffers-overview.md b/resources/ai/research/data-streaming/02-flatbuffers-overview.md new file mode 100644 index 0000000..916b423 --- /dev/null +++ b/resources/ai/research/data-streaming/02-flatbuffers-overview.md @@ -0,0 +1,183 @@ +# 02 — FlatBuffers Overview + +## What it is + +FlatBuffers is a **cross-platform serialization library** created by Google, +originally for game development and other performance-critical applications. Its +defining feature is **zero-copy data access**: you can read fields directly out +of the serialized byte buffer without parsing, unpacking, or allocating objects. + +- **Homepage**: https://flatbuffers.dev +- **GitHub**: https://github.com/google/flatbuffers +- **License**: Apache 2.0 +- **Schema language**: `.fbs` files (similar to `.proto` in spirit, but different) +- **Current version**: 25.12.19 (Dec 2025 on crates.io — ships with `flatc`) + +## Core guarantees + +| Property | Description | +|---|---| +| Zero-copy reads | Fields are accessed in-place via offsets — no deserialization step | +| Zero-alloc reads | Reading a buffer allocates nothing | +| Cross-language | Same schema compiles to C++, C, Rust, Go, Java, C#, JS/TS, Python, Swift, Dart, PHP, Lobster, Lua | +| Forward compatible | Code compiled against schema V1 reads data written with schema V2 (ignores unknown fields) | +| Backward compatible | Code compiled against schema V2 reads data written with schema V1 (returns defaults for missing fields) | +| Little-endian, fixed-width | Deterministic cross-platform serialization (offsets are u32 LE) | +| mmap-friendly | Works with memory-mapped files; only touched bytes are paged in | +| Thread-safe reads | Reading requires no synchronization; tables implement `Send + Sync` | + +## How the format works internally + +A FlatBuffer is a **contiguous binary blob** of nested objects arranged so the +data can be traversed in place like a pointer-based structure. The key concepts: + +### Tables + vtables + +- A **table** is an object whose fields are optional. It carries an inline + `soffset_t` (signed 32-bit) pointing at a **vtable**. +- The **vtable** maps field slots to (offset, presence) pairs. If a field is + absent (written by older code), the accessor returns the schema's default. +- Multiple tables with the same layout **share the same vtable** — this is where + the zero-copy "no per-object metadata" win comes from. +- Tables are the *only* construct that supports schema evolution. + +### Structs + +- A **struct** is a "naked" fixed-layout record with **no vtable** and **no + per-field optionality** (every field required). +- Layout is deterministic across platforms: scalars aligned to their own size; + struct aligned to its largest member. +- Smaller and faster than tables, but **cannot evolve** — fields can't be added + or removed. +- Perfect for things like `Vec3`, `Color`, or a coordinate record that will + never change. Example: DESI's `ra`/`dec`/`redshift` triplet could be a struct, + or (for bulk data) arrays of raw doubles. + +### Vectors + +- Native vector type (unlike protobuf's `repeated`). Length-prefixed, contiguous, + and for scalars/structs can be accessed via `safe_slice()` returning a + `&[T]` view (on little-endian machines) — **zero-copy slice access**. +- This is the single most important feature for DESI render workloads: + `table { ra: [double]; dec: [double]; z: [double] }` gives you three + `&[f64]` slices straight from the wire buffer. + +### Unions + +- A `union` is a type-discriminated tagged field (a `Type` enum + value). +- Used for things like `ServerMessage` that can carry different payloads + (catalog list, object chunk, error). + +### File identifiers + size prefixes + +- **File identifier**: up to 4 ASCII chars stored at offset 4, enabled in the + schema with `file_identifier "DESI"`. Lets a reader check "is this buffer a + DESI buffer" before touching it. +- **Size prefix**: a `u32` LE length written at the start of the buffer, + enabling stream framing (`size_prefixed_root_*`). This is the mechanism you + use to frame a stream of FlatBuffer messages. + +## Schema evolution rules (summary) + +These are the "thou shalt" rules for keeping buffers compatible: + +1. Add new fields **only at the end** of a `table`. (Or use explicit `id` + attributes on every field to allow reordering.) +2. Never remove a field — mark it `(deprecated)` instead. +3. Never change a field's type unless the width is identical *and* there are no + negative-value edge cases. +4. Never change a field's `default` after shipping (the default is baked into + generated code, not the buffer). +5. Enums: only ever add values, never remove. Handle unknown values defensively. +6. Structs: **cannot** gain or lose fields. Ever. +7. `required` on a non-scalar field: the verifier fails if it's absent. Adding + or removing `required` is a compatibility break. +8. Renaming fields/tables is safe for binary compat (buffers address by id, not + name) but breaks code that uses the old names. +9. Run `flatc --conform schema_v1.fbs schema_v2.fbs` to mechanically check that + `schema_v2` properly evolves from `schema_v1`. + +> A subtle trap from the official docs: **scalar fields equal to their default +> are NOT written to the buffer**. Non-presence is indistinguishable from "wrote +> the default". If presence matters (e.g. `object_count: Option` where +> `Some(0)` is meaningful) use **optional scalars** (`default: null`) or wrap the +> scalar in a struct. + +## Reading a buffer (conceptual) + +Buffer layout: + +```mermaid +flowchart LR + subgraph BUF["bytes: &[u8]"] + O["uoffset
root table offset"] + FI["file_identifier
(optional)"] + D["tables · vtables · data"] + end + O --> FI --> D +``` + +Access sequence — each field is a few offset dereferences and a read: + +```mermaid +flowchart TD + R["root = follow(bytes)
jump to root table via uoffset"] + V["vtable = root − root.vtable_off
locate vtable for this table"] + Q{"field slot present?"} + R --> V --> Q + Q -- "no" --> DEF["use schema default"] + Q -- "yes" --> RD["ra = read_f64(bytes, root + slot_ra)"] +``` + +There is **no parsing loop**. Each accessor is a few offset dereferences and a +read. The Rust runtime exposes this via a `Follow` trait that compiles to near-zero +code after optimization. + +## Verification (security) + +- The **verifier** walks a buffer and checks offsets, alignment, required fields, + and structural sanity *before* you access it. Protects against malicious or + corrupted buffers and out-of-bounds reads. +- Rust: `flatbuffers::root::(&opts, bytes)` runs the verifier (default + options); `root_unchecked::` skips it for trusted data. +- Available since the 25.x era in Rust; the C port (flatcc) has long had full + verifier support; vitally, Rust verification is now mature, so the "Rust has + no verifier" caveat from old docs is outdated. +- For untrusted network data (which is exactly your WebSocket case), **always + verify** before zero-copy access. +- Verifier limits (`VerifierOptions`) can cap `max_tables`, `max_depth`, etc. to + defend against resource-exhaustion attacks. + +## Buffer size limits + +- FlatBuffers uses `u32` offsets internally; a single buffer **cannot exceed + 2 GiB**. `FlatBufferBuilder` enforces `FLATBUFFERS_MAX_BUFFER_SIZE`. +- For DESI catalogs with millions of objects, this means **batch** (chunk) the + data — e.g. 100k objects per buffer — rather than one giant buffer. That + aligns naturally with streaming/pagination anyway. + +## Why it fits DESI Explorer + +- **Render-loop reads**: 100k Galaxy points → drop the network bytes into a + buffer, call `verify()`, then per frame iterate `ra[i]`, `dec[i], z[i]` as raw + `f64` slices. Zero allocations, zero GC pressure, cache-friendly contiguous + arrays. +- **Schema versioning across DESI releases**: `edr` data vs `dr1` vs `dr2` + differ in coverage/targets. Add `object_id` in release 2 at the end of the + table; release-1 clients still read old buffers fine. +- **Cross-language integrity**: the same `.fbs` compiles to Rust (API) and C/Odin + (GUI) with byte-identical wire format — no hand-maintained JSON contract drift, + which is exactly the `data.odin` mirror-struct problem today. +- **Single obvious streaming primitives**: size-prefixed buffers make framing + trivial; vectors make bulk data compact. + +## Official resources + +- Tutorial: https://flatbuffers.dev/tutorial/ +- Schema guide: https://flatbuffers.dev/schema/ +- Evolution rules: https://flatbuffers.dev/evolution/ +- Rust usage: https://flatbuffers.dev/languages/rust/ +- C (flatcc) usage: https://flatbuffers.dev/languages/c/ +- White paper: https://flatbuffers.dev/white_paper/ +- API docs (Rust): https://docs.rs/flatbuffers +- crates.io: https://crates.io/crates/flatbuffers \ No newline at end of file diff --git a/resources/ai/research/data-streaming/03-rust-integration.md b/resources/ai/research/data-streaming/03-rust-integration.md new file mode 100644 index 0000000..acc0b9d --- /dev/null +++ b/resources/ai/research/data-streaming/03-rust-integration.md @@ -0,0 +1,279 @@ +# 03 — Rust / API Integration + +## Crate landscape + +### `flatbuffers` (runtime, official) + +- **crate**: `flatbuffers` — https://crates.io/crates/flatbuffers +- **Version**: 25.12.19 (Dec 2025). Note: project moved to CalVer in 2025 + (24.12, 25.1, 25.2, 25.9, 25.12.19). The old `2.1.x` line is legacy. +- **Support level**: labeled "experimental" historically, but is the official + Google runtime; used in production at Google and elsewhere. API stable enough + for real usage. +- **Features**: `serde` (optional), `std` (default), also supports `no_std`. +- **Dependencies**: `bitflags` only. +- **Downloads**: ~17M/month, 174 dependents — well maintained. + +### `flatbuffers-build` (build-time codegen, community) + +- **crate**: `flatbuffers-build` — wraps `flatc` so `.fbs → .rs` happens in + `build.rs` instead of manually. Requires `flatc` on PATH (or `FLATC_PATH`). +- **Caveat**: it writes generated code into a **symlinked directory under + `src/`** (`set_symlink_directory("src/generated")`) because Cargo can't include + `OUT_DIR` output as a module. This is a known hack; works but slightly ugly. +- **Alternative**: the `flatc` crate (`https://crates.io/crates/flatc`) vendors + a prebuilt `flatc` binary as a dependency. + +### `flatc` (vendored binary, community) + +- Crate `flatc 0.2.2+23.5.26` bundles the `flatc` executable. Let's you invoke + `flatc` from `build.rs` without requiring it system-wide. + +### `flatstream` (framing layer, community — very relevant) + +- **crate**: `flatstream` — https://github.com/dallasmarlow/flatstream-rs +- Adds a **framing layer** for streams of FlatBuffers: each frame is + `[4-byte LE length][optional checksum][payload]`. +- Zero-copy read via `StreamReader::process_all(&[u8])`. +- Originated as a high-frequency telemetry capture format. Composability: + bounded payload caps, checksum via xxhash64/crc32, observers, validators. +- **Sync IO only** (std `Read`/`Write`). Not async — you'd pair it with + `tokio::task::spawn_blocking` for network use, or just write your own framing + (it's ~20 lines). +- Note: the crate is young (2025) and small. Treat as a reference pattern, not + a hard dependency. A size-prefixed FlatBuffer + (`builder.finish_size_prefixed(root, Some("ID"))`) already gives you framing + for free if you only need one message type per stream. + +### Choice matrix + +| Need | Crate | Notes | +|---|---|---| +| Runtime read/write | `flatbuffers` | Required regardless | +| `.fbs → .rs` manually | `flatc` binary | Simple, deterministic, scriptable | +| `.fbs → .rs` in build.rs | `flatbuffers-build` | Needs flatc on PATH; symlink hack | +| Vendored flatc binary | `flatc` crate | Avoids system install | +| Stream framing | `flatstream` (or hand-rolled) | 4-byte LE length prefix pattern | +| Testing support | `flatbuffers` verification + `cargo test`, `proptest`, `quickcheck` | See testing doc | + +## Codegen options for the API + +### Option A — Commit generated code (recommended starting point) + +```bash +flatc --rust -o api/src/generated catalog.fbs +``` + +Then `mod catalog_generated;` in `lib.rs`. Pros: dead simple, zero build +scripts, works with `cargo test` out of the box, diffable in code review. Cons: +regenerate manually on schema change, risk of stale generated code. + +### Option B — build.rs codegen + +```rust +// api/build.rs +fn main() { + let schemas = ["../schema/catalog.fbs"]; + flatbuffers_build::BuilderOptions::new_with_files(&schemas) + .set_symlink_directory("src/generated") + .compile() + .expect("flatbuffer compilation failed"); +} +``` + +Pros: automatic regeneration. Cons: symlink hack, `flatc` must be installed +(CI needs it too), `cargo::rerun-if-changed` wiring is manual-ish. For a repo +that already pins toolchains in CI, adding `flatc` to `.gitea/workflows/ci.yml` +is mechanical. + +### Option C — shared schema + make target + +The project's build is `make`-based and mono-repo style. A root target like + +```make +schema-gen: + flatc --rust -o api/src/generated ../schema/catalog.fbs + flatcc --c -o gui/src/generated ../schema/catalog.fbs +``` + +run before `make build`/`make test` keeps both sides in sync from **one +schema file**. This fits the existing "root Makefile delegates" pattern well. + +## Recipe: serving FlatBuffers from axum + +### Dependencies + +```toml +[dependencies] +axum = { version = "0.8", features = ["ws"] } # ws feature for WebSocket +flatbuffers = "25.12.19" # runtime +# for build.rs codegen (if you choose Option B): +[build-dependencies] +flatbuffers-build = "0.2" # or use `flatc` binary option +``` + +### Endpoint that returns a FlatBuffer body (content-negotiated) + +```rust +use axum::{body::Bytes, extract::Query, http::StatusCode, response::{IntoResponse, Response}}; +use flatbuffers::{FlatBufferBuilder, WIPOffset}; + +mod catalog_generated; +use catalog_generated::desi::{CatalogObject, CatalogObjectArgs, ObjectBatch, ObjectBatchArgs, + finish_object_batch_buffer, root_as_object_batch}; + +async fn objects(query: Query) -> Response { + let rows = fetch_desi_objects(&query.catalog, query.limit.clamp(1, 100_000)).await; + + let mut fbb = FlatBufferBuilder::with_capacity(1 << 20); + let ra = fbb.create_vector(&rows.ra); + let dec = fbb.create_vector(&rows.dec); + let z = fbb.create_vector(&rows.z); + let ids = fbb.create_vector(&rows.id_offsets); + + let batch = ObjectBatch::create(&mut fbb, &ObjectBatchArgs { + catalog: Some(fbb.create_string(&rows.catalog)), + n: rows.len as u64, + ra: Some(ra), + dec: Some(dec), + redshift: Some(z), + ids: Some(ids), + ..Default::default() + }); + finish_object_batch_buffer(&mut fbb, batch); + + ([(header::CONTENT_TYPE, "application/octet-stream")], Bytes::copy_from_slice(fbb.finished_data())) + .into_response() +} +``` + +### WebSocket endpoint + +```rust +use axum::{ + extract::ws::{Message, WebSocket, WebSocketUpgrade}, + response::Response, + routing::any, +}; +use futures_util::{stream::StreamExt, sink::SinkExt}; + +async fn ws_handler(ws: WebSocketUpgrade) -> Response { + ws.on_upgrade(handle_socket) +} + +async fn handle_socket(mut socket: WebSocket) { + // On connect, stream catalog batches as Binary(Message::Binary(...)) frames: + // each frame's payload is a size-prefixed or length-prefixed FlatBuffer. + while let Some(Ok(msg)) = socket.recv().await { + match msg { + Message::Text(query) => { + // Client asked for a catalog region; stream batch FlatBuffers back. + let rows = fetch_region(&query).await; + for chunk in rows.chunks(50_000) { + let buf = build_object_batch_flatbuffer(chunk); + socket.send(Message::Binary(buf.into())).await.unwrap(); + } + } + Message::Close(_) => break, + _ => {} + } + } +} +``` + +Notes: +- WebSocket **binary messages** (`Message::Binary`) are exactly-typed `Vec`/ + `Bytes` — ideal for FlatBuffer payloads with no extra encoding. +- Use `WebSocketUpgrade.max_message_size(...)` to cap client message size + (default 64 MB). +- For a single server pushing to many clients, use `tokio::sync::broadcast` and + split the socket (`socket.split()` → `SinkExt` for sending, `StreamExt` for + receiving) — see `05-streaming-protocols.md`. +- If the Odin client is WASM (browser), CORS matters: add `tower-http`'s + `CorsLayer` since the WASM origin differs from the API origin. WebSockets + aren't CORS-restricted the same way HTTP is, but the HTTP upgrade handshake + still goes through the middleware stack. + +## Building buffers efficiently (Rust specifics) + +Build order is **back-to-front** (children before parents): + +```mermaid +flowchart TB + A["create_string / create_vector
children first"] + B["create nested child tables"] + C["create parent table
ObjectBatch::create(&args)"] + D["builder.finish(root, Some("DESI"))"] + E["Bytes::copy_from_slice(fbb.finished_data())
→ HTTP / WebSocket response"] + A --> B --> C --> D --> E +``` + +- `FlatBufferBuilder::with_capacity(n)` pre-allocates; `reset()` reuses the + buffer across messages. In a loop streaming batches, create one builder, reuse + it — avoid repeated reallocation. +- Builder methods have `try_*` counterparts (`try_create_string`, + `try_push`, `try_finish`) that return `Result` instead of panicking on + out-of-capacity — useful on the server under memory pressure. +- `with_internal_capacity(size, field_locs, vtables, strings)` preallocates all + internal data structures — removes allocation-related latency spikes in a + latency-sensitive server path. Combined with `reset()`, a long-lived stream + handler does **zero allocations** after startup. +- For bulk numeric data, prefer **vectors of scalars** (columnar layout) over + `create_vector` of tables. `create_vector` on `&[f64]` is a fast path; on + little-endian it back-fills with scalar offsets, enabling `safe_slice` + reads. Result: one contiguous `&[f64]` slice per column on the client. +- Building is **back-to-front**: children (strings, vectors, nested tables) + must be created before the parent table referencing them. The `*Args` struct + pattern (`ObjectBatchArgs { .. }`) makes this manageable, but be deliberate + about creation order. +- `builder.finished_data()` returns `&[u8]` borrowing the builder — copy to an + owned `Bytes` for the response, then `reset()`. + +## Serialization on the server (axes to optimize) + +- **Batch size tradeoff**: each batch carries one vtable + one table header; + amortize over more rows. Choose batch size so the buffer is well under the + 2 GiB limit and matches your streaming cadence (e.g. 50k objects ≈ ~3 MB). +- **CPU cost**: FlatBuffers serialization is fast but not free (it's the + read-side where FlatBuffers shines). Measure with criterion; if serialization + becomes the bottleneck, move builder work to a dedicated worker thread / + `spawn_blocking` and hand finished batches to the WebSocket sender via a + channel. +- **Vtables**: tables with identical field layouts share vtables automatically — + batching objects with the same schema shape is free (they all share). + +## Integration with existing serde models + +You don't have to delete serde. Pragmatic path: + +- Keep `CatalogObject` (serde) for JSON endpoints / debugging / curl-ability. +- Add a **separate** FlatBuffers representation for the binary path, generated + from the shared `.fbs`. +- Add a `From for ObjectBatchArgs`-style conversion, or better: + build FlatBuffers **directly from the raw DESI rows** (skip the serde struct + entirely, avoiding a serialize-then-rebuild pass). The DESI store should hand + you vectors of `f64`; feed those straight into `create_vector`. + +This preserves today's test shape (`api/tests/health.rs` parses JSON) and adds +the binary path without churn. + +## CI / build concerns for this repo + +- The Gitea workflow (`ci.yml`) runs `cargo test` + clippy. If you switch to + build.rs codegen, CI needs `flatc` installed (or use the vendored `flatc` + crate). If you commit generated code, CI needs nothing extra — one more reason + Option A (commit generated code) is the lowest-friction start. +- Pin the **same schema file** for both API and GUI. If the schema lives in a + shared `schema/` directory and both sides' Makefiles depend on it, `make test` + stays the single verification step: Rust tests + Odin tests + Go tests each + validate their own pipeline against the shared contract. +- Mind the repo convention: `make -C api check` is the passthrough for + `cargo test` + clippy. Keep the FlatBuffers stack incrementally addable here. + +## References + +- https://flatbuffers.dev/languages/rust/ — official Rust usage doc +- https://docs.rs/flatbuffers — runtime API reference +- https://github.com/frol/flatc-rust — flatc-as-build-API (alternative) +- https://github.com/dallasmarlow/flatstream-rs — framing reference +- https://crates.io/crates/flatbuffers-build — codegen crate \ No newline at end of file diff --git a/resources/ai/research/data-streaming/04-odin-client-integration.md b/resources/ai/research/data-streaming/04-odin-client-integration.md new file mode 100644 index 0000000..c406bd6 --- /dev/null +++ b/resources/ai/research/data-streaming/04-odin-client-integration.md @@ -0,0 +1,235 @@ +# 04 — Odin / GUI Integration + +**This is the highest-risk area of the whole research effort.** FlatBuffers has +**no first-party Odin binding**. You must choose one of several integration +paths. This doc lays out the realistic options with tradeoffs, plus the +networking story for the GUI. + +## The core problem + +Odin can talk to C via `foreign` blocks (the C ABI is first-class). FlatBuffers +has two C-related access points: + +1. **Google's official `flatc`** generates C++ and C (C#/Java/etc.) — but its + C support is a *separate project*, **FlatCC**. +2. **FlatCC** (`dvidelabs/flatcc`) is an independent FlatBuffers compiler + + runtime for **pure C**. It generates `_reader.h`/`_builder.h` headers per + schema plus a small `libflatccrt.a` runtime. Works via the C ABI, so Odin's + `foreign import` can consume it. + +Choosing a path: + +```mermaid +flowchart TD + NAT{"primary target is native desktop?"} + NAT -- "yes" --> CGO{"want to avoid C in the build?"} + CGO -- "yes" --> PATHA["Path B · pure-Odin reader
hand-rolled, no C dependency"] + CGO -- "no" --> PATHA + CGO -- "prefer proven lib / less maintenance" --> PATHC["Path A · FFI to FlatCC
bind generated C headers"] + PATHC --> REUSE["Path C · OdinArrow reuse
or borrow its decode patterns"] + NAT -- "no · browser/WASM" --> PATHD["Path D · TS/JS interop
official JS lib → typed arrays into WASM"] +``` + +Index of paths: + +| Path | Effort | Zero-copy on reads | Notes | +|---|---|---|---| +| A: FFI to FlatCC (C runtime) | Medium | ✅ | Bind generated C headers to Odin `foreign` | +| B: Pure-Odin reader (hand-rolled) | High | ✅ | Port a minimal reader; no C dependency | +| C: OdinArrow reuse | Medium-High | ✅ | OdinArrow already has hand-rolled FlatBuffers encoder/decoder | +| D: TS/JS interop (WASM-only) | Low-N/A | Partial | Browser builds can read FlatBuffers via JS lib instead | + +### Path A — FFI to FlatCC (recommended starting point) + +**Mechanics.** FlatCC generates per-schema C headers. You: + +1. Install/build flatcc (it's a small C project, `flatbuffer`-compatible). +2. Generate C reader+builder headers from your `.fbs`: + ```bash + flatcc --common -a schema/catalog.fbs -o gui/src/generated + ``` + yields `catalog_reader.h`, `catalog_builder.h`, `catalog_verifier.h`, plus a + `flatccrt.h`/`libflatccrt.a` runtime. +3. Write (or generate with `odin-c-bindgen` / + `Breush/odin-binding-generator`) Odin foreign bindings for the handful of + functions your GUI actually uses. + +Illustrative Odin binding shape (rough, not final API): + +```odin +package catalog_fb + +foreign import flatcc "libflatccrt.a" + +@(default_calling_convention = "c") +foreign flatcc { + // table accessors generated by flatcc look like: + flatbuffers_verify_buffer :: proc(buf: rawptr, size: uint, id: ^byte) -> c.int --- + catalog_Catalog_object_count :: proc(t: ^catalog_Catalog_table) -> u64 --- + catalog_CatalogObject_ra :: proc(t: ^catalog_CatalogObject_table) -> f64 --- + // etc. +} +``` + +Zero-copy: flatcc's generated **reader** macros operate directly on the buffer — +`ra()` is a macro expanding to a bounds-checked buffer read. That maps naturally +to Odin's `#foreign` + cstring/`^f64` access. + +**Downsides:** +- Odin bindings must **track schema regeneration**. Every time you add a field + to the `.fbs`, the C headers change and the Odin foreign decls (or the + generated bindings) must be refreshed. +- flatcc's API is oriented to C macros; binding it faithfully through Odin + `foreign` is doable but fiddly (macros don't transfer — you translate each + macro into the equivalent C function or hand-roll the offset arithmetic). +- You now have a C runtime (static lib) in the GUI build. For the **WASM + target** this must compile under Emscripten — flatcc is plain C and does build + for WASM, but adds to WASM binary size and toolchain coupling. + +**When to choose:** when you want a *proven* library, don't mind C in your +Odin build, and want to avoid writing and maintaining a reader yourself. + +### Path B — Pure-Odin reader (hand-rolled, most effort, most control) + +FlatBuffers read access is genuinely simple — as the ODINARROW project proves. +OdinArrow ships a "hand-rolled FlatBuffers encoder/decoder" for the Arrow IPC +header format. A minimal FlatBuffers table reader in Odin is ~100-300 lines: +follow `u32` offsets, deref vtables, read little-endian scalars. + +What you'd implement: +- `ReadRoot(root: ^u8, size: uint) -> root_offset` (uoffset at byte 0; skip + 4-byte length prefix if size-prefixed). +- Vtable lookup: given a table addr, read `uoffset` back to vtable, scan slots + for a field id, read field offset (or treat as absent → default). +- Vectors: u32 length + element stride; for `f64`/`f32`/structs this is a direct + `^f64` slice after bounds check. +- Verification: walk offsets checking bounds/alignment (or skip for trusted data + — but you're on a **network stream**; verify, at least bounds, before use). + +**Downsides:** +- You own correctness, update discipline, and testing. Every FlatBuffers format + nuance (file identifiers, size prefixes, unions) must be re-implemented. +- Risk of subtle divergence from the C++/Rust/Java implementations + (endianness, alignment, default-value semantics). +- Cross-language conformance tests (see `08-testing-strategies.md`) absolutely + required — you're re-implementing a spec. + +**When to choose:** when you want zero C in the Odin build, plan long-term +maintenance, and value full control (and you can lean on OdinArrow's already +proven patterns). + +### Path C — OdinArrow reuse (hybrid) + +- https://github.com/TimeLord/OdinArrow — a mature-ish Odin implementation of + Apache Arrow's IPC format, **including** a hand-rolled FlatBuffers + encoder/decoder (Arrow IPC metadata is FlatBuffers). +- You could extract/adapt OdinArrow's FlatBuffers decode machinery for your own + schema, or (bolder) adopt Arrow IPC entirely for the data path (Arrow IPC + *is* FlatBuffers-framed + columnar buffers — arguably an excellent fit for + streaming galaxy positions/redshifts). +- OdinArrow is a small, MIT-style community project (TimeLord). Verify license + and maintenance before depending on it. +- If you go pure Arrow IPC, you get batch semantics for free (schema message → + record batch messages) — same framing pattern as FlatBuffers with the + columnar layout built in. +- Arrow IPC stream = length-prefixed (u32 LE) messages, with a + `continuation marker 0xFFFFFFFF` for 4-byte alignment. This is a well-specified + framing you can reuse *without* adopting Arrow's data model. + +**When to choose:** when Arrow-style columnar data is actually what you want +(millions of numeric rows — it is a great fit), and you're okay depending on / +contributing to OdinArrow. + +### Path D — WASM/JS interop (browser build only) + +- The Web GUI (`gui/www/`) builds Odin to WASM and runs alongside JS. +- FlatBuffers has **official JS/TS support** (`flatbuffers` npm package). For + the web build you could do the parsing/decoding in JS (or TypeScript) and hand + plain arrays (`Float64Array`) to the Odin WASM side — losing zero-copy at the + WASM boundary but gaining ecosystem-tested parsing. +- Practical hybrid: **WASM build path**: keep the WebSocket in JS, decode + FlatBuffers in JS (official lib), then transfer typed arrays into WASM memory + (single `Emscripten.HEAPF64.set(...)` copy). Zero-copy is not preserved across + the WASM boundary, but the *raw-bytes → arrays* decode in JS is still far + cheaper than JSON and uses a battle-tested library. +- Native desktop build (`odin build` + raylib, the primary target): need one of + A/B/C. The WEB GUI is secondary. + +**Recommendation for this repo's roadmap:** Start with **Path B or A** for the +native Odin build (the primary `make run` target), and use **Path D** for the +WASM build if/when it becomes a shipping concern. The conformance-test suite +(shared static `.mon`/`.bin` fixture files read by both Rust and Odin) is the +safety net that makes the hand-rolled Path B safe. + +## Networking from Odin + +There is **no networking code in the GUI today**. Options for receiving +FlatBuffers from the Rust API: + +| Option | Fit | Notes | +|---|---|---| +| `core:net` (Odin stdlib) | Native desktop | Built-in `core:net` module has socket APIs; HTTP is manual or minimal — fine for `GET` of a binary body; WebSocket requires hand-rolling the upgrade + frame handling (doable, ~200 lines) | +| Curl FFI (`libcurl`) | Native desktop | Battle-tested HTTP, easy `buffer` callback for `application/octet-stream`; Odin `#foreign` to curl is well-trodden (e.g., furbs). Adds libcurl dep to native build | +| Emscripten `fetch` bridge | WASM | In the browser build, JS owns the network; call `fetch` from JS or via Odin's Emscripten bindings, then `HEAP`-copy | +| WebSocket via JS | WASM | Same as above for the browser | +| Community libs | Both | e.g. various `core:net`-based or `thirdparty` HTTP clients; vet for maturity | + +`gui/src/data.odin` already has the intended procedure signatures: + +```odin +get_catalogs :: proc(url: string) -> ([dynamic]Catalog, ^APIError) +get_catalog_objects :: proc(url: string, catalog_name: string) -> ([dynamic]CatalogObject, ^APIError) +``` + +When the transport is in place, these become the seam between the network layer +and the FlatBuffers decode layer: fetch bytes → verify → read fields → return +typed Odin slices. + +## ZSS / zero-copy in the render loop + +The rendering win only materializes if data stays zero-copy **into the frame +loop**: + +```mermaid +flowchart LR + A["fetch frame bytes → [dynamic]u8
or a slice pinned for the frame"] + B["verify the buffer once"] + C["ObjectBatch.ra(&buf) → []f64 view"] + D["per object in update()/draw()
ra[i] · dec[i] · z[i] → rl.Vector3 → DrawPoint3D"] + A --> B --> C --> D +``` + +No per-object allocation. The current `Galaxy { position, color }` dynamic array +in `main.odin` is the data structure you'd replace with *slices into the +FlatBuffer*. + +## WASM memory-model caveats + +- If JS decodes FlatBuffers and hands typed arrays to Odin/WASM, the copy into + WASM linear memory is one `HEAPF64.set()` — a single memcpy, not per-field. +- If Odin/WASM itself decodes FlatBuffers over its own `core:net` binding, it + reads directly from WASM heap — same as native, but you're now maintaining the + hand-rolled decoder in WASM too (subject to WASM's 32-bit indexing, still fine + for sub-2GiB buffers). +- Emscripten `-sALLOW_MEMORY_GROWTH` and 4GB heap settings matter if you plan + to hold multi-GB catalogs; keep to batched frames (e.g. ≤ 64 MB) and free per + chunk. + +## Build integration in this repo + +The `gui/Makefile` owns the Odin targets. Adding FlatBuffers means: +- A `-collection:lib=lib/local` (or a vendored submodule under `gui/lib/`) for + either the `flatcc` runtime lib or the hand-rolled Odin package. +- A `schema-gen` source of truth: run flatc for Rust + C (Path A) or otherwise + regenerate, as a make target (see README root / `03-rust-integration.md`). +- CI (`odin test` in the Gitea workflow) must pick up the new collection and any + generated files. Commit generated files to avoid CI toolchain surprises. + +## References + +- Odin FFI docs: https://odin-lang.org/docs/ffi/ (foreign blocks, calling conventions) +- Odin binding to C (official news post): https://odin-lang.org/news/binding-to-c/ +- FlatCC: https://github.com/dvidelabs/flatcc (and `flatbuffers.dev/languages/c/`) +- OdinArrow: https://github.com/TimeLord/OdinArrow (owned FlatBuffers + Arrow IPC) +- Odin binding generator (community): https://github.com/karl-zylinski/odin-c-bindgen +- msgpack-odin (reference for a hand-rolled binary codec in Odin, small): https://github.com/tgolsson/msgpack-odin \ No newline at end of file diff --git a/resources/ai/research/data-streaming/05-streaming-protocols.md b/resources/ai/research/data-streaming/05-streaming-protocols.md new file mode 100644 index 0000000..76b8f48 --- /dev/null +++ b/resources/ai/research/data-streaming/05-streaming-protocols.md @@ -0,0 +1,295 @@ +# 05 — Streaming Protocols & Transport + +The wire isn't just the serialization — it's the **transport + framing**. This +doc covers how to move framed FlatBuffer messages from the Rust/axum API to the +Odin GUI. + +## High-level options + +| Transport | Full-duplex | Server push | Browser (WASM) support | Native Odin support | Fit for DESI Explorer | +|---|---|---|---|---|---| +| HTTP GET + paginated responses | No | No | Native `fetch` (works via JS bridging) | `core:net` / curl | Baseline; simplest start | +| HTTP chunked streaming (SSE-like) | No | One-way stream | `fetch` streaming / SSE | `core:net` | Good for one-shot bulk pushes, no client→server mid-stream control | +| WebSocket | Yes | Yes | Native browser API | Hand-rolled or FFI | **Best overall** for interactive pan/zoom + chunked object streaming | +| Raw TCP | Yes | Yes | N/A (browser can't do raw TCP) | `core:net` | Best perf but breaks the web target | + +**Recommendation: WebSocket for the interactive streaming path.** It supports +bidirectional messaging (client asks for a catalog region; server pushes chunks), +works in both native Odin and the browser, and axum has first-class `ws` support. + +## Framing — where one message ends and the next begins + +A byte stream gives you no message boundaries. You need framing. FlatBuffers +offers two built-in mechanisms plus the community pattern: + +### Option 1: Size-prefixed FlatBuffers (built-in) + +```mermaid +flowchart LR + A["u32 LE
total buffer len
size prefix"] + B["u32 LE
root table offset"] + C["file identifier
(4 bytes)"] + D["tables · vtables · data"] + A --> B --> C --> D +``` + +```rust +builder.finish_size_prefixed(root, Some("DESI")); +``` + +Reader side: + +```rust +// Rust +let msg = flatbuffers::size_prefixed_root::(&buf).unwrap(); +``` + +```odin +// Odin (hand-rolled) +length := read_u32_le(buf) // skip this many bytes for the payload +root_offset := read_u32_le(buf + 4) // root table starts here +``` + +Pros: zero framing code; built into the format. Cons: doesn't carry an explicit +checksum; one message type per stream (fine here unless you multiplex multiple +message kinds). + +### Option 2: Custom length-prefix framing (like `flatstream`) + +```mermaid +flowchart LR + A["u32 LE
message_len"] + B["optional checksum
(u32 crc / xxhash)"] + C["FlatBuffer payload"] + A --> B --> C +``` + +- `flatstream-rs` (see `03-rust-integration.md`) is a reference implementation + of exactly this pattern: length prefix + optional checksum + zero-copy reads. +- For a WebSocket you *could* rely on WebSocket frames as boundaries — each + `Message::Binary` is already a discrete frame. Simpler than custom framing: + **one WebSocket message = one FlatBuffer**. This is the lowest-effort and + totally viable option. FlatBuffers even recommends size-prefixed buffers for + streams, but if your transport already frames (WebSocket, HTTP chunked), the + extra size prefix is optional — though harmless to keep for uniformity. + +### Option 3: Batch as a single buffer (no framing needed) + +If each HTTP response / WebSocket message is a **complete** `ObjectBatch` +(columnar) FlatBuffer, you don't need in-band framing at all — the response +*is* one message. Framing only matters when you stream *multiple* messages per +connection. For DESI Explorer: +- **HTTP path**: one response = one batch → no extra framing. +- **WebSocket path**: one `Binary` message = one batch → no extra framing + (WebSocket frames give you boundaries). +- **Raw TCP path**: you need framing (Option 1 or 2). + +## Transport deep-dive + +### WebSocket (axum) + +Server: + +```rust +use axum::{ + extract::ws::{Message, WebSocket, WebSocketUpgrade}, + extract::State, + routing::any, + Router, +}; + +async fn ws_handler(ws: WebSocketUpgrade, State(state): State>) -> Response { + ws.max_message_size(64 * 1024 * 1024) // 64 MB + .on_upgrade(|socket| handle_socket(socket, state)) +} + +async fn handle_socket(socket: WebSocket, state: Arc) { + let (mut sender, mut receiver) = socket.split(); + + // Writer task: sends FlatBuffer batches from a broadcast channel + let mut rx = state.tx.subscribe(); + let send_task = tokio::spawn(async move { + while let Ok(batch) = rx.recv().await { + if sender.send(Message::Binary(batch.into())).await.is_err() { + break; // client disconnected + } + } + }); + + // Reader task: handles client requests (region query, filters) + while let Some(Ok(msg)) = receiver.next().await { + match msg { + Message::Text(q) => handle_query(q, &state), // triggers a push + Message::Close(_) => break, + _ => {} + } + } + send_task.abort(); +} +``` + +Key points: +- `Message::Binary(Bytes)` is the native carrier for FlatBuffer payloads. +- Use `tokio::sync::broadcast` for fan-out to many clients (window `Lagged` + errors; a slow client falls behind → notify instead of blocking). +- `max_message_size` and `max_frame_size` tunables matter for large batches. +- `WebSocketUpgrade` works over HTTP/1.1 `GET` upgrades; in a browser, the + connection is `ws://`/`wss://`. + +Client (Odin, native): + +```odin +import "core:net" + +// 1. Open TCP to host:port +sock, err := net.dial_tcp(endpoint) +// 2. Send the HTTP upgrade handshake: +// GET /ws HTTP/1.1 +// Host: ... +// Upgrade: websocket +// Connection: Upgrade +// Sec-WebSocket-Key: +// Sec-WebSocket-Version: 13 +// 3. Parse 101 Switching Protocols response. +// 4. Frame format (client→server): [fin|opcode=2 (binary)] [mask=1] [len] [mask-4B] [payload] +// 5. Server→client frames: [fin|opcode=2] [len] [payload] (no mask requirement) +``` + +- Odin `core:net` has no WebSocket built-in; hand-rolling the upgrade + simple + frame reader is ~150-250 lines for the *subset* you need (text in, binary + out). There are community implementations (e.g. furbs has a networking lib). +- For the WASM build, JS owns the WebSocket (`new WebSocket(url)`), delivers + `ArrayBuffer`s — you then copy the byte slice into WASM heap. See + `04-odin-client-integration.md` Path D. + +### HTTP streaming (alternative, simpler) + +Server: + +```rust +use axum::body::Body; +use axum::http::header::CONTENT_TYPE; +use futures_util::StreamExt; +use tokio::sync::mpsc; + +async fn stream_objects(Query(q): Query) -> Response { + let (tx, rx) = mpsc::channel::>(8); + tokio::spawn(async move { + for chunk in paginate_desi(q).await.unwrap().chunks(50_000) { + let _ = tx.send(build_object_batch_flatbuffer(chunk)).await; // backpressure via channel + } + }); + let stream = tokio_stream::wrappers::ReceiverStream::new(rx) + .map(|bytes| Ok::<_, std::io::Error>(Bytes::from(bytes))); + Response::new(Body::from_stream(stream)) +} +``` + +- Client reads length-prefixed messages from the response body. +- Simpler than WebSocket (no handshake/frames), but one-directional and no + mid-stream client control. + +### Raw TCP (maximum perf, breaks browser) + +- `core:net` TCP + length-prefixed frames. Low overhead, but the web GUI can't + participate. Only worth it if the native desktop build must outperform + WebSocket by a large margin (it won't in practice for this workload — network + bandwidth dominates, and WebSocket adds ~2-14 bytes/frame depending on size). + +## Message envelope design + +Even with transport framing, you'll likely multiplex message **kinds** over the +same stream (catalog list, object batch, error, ping). Two clean patterns: + +### Pattern A — size-prefixed buffer + `ServerMessage` union (fully self-describing) + +```fbs +union Message { + CatalogList, + ObjectBatch, + APIError, +} + +table ServerMessage { + msg: Message; // disc + catalog_list: CatalogList; + object_batch: ObjectBatch; + error: APIError; +} + +root_type ServerMessage; +``` + +The client `verify`s the whole buffer and pattern-matches the union tag. One +buffer format for everything. + +### Pattern B — per-kind size-prefixed buffers (simpler decoder) + +```fbs +root_type CatalogList; // separate finish buffer +root_type ObjectBatch; // separate finish buffer +root_type APIError; // separate finish buffer +``` + +Each message is a different root type. The client sniffs the file identifier +(sector size prefix `size_prefixed_root_with_opts`) to dispatch. Slightly less +structural safety, but each decoder is trivial. + +Recommendation: **Pattern A** if the stream carries mixed traffic (interactive +control + data + errors); **Pattern B** if the path is a pure object stream +(e.g. dedicated `/ws/objects` endpoints per kind). + +## Backpressure & chunking + +- **Chunk size**: batch ~ tens of thousands of objects (~1-4 MB) keeps latency + tight for interactive pan/zoom while amortizing per-buffer overhead. +- **mpsc channel**: cap in-flight buffered batches (e.g. 8) so a slow client + never piles up unbounded memory server-side. `tokio::sync::broadcast` instead + for fan-out, with `Lagged` handling. +- **Client**: buffer at most N in-flight frames; drop old detail when a new + region query supersedes an in-flight one. With zero-copy reads this is cheap: + each frame is one buffer you can free wholesale. +- **Cadence**: push an initial batch immediately on connect; then respond to + client region moves, not a blind rate-limited firehose. (Unless "streaming" + means full-survey playback — then a T-state machine with cursor/seq is better.) + +## HTTP-range / mmap alternative (worth remembering) + +If catalogs become **static cloud files** (like Apache Arrow / FlatCityBuf's +model), you can skip the API entirely for bulk data: ship pre-built `.fbs` files +and let the *client* do HTTP range requests straight from object storage. The +FlatCityBuf project (see references) does exactly this and is a working proof of +concept: spatial index stored beside the FlatBuffer file, `Range` requests fetch +only the slabs you need. + +This is probably a later-stage architecture, but it's the strongest argument for +FlatBuffers long-term (mmap-friendly, page-in-what-you-touch). + +## Decision summary for this repo + +```mermaid +flowchart TD + A["HTTP GET → one FlatBuffer body per batch
validate Rust builder + Odin reader"] + B["WebSocket → one Binary message per batch
interactive path · no custom framing"] + C["Self-identifying messages
file_identifier "DESI""] + D["size-prefixed / flatstream-style framing
or HTTP-range + mmap for static catalogs"] + A --> B --> C + C -. "later, if needed" .-> D +``` + +1. Start with **HTTP GET → one FlatBuffer body per batch** to validate the Rust + builder + Odin reader (no protocol work at all). +2. Then add **WebSocket** with one `Binary` message per batch (no custom framing) + for the interactive path. +3. Keep messages **self-identifying** via file identifiers (`file_identifier "DESI"`). +4. Broaden later: hot spots can move to size-prefixed/`flatstream`-style framing + or even HTTP-range + mmap if catalogs go static. + +## References + +- axum ws docs: https://docs.rs/axum/latest/axum/extract/ws/index.html +- flatstream-rs (framing reference): https://github.com/dallasmarlow/flatstream-rs +- FlatCityBuf (HTTP-range + FlatBuffers + WASM proof-of-concept): + https://github.com/cityjson/flatcitybuf +- Apache Arrow IPC framing spec (continuation + length prefix pattern): + https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format \ No newline at end of file diff --git a/resources/ai/research/data-streaming/06-schema-design.md b/resources/ai/research/data-streaming/06-schema-design.md new file mode 100644 index 0000000..adaeb88 --- /dev/null +++ b/resources/ai/research/data-streaming/06-schema-design.md @@ -0,0 +1,240 @@ +# 06 — Schema Design for DESI Data + +This doc sketches a FlatBuffers schema for DESI Explorer and the rationale +behind each decision. It's a *proposal* — validate against real DESI catalog +columns before finalizing. + +## Design goals + +1. **Compact bulk numeric streaming** — the dominant payload is millions of + (ra, dec, redshift) triples. Use **columnar vectors of scalars**, not one + table per object. This is what makes the performance win real (see + `10-performance-benchmarks.md`) — vectors of scalars are contiguous and can be + `safe_slice`'d into `&[f64]` views. +2. **Schema evolution** for future DESI releases (edr → dr1 → dr2, added + columns like `petro_mag`, `spectroscopic_class`, coverage tiles). +3. **Self-describing streams** via file identifiers and a message union. +4. **Keep the existing API contract** (`Catalog`, `CatalogObject` from + `api/src/models.rs`) as the JSON-facing mirror, so `curl`/debug paths stay. + +## Proposed schema + +```fbs +// schema/catalog.fbs +namespace desi; + +file_identifier "DESI"; + +// ---------- Catalog metadata ---------- + +table Catalog { + name: string; // "edr", "dr1", ... + release: string; // "Early Data Release" + description: string; + object_count: uint64; // Some(0) problem → see note below +} + +// ---------- Bulk object data (columnar) ---------- + +// A single point, for small results / per-row paths. +struct Point { + ra: double; // ICRS, degrees, [0, 360) + dec: double; // ICRS, degrees, [-90, 90] + redshift: double; // dimensionless + object_type: byte; // 0=galaxy 1=quasar 2=star (enum → byte) +} + +table ObjectBatch { + catalog: string; // catalog this batch belongs to + batch_seq: uint32; // for reassembling a paged stream in order + n: uint64; // number of objects in this batch + + // Columnar arrays — zero-copy friendly, contiguous per column. + ids: [uint64]; // object ids (or bytes for survey ids) + object_types: [byte]; // enum per object (galaxy/quasar/star) + ra: [double]; // degrees + dec: [double]; // degrees + redshift: [double]; // dimensionless + // future columns (release-2): petro_mag_g/r/z (optional vectors), + // is_spectroscopic: [bool], ... +} + +// ---------- Server → client envelope ---------- + +table APIError { + code: int32; + message: string; +} + +table CatalogList { + catalogs: [Catalog]; +} + +union Message { + CatalogList, + Catalog, + ObjectBatch, + APIError, +} + +table ServerMessage { + msg: Message; + catalog_list: CatalogList; + catalog: Catalog; + object_batch: ObjectBatch; + error: APIError; +} + +root_type ServerMessage; +``` + +## Rationale per decision + +### Columnar `ObjectBatch` vs. per-object `CatalogObject` tables + +| | Per-object tables | Columnar arrays | +|---|---|---| +| Size | ~vtable (≥4-16B) + per-row overhead per object | ~8 bytes per f64, zero per-row metadata | +| Speed | vtable deref per field per object | Linear memcpy / slice iter | +| Odin reads | method call per field per object | `[]f64` direct slice | +| Evolution | per-field evolveable | vector types can't structurally evolve (add new vectors instead) | +| Fit for render loop | ok for thousands | **ideal for 100k+** | + +3-5 columns of raw doubles per row dominate the payload. Columnar wins by a lot. +Rust builds with `create_vector(&flat_array_of_f64)` — trivial. Odin reads with +`ObjectBatch_ra(obj)` → `[]f64`. + +Caveat: in the columnar layout, an individual object is identified by *index* +across columns (`ra[i]`, `dec[i]`, `redshift[i]`). That's fine for bulk +rendering, and `ids[i]` links back to a catalog row when needed. + +### `file_identifier "DESI"` + +- Enables `flatbuffers::size_prefixed_root_with_opts::(opts, buf)` + and (on the Odin side) cheap "is this our message?" pre-checks. +- Without a file identifier, verification accepts buffers that are structurally + valid but a *different* schema — dangerous across stream kinds. + +### `object_count: uint64` — the `Option` presence trap + +Current Rust model has `object_count: Option`. In FlatBuffers: + +- A `uint64` with `default: 0` **is not written when 0** → reader can't tell + "zero" from "absent". If `Some(0)` is meaningful, use an **optional scalar**: + `object_count: uint64 (nullable)`? No — the FBS way is: + +```fbs +table Catalog { + object_count: uint64 (id: 0, default: null); // optional scalar — presence preserved +} +``` + +Optional scalars (`default: null`) are always written when set, and the accessor +returns the language's optional type. Rust: `object_count: Option`; +Odin (hand-rolled): an `is_set` flag. + +Alternatively wrap in a struct or keep as raw `uint64` if you never ship a +meaningful zero. + +### `enum` for object_type + +```fbs +enum ObjectType : byte { + Galaxy = 0, + Quasar = 1, + Star = 2, +} +``` + +- Table fields can be enums; the accessor returns typed enum. +- Keep it a `byte`/`ubyte` — flat buffering stores the underlying integer; a + vector of enums (`object_types: [ObjectType]`) is just a `&[u8]`/`[]byte` view + on the wire (dense — nice for thousands of classifications). + +### Struct `Point` + +- Only correct if a coordinate + type is *structurally fixed forever* (a struct + cannot gain fields). Risky: DESI might add errors/ellipsoid fields later. For + the bulk path prefer the columnar batch (add a new vector). Keep `Point` + struct only if you need tiny random-access records that will never change — + otherwise a `table` is safer for any "object record" shape. + +### Batch sequence number + +- `batch_seq: uint32` lets the client detect drops/reordering/restarts when + paging/straming through a catalog. Server increments per (catalog, region) + session. Cheap insurance on an unordered transport. + +### Union envelop `ServerMessage` + +- Lets **one** WebSocket carry catalog lists, object batches, errors, and + future message kinds, with the type tag built into the buffer (no guessing by + file identifier). +- Rust: generated `msg_type()` + `msg_as_object_batch()` etc. +- Odin: hand-rolled union-tag read + payload access. + +If you'd rather keep streams homogeneous, skip the union and use separate root +types per endpoint (see `05-streaming-protocols.md` Pattern B). + +## Evolution roadmap (defensive) + +- **v1 (this schema)**: as above. +- **v2 (next release)**: append `petro_mag_g/z/r: [float]` (or per-band uint16 + scaled), `spectral_class: [byte]`, `is_spec_selected: [bool]` to `ObjectBatch` + (all **at the end** of the table — v1 decoders ignore them, v2 decoders get + defaults/absent for v1 data). +- **v3**: `Catalog` gains `coverage_footprint: string` (WKT) at the end. +- Do **not** reorder, remove, or deprecate anything unless the schema owner + signs off; run `flatc --conform` in CI. + +## Compatibility with existing Rust/JSON models + +Keep the FlatBuffers schema **in sync** with `api/src/models.rs` (serde) and +`gui/src/data.odin` (mirrors) during the transition: + +- Add a `From for ObjectBatch`-ish conversion inside the API, or + (better) have the API build FlatBuffers **directly from the DESI row fetch**, + bypassing serde for the binary path. +- The JSON endpoints can stay for debugging/curl and for the initial catalog + list (tiny). Document in `api/` that `/api/v1/objects` will switch to + `application/flatbuffer` (or gain a `/json` variant) once the client is wired. + +## Where the schema lives & codegen ownership + +Proposal: a top-level `schema/catalog.fbs` is the single source of truth. + +``` +schema/ + catalog.fbs <- cross-language contract +``` + +- Rust: flatc → `api/src/generated/catalog_generated.rs` +- Odin (Path A): flatcc → `gui/src/generated/catalog_reader.h` (+ bindings) +- Odin (Path B): hand-rolled reader must be tested against fixtures built from + *this* schema (see `08-testing-strategies.md`). + +Add a make target: + +```make +schema-gen: + flatc --rust -o api/src/generated ../schema/catalog.fbs + flatcc --c --common -o gui/src/generated ../schema/catalog.fbs +``` + +and (optionally) a CI check that committed generated files match a fresh +regeneration (`git diff --exit-code` after regen). + +## Open questions to resolve with real DESI data + +1. DESI object IDs: u64 internal, or string (survey-id like "DESI-1234")? + If string, `ids: [string]` (each a mini-table) costs far more than `[uint64]` + — decide based on the actual catalog schema. +2. Coordinate range conventions: ra in `[0,360)` vs `[-180,180)`? Pick once, + document, optionally store a normalized flag in the schema. +3. Redshift scale/precision: double vs float? For rendering, `f32` halves the + payload and is plenty; for science fidelity, keep `f64` as in the model today. +4. Does the renderer need per-object colors today? If yes, precompute a packed + `[uint32]` RGBA vector in the API — better than the client computing per frame. +5. In-stream metadata: do you need a `Metadata` message (region bounds, epoch, + projection) before the first batch? Add now if so — retrofitting a union arm + mid-release is possible but adds churn. \ No newline at end of file diff --git a/resources/ai/research/data-streaming/07-alternatives.md b/resources/ai/research/data-streaming/07-alternatives.md new file mode 100644 index 0000000..d6761ff --- /dev/null +++ b/resources/ai/research/data-streaming/07-alternatives.md @@ -0,0 +1,159 @@ +# 07 — Alternatives to FlatBuffers + +FlatBuffers isn't the only option. This doc compares the realistic alternatives +for a Rust API → Odin GUI streaming pipeline. The evaluation lens: **cross- +language support (Rust + Odin), zero-copy reads, schema evolution, streaming +fitness, and ecosystem maturity.** + +## Quick comparison table + +| Format | Zero-copy reads | Schema | Wire size (vs JSON) | Serialize | Deser/read | Streaming | Rust support | Odin support | +|---|---|---|---|---|---|---|---|---| +| JSON (current) | No | No | 1.00× (268B) | 5.2 µs | 8.7 µs | SSE/chunk | serde | core:json | +| MessagePack | No | No | 0.60× | 2.1 µs | 3.8 µs | chunk | rmp-serde | tgolsson/msgpack-odin (small) | +| Protocol Buffers | No | **Yes** (proto) | **0.35×** | 1.8 µs | 2.4 µs | gRPC streaming | **prost/tonic** | none official (C via protobuf-c) | +| **FlatBuffers** | **Yes** | **Yes** (fbs) | 0.47× | 0.9 µs | **0.03 µs** | size-prefix/WS | **flatbuffers crate** | none official (C via flatcc) | +| Cap'n Proto | **Yes** | Yes | 0.54× | **0.5 µs** | **0.02 µs** | structured RPC | capnp crate | none official (C via capnproto-c) | +| Apache Arrow IPC | File/stream batch | Yes (FlatBuffers-based) | ~0.42× (typed cols) | batch-append | ~zero-cost column slices | **native stream format** | **arrow crate (official)** | **OdinArrow (community)** | + +(Benchmarks: Mechanical Snail wire-format study, Nov 2025, Rust, 1-MB +median message; 0.47× etc. are compression ratios; absolute numbers vary by +message shape — see `10-performance-benchmarks.md`.) + +## Deeper per-format notes + +### JSON (status quo) + +- **Pros**: human-readable, debug-friendly, universal, no schema burden, + trivially works with curl, streaming via SSE/chunked HTTP. +- **Cons**: slowest (5-14× slower than binary on read-heavy paths), verbose + (2-3× payload size), no type integrity, allocates per-field on parse. +- **In this repo**: already stubbed. Could stay for the *HTTP diagnostics* path, + but for streaming 100k+ points per frame to a renderer, JSON is the wrong tool. +- Verdict: keep for curl/debug endpoints; not the streaming wire. + +### MessagePack + +- **Pros**: binary offshoot of JSON — schema-less, faster, 40% smaller. Rust + (rmp-serde) is mature. There's a small pure-Odin msgpack impl (tgolsson/ + msgpack-odin — timestamps included, no ext types). +- **Cons**: **no schema** → no forward/backward compatibility guarantees, no + zero-copy reads (you parse into Odin structs per field), no type end-to-end + integrity. On reads it's only ~2× faster than JSON — the renderer still pays + a per-field decode. +- Verdict: good "fast JSON" but doesn't solve the zero-copy/schema-evolution + problems. + +### Protocol Buffers (protobuf) + +- **Pros**: smallest wire (varint encoding); gRPC for streaming; **smallest + payloads**; excellent Rust (prost/tonic, prost-build); schema-evolution via + field numbers; huge ecosystem. +- **Cons**: **no zero-copy reads** — deserialization allocates the full object + graph. For 100k objects/frame that's the exact cost FlatBuffers eliminates. + No first-party Odin; C via protobuf-c is clunky. +- Verdict: ideal for "fat service-to-service RPC" where you decode once and + hold objects. Wrong shape for a per-frame high-frequency render workload + unless you pre-decode into the renderer's own buffer. + +### Cap'n Proto + +- **Pros**: true zero-copy (wire layout ≈ memory layout), pointer-based lazy + access, built-in RPC, slightly simpler encode than FlatBuffers (no backward + construction). Serialize ~0.5 µs, deserialize ~0.2 µs (fastest table entries + in the Mechanical Snail study). +- **Cons**: **8-byte alignment makes it ~20-50% larger than FlatBuffers** on + the wire; **no Robert-native Odin**; lower ecosystem/maturity than + FlatBuffers; fewer language bindings (no official WASM/JS story as rich). +- Verdict: a genuinely strong competitor, but for this repo the wiresize penalty + (bandwidth for millions of points) and weaker Odin/JS ecosystem tilt toward + FlatBuffers. + +### Apache Arrow IPC (columnar, used as transport) + +- **Pros**: **the canonical columnar data format**; the *only* alternative with a + **native Odin implementation** (TimeLord/OdinArrow) already shipping a + Zero-copy IPC reader (memory-mapped, pyarrow-interopable). Streaming file/ + stream formats are purpose-built for bulk numeric data. Rust side gets the + official `arrow` crate. +- **Cons**: bigger dependency surface (arrow crate is heavy); schema model is + Arrow's own (Field/RecordBatch) rather than your own; IPC header uses + FlatBuffers internally (Arrow metadata *is* FlatBuffers), which is a point in + FlatBuffers' favor; existing `CatalogObject` models don't map 1:1. +- Verdict: **strongest technical fit for "millions of numeric rows, columnar, + zero-copy, streaming"**. If the DESI dataset is *really* millions-of-points + columnar, Arrow IPC deserves a serious look — but you'd be adopting Arrow's + framing + OdinArrow dependency rather than your own schema/format, which reads + as more risk for a repo this young. + +### Flexbuffers / schemaless (bonus) + +- FlatBuffers ships **Flexbuffers** — its own schemaless, self-describing, + variable-typed format (kind of a fast MessagePack). Rust+JS+Python etc. + supported; Odin via same C ABI story as FlatBuffers. +- Not a fit here: the point of going FlatBuffers is the *shared compiled + schema* keeping Rust/Odin coherent. Flexbuffers gives up exactly that. + +## Decision matrix mapped to DESI Explorer + +| Requirement | FlatBuffers | Cap'n Proto | Protobuf | Arrow | MsgPack | +|---|---|---|---|---|---| +| Zero-copy per-frame reads (100k+ pts) | ✅✅ | ✅✅ | ❌ | ✅✅ | ❌ | +| Compact wire for many f64 columns | ✅ (f64 vectors) | ⚠️ (8B-aligned, larger) | ✅ (varint packed) | ✅✅ (chunked) | ⚠️ | +| Schema evolution (edr→dr1→dr2) | ✅ | ✅ | ✅✅ | ⚠️ (Arrow schema is rigid) | ❌ | +| Rust (API) maturity | ✅ (official crate) | ✅ (capnp crate) | ✅✅ (prost/tonic) | ✅✅ (official) | ✅ | +| Odin (GUI) availability | ⚠️ C-ABI/flatcc or hand-rolled | ⚠️ C-ABI or hand-rolled | ❌ (protobuf-c barely) | ✅ (OdinArrow) | ⚠️ (small impl) | +| WASM/JS story | ✅ (official npm) | ⚠️ | ✅ | ⚠️ | ✅ | +| Streaming primitives | ⚠️ (BYO framing; flatstream-rs exists) | ✅ (RPC) | ✅ (gRPC) | ✅✅ (IPC stream) | ⚠️ (chunking only) | + +## Bottom line + +```mermaid +flowchart TD + Q1{"zero-copy reads
in the per-frame render loop?"} + Q1 -- "no" --> PB["Protobuf / gRPC
decode once into draw buffers"] + Q1 -- "yes" --> Q2{"truly columnar?
millions of rows"} + Q2 -- "yes" --> ARR["Apache Arrow IPC
via OdinArrow"] + Q2 -- "no · batched vectors" --> FB["FlatBuffers · this proposal"] +``` + +- **FlatBuffers is the best default** for this project: the zero-copy read model + matches the render loop, the wire format is compact for numeric vectors, schema + evolution fits DESI's release cadence, and the Rust + WASM/JS official story + is strong. The Odin gap is real but solvable (C-ABI via flatcc, hand-rolled + reader, or OdinArrow patterns). +- **Apache Arrow IPC is the strongest alternative** if the data is truly + "columnar millions-of-rows" and you accept OdinArrow + the Arrow header/ + framing dependency. It even uses FlatBuffers internally for its metadata, so + it's not an either/or at the format level. +- **Protobuf/gRPC** wins only if you'd rather have official Rust streaming + + smallest payloads and are OK decoding into objects once (not per frame) — the + renderer then builds its own draw buffers. For 100k objects/frame the + "decode-once-into-draw-buffer" model is actually defensible; measure before + over-engineering. + +## Test-before-commit suggestion + +Run a micro-benchmark (criterion) on your **actual** payload shape (e.g. 10k +objects × 3 f64 columns / batch) comparing: +1. serde JSON → parse +2. FlatBuffers columnar batch (this proposal) +3. protobuf (prost) → decode to Vec +4. Arrow IPC record batch (arrow crate) → record batch slices + +Numbers will decide the finer trade-offs; the qualitative table above is the +strategic direction. + +## References + +- Mechanical Snail, "Wire Formats for High-Volume Service Communication" (2025-11): + https://mechanicalsnail.com/posts/wire-formats-rust/ +- Youngju, "Binary Serialization Complete Guide" (2025-2026): + https://www.youngju.dev/blog/...-binary-serialization-... +- Maltsev & others, "Impact of Serialization Format on Inter-Service Latency" + (2024): https://science.lpnu.ua/sites/default/files/journal-paper/2024/dec/... +- Opentraffic/DEEPWIKI Flatbuffers deep dive: + https://deepwiki.com/opentraffic/flatbuffers +- Arrow IPC / OdinArrow: + https://arrow.apache.org/docs/format/Columnar.html + https://github.com/TimeLord/OdinArrow +- msgpack-odin: https://github.com/tgolsson/msgpack-odin \ No newline at end of file diff --git a/resources/ai/research/data-streaming/08-testing-strategies.md b/resources/ai/research/data-streaming/08-testing-strategies.md new file mode 100644 index 0000000..67bb3c0 --- /dev/null +++ b/resources/ai/research/data-streaming/08-testing-strategies.md @@ -0,0 +1,256 @@ +# 08 — Unit & Integration Testing Strategies + +FlatBuffers introduces a **cross-language wire contract**. The single most +important testing principle: **both sides must prove they talk the identical +bytes**, not just that each side round-trips with itself. + +## Testing layers + +### 0. Schema hygiene (fastest feedback) + +- **Schema lint / conformance**: run `flatc --conform base.fbs candidate.fbs` + in CI to catch accidental schema-evolution violations (fields added in the + middle, removals, type changes). Early warning before any of the below. +- **Commit-and-diff**: check generated code (`.rs`, `.h`) into the repo; CI + re-runs codegen and `git diff --exit-code`s — catches "you forgot to + regenerate" drift. + +### 1. Rust-side unit tests (API crate) + +The `flatbuffers` crate's own tests live in +`tests/rust_usage_test/tests/integration_test.rs` — copy the patterns: + +**Round-trip test** (write → read own buffer): + +```rust +#[cfg(test)] +mod object_batch_tests { + use crate::generated::desi::{ObjectBatch, ObjectBatchArgs, finish_object_batch_buffer, root_as_object_batch}; + use flatbuffers::FlatBufferBuilder; + + fn build_sample_batch() -> Vec { + let mut fbb = FlatBufferBuilder::with_capacity(4096); + let ra = fbb.create_vector(&[0.1, 25.3, 99.9]); + let dec = fbb.create_vector(&[-5.0, 44.0, -77.2]); + let z = fbb.create_vector(&[0.5, 1.2, 2.8]); + let ids = fbb.create_vector(&[1u64, 2, 3]); + let batch = ObjectBatch::create( + &mut fbb, &ObjectBatchArgs { + catalog: Some(fbb.create_string("dr1")), + batch_seq: 0, n: 3, + ids: Some(ids), ra: Some(ra), dec: Some(dec), redshift: Some(z), + ..Default::default() + }); + finish_object_batch_buffer(&mut fbb, batch); + fbb.finished_data().to_vec() + } + + #[test] + fn roundtrip_columnar_batch() { + let bytes = build_sample_batch(); + let batch = root_as_object_batch(&bytes).unwrap(); + assert_eq!(batch.catalog(), Some("dr1")); + assert_eq!(batch.n(), 3); + assert_eq!(batch.ra().unwrap()[1], 25.3); + assert_eq!(batch.dec().unwrap()[2], -77.2); + assert_eq!(batch.redshift().unwrap()[0], 0.5); + } + + #[test] + fn verifier_rejects_corruption() { + let bytes = build_sample_batch(); + for idx in [0usize, 1, bytes.len() - 1] { + let mut bad = bytes.clone(); + bad[idx] ^= 0xFF; + assert!(root_as_object_batch(&bad).is_err(), "byte mutate at {idx} should fail verification"); + } + } +} +``` + +Key Rust unit-test patterns: +- Values equal **defaults are not serialized** → test that explicitly: + `assert_eq!(batch.missing_field_u64(), 0)` when field absent, and that an + explicitly-set 0 is indistinguishable (unless optional scalar). Cf. the + `optional_scalars_test.rs` macro in the flatbuffers repo. +- **Alignment tests**: assert struct/table accessor pointers are properly + aligned relative to buffer start (the flatbuffers repo has + `generated_code_alignment_and_padding` tests — replicate for `Point`/ + columnar slices). +- **Fuzz** (proptest/quickcheck): roundtrip random vectors of `f64`/`u64` + through the builder and back (fb's `roundtrip_vectors` uses quickcheck). Add + property tests: for any `&[f64]` built with `create_vector`, `ra().unwrap()` + equals the input exactly. + +### 2. Rust-side integration tests (HTTP/WS layer) + +The repo already has `api/tests/health.rs` (spins the app against real routes +via axum). Add an `api/tests/catalog_fb.rs`: + +```rust +// Uses the same axum::Router::app() as health.rs, hits the new WS or +// application/flatbuffer endpoint with tower::ServiceExt::oneshot(), +// collects the body Bytes, and verifies + reads it with the generated types. +#[tokio::test] +async fn objects_endpoint_returns_valid_flatbuffers() { + let app = desi_explorer_api::app(); // or routes::app() + let resp = app.oneshot(Request::builder().uri("/api/v1/objects?catalog=dr1&limit=100").body(Body::empty()).unwrap()).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + let batch = crate::generated::desi::root_as_object_batch(&body).unwrap(); + assert_eq!(batch.n(), 100); + assert_eq!(batch.ra().unwrap().len(), 100); +} +``` + +For WebSocket integration, spawn the server on `127.0.0.1:0` in a test (tokio) +and use `tungstenite`/`tokio-tungstenite` as a test client, or call the endpoint +through axum's mock. At minimum: open a WS, receive N binary frames, verify each +frame parses + verifies + row counts match expectations. + +### 3. Odin-side tests (GUI crate) + +`odin test` exists in this repo (`make test` → `odin test` in gui/). Structure: +- **Reader unit tests** in the Odin package: given a hard-coded byte blob (from + a fixture file), `verify` and read expected field values. +- **Round-trip** against a builder written in Odin (if you hand-roll a builder + for Path B) — mirror the Rust patterns. + +```odin +// gui/src_tests/catalog_fb_test.odin (whatever path `odin test` picks up) +package catalog_fb + +import "core:testing" + +@(test) +test_read_static_batch :: proc(t: ^testing.T) { + data := #load("../testdata/catalog_sample.bin", []u8) + // Read root, iterate ra/dec/redshift via the reader, assert values. + batch := root_as_object_batch(data) + testing.expect(t, batch.n == 3) + testing.expect(t, batch.ra[1] == 25.3) + testing.expect(t, batch.dec[2] == -77.2) + testing.expect(t, batch.redshift[0] == 0.5) +} +``` + +The **critical** test here is the one reading a fixture *produced by the Rust +side / flatc* — see the "cross-language fixture" section below. + +### 4. Cross-language conformance suite (THE key integration test) + +This is the test that actually catches incompatibility. Pipeline: + +```mermaid +flowchart LR + S["schema/catalog.fbs"] + J["testdata/catalog_sample.json"] + S --> F["flatc --binary"] + J --> F + F --> BIN["committed .bin fixtures
repo-checked-in"] + BIN --> OT["Odin tests
assert identical values"] + BIN --> RT["Rust tests
assert expected values"] + RT -. "deterministic builder" .-> PAR["byte-for-byte parity"] + OT -. "reads it" .-> PAR + PAR -. "catch drift" .-> F +``` + +Design: + +1. **Static fixtures, committed to the repo** (`testdata/*.bin`): + - Built once by `flatc --binary .fbs .json` (deterministic, + reproducible in CI with the pinned `flatc`), or by the Rust builder in a + small crate that writes testdata. + - Cover every field type: catalogs, list, empty batch, batch with defaults + omitted, optional scalars present/absent, enum variants, nested strings. +2. **Rust reads fixtures** + asserts expected values. +3. **Odin reads the same fixtures** + asserts identical expected values. +4. **Both sides parse each other's output**: a Rust test writes a buffer and a + companion Odin test reads it (and vice versa). Practically: Rust generates a + batch during `cargo test`, dumps to `OUT_DIR`, Odin test loads it via + `#load` — but static committed fixtures are simpler and just as strong for + the schema-versioned contract. +5. **Byte-for-byte equality on builders**: for deterministic input, the Rust + builder output should byte-identical to flatc's. Assert on fixtures. + +Generate fixture data from a **JSON descriptor** in the repo +(`testdata/catalog_sample.json`) via: + +```bash +flatc --binary --schema schema/catalog.fbs -- testdata/catalog_sample.json +``` + +Committed `.bin` files mean: no toolchain needed to run the test suite; CI can +regenerate and diff to catch drift. + +**A pure-Odin reader on Path B lives or dies by this suite** — it's the only +guarantee your hand-rolled offset/jtable/vtable arithmetic matches Google's. + +### 5. Transport-level integration (end-to-end) + +Two-app test: run the **real** axum server with the **real** DESI fixture data, +drive it from an **Odin test binary** (or a Rust test client for parity): + +```rust +// api/tests/ws_stream.rs — spawn server on ephemeral port, connect WS +// as the client would, assert: schema message first, then N batches, +// then EOS; each batch verifies and cumulative n == expected. +``` + +This closes the loop on the exact runtime path (frame boundaries, file +identifier, union tags) that unit tests miss. + +### 6. Property / fuzz testing + +- **Rust**: `proptest` generating arbitrary `&[f64]`/`Vec` row sets → + build batch → read → compare; plus fuzz the *verifier* with mutated buffers + (the flatbuffers repo does a one-byte-corruption sweep on its own tests — + replicate: for many positions, flip byte, assert verifier either accepts + *correctly* (data still consistent) or rejects; never panics / OOB). +- **Odin**: property tests are harder (no built-in fuzzer); rely on the Rust + side for mutation fuzzing and keep Odin tests as fixed-fixture + round-trip + sanity checks. +- **Corrupt-length attack**: a stream with `length = 0xFFFFFFFF` must be + rejected at framing, not OOM. Test the length-prefix path explicitly + (size prefix + WebSocket frame-size caps). + +### 7. CI wiring in this repo + +`.gitea/workflows/ci.yml` currently: `odin test` + build, `cargo test` + +clippy, `go test` + vet. + +Additions when FlatBuffers lands: +- A **shared schema lane**: `flatc --conform` check + `git diff --exit-code` + after regen (catches schema drift). +- Rust lane: compile generated code included from `api/src/generated`, run new + unit/integration tests (no extra CI deps if generated code is committed). +- Odin lane: run the conformance tests reading committed `.bin` fixtures (no + extra CI deps). +- Optional: the end-to-end WS test (server + client) in the cargo lane. + +Keep `make test` as the single gate: `make test` = odin test + cargo test + +go test, and each now includes the FlatBuffers conformance pieces. + +## Testing checklist (implementation-ready) + +- [ ] `flatc --conform` ran against schema history in CI +- [ ] Committed generated code matches a fresh regen (CI diff check) +- [ ] Rust: round-trip builder↔reader unit tests (all field types / enum / + optional scalar / default-omission cases) +- [ ] Rust: verifier-safety & corruption fuzz +- [ ] Rust: HTTP + WS integration tests against live router +- [ ] Odin: reader unit tests on static fixtures +- [ ] Cross-language: Rust reads fixtures ✓; Odin reads same fixtures ✓; + byte-for-byte parity on builder output +- [ ] End-to-end: real server + real client (WS) with DESI-shaped rows +- [ ] Length/frame-stealing attack tests (huge length, truncated frames) +- [ ] No regression to existing JSON health/catalogs tests during migration + +## References + +- flatbuffers Rust test suite (patterns to copy): + https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/tests/integration_test.rs +- optional_scalars_test.rs (default-omission semantics): + https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/tests/optional_scalars_test.rs +- `flatc --conform` docs: https://flatbuffers.dev/evolution/ +- Existing repo test conventions: `api/tests/health.rs`, `make test`, `.gitea/workflows/ci.yml` \ No newline at end of file diff --git a/resources/ai/research/data-streaming/09-pain-points.md b/resources/ai/research/data-streaming/09-pain-points.md new file mode 100644 index 0000000..4ab4023 --- /dev/null +++ b/resources/ai/research/data-streaming/09-pain-points.md @@ -0,0 +1,199 @@ +# 09 — Cons, Gotchas, & Pain Points + +A candid list of everything that will hurt if you adopt FlatBuffers for this +project — and how to mitigate each. + +## 1. No first-party Odin binding (the biggest risk) + +- FlatBuffers officially supports many languages; **Odin is not one**. The Odin + side requires either (a) C ABI FFI to the third-party **FlatCC** runtime, (b) + a **hand-rolled pure-Odin reader** (100-300 lines, plus maintenance forever), + or (c) reuse of **OdinArrow's** hand-rolled FlatBuffers decoder. +- **Impact**: schema changes ripple into *your* Odin code, not generated code. + Update discipline + the conformance suite are mandatory, not optional. +- **Mitigation**: commit static `.bin` fixtures and hard-code the conformance + tests; keep the schema churn low; consider a tiny code generator if the + schema grows fast. + +## 2. Serialization is harder than JSON/protobuf (back-to-front building) + +- Objects are built **children-before-parents**: create all strings/vectors/ + nested tables first, then the parent that references them. FlatCC's builder + is oriented differently (start/end + stack) and also non-obvious. +- Error-prone ordering bugs are caught only by round-trip tests. +- **Impact**: server-side code is more verbose; onboarding cost. +- **Mitigation**: wrap builder logic in small helper functions per message type; + comprehensive round-trip tests; the columnar layout keeps touching-code + minimal (just `create_vector` × columns). + +## 3. Larger on the wire than protobuf + +- vtables add 4-16 bytes per table; scalar vectors are NOT varint-packed + (every f64 is 8 bytes regardless). FlatBuffers is ~20-50% larger than + protobuf on comparable payloads; in the Go benchmark, proto 82B vs FB 192B on + a small object. +- **Impact**: for millions of objects over a constrained link this is real + bandwidth. For a LAN/k3s homelab renderer, negligible. +- **Mitigation**: columnar vectors eliminate per-row vtable overhead; omit + default-valued fields (free savings); compress with gzip/zstd at the + transport (WebSocket payloads are compressible — but you lose zero-copy if + you decompress into another buffer; keep compressed frames bounded). + +## 4. Not self-describing / schema coupling + +- A FlatBuffer is meaningless without its `.fbs`. Clients (and your future + self debugging with `curl`) cannot inspect a buffer without the schema. +- Schema must be **shared and version-disciplined** between api/ and gui/ — + exactly the drift problem that `data.odin`'s JSON mirrors already suffer. +- **Impact**: debugging is harder; schema changes are coordinated releases. +- **Mitigation**: file identifiers (`file_identifier "DESI"`) for self-checking; + generate JSON from buffers (`flatc --json --schema foo.fbs -- foo.bin`) for + debugging/inspection; keep a `schema/` in-repo single source of truth; + `flatc --conform` in CI. + +## 5. Immutable buffers / no in-place mutation + +- Once built, you cannot edit a FlatBuffer in place (the old C++ JSON path had + reflective mutation; Rust's reflection/resizing is **experimental**). "Fix a + field" → rebuild the whole buffer. +- For the renderer this is fine (it's read-only). For the **server** it means + any per-client customization (filtering, projections) happens *before* build, + or you build per-client variants (expensive). +- **Mitigation**: build canonical buffers; push client-side filtering to the + client (it's zero-copy cheap for the client to skip rows). + +## 6. 2 GiB buffer ceiling + +- Single FlatBuffer cannot exceed 2 GiB (`u32` offsets). A full DESI survey is + millions of rows × ~24-40 bytes = gigabytes. You **must batch**. +- **Impact**: streaming design becomes mandatory (fine — it's the plan). +- **Mitigation**: batches of ≤ 1-4 MB; `batch_seq` to reassemble/order; client + holds N buffers, not one gigantic one. + +## 7. Verification is required for untrusted input + +- Without verification, field reads on a malformed buffer → OOB (Rust: panic / + UB with `_unchecked`; Odin: guaranteed memory unsafety). Your data comes from + **a network** — verify. +- Rust's verifier is good but "experimental" historically: configurable + `VerifierOptions` (max_depth/max_tables) help against DoS via deep buffers. +- **Impact**: an extra O(n) pass per frame. For a 4 MB batch this is sub-ms — + acceptable; still, do it **once per batch**, not per accessor. +- **Mitigation**: verify on ingest for untrusted streams; for *trusted* replay + (local fixtures) you may use `_unchecked` for the benchmark path; add + `max_*` caps. + +## 8. Cross-build/toolchain friction + +- flatc/flatcc development video: version pinning matters — generated code must + match the runtime crate version. Mismatched `flatc` ↔ `flatbuffers` crate → + subtle incompatibilities. +- The repo pins Odin version and Rust version; now you pin **flatbuffers too** + (schema → generator → runtime all consistent), plus (Path A) compile flatcc + under Emscripten for WASM. +- **Impact**: build-system surface area grows. +- **Mitigation**: commit generated code (skip codegen in CI); pin exact crate/ + tool versions in `api/Cargo.toml`, gui Makefile, and CI YAML; centralize flatc + install (scripts/ like `install_odin.sh` pattern). + +## 9. WASM/Emscripten path is the fuzziest area + +- Zero-copy across the JS↔WASM boundary doesn't exist; JS decodes FlatBuffers + (official lib) then hands typed arrays into WASM heap (one memcpy) — you've + given up the headline feature for the web build. +- Or the Odin WASM code reads frames delivered by JS; but then JS→WASM handoff + doubles as a copy anyway (unless you use `WebAssembly.Memory` shared views and + careful ownership). +- **Impact**: web build behaves *worse* than native (memcpy + JS decode), which + is expected but worth stating. Native desktop (`make run`) is unaffected. +- **Mitigation**: accept one memcpy for web; keep the native zero-copy path + clean; test both. + +## 10. Odin-type ergonomics: optional scalars, enums, slices + +- Odin has no `Option`/`Result` sugar. Optional scalars (`default: null`) + surface as "present? + value" pairs you hand-roll: an `is_set bool` flag (or + NaN sentinel for f64) alongside `value: f64`. +- Enums are fine (Odin `enum` with explicit values) but vector-of-enum reads as + `[]byte` with manual cast. +- Reading column vectors → `[^]f64`/`[]f64` requires the **buffer outlive the + view**: zero-copy slicing borrows the frame buffer. Freeing the frame before + the render pass = dangling `[]f64`. Ownership discipline (defer-free) needed. +- **Mitigation**: document the "frame buffer must outlive render pass" rule; + write copy-free helper procs that take the buffer explicitly; test with + over-release patterns. + +## 11. Debugging binary data + +- No `curl | jq`. Binary payloads are opaque without tooling: `flatc --json` + (official) and flatcc's JSON printer (per-schema C) are your tools. +- IDE display of buffers is limited; hex dumps are the fallback. +- **Impact**: slower onboarding for new contributors; harder support/debug + sessions. +- **Mitigation**: keep the JSON endpoints (health/catalogs) for humans; add a + tiny dev CLI (or make target) that dumps a `.bin` to JSON via flatc; log + length/hash per frame instead of contents. + +## 12. Ecosystem / library maturity wrinkles + +- Rust flatbuffers crate is labeled **"experimental"** in docs although widely + used; API churn across minor versions historically (see docs.rs). Pin + precisely. +- `flatbuffers-build` uses a **symlink-in-src** hack to surface generated code + to build.rs — a smell that can break with Cargo changes. Committing generated + code avoids it (recommended here). +- No official Odin, no official C# for web-unity-esque paths, no protobuf-style + RFC coverage. Small but real gaps. + +## 13. Opinionated schema discipline + +- Teams that skip evolution rules (reordering, deleting, default changes) get + silent data corruption — *worse* than an explicit break because reads don't + error, they return wrong values. +- Rule: new fields only at table end; deprecate (don't delete); don't change + defaults; enums only grow; structs are frozen forever. Enforce with + `flatc --conform` + review discipline. + +## 14. Odin ecosystem: borrowing from OdinArrow is a dependency decision + +- OdinArrow is community-maintained (TimeLord). If you extract only its + FlatBuffers decoder, you own a fork; if you depend on it wholesale, you hold + a third-party Odin dependency in `gui/lib/` with its own update cadence. +- License + maintenance verification required before adoption (the repo's + `add-dep` flow exists for exactly this: submodule into `gui/lib/local/`). + +## 15. Performance expectation management + +- Benchmarks are message-shape-dependent. FlatBuffers' deserialization reads + are near-zero, but **serialization is not fastest** (Mechanical Snail: FB + serialize 0.9µs vs Cap'n 0.5µs). The distributed-latency study even found + protobuf/avro/thrift beat FlatBuffers on end-to-end latency in some + microservice topologies because **payload size dominates network time**. +- For LAN renderer workloads, FlatBuffers' win is real (read-side) but don't + over-promise on shrink: it's bigger than protobuf. +- Measure with your own criterion bench before/after (see `07-alternatives.md` + suggestion). + +## Pain-point severity ranking + +| # | Pain | Severity | Mitigation exists? | +|---|---|---|---| +| 1 | No Odin binding (FFI/hand-roll) | High | Yes (flatcc / OdinArrow / conformance) | +| 2 | Back-to-front serialization complexity | Medium | Yes (helpers + tests) | +| 3 | Bigger wire than proto | Low-Medium | Yes (columnar + omit defaults) | +| 4 | Not self-describing | Medium | Yes (fixture files + file id) | +| 5 | Immutability | Low | Yes (rebuild per batch) | +| 6 | 2GiB ceiling → batching | Low | Yes (batch streaming) | +| 7 | Verifier needed for untrusted input | Medium | Yes (verify once per batch) | +| 8 | Toolchain/version pinning | Medium | Yes (commit generated, pin versions) | +| 9 | WASM zero-copy loss | Low (web is secondary) | Yes (accept 1 memcpy) | +| 10 | Odin ergonomics (Option/slices/borrow) | Medium | Yes (patterns + tests) | +| 11 | Debugging binary | Low | Yes (flatc --json + keep JSON endpoints) | +| 12 | Library maturity wrinkles | Low | Yes (pin, commit generated) | +| 13 | Schema discipline | Medium | Yes (flatc --conform in CI) | +| 14 | OdinArrow dependency | Low | Yes (license/maintenance review) | +| 15 | Performance expectation | Low | Yes (own benchmarks) | + +**Overall**: all pain points have identified mitigations; the two that require +active, ongoing investment are **(1) the Odin reader** and **(13) schema +discipline**. Budget for them in the implementation plan. \ No newline at end of file diff --git a/resources/ai/research/data-streaming/10-performance-benchmarks.md b/resources/ai/research/data-streaming/10-performance-benchmarks.md new file mode 100644 index 0000000..18cb2c0 --- /dev/null +++ b/resources/ai/research/data-streaming/10-performance-benchmarks.md @@ -0,0 +1,147 @@ +# 10 — Performance Benchmarks (published data) + +Numbers vary a lot by benchmark harness, message shape, and library version. +This doc collects representative published results, normalizes what can be +normalized, and interprets them **for the DESI Explorer use case** (streaming +bulk `ra`/`dec`/`redshift` columns to a renderer). + +## Summary table (large/mixed messages, Rust, per-op medians) + +From Mechanical Snail's Rust wire-format study (Nov 2025, c6i.4xlarge, +UserProfile sample ~280B JSON; 1M cycles; versions noted in `03`): + +| Format | Serialize (µs) | Deserialize (µs) | Size (bytes) | Rel. size | Notes | +|---|---|---|---|---|---| +| JSON (serde_json) | 5.2 | 8.7 | 280 | 1.00× | baseline | +| MessagePack (rmp_serde) | 2.1 | 3.8 | 168 | 0.60× | schema-less | +| Protobuf (prost) | 1.8 | 2.4 | 98 | 0.35× | smallest | +| **FlatBuffers** | **0.9** | **0.3*** | 132 | 0.47× | *single-field access, zero-copy | +| Cap'n Proto (capnp) | **0.5** | **0.2*** | 152 | 0.54× | *single-field access, zero-copy | +| Avro | 3.1 | 4.5 | 105 | 0.38× | | + +> *For zero-copy formats, "deserialize" here means *accessing a single field* +> without traversal. Full traversal ≈ 2-3 µs for both — still ~3-30× faster +> than decoding JSON to objects. + +Key takeaways: +- **Undisputed**: binary formats are 2-14× faster than JSON; zero-copy formats + are the fastest on the read path. +- **Size leader**: protobuf (varint). FlatBuffers and Cap'n Proto are 30-55% + larger than proto at this message shape. +- **Write leader**: Cap'n (+40% vs FlatBuffers at this shape). Read leader: + Cap'n/FB both near-free for single field. + +## Second benchmark (Go, small objects, `format-wars.go`) + +| Format | Marshal ns/op | Unmarshal ns/op | Size bytes | Allocs (marshal) | +|---|---|---|---|---| +| MessagePack (msgp) | 69-205 | 155-336 | 157-323 | 1 | +| **FlatBuffers** | **183-414** | **192-450** | 192-448 | **0** | +| Protobuf | 156-1033 | 315-1173 | 82-276 | 1-21 | +| CBOR | 253-453 | 751-1056 | 141-324 | 1-3 | +| JSON (std/v2) | 463-946 | 842-2819 | 213-1940K | 1-3+ | + +Highlights: **FlatBuffers is the only format with 0 allocations on marshal** — +its CSR for the renderer (no per-frame GC/allocation churn) is genuinely +unique among these. + +## Third benchmark (distributed latency, Lviv Polytechnic 2024/2025) + +- **Serialize speed**: FlatBuffers fastest across all message sizes (med <2µs; + vs proto 2.8-27µs). +- **Deserialize speed**: FlatBuffers and Cap'n (unpacked) overwhelming leaders + (FlatBuffers med 0.02-0.03µs). +- **End-to-end distributed latency**: Avro/Protobuf/Thrift beat FlatBuffers + (med reductions vs JSON: Avro -84%, Proto -82%, Thrift -80%, Cap'n packed + -71%, **FlatBuffers only -17%**). Reason: **payload size dominates** in + distributed hops; FlatBuffers isn't the most compact. +- **Interpretation**: if your bottleneck is *network* (large hops over WAN), + compact formats (Avro/proto) win end-to-end. If your bottleneck is + *CPU/allocation on massive reads* (a renderer ingesting the same bytes + repeatedly), zero-copy formats win. DESI Explorer is the latter. + +## Fourth benchmark (C++, embedded/small messages, CppSerialization 2025) + +| Protocol | Message size | Serialize | Deserialize | +|---|---|---|---| +| SBE | 138 B | 35 ns | 52 ns | +| zpp::bits | 130 B | 34 ns | 37 ns | +| Cap'n Proto | 208 B | 247 ns | 184 ns | +| FlatBuffers | 280 B | 272 ns | **81 ns** | +| Protobuf | 120 B | 322 ns | 351 ns | +| JSON | 301 B | 696 ns | 291 ns | + +## What these numbers mean for DESI Explorer specifically + +### Reads are the whole point + +The renderer will read every point **every frame** (projection + color). Even +at a modest 60 fps with 100k visible objects: + +| Approach | Per-object read cost (approx) | Per-frame cost @100k | +|---|---|---| +| JSON parse (serde_json to objects) | ~87 ns (1ms/100k) | ~8.7 ms — 52% of a 16.7ms frame | +| protobuf decode to objects | ~24 ns | ~2.4 ms — 14% | +| FlatBuffers (safe_slice into `&[f64]` columns) | ~0.3 ns (slice view) | **~0.03 ms — 0.2%** | + +The gap is a **100×** per-frame savings against JSON — the difference between +"render a 100k+ galaxy view comfortably" and "frame budget eaten by parsing." + +- The reason the win is so large for your workload: **bulk reads of the same + buffer reconstituted each frame**. Allocating + copying 100k objects per + frame is what you're avoiding. +- If instead you decode protobuf **once** into a renderer-owned + `[]rl.Vector3` buffer (not per frame), protobuf becomes competitive — you're + paying allocation once, not per frame. That's the strongest counter-case to + FlatBuffers, and worth benchmarking before you commit (see below). + +### Writes (server side) are a non-issue + +Server serialization of a 50k-object batch, even at FlatBuffers' ~1-3µs/MB +shape, is negligible next to DB query + network. If it ever matters, reuse the +`FlatBufferBuilder` (with `reset()`) and use `with_internal_capacity` to avoid +realloc spikes. FlatBuffers' *builder* is not the fastest writer, but writers +are amortized; **readers are per-frame**. Optimize where the frame is. + +### Wire size tradeoff is acceptable + +On ~24 MB of raw f64 columns per 100k objects, FlatBuffers columnar adds only +vtables per batch (≈ tens of bytes), not per row — dropping closer to the raw +size than the "132 vs 98 vs proto" small-object ratio suggests. Columnar **is** +where the size concern basically evaporates. + +Bandwidth-wise: LAN/k3s homelab is not bandwidth-constrained; a 3 MB batch +arrives in ~ms. + +## Recommended local benchmark (before committing) + +Write a criterion benchmark on *your* exact shape: + +1. 100k rows × columns `ra/dec/redshift: f64` (+optional `ids: u64`). +2. Variants: + - serde JSON (current contract) → parse + map to Vec + - FlatBuffers **columnar** batch → `safe_slice` reads + - protobuf (prost) → decode to Vec + - Arrow IPC record batch → batch slices +3. Metric: **per-frame read cost** (60fps frame budget) + wire bytes + first-batch + latency. +4. Also measure: build time (server), verify time at ingest, delta vs `_unchecked`. + +Reality check: the numbers here point to FlatBuffers (or Arrow IPC) winning the +*frame-cost* metric decisively; protobuf wins the *wire-size* metric; JSON wins +only developer ergonomics. Decide based on what frame cost is worth to you. + +## Sources + +- Mechanical Snail, "Wire Formats for High-Volume Service Communication" (2025-11-24): + https://mechanicalsnail.com/posts/wire-formats-rust/ +- kapetan-io/format-wars.go (Go, small objects, allocs): + https://github.com/kapetan-io/format-wars.go +- Maltsev et al., "Impact of Serialization Format on Inter-Service Latency" + (ACPS 2024): https://science.lpnu.ua/sites/default/files/journal-paper/2024/dec/36976/... +- chronoxor/CppSerialization (C++, embedded/micro): + https://github.com/chronoxor/CppSerialization +- Shekhar Manna, "Binary Serialization Formats — A Technical Benchmark & Decision Guide" (2026-02): + https://medium.com/@shekhar.manna83/binary-serialization-formats-e2703f053010 +- ADHDecode, "Protobuf vs FlatBuffers vs Cap'n Proto" (2026-03): + https://adhdecode.com/api-architecture/grpc-deep-dive/protobuf-vs-flatbuffers-vs-capn-proto/ \ No newline at end of file diff --git a/resources/ai/research/data-streaming/README.md b/resources/ai/research/data-streaming/README.md new file mode 100644 index 0000000..f308b34 --- /dev/null +++ b/resources/ai/research/data-streaming/README.md @@ -0,0 +1,95 @@ +# Data Streaming Research: FlatBuffers for DESI Explorer + +This directory aggregates research on using **FlatBuffers** as the wire protocol +between the Rust/axum API (`api/`) and the Odin + raylib GUI (`gui/`). The goal +is to provide implementation-ready knowledge — not to build anything yet. + +## Context + +DESI Explorer currently serves placeholder JSON from a Rust/axum API and renders +procedurally generated points in an Odin + raylib client. The two halves are +**disconnected**: the API has stubbed `/api/v1/catalogs` and `/api/v1/objects` +endpoints returning JSON; the GUI has hand-written mirror structs in +`gui/src/data.odin` with stub `get_catalogs()` / `get_catalog_objects()` +procedures. Real DESI catalog data (galaxies, quasars, stars with `ra`, `dec`, +`redshift`) will eventually stream server → client. + +FlatBuffers is attractive here because: +- **Zero-copy reads**: the GUI can access fields directly from the network buffer + with no parsing or allocation — critical when rendering tens of thousands of + points per frame. +- **Schema-versioned**: forward/backward compatibility as the DESI catalog + schema evolves across releases (edr → dr1 → dr2 → ...). +- **Cross-language**: Rust and Odin can both generate code from the same `.fbs` + schema. +- **Compact binary**: smaller payloads than JSON over the wire. + +## Research Documents + +| # | Document | Purpose | +|---|----------|---------| +| 01 | [architecture.md](01-architecture.md) | Current system architecture and where FlatBuffers fits in | +| 02 | [flatbuffers-overview.md](02-flatbuffers-overview.md) | What FlatBuffers is, how the format works internally | +| 03 | [rust-integration.md](03-rust-integration.md) | Rust/axum API integration: crates, build tooling, codegen pipelines | +| 04 | [odin-client-integration.md](04-odin-client-integration.md) | Odin GUI integration: FFI, flatcc, OdinArrow, HTTP/WebSocket clients | +| 05 | [streaming-protocols.md](05-streaming-protocols.md) | WebSocket + HTTP + framing options for streaming FlatBuffers | +| 06 | [schema-design.md](06-schema-design.md) | DESI-specific schema proposal and evolution rules | +| 07 | [alternatives.md](07-alternatives.md) | Cap'n Proto, Protocol Buffers, MessagePack, Apache Arrow comparison | +| 08 | [testing-strategies.md](08-testing-strategies.md) | Unit, integration, cross-language, fuzz, and conformance testing | +| 09 | [pain-points.md](09-pain-points.md) | Cons, gotchas, and pain points to watch out for | +| 10 | [performance-benchmarks.md](10-performance-benchmarks.md) | Published benchmarks (serialize/deserialize/size) and analysis | + +## TL;DR Recommendation + +**FlatBuffers is a strong fit** for the DESI Explorer streaming use case, +specifically for the *server → client* bulk data path (catalog objects). The +zero-copy read model matches the render loop perfectly: the GUI ingests a binary +blob over WebSocket or HTTP, verifies it once, and reads `ra`/`dec`/`redshift` +directly from the buffer each frame without allocations. + +**Key caveats to weigh before committing:** + +1. **Odin has no first-party FlatBuffers binding.** You must go through the + C ABI (via `flatcc` headers + Odin `foreign` blocks) or hand-roll a minimal + reader. [OdinArrow](https://github.com/TimeLord/OdinArrow) already hand-rolls + a FlatBuffers encoder/decoder for the Arrow IPC header — proof the pattern is + viable in pure Odin. +2. **Serialization is more complex than JSON/protobuf.** The builder API builds + buffers back-to-front (children before parents). This is a server-side cost + you pay once per batch, not per client — acceptable. +3. **Not self-describing.** Binary buffers are opaque without the schema. The + API and GUI must share the same `.fbs` file and version discipline. Add file + identifiers and keep both sides in lockstep via a shared schema checkout. +4. **FlatBuffers is larger on the wire than protobuf** (estimated 20-50% larger + on small messages due to vtable overhead), but ~30-100x faster on reads. For + a read-heavy renderer this trade is worth it. +5. **WASM considerations.** The Web GUI build (Emscripten/WASM) uses raylib via + Odin. If FlatBuffers keep-alive buffers share memory between the Odin side and + the JS/WebSocket glue, you need to manage Emscripten memory carefully. This + is the least-researched area of this report. + +## Immediate Next Steps (when you're ready to implement) + +```mermaid +flowchart TD + A["Prototype schema
catalog.fbs: Catalog · CatalogObject · ServerMessage union"] + B["Generate Rust code
flatc --rust → api/build.rs · serve WS via axum"] + C["Prototype the Odin reader
flatcc FFI · pure-Odin · OdinArrow"] + D["Static fixture files
flatc --binary → committed .bin"] + E["Cross-language tests
Rust + Odin read the same fixtures identically"] + A --> B --> C --> D --> E +``` + +1. **Prototype schema first.** Write `catalog.fbs` covering `Catalog`, `CatalogObject`, + and a `ServerMessage` union (handshake / catalog list / chunk of objects / end). +2. **Generate Rust code** via `flatc --rust` in an `api/build.rs` (see + `03-rust-integration.md`) and serve over WebSocket using axum's `ws` module + (tower-http CORS + `axum::extract::ws`). +3. **Prototype the Odin reader.** Pick one of three paths (FFI to `flatcc`, + pure-Odin reader port, or OdinArrow reuse) and read a FlatBuffer produced by + the Rust side to prove interop. A single `CatalogObject` with `ra`, `dec`, + `redshift` is enough to validate the entire pipeline. +4. **Static fixture files.** Generate `.fbs` → FlatBuffer binaries once with + `flatc --binary`, check them into the repo, and write a cross-language test + that both Rust and Odin read the same fixture identically. This is the + backbone of your integration test story (see `08-testing-strategies.md`). \ No newline at end of file diff --git a/resources/dev/api.env.example b/resources/dev/api.env.example new file mode 100644 index 0000000..4101f60 --- /dev/null +++ b/resources/dev/api.env.example @@ -0,0 +1,8 @@ +# Example API environment — passed via API_ENV_FILE (see root Makefile). +# +# The API also reads API_DESI_DATA from the environment (set by the root +# Makefile to resources/dev/desi_subset.json by default), so it is not +# repeated here. Values here win unless the same key is already set in the +# real process environment. + +API_BIND_ADDR=127.0.0.1:8080 \ No newline at end of file diff --git a/resources/dev/desi_subset.json b/resources/dev/desi_subset.json new file mode 100644 index 0000000..3df3715 --- /dev/null +++ b/resources/dev/desi_subset.json @@ -0,0 +1,58 @@ +{ + "catalogs": [ + { + "name": "edr", + "release": "EDR", + "description": "DESI Early Data Release (local dev subset)", + "object_count": 3 + }, + { + "name": "dr1", + "release": "DR1", + "description": "DESI Data Release 1 (local dev subset)", + "object_count": 2 + } + ], + "objects": [ + { + "id": "DESI_EDR_000000001", + "catalog": "edr", + "object_type": "GALAXY", + "ra": 150.123456, + "dec": 2.345678, + "redshift": 0.5521 + }, + { + "id": "DESI_EDR_000000002", + "catalog": "edr", + "object_type": "GALAXY", + "ra": 254.987654, + "dec": -15.203041, + "redshift": 1.1045 + }, + { + "id": "DESI_EDR_000000003", + "catalog": "edr", + "object_type": "QSO", + "ra": 75.001234, + "dec": 38.765432, + "redshift": 2.8756 + }, + { + "id": "DESI_DR1_000000001", + "catalog": "dr1", + "object_type": "STAR", + "ra": 188.556677, + "dec": 47.112233, + "redshift": 0.0001 + }, + { + "id": "DESI_DR1_000000002", + "catalog": "dr1", + "object_type": "GALAXY", + "ra": 300.445566, + "dec": 12.778899, + "redshift": 0.7742 + } + ] +} \ No newline at end of file diff --git a/resources/dev/gui.env.example b/resources/dev/gui.env.example new file mode 100644 index 0000000..7d1dabc --- /dev/null +++ b/resources/dev/gui.env.example @@ -0,0 +1,7 @@ +# Example GUI environment — passed via GUI_ENV_FILE (see root Makefile). +# +# Point the renderer at the local dev API: `make run` also starts the API +# server, so http://127.0.0.1:8080 is the default. Values here win unless +# the same key is already set in the real process environment. + +API_URL=http://127.0.0.1:8080 \ No newline at end of file diff --git a/scripts/install_odin.sh b/scripts/install_odin.sh index 02670e3..be5e252 100755 --- a/scripts/install_odin.sh +++ b/scripts/install_odin.sh @@ -18,10 +18,19 @@ case "$ARCH" in ;; esac -curl -fL -o /tmp/odin.tar.gz \ - "https://github.com/odin-lang/Odin/releases/download/${VERSION}/odin-linux-${OBJ_ARCH}-${VERSION}.tar.gz" -mkdir -p /tmp/odin -tar -xzf /tmp/odin.tar.gz -C /tmp/odin --strip-components=1 +# If a prior job restored the install from cache, reuse it instead of +# re-downloading. The extracted Odin binary already on PATH is authoritative; +# otherwise fetch the version (plus the raylib workaround) fresh. +if [ -x /tmp/odin/odin ]; then + echo "Odin ${VERSION} found in cache (host arch: ${ARCH}, release arch: ${OBJ_ARCH})" + BIN_DIR=/tmp/odin +else + curl -fL -o /tmp/odin.tar.gz \ + "https://github.com/odin-lang/Odin/releases/download/${VERSION}/odin-linux-${OBJ_ARCH}-${VERSION}.tar.gz" + mkdir -p /tmp/odin + tar -xzf /tmp/odin.tar.gz -C /tmp/odin --strip-components=1 + BIN_DIR=/tmp/odin +fi # Work around an Odin binding bug: for ODIN_ARCH == .arm64 the vendored # raylib references `vendor/raylib/linux-arm/libraylib.a`, but the release @@ -33,7 +42,7 @@ if [ "${OBJ_ARCH}" = "arm64" ] \ fi if [ -n "${GITHUB_PATH:-}" ]; then - echo "/tmp/odin" >> "$GITHUB_PATH" + echo "${BIN_DIR}" >> "$GITHUB_PATH" fi -echo "Installed Odin ${VERSION} (host arch: ${ARCH}, release arch: ${OBJ_ARCH})" +echo "Odin ${VERSION} ready (host arch: ${ARCH}, release arch: ${OBJ_ARCH})"