add dev resources; plumb GUI_ENV_FILE/API_ENV_FILE/API_DESI_DATA into run targets
CI / Detect changed paths (pull_request) Failing after 33m59s
CI / Infra unit tests, vet, and preview (pull_request) Has been skipped
CI / API unit tests and lint (pull_request) Failing after 31m0s
CI / Odin unit tests and build (pull_request) Failing after 31m9s

This commit is contained in:
2026-09-07 14:55:49 -06:00
parent ba24005bfb
commit 1443369f6c
18 changed files with 490 additions and 58 deletions
+12 -4
View File
@@ -9,6 +9,14 @@ GUI := gui
API := api
INFRA := infra
# Local dev/test assets live under resources/dev. These are the defaults for
# the run/*-web targets; override any of them on the command line, e.g.
# make run GUI_ENV_FILE=/path/to/gui.env API_DESI_DATA=/path/to/data.json
RESOURCE_DIR := $(CURDIR)/resources/dev
GUI_ENV_FILE ?= $(RESOURCE_DIR)/gui.env.example
API_ENV_FILE ?= $(RESOURCE_DIR)/api.env.example
API_DESI_DATA ?= $(RESOURCE_DIR)/desi_subset.json
.PHONY: help setup run run-web build build-debug build-web test clean fmt \
renovate-validate
@@ -32,9 +40,9 @@ setup: ## Setup all sub-projects (submodules, gui deps, api deps, infra deps)
## ---- Renderer (Odin) -----------------------------------------------------
run: ## Run the native app (gui/)
@$(MAKE) -C $(API) run & api_pid=$$!; \
@API_ENV_FILE="$(API_ENV_FILE)" API_DESI_DATA="$(API_DESI_DATA)" $(MAKE) -C $(API) run & api_pid=$$!; \
trap 'kill $$api_pid 2>/dev/null' INT TERM EXIT; \
$(MAKE) -C $(GUI) run; \
GUI_ENV_FILE="$(GUI_ENV_FILE)" $(MAKE) -C $(GUI) run; \
kill $$api_pid 2>/dev/null
build: ## Release build (gui/ + api/)
@@ -49,9 +57,9 @@ build-web: ## WebAssembly build -> build/web (gui/, needs emscripten)
@$(MAKE) -C $(GUI) build-web
run-web: ## Start WASM build + API server for web dev
@$(MAKE) -C $(API) run & api_pid=$$!; \
@API_ENV_FILE="$(API_ENV_FILE)" API_DESI_DATA="$(API_DESI_DATA)" $(MAKE) -C $(API) run & api_pid=$$!; \
trap 'kill $$api_pid 2>/dev/null' INT TERM EXIT; \
$(MAKE) -C $(GUI) build-web; \
GUI_ENV_FILE="$(GUI_ENV_FILE)" $(MAKE) -C $(GUI) build-web; \
kill $$api_pid 2>/dev/null
## ---- Aggregates ----------------------------------------------------------
+16
View File
@@ -79,6 +79,22 @@ make clean # remove build artifacts from all projects
make fmt # format all projects
```
`make run` / `make run-web` configure the API and renderer from the example
assets under `resources/dev/` (see the dotenv lib in `gui/lib/local/dotenv`).
Override any of them on the command line:
```sh
make run \
GUI_ENV_FILE=/path/to/gui.env \
API_ENV_FILE=/path/to/api.env \
API_DESI_DATA=/path/to/desi_data.json
```
The defaults point at `resources/dev/gui.env.example` (renderer's `API_URL`),
`resources/dev/api.env.example` (API `API_BIND_ADDR`), and
`resources/dev/desi_subset.json` (a small JSON catalog subset served by the
API's `/api/v1/catalogs` and `/api/v1/objects` endpoints).
Project-specific targets live in their own `Makefile` and are reached with
`make -C <dir> <target>`:
+7
View File
@@ -99,6 +99,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"dotenvy",
"serde",
"serde_json",
"tokio",
@@ -108,6 +109,12 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "errno"
version = "0.3.14"
+5 -4
View File
@@ -5,17 +5,18 @@ edition = "2021"
description = "Backend API for DESI Explorer — serves DESI survey catalog data"
[dependencies]
anyhow = "1"
axum = "0.8"
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
dotenvy = "0.15"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
tower-http = { version = "0.7", features = ["cors", "trace"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tower-http = { version = "0.7", features = ["cors", "trace"] }
anyhow = "1"
[dev-dependencies]
tower = { version = "0.5", features = ["util"] }
serde_json = "1"
[profile.release]
lto = true
+14 -1
View File
@@ -1,5 +1,10 @@
use std::path::PathBuf;
pub struct Config {
pub bind_addr: String,
/// Path to a DESI data file (JSON) to serve; `None` falls back to the
/// built-in placeholder catalogs.
pub desi_data: Option<PathBuf>,
}
impl Config {
@@ -7,6 +12,14 @@ impl Config {
let bind_addr =
std::env::var("API_BIND_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string());
Ok(Self { bind_addr })
let desi_data = std::env::var("API_DESI_DATA")
.ok()
.filter(|s| !s.is_empty())
.map(PathBuf::from);
Ok(Self {
bind_addr,
desi_data,
})
}
}
+1
View File
@@ -1,3 +1,4 @@
pub mod config;
pub mod models;
pub mod routes;
pub mod store;
+40 -2
View File
@@ -1,4 +1,7 @@
use desi_explorer_api::{config, routes};
use std::path::Path;
use std::sync::Arc;
use desi_explorer_api::{config, routes, store};
use tracing_subscriber::EnvFilter;
@@ -11,8 +14,22 @@ async fn main() -> anyhow::Result<()> {
)
.init();
load_env_file()?;
let config = config::Config::from_env()?;
let app = routes::app();
let catalog_store = match &config.desi_data {
Some(path) => {
tracing::info!(path = %path.display(), "loading DESI data");
store::CatalogStore::load(Path::new(path))?
}
None => {
tracing::warn!("API_DESI_DATA not set, serving placeholder catalogs");
store::CatalogStore::placeholder()
}
};
let app = routes::app_with_state(Arc::new(catalog_store));
let listener = tokio::net::TcpListener::bind(&config.bind_addr).await?;
tracing::info!("DESI Explorer API listening on {}", config.bind_addr);
@@ -24,6 +41,27 @@ async fn main() -> anyhow::Result<()> {
Ok(())
}
/// Loads the env file named by `API_ENV_FILE` (if set) into the process
/// environment. Existing env vars are not overridden, so values passed
/// directly on the command line or by the Makefile take precedence.
fn load_env_file() -> anyhow::Result<()> {
let path = std::env::var("API_ENV_FILE").unwrap_or_default();
if path.is_empty() {
return Ok(());
}
match dotenvy::from_path(&path) {
Ok(_) => tracing::info!(%path, "loaded env file"),
Err(err) => {
return Err(anyhow::anyhow!(
"failed to load API_ENV_FILE {path:?}: {err}"
))
}
}
Ok(())
}
async fn shutdown_signal() {
let _ = tokio::signal::ctrl_c().await;
tracing::info!("shutting down");
+4 -4
View File
@@ -1,17 +1,17 @@
use serde::Serialize;
use serde::{Deserialize, Serialize};
/// Catalog metadata for a DESI data release/survey.
#[derive(Debug, Clone, Serialize)]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Catalog {
pub name: String,
pub release: String,
pub description: &'static str,
pub description: String,
pub object_count: Option<u64>,
}
/// A single catalog object (galaxy / quasar / star) with its survey
/// coordinates. `ra` and `dec` are in degrees; `redshift` is dimensionless.
#[derive(Debug, Serialize)]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct CatalogObject {
pub id: String,
pub catalog: String,
+24 -30
View File
@@ -1,34 +1,16 @@
use std::sync::Arc;
use axum::{
extract::Query,
http::StatusCode,
response::{IntoResponse, Response},
extract::{Query, State},
Json,
};
use serde::Deserialize;
use std::sync::LazyLock;
use crate::models::{Catalog, CatalogObject};
use crate::store::CatalogStore;
/// Placeholder catalogs until real DESI EDR/DR1 ingestion lands.
static CATALOGS: LazyLock<Vec<Catalog>> = LazyLock::new(|| {
vec![
Catalog {
name: "edr".to_string(),
release: "EDR".to_string(),
description: "DESI Early Data Release",
object_count: None,
},
Catalog {
name: "dr1".to_string(),
release: "DR1".to_string(),
description: "DESI Data Release 1",
object_count: None,
},
]
});
pub async fn list_catalogs() -> Json<Vec<Catalog>> {
Json(CATALOGS.clone())
pub async fn list_catalogs(State(state): State<Arc<CatalogStore>>) -> Json<Vec<Catalog>> {
Json(state.catalogs.clone())
}
#[derive(Debug, Deserialize)]
@@ -38,17 +20,29 @@ pub struct ObjectQuery {
limit: Option<usize>,
}
/// Placeholder object query. Real implementation will page through the
/// centralized DESI catalog store rather than return an empty result set.
pub async fn list_objects(Query(query): Query<ObjectQuery>) -> Response {
pub async fn list_objects(
State(state): State<Arc<CatalogStore>>,
Query(query): Query<ObjectQuery>,
) -> Json<Vec<CatalogObject>> {
let limit = query.limit.unwrap_or(100).min(10_000);
tracing::debug!(
%limit,
catalog = query.catalog.as_deref().unwrap_or("all"),
"querying catalog objects (placeholder)"
objects = state.objects.len(),
"querying catalog objects"
);
let objects: Vec<CatalogObject> = Vec::new();
(StatusCode::OK, Json(objects)).into_response()
let objects: Vec<CatalogObject> = match &query.catalog {
Some(catalog) => state
.objects
.iter()
.filter(|o| &o.catalog == catalog)
.take(limit)
.cloned()
.collect(),
None => state.objects.iter().take(limit).cloned().collect(),
};
Json(objects)
}
+12 -2
View File
@@ -1,13 +1,23 @@
pub mod catalogs;
pub mod health;
use std::sync::Arc;
use axum::{routing::get, Router};
/// Builds the application router. Kept separate from `main` so tests can
/// construct it without binding a socket.
use crate::store::CatalogStore;
/// Builds the application router with a static placeholder store. Kept
/// separate from `main` so tests can construct it without binding a socket.
pub fn app() -> Router {
app_with_state(Arc::new(CatalogStore::placeholder()))
}
/// Builds the application router serving the given catalog store.
pub fn app_with_state(state: Arc<CatalogStore>) -> Router {
Router::new()
.route("/health", get(health::health))
.route("/api/v1/catalogs", get(catalogs::list_catalogs))
.route("/api/v1/objects", get(catalogs::list_objects))
.with_state(state)
}
+110
View File
@@ -0,0 +1,110 @@
use std::path::Path;
use serde::Deserialize;
use crate::models::{Catalog, CatalogObject};
/// In-memory catalog store, shared (via `Arc`) across routes. Populated either
/// from a DESI data file loaded at startup or from `placeholder`.
#[derive(Debug, Clone, Default)]
pub struct CatalogStore {
pub catalogs: Vec<Catalog>,
pub objects: Vec<CatalogObject>,
}
/// JSON layout of the DESI data file referenced by `API_DESI_DATA`.
#[derive(Debug, Deserialize)]
pub struct DataFile {
pub catalogs: Vec<Catalog>,
#[serde(default)]
pub objects: Vec<CatalogObject>,
}
impl CatalogStore {
/// Static fallback catalogs used when no `API_DESI_DATA` file is given
/// (and by the `routes::app()` test helper).
pub fn placeholder() -> Self {
Self {
catalogs: vec![
Catalog {
name: "edr".to_string(),
release: "EDR".to_string(),
description: "DESI Early Data Release".to_string(),
object_count: None,
},
Catalog {
name: "dr1".to_string(),
release: "DR1".to_string(),
description: "DESI Data Release 1".to_string(),
object_count: None,
},
],
objects: Vec::new(),
}
}
/// Loads catalogs and objects from a JSON data file. Errors on unreadable
/// files or malformed JSON so the caller can fail loudly instead of
/// silently serving empty data.
pub fn load(path: &Path) -> anyhow::Result<Self> {
let text = std::fs::read_to_string(path)?;
let file: DataFile = serde_json::from_str(&text)?;
Ok(Self {
catalogs: file.catalogs,
objects: file.objects,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn load_parses_catalogs_and_objects() {
let json = r#"{
"catalogs": [
{"name":"edr","release":"EDR","description":"test","object_count":2}
],
"objects": [
{"id":"o1","catalog":"edr","object_type":"GALAXY","ra":1.5,"dec":2.5,"redshift":0.8}
]
}"#;
let dir = std::env::temp_dir().join(format!(
"desi_explorer_store_{}_{}",
std::process::id(),
line!()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("data.json");
std::fs::write(&path, json).unwrap();
let store = CatalogStore::load(&path).unwrap();
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(store.catalogs.len(), 1);
assert_eq!(store.catalogs[0].name, "edr");
assert_eq!(store.catalogs[0].object_count, Some(2));
assert_eq!(store.objects.len(), 1);
assert_eq!(store.objects[0].id, "o1");
assert_eq!(store.objects[0].ra, 1.5);
}
#[test]
fn load_rejects_malformed_json() {
let dir = std::env::temp_dir().join(format!(
"desi_explorer_store_{}_{}",
std::process::id(),
line!()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("data.json");
std::fs::write(&path, "not json").unwrap();
let result = CatalogStore::load(&path);
let _ = std::fs::remove_dir_all(&dir);
assert!(result.is_err());
}
}
+93
View File
@@ -0,0 +1,93 @@
use axum::body::{to_bytes, Body};
use axum::http::{Request, StatusCode};
use std::sync::Arc;
use tower::ServiceExt;
use desi_explorer_api::models::CatalogObject;
use desi_explorer_api::routes;
use desi_explorer_api::store::CatalogStore;
fn sample_store() -> CatalogStore {
CatalogStore {
catalogs: Vec::new(),
objects: vec![
CatalogObject {
id: "o1".to_string(),
catalog: "edr".to_string(),
object_type: "GALAXY".to_string(),
ra: 1.5,
dec: 2.5,
redshift: 0.8,
},
CatalogObject {
id: "o2".to_string(),
catalog: "dr1".to_string(),
object_type: "STAR".to_string(),
ra: 3.5,
dec: 4.5,
redshift: 0.0,
},
],
}
}
#[tokio::test]
async fn objects_returns_all_when_no_filter() {
let app = routes::app_with_state(Arc::new(sample_store()));
let response = app
.oneshot(
Request::builder()
.uri("/api/v1/objects")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let objects: Vec<CatalogObject> = serde_json::from_slice(&body).unwrap();
assert_eq!(objects.len(), 2);
}
#[tokio::test]
async fn objects_filters_by_catalog() {
let app = routes::app_with_state(Arc::new(sample_store()));
let response = app
.oneshot(
Request::builder()
.uri("/api/v1/objects?catalog=edr")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let objects: Vec<CatalogObject> = serde_json::from_slice(&body).unwrap();
assert_eq!(objects.len(), 1);
assert_eq!(objects[0].id, "o1");
}
#[tokio::test]
async fn objects_respects_limit() {
let app = routes::app_with_state(Arc::new(sample_store()));
let response = app
.oneshot(
Request::builder()
.uri("/api/v1/objects?limit=1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let objects: Vec<CatalogObject> = serde_json::from_slice(&body).unwrap();
assert_eq!(objects.len(), 1);
}
+22 -8
View File
@@ -80,13 +80,26 @@ destroy :: proc(env: map[string]string) {
delete(env)
}
// env_key returns the env key that should bind to a struct field. It prefers
// an explicit `env:"NAME"` tag; when the tag is absent or empty it falls
// back to the field's name. Matching against the parsed map is
// case-insensitive, so API_URL maps onto api_url (or an `env:"API_URL"` tag).
@(private)
env_key_for_field :: proc(field: reflect.Struct_Field) -> string {
if tag_key, ok := reflect.struct_tag_lookup(field.tag, "env");
ok && tag_key != "" {
return tag_key
}
return field.name
}
// decode populates dest's fields from env, matching each field by name
// (case-insensitively, so API_URL maps onto api_url). Values are converted
// to the field's type: string is cloned as-is into allocator, integers are
// parsed with strconv.parse_int (decimal/hex/negative), booleans with
// strconv.parse_bool, and floats with strconv.parse_f64. Keys missing from
// env leave the field at its zero value. It returns false if a present value
// cannot be converted to the field's type.
// (or by an `env:"NAME"` struct tag). Values are converted to the field's
// type: string is cloned as-is into allocator, integers are parsed with
// strconv.parse_int (decimal/hex/negative), booleans with strconv.parse_bool,
// and floats with strconv.parse_f64. Keys missing from env leave the field
// at its zero value. It returns false if a present value cannot be converted
// to the field's type.
decode :: proc(
env: map[string]string,
dest: ^$T,
@@ -98,11 +111,12 @@ decode :: proc(
return false
}
name: string
value: string
field_ptr := rawptr(dest)
st: reflect.Struct_Field
for _, i in fields.names[:fields.field_count] {
name = fields.names[i]
st = reflect.struct_field_at(T, i)
name := env_key_for_field(st)
value = ""
found := false
for key, v in env {
@@ -12,6 +12,16 @@ Test_Config :: struct {
ratio: f64,
}
Tagged_Config :: struct {
api_url: string `env:"API_URL"`,
port: int `env:"PORT"`,
debug: bool `env:"DEBUG"`,
}
Tagged_Empty :: struct {
api_url: string `env:""`,
}
// load_env writes src to a unique temp file and parses it via parse_file.
// The returned map owns its strings; callers must destroy it.
load_env :: proc(t: ^testing.T, src: string) -> map[string]string {
@@ -199,3 +209,37 @@ test_decode_hex_and_negative_ints :: proc(t: ^testing.T) {
testing.expect(t, dotenv.decode(env, &cfg))
testing.expect(t, cfg.port == 31, "hex int should decode")
}
@(test)
test_decode_honors_env_tags :: proc(t: ^testing.T) {
env := load_env(t, "API_URL=http://127.0.0.1:8080\nPORT=9090\nDEBUG=true\n")
defer dotenv.destroy(env)
cfg := Tagged_Config{}
testing.expect(t, dotenv.decode(env, &cfg))
defer delete(cfg.api_url)
testing.expect(
t,
cfg.api_url == "http://127.0.0.1:8080",
"env tag should bind API_URL",
)
testing.expect(t, cfg.port == 9090, "env tag should bind PORT")
testing.expect(t, cfg.debug == true, "env tag should bind DEBUG")
}
@(test)
test_decode_empty_env_tag_falls_back_to_field_name :: proc(t: ^testing.T) {
env := load_env(t, "api_url=http://fallback\n")
defer dotenv.destroy(env)
cfg := Tagged_Empty{}
testing.expect(t, dotenv.decode(env, &cfg))
defer delete(cfg.api_url)
testing.expect(
t,
cfg.api_url == "http://fallback",
"empty env tag should use field name",
)
}
+12 -2
View File
@@ -4,11 +4,21 @@ import "core:os"
import dotenv "lib:dotenv/src"
Config :: struct {
api_url: string,
api_url: string `env:"API_URL"`,
}
get_config :: proc(env_file: ^string = nil) -> (^Config, ^Error) {
env, _ := dotenv.parse_file(".env", context.temp_allocator)
path := ".env"
if env_file != nil && env_file^ != "" {
path = env_file^
} else if from_env, ok := os.lookup_env(
"GUI_ENV_FILE",
context.temp_allocator,
); ok {
path = from_env
}
env, _ := dotenv.parse_file(path, context.temp_allocator)
defer dotenv.destroy(env)
c := new(Config)
+8
View File
@@ -0,0 +1,8 @@
# Example API environment — passed via API_ENV_FILE (see root Makefile).
#
# The API also reads API_DESI_DATA from the environment (set by the root
# Makefile to resources/dev/desi_subset.json by default), so it is not
# repeated here. Values here win unless the same key is already set in the
# real process environment.
API_BIND_ADDR=127.0.0.1:8080
+58
View File
@@ -0,0 +1,58 @@
{
"catalogs": [
{
"name": "edr",
"release": "EDR",
"description": "DESI Early Data Release (local dev subset)",
"object_count": 3
},
{
"name": "dr1",
"release": "DR1",
"description": "DESI Data Release 1 (local dev subset)",
"object_count": 2
}
],
"objects": [
{
"id": "DESI_EDR_000000001",
"catalog": "edr",
"object_type": "GALAXY",
"ra": 150.123456,
"dec": 2.345678,
"redshift": 0.5521
},
{
"id": "DESI_EDR_000000002",
"catalog": "edr",
"object_type": "GALAXY",
"ra": 254.987654,
"dec": -15.203041,
"redshift": 1.1045
},
{
"id": "DESI_EDR_000000003",
"catalog": "edr",
"object_type": "QSO",
"ra": 75.001234,
"dec": 38.765432,
"redshift": 2.8756
},
{
"id": "DESI_DR1_000000001",
"catalog": "dr1",
"object_type": "STAR",
"ra": 188.556677,
"dec": 47.112233,
"redshift": 0.0001
},
{
"id": "DESI_DR1_000000002",
"catalog": "dr1",
"object_type": "GALAXY",
"ra": 300.445566,
"dec": 12.778899,
"redshift": 0.7742
}
]
}
+7
View File
@@ -0,0 +1,7 @@
# Example GUI environment — passed via GUI_ENV_FILE (see root Makefile).
#
# Point the renderer at the local dev API: `make run` also starts the API
# server, so http://127.0.0.1:8080 is the default. Values here win unless
# the same key is already set in the real process environment.
API_URL=http://127.0.0.1:8080