diff --git a/README.md b/README.md index 5423b42..0aa9a58 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ A Star Citizen item catalog built from the [scunpacked](https://github.com/richardthombs/scunpacked) game data (mirrored at [StarCitizenWiki/scunpacked-data](https://github.com/StarCitizenWiki/scunpacked-data)). -Ships, ship equipment, and FPS items are browsable with search, manufacturer / -type / size filters, pagination, and a detail view per item. +Ships, ship equipment, FPS items, and cargo commodities (ores, refined metals, +RMC, construction materials, …) are browsable with search, manufacturer / type / +size filters, pagination, and a detail view per item. Users can optionally create an account and build a **personal inventory**: items from the catalog are added to their own list with a quantity and notes. The @@ -22,8 +23,10 @@ dataset can be replaced wholesale without touching user data. | database | PostgreSQL 16 | Accounts, sessions, and per-user inventory rows | | deploy | Ansible + Podman (containers.podman) | Builds images and runs the three containers | -The backend image downloads the full scunpacked dataset (~83 MB) at build time, -so the catalog needs no database — only user data does. +The backend image downloads the full scunpacked dataset at build time, then +reduces `items.json` to a small `commodities.json` (one entry per commodity) and +discards the ~68 MB raw file, so the catalog needs no database — only user data +does. ## Layout @@ -172,7 +175,7 @@ curl -fsS -b cookies.txt http://localhost:8080/api/v1/me/inventory cd trade-hub/backend python3 -m venv .venv .venv/bin/pip install -r requirements.txt -r requirements-dev.txt -.venv/bin/python -m pytest tests/ -q # 59 tests, no database needed +.venv/bin/python -m pytest tests/ -q # 65 tests, no database needed DATA_DIR=./data/samples .venv/bin/uvicorn main:app --port 8000 ``` @@ -182,6 +185,11 @@ so no PostgreSQL server is required for the suite. Production uses PostgreSQL. - `data/samples/` — small real-schema subset for local dev/tests - `scripts/fetch_data.py` — downloads the full dataset (used by the Dockerfile) +Commodities are deduplicated to one catalog entry per name (the 1 SCU container +variant is canonical); quantity lives on the inventory row, so 1 SCU, 8 SCU and +32 SCU crates of the same commodity share a single entry. Refined and unrefined +materials stay separate (`Iron` vs `Iron (Ore)`). + Tables are created on startup with an idempotent `create_all` and a connection retry loop, so a fresh volume bootstraps itself and a slow PostgreSQL start does not crash the backend. Additive columns for existing deployments (currently @@ -213,7 +221,8 @@ GET /api/v1/available?category=&search=&sort=&page=&page_size= `/api/v1/me/*` requires a valid session cookie and returns `401` otherwise. `/api/v1/available` is public and read-only; it accepts `sort=name|category|quantity|updated` -(default `updated`). +(default `updated`). Catalog `category` values are `ships`, `ship-items`, +`fps-items` and `commodities`. ### Configuration diff --git a/trade-hub/backend/Dockerfile b/trade-hub/backend/Dockerfile index b5e4686..f8fcc87 100644 --- a/trade-hub/backend/Dockerfile +++ b/trade-hub/backend/Dockerfile @@ -11,8 +11,8 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -# Download the full scunpacked dataset (~83 MB) at build time so the image -# is self-contained; DATA_DIR points the API at it. +# Download the full scunpacked dataset at build time so the image is +# self-contained; fetch_data.py reduces items.json to commodities.json. RUN python scripts/fetch_data.py --output-dir /app/data/full ENV DATA_DIR=/app/data/full diff --git a/trade-hub/backend/app/api.py b/trade-hub/backend/app/api.py index e0a4726..93495bc 100644 --- a/trade-hub/backend/app/api.py +++ b/trade-hub/backend/app/api.py @@ -48,7 +48,7 @@ app.include_router(auth_router) app.include_router(inventory_router) app.include_router(available_router) -VALID_CATEGORIES = {"ships", "ship-items", "fps-items"} +VALID_CATEGORIES = {"ships", "ship-items", "fps-items", "commodities"} def _filter_items(category, search, manufacturer, item_type, size): diff --git a/trade-hub/backend/app/loader.py b/trade-hub/backend/app/loader.py index 64666db..90235ef 100644 --- a/trade-hub/backend/app/loader.py +++ b/trade-hub/backend/app/loader.py @@ -25,6 +25,7 @@ PLACEHOLDER = "<= PLACEHOLDER =>" CATEGORY_SHIPS = "ships" CATEGORY_SHIP_ITEMS = "ship-items" CATEGORY_FPS_ITEMS = "fps-items" +CATEGORY_COMMODITIES = "commodities" def _load_json(data_dir, filename): @@ -111,6 +112,8 @@ def _normalize_item(item, category): "description": std.get("Description") or std.get("DescriptionText") or "", "tags": tags, } + if category == CATEGORY_COMMODITIES: + summary["size"] = None details = { "ports": std.get("Ports") or [], "mass": std.get("Mass"), @@ -133,6 +136,9 @@ class Store: CATEGORY_SHIPS: sum(1 for i in items if i["category"] == CATEGORY_SHIPS), CATEGORY_SHIP_ITEMS: sum(1 for i in items if i["category"] == CATEGORY_SHIP_ITEMS), CATEGORY_FPS_ITEMS: sum(1 for i in items if i["category"] == CATEGORY_FPS_ITEMS), + CATEGORY_COMMODITIES: sum( + 1 for i in items if i["category"] == CATEGORY_COMMODITIES + ), } @@ -157,6 +163,11 @@ def load_store(data_dir=None): items.append(summary) details[summary["class_name"]] = detail + for entry in _load_json(data_dir, "commodities.json"): + summary, detail = _normalize_item(entry, CATEGORY_COMMODITIES) + items.append(summary) + details[summary["class_name"]] = detail + # Deterministic order so pagination is stable. items.sort(key=lambda i: (i["category"], i["name"].lower(), i["class_name"])) diff --git a/trade-hub/backend/scripts/fetch_data.py b/trade-hub/backend/scripts/fetch_data.py index 5d65f8d..ee0582d 100644 --- a/trade-hub/backend/scripts/fetch_data.py +++ b/trade-hub/backend/scripts/fetch_data.py @@ -6,6 +6,7 @@ Usage: Default output: backend/data/full (used by the Docker image build). """ import argparse +import json import os import sys import urllib.request @@ -16,7 +17,38 @@ FILES = [ "ship-items.json", "fps-items.json", "manufacturers.json", + "items.json", ] +COMMODITY_KEYS = ("className", "name", "type", "subType", "reference") +PLACEHOLDER = "<= PLACEHOLDER =>" + + +def _commodity_rank(entry): + class_name = entry.get("className") or "" + return (0 if "_1SCU_" in class_name else 1, entry.get("size") or 0, class_name) + + +def build_commodities(items_path, out_path): + with open(items_path, encoding="utf-8") as f: + items = json.load(f) + + grouped = {} + for entry in items: + class_name = (entry.get("className") or "").lower() + name = (entry.get("name") or "").strip() + if entry.get("type") != "Cargo" or "commodity" not in class_name: + continue + if not name or name == PLACEHOLDER: + continue + grouped.setdefault(name, []).append(entry) + + commodities = [ + {key: best.get(key) for key in COMMODITY_KEYS} + for best in (min(grouped[name], key=_commodity_rank) for name in sorted(grouped)) + ] + with open(out_path, "w", encoding="utf-8") as f: + json.dump(commodities, f, ensure_ascii=False, separators=(",", ":")) + return len(commodities) def main(): @@ -35,6 +67,13 @@ def main(): urllib.request.urlretrieve(url, dest) size = os.path.getsize(dest) print(f" done ({size / 1_000_000:.1f} MB)", flush=True) + + items_path = os.path.join(args.output_dir, "items.json") + count = build_commodities( + items_path, os.path.join(args.output_dir, "commodities.json") + ) + os.remove(items_path) + print(f"Wrote {count} commodities (items.json discarded)", flush=True) return 0 diff --git a/trade-hub/backend/tests/test_api.py b/trade-hub/backend/tests/test_api.py index d5eed93..629bb2c 100644 --- a/trade-hub/backend/tests/test_api.py +++ b/trade-hub/backend/tests/test_api.py @@ -11,14 +11,24 @@ def test_healthz(): assert resp.status_code == 200 body = resp.json() assert body["status"] == "ok" - assert body["counts"] == {"ships": 2, "ship-items": 2, "fps-items": 1} + assert body["counts"] == { + "ships": 2, + "ship-items": 2, + "fps-items": 1, + "commodities": 4, + } def test_stats(): resp = client.get("/api/v1/stats") assert resp.status_code == 200 body = resp.json() - assert body["categories"] == {"ships": 2, "ship-items": 2, "fps-items": 1} + assert body["categories"] == { + "ships": 2, + "ship-items": 2, + "fps-items": 1, + "commodities": 4, + } assert body["manufacturers"] == 4 @@ -33,11 +43,11 @@ def test_items_default_pagination(): resp = client.get("/api/v1/items") assert resp.status_code == 200 body = resp.json() - assert body["total"] == 5 + assert body["total"] == 9 assert body["page"] == 1 assert body["page_size"] == 25 assert body["total_pages"] == 1 - assert len(body["items"]) == 5 + assert len(body["items"]) == 9 def test_items_page_size_capped(): @@ -155,3 +165,29 @@ def test_deterministic_sort_order(): body = client.get("/api/v1/items", params={"page_size": 100}).json() keys = [(i["category"], i["name"].lower()) for i in body["items"]] assert keys == sorted(keys) + + +def test_items_category_filter_commodities(): + resp = client.get("/api/v1/items", params={"category": "commodities"}) + assert resp.status_code == 200 + body = resp.json() + assert body["total"] == 4 + assert all(i["category"] == "commodities" for i in body["items"]) + + +def test_items_search_matches_refined_and_ore_iron(): + resp = client.get("/api/v1/items", params={"search": "iron"}) + body = resp.json() + assert body["total"] == 2 + assert {i["name"] for i in body["items"]} == {"Iron", "Iron (Ore)"} + + +def test_item_detail_commodity(): + resp = client.get("/api/v1/items/Carryable_TBO_FL_1SCU_Commodity_Metal_Ore_Iron") + assert resp.status_code == 200 + body = resp.json() + assert body["name"] == "Iron (Ore)" + assert body["category"] == "commodities" + assert body["type"] == "Cargo" + assert body["size"] is None + assert isinstance(body["details"], dict) diff --git a/trade-hub/backend/tests/test_inventory.py b/trade-hub/backend/tests/test_inventory.py index 5ced3d1..bf8cd2b 100644 --- a/trade-hub/backend/tests/test_inventory.py +++ b/trade-hub/backend/tests/test_inventory.py @@ -6,6 +6,7 @@ from app.api import app PASSWORD = "correct horse battery staple" CARROCK = "ANVL_Carrack" RIFLE = "BEHR_Rifle_Ballistic_01" +IRON_ORE = "Carryable_TBO_FL_1SCU_Commodity_Metal_Ore_Iron" def authed_client(username: str = "alice") -> TestClient: @@ -251,3 +252,15 @@ def test_sort_defaults_to_name_and_rejects_unknown_values(): listing = client.get("/api/v1/me/inventory").json() assert [i["class_name"] for i in listing["items"]] == [CARROCK] assert client.get("/api/v1/me/inventory?sort=bogus").status_code == 422 + + +def test_add_commodity_uses_catalog_name_and_quantity(): + client = authed_client("refiner") + resp = add(client, IRON_ORE, quantity=12, notes="from Aaron Halo") + assert resp.status_code == 201 + entry = client.get("/api/v1/me/inventory").json()["items"][0] + assert entry["name"] == "Iron (Ore)" + assert entry["category"] == "commodities" + assert entry["quantity"] == 12 + assert entry["available"] is True + assert entry["visibility"] == "private" diff --git a/trade-hub/frontend/src/app.js b/trade-hub/frontend/src/app.js index 79952a8..42b0600 100644 --- a/trade-hub/frontend/src/app.js +++ b/trade-hub/frontend/src/app.js @@ -107,7 +107,7 @@ async function initHeader() { const c = health.counts || {}; els["stats-line"].textContent = `${c.ships ?? 0} ships · ${c["ship-items"] ?? 0} ship items · ` + - `${c["fps-items"] ?? 0} FPS items`; + `${c["fps-items"] ?? 0} FPS items · ${c.commodities ?? 0} commodities`; } catch { els["api-status"].textContent = "API OFFLINE"; els["api-status"].className = "status-pill offline"; @@ -239,7 +239,12 @@ function buildQuery() { } function categoryBadge(category) { - const labels = { ships: "Ship", "ship-items": "Ship Item", "fps-items": "FPS Item" }; + const labels = { + ships: "Ship", + "ship-items": "Ship Item", + "fps-items": "FPS Item", + commodities: "Commodity", + }; return `${labels[category] || category}`; } @@ -457,7 +462,12 @@ async function refreshSummary() { } function renderInventorySummary(summary) { - const labels = { ships: "ships", "ship-items": "ship items", "fps-items": "FPS items" }; + const labels = { + ships: "ships", + "ship-items": "ship items", + "fps-items": "FPS items", + commodities: "commodities", + }; const parts = Object.entries(summary.categories || {}) .map(([key, qty]) => `${qty} ${labels[key] || key}`) .join(" · "); diff --git a/trade-hub/frontend/src/index.html b/trade-hub/frontend/src/index.html index f30b381..cef6856 100644 --- a/trade-hub/frontend/src/index.html +++ b/trade-hub/frontend/src/index.html @@ -28,6 +28,7 @@ + @@ -68,6 +69,7 @@ +