diff --git a/README.md b/README.md
index 5423b42..0aa9a58 100644
--- a/README.md
+++ b/README.md
@@ -4,8 +4,9 @@ A Star Citizen item catalog built from the
[scunpacked](https://github.com/richardthombs/scunpacked) game data (mirrored at
[StarCitizenWiki/scunpacked-data](https://github.com/StarCitizenWiki/scunpacked-data)).
-Ships, ship equipment, and FPS items are browsable with search, manufacturer /
-type / size filters, pagination, and a detail view per item.
+Ships, ship equipment, FPS items, and cargo commodities (ores, refined metals,
+RMC, construction materials, …) are browsable with search, manufacturer / type /
+size filters, pagination, and a detail view per item.
Users can optionally create an account and build a **personal inventory**: items
from the catalog are added to their own list with a quantity and notes. The
@@ -22,8 +23,10 @@ dataset can be replaced wholesale without touching user data.
| database | PostgreSQL 16 | Accounts, sessions, and per-user inventory rows |
| deploy | Ansible + Podman (containers.podman) | Builds images and runs the three containers |
-The backend image downloads the full scunpacked dataset (~83 MB) at build time,
-so the catalog needs no database — only user data does.
+The backend image downloads the full scunpacked dataset at build time, then
+reduces `items.json` to a small `commodities.json` (one entry per commodity) and
+discards the ~68 MB raw file, so the catalog needs no database — only user data
+does.
## Layout
@@ -172,7 +175,7 @@ curl -fsS -b cookies.txt http://localhost:8080/api/v1/me/inventory
cd trade-hub/backend
python3 -m venv .venv
.venv/bin/pip install -r requirements.txt -r requirements-dev.txt
-.venv/bin/python -m pytest tests/ -q # 59 tests, no database needed
+.venv/bin/python -m pytest tests/ -q # 65 tests, no database needed
DATA_DIR=./data/samples .venv/bin/uvicorn main:app --port 8000
```
@@ -182,6 +185,11 @@ so no PostgreSQL server is required for the suite. Production uses PostgreSQL.
- `data/samples/` — small real-schema subset for local dev/tests
- `scripts/fetch_data.py` — downloads the full dataset (used by the Dockerfile)
+Commodities are deduplicated to one catalog entry per name (the 1 SCU container
+variant is canonical); quantity lives on the inventory row, so 1 SCU, 8 SCU and
+32 SCU crates of the same commodity share a single entry. Refined and unrefined
+materials stay separate (`Iron` vs `Iron (Ore)`).
+
Tables are created on startup with an idempotent `create_all` and a connection
retry loop, so a fresh volume bootstraps itself and a slow PostgreSQL start does
not crash the backend. Additive columns for existing deployments (currently
@@ -213,7 +221,8 @@ GET /api/v1/available?category=&search=&sort=&page=&page_size=
`/api/v1/me/*` requires a valid session cookie and returns `401` otherwise.
`/api/v1/available` is public and read-only; it accepts `sort=name|category|quantity|updated`
-(default `updated`).
+(default `updated`). Catalog `category` values are `ships`, `ship-items`,
+`fps-items` and `commodities`.
### Configuration
diff --git a/trade-hub/backend/Dockerfile b/trade-hub/backend/Dockerfile
index b5e4686..f8fcc87 100644
--- a/trade-hub/backend/Dockerfile
+++ b/trade-hub/backend/Dockerfile
@@ -11,8 +11,8 @@ RUN pip install --no-cache-dir -r requirements.txt
COPY . .
-# Download the full scunpacked dataset (~83 MB) at build time so the image
-# is self-contained; DATA_DIR points the API at it.
+# Download the full scunpacked dataset at build time so the image is
+# self-contained; fetch_data.py reduces items.json to commodities.json.
RUN python scripts/fetch_data.py --output-dir /app/data/full
ENV DATA_DIR=/app/data/full
diff --git a/trade-hub/backend/app/api.py b/trade-hub/backend/app/api.py
index e0a4726..93495bc 100644
--- a/trade-hub/backend/app/api.py
+++ b/trade-hub/backend/app/api.py
@@ -48,7 +48,7 @@ app.include_router(auth_router)
app.include_router(inventory_router)
app.include_router(available_router)
-VALID_CATEGORIES = {"ships", "ship-items", "fps-items"}
+VALID_CATEGORIES = {"ships", "ship-items", "fps-items", "commodities"}
def _filter_items(category, search, manufacturer, item_type, size):
diff --git a/trade-hub/backend/app/loader.py b/trade-hub/backend/app/loader.py
index 64666db..90235ef 100644
--- a/trade-hub/backend/app/loader.py
+++ b/trade-hub/backend/app/loader.py
@@ -25,6 +25,7 @@ PLACEHOLDER = "<= PLACEHOLDER =>"
CATEGORY_SHIPS = "ships"
CATEGORY_SHIP_ITEMS = "ship-items"
CATEGORY_FPS_ITEMS = "fps-items"
+CATEGORY_COMMODITIES = "commodities"
def _load_json(data_dir, filename):
@@ -111,6 +112,8 @@ def _normalize_item(item, category):
"description": std.get("Description") or std.get("DescriptionText") or "",
"tags": tags,
}
+ if category == CATEGORY_COMMODITIES:
+ summary["size"] = None
details = {
"ports": std.get("Ports") or [],
"mass": std.get("Mass"),
@@ -133,6 +136,9 @@ class Store:
CATEGORY_SHIPS: sum(1 for i in items if i["category"] == CATEGORY_SHIPS),
CATEGORY_SHIP_ITEMS: sum(1 for i in items if i["category"] == CATEGORY_SHIP_ITEMS),
CATEGORY_FPS_ITEMS: sum(1 for i in items if i["category"] == CATEGORY_FPS_ITEMS),
+ CATEGORY_COMMODITIES: sum(
+ 1 for i in items if i["category"] == CATEGORY_COMMODITIES
+ ),
}
@@ -157,6 +163,11 @@ def load_store(data_dir=None):
items.append(summary)
details[summary["class_name"]] = detail
+ for entry in _load_json(data_dir, "commodities.json"):
+ summary, detail = _normalize_item(entry, CATEGORY_COMMODITIES)
+ items.append(summary)
+ details[summary["class_name"]] = detail
+
# Deterministic order so pagination is stable.
items.sort(key=lambda i: (i["category"], i["name"].lower(), i["class_name"]))
diff --git a/trade-hub/backend/scripts/fetch_data.py b/trade-hub/backend/scripts/fetch_data.py
index 5d65f8d..ee0582d 100644
--- a/trade-hub/backend/scripts/fetch_data.py
+++ b/trade-hub/backend/scripts/fetch_data.py
@@ -6,6 +6,7 @@ Usage:
Default output: backend/data/full (used by the Docker image build).
"""
import argparse
+import json
import os
import sys
import urllib.request
@@ -16,7 +17,38 @@ FILES = [
"ship-items.json",
"fps-items.json",
"manufacturers.json",
+ "items.json",
]
+COMMODITY_KEYS = ("className", "name", "type", "subType", "reference")
+PLACEHOLDER = "<= PLACEHOLDER =>"
+
+
+def _commodity_rank(entry):
+ class_name = entry.get("className") or ""
+ return (0 if "_1SCU_" in class_name else 1, entry.get("size") or 0, class_name)
+
+
+def build_commodities(items_path, out_path):
+ with open(items_path, encoding="utf-8") as f:
+ items = json.load(f)
+
+ grouped = {}
+ for entry in items:
+ class_name = (entry.get("className") or "").lower()
+ name = (entry.get("name") or "").strip()
+ if entry.get("type") != "Cargo" or "commodity" not in class_name:
+ continue
+ if not name or name == PLACEHOLDER:
+ continue
+ grouped.setdefault(name, []).append(entry)
+
+ commodities = [
+ {key: best.get(key) for key in COMMODITY_KEYS}
+ for best in (min(grouped[name], key=_commodity_rank) for name in sorted(grouped))
+ ]
+ with open(out_path, "w", encoding="utf-8") as f:
+ json.dump(commodities, f, ensure_ascii=False, separators=(",", ":"))
+ return len(commodities)
def main():
@@ -35,6 +67,13 @@ def main():
urllib.request.urlretrieve(url, dest)
size = os.path.getsize(dest)
print(f" done ({size / 1_000_000:.1f} MB)", flush=True)
+
+ items_path = os.path.join(args.output_dir, "items.json")
+ count = build_commodities(
+ items_path, os.path.join(args.output_dir, "commodities.json")
+ )
+ os.remove(items_path)
+ print(f"Wrote {count} commodities (items.json discarded)", flush=True)
return 0
diff --git a/trade-hub/backend/tests/test_api.py b/trade-hub/backend/tests/test_api.py
index d5eed93..629bb2c 100644
--- a/trade-hub/backend/tests/test_api.py
+++ b/trade-hub/backend/tests/test_api.py
@@ -11,14 +11,24 @@ def test_healthz():
assert resp.status_code == 200
body = resp.json()
assert body["status"] == "ok"
- assert body["counts"] == {"ships": 2, "ship-items": 2, "fps-items": 1}
+ assert body["counts"] == {
+ "ships": 2,
+ "ship-items": 2,
+ "fps-items": 1,
+ "commodities": 4,
+ }
def test_stats():
resp = client.get("/api/v1/stats")
assert resp.status_code == 200
body = resp.json()
- assert body["categories"] == {"ships": 2, "ship-items": 2, "fps-items": 1}
+ assert body["categories"] == {
+ "ships": 2,
+ "ship-items": 2,
+ "fps-items": 1,
+ "commodities": 4,
+ }
assert body["manufacturers"] == 4
@@ -33,11 +43,11 @@ def test_items_default_pagination():
resp = client.get("/api/v1/items")
assert resp.status_code == 200
body = resp.json()
- assert body["total"] == 5
+ assert body["total"] == 9
assert body["page"] == 1
assert body["page_size"] == 25
assert body["total_pages"] == 1
- assert len(body["items"]) == 5
+ assert len(body["items"]) == 9
def test_items_page_size_capped():
@@ -155,3 +165,29 @@ def test_deterministic_sort_order():
body = client.get("/api/v1/items", params={"page_size": 100}).json()
keys = [(i["category"], i["name"].lower()) for i in body["items"]]
assert keys == sorted(keys)
+
+
+def test_items_category_filter_commodities():
+ resp = client.get("/api/v1/items", params={"category": "commodities"})
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["total"] == 4
+ assert all(i["category"] == "commodities" for i in body["items"])
+
+
+def test_items_search_matches_refined_and_ore_iron():
+ resp = client.get("/api/v1/items", params={"search": "iron"})
+ body = resp.json()
+ assert body["total"] == 2
+ assert {i["name"] for i in body["items"]} == {"Iron", "Iron (Ore)"}
+
+
+def test_item_detail_commodity():
+ resp = client.get("/api/v1/items/Carryable_TBO_FL_1SCU_Commodity_Metal_Ore_Iron")
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["name"] == "Iron (Ore)"
+ assert body["category"] == "commodities"
+ assert body["type"] == "Cargo"
+ assert body["size"] is None
+ assert isinstance(body["details"], dict)
diff --git a/trade-hub/backend/tests/test_inventory.py b/trade-hub/backend/tests/test_inventory.py
index 5ced3d1..bf8cd2b 100644
--- a/trade-hub/backend/tests/test_inventory.py
+++ b/trade-hub/backend/tests/test_inventory.py
@@ -6,6 +6,7 @@ from app.api import app
PASSWORD = "correct horse battery staple"
CARROCK = "ANVL_Carrack"
RIFLE = "BEHR_Rifle_Ballistic_01"
+IRON_ORE = "Carryable_TBO_FL_1SCU_Commodity_Metal_Ore_Iron"
def authed_client(username: str = "alice") -> TestClient:
@@ -251,3 +252,15 @@ def test_sort_defaults_to_name_and_rejects_unknown_values():
listing = client.get("/api/v1/me/inventory").json()
assert [i["class_name"] for i in listing["items"]] == [CARROCK]
assert client.get("/api/v1/me/inventory?sort=bogus").status_code == 422
+
+
+def test_add_commodity_uses_catalog_name_and_quantity():
+ client = authed_client("refiner")
+ resp = add(client, IRON_ORE, quantity=12, notes="from Aaron Halo")
+ assert resp.status_code == 201
+ entry = client.get("/api/v1/me/inventory").json()["items"][0]
+ assert entry["name"] == "Iron (Ore)"
+ assert entry["category"] == "commodities"
+ assert entry["quantity"] == 12
+ assert entry["available"] is True
+ assert entry["visibility"] == "private"
diff --git a/trade-hub/frontend/src/app.js b/trade-hub/frontend/src/app.js
index 79952a8..42b0600 100644
--- a/trade-hub/frontend/src/app.js
+++ b/trade-hub/frontend/src/app.js
@@ -107,7 +107,7 @@ async function initHeader() {
const c = health.counts || {};
els["stats-line"].textContent =
`${c.ships ?? 0} ships · ${c["ship-items"] ?? 0} ship items · ` +
- `${c["fps-items"] ?? 0} FPS items`;
+ `${c["fps-items"] ?? 0} FPS items · ${c.commodities ?? 0} commodities`;
} catch {
els["api-status"].textContent = "API OFFLINE";
els["api-status"].className = "status-pill offline";
@@ -239,7 +239,12 @@ function buildQuery() {
}
function categoryBadge(category) {
- const labels = { ships: "Ship", "ship-items": "Ship Item", "fps-items": "FPS Item" };
+ const labels = {
+ ships: "Ship",
+ "ship-items": "Ship Item",
+ "fps-items": "FPS Item",
+ commodities: "Commodity",
+ };
return `${labels[category] || category}`;
}
@@ -457,7 +462,12 @@ async function refreshSummary() {
}
function renderInventorySummary(summary) {
- const labels = { ships: "ships", "ship-items": "ship items", "fps-items": "FPS items" };
+ const labels = {
+ ships: "ships",
+ "ship-items": "ship items",
+ "fps-items": "FPS items",
+ commodities: "commodities",
+ };
const parts = Object.entries(summary.categories || {})
.map(([key, qty]) => `${qty} ${labels[key] || key}`)
.join(" · ");
diff --git a/trade-hub/frontend/src/index.html b/trade-hub/frontend/src/index.html
index f30b381..cef6856 100644
--- a/trade-hub/frontend/src/index.html
+++ b/trade-hub/frontend/src/index.html
@@ -28,6 +28,7 @@
+
@@ -68,6 +69,7 @@
+