created new role

roles/trade_hub/files/trade-hub/backend/scripts/fetch_data.py

@@ -0,0 +1,81 @@
+"""Download the full scunpacked dataset into a target directory.
+
+Usage:
+    python scripts/fetch_data.py [--output-dir DIR]
+
+Default output: backend/data/full (used by the Docker image build).
+"""
+import argparse
+import json
+import os
+import sys
+import urllib.request
+
+BASE_URL = "https://raw.githubusercontent.com/StarCitizenWiki/scunpacked-data/master"
+FILES = [
+    "ships.json",
+    "ship-items.json",
+    "fps-items.json",
+    "manufacturers.json",
+    "items.json",
+]
+COMMODITY_KEYS = ("className", "name", "type", "subType", "reference")
+PLACEHOLDER = "<= PLACEHOLDER =>"
+
+
+def _commodity_rank(entry):
+    class_name = entry.get("className") or ""
+    return (0 if "_1SCU_" in class_name else 1, entry.get("size") or 0, class_name)
+
+
+def build_commodities(items_path, out_path):
+    with open(items_path, encoding="utf-8") as f:
+        items = json.load(f)
+
+    grouped = {}
+    for entry in items:
+        class_name = (entry.get("className") or "").lower()
+        name = (entry.get("name") or "").strip()
+        if entry.get("type") != "Cargo" or "commodity" not in class_name:
+            continue
+        if not name or name == PLACEHOLDER:
+            continue
+        grouped.setdefault(name, []).append(entry)
+
+    commodities = [
+        {key: best.get(key) for key in COMMODITY_KEYS}
+        for best in (min(grouped[name], key=_commodity_rank) for name in sorted(grouped))
+    ]
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(commodities, f, ensure_ascii=False, separators=(",", ":"))
+    return len(commodities)
+
+
+def main():
+    default_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..", "data", "full")
+    )
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output-dir", default=default_dir)
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    for name in FILES:
+        url = f"{BASE_URL}/{name}"
+        dest = os.path.join(args.output_dir, name)
+        print(f"Downloading {url} -> {dest}", flush=True)
+        urllib.request.urlretrieve(url, dest)
+        size = os.path.getsize(dest)
+        print(f"  done ({size / 1_000_000:.1f} MB)", flush=True)
+
+    items_path = os.path.join(args.output_dir, "items.json")
+    count = build_commodities(
+        items_path, os.path.join(args.output_dir, "commodities.json")
+    )
+    os.remove(items_path)
+    print(f"Wrote {count} commodities (items.json discarded)", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())