82 lines
2.5 KiB
Python
82 lines
2.5 KiB
Python
"""Download the full scunpacked dataset into a target directory.
|
|
|
|
Usage:
|
|
python scripts/fetch_data.py [--output-dir DIR]
|
|
|
|
Default output: backend/data/full (used by the Docker image build).
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
|
|
BASE_URL = "https://raw.githubusercontent.com/StarCitizenWiki/scunpacked-data/master"
|
|
FILES = [
|
|
"ships.json",
|
|
"ship-items.json",
|
|
"fps-items.json",
|
|
"manufacturers.json",
|
|
"items.json",
|
|
]
|
|
COMMODITY_KEYS = ("className", "name", "type", "subType", "reference")
|
|
PLACEHOLDER = "<= PLACEHOLDER =>"
|
|
|
|
|
|
def _commodity_rank(entry):
|
|
class_name = entry.get("className") or ""
|
|
return (0 if "_1SCU_" in class_name else 1, entry.get("size") or 0, class_name)
|
|
|
|
|
|
def build_commodities(items_path, out_path):
|
|
with open(items_path, encoding="utf-8") as f:
|
|
items = json.load(f)
|
|
|
|
grouped = {}
|
|
for entry in items:
|
|
class_name = (entry.get("className") or "").lower()
|
|
name = (entry.get("name") or "").strip()
|
|
if entry.get("type") != "Cargo" or "commodity" not in class_name:
|
|
continue
|
|
if not name or name == PLACEHOLDER:
|
|
continue
|
|
grouped.setdefault(name, []).append(entry)
|
|
|
|
commodities = [
|
|
{key: best.get(key) for key in COMMODITY_KEYS}
|
|
for best in (min(grouped[name], key=_commodity_rank) for name in sorted(grouped))
|
|
]
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
json.dump(commodities, f, ensure_ascii=False, separators=(",", ":"))
|
|
return len(commodities)
|
|
|
|
|
|
def main():
|
|
default_dir = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), "..", "data", "full")
|
|
)
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output-dir", default=default_dir)
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
for name in FILES:
|
|
url = f"{BASE_URL}/{name}"
|
|
dest = os.path.join(args.output_dir, name)
|
|
print(f"Downloading {url} -> {dest}", flush=True)
|
|
urllib.request.urlretrieve(url, dest)
|
|
size = os.path.getsize(dest)
|
|
print(f" done ({size / 1_000_000:.1f} MB)", flush=True)
|
|
|
|
items_path = os.path.join(args.output_dir, "items.json")
|
|
count = build_commodities(
|
|
items_path, os.path.join(args.output_dir, "commodities.json")
|
|
)
|
|
os.remove(items_path)
|
|
print(f"Wrote {count} commodities (items.json discarded)", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|