43 lines
1.1 KiB
Python
43 lines
1.1 KiB
Python
"""Download the full scunpacked dataset into a target directory.
|
|
|
|
Usage:
|
|
python scripts/fetch_data.py [--output-dir DIR]
|
|
|
|
Default output: backend/data/full (used by the Docker image build).
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
|
|
BASE_URL = "https://raw.githubusercontent.com/StarCitizenWiki/scunpacked-data/master"
|
|
FILES = [
|
|
"ships.json",
|
|
"ship-items.json",
|
|
"fps-items.json",
|
|
"manufacturers.json",
|
|
]
|
|
|
|
|
|
def main():
|
|
default_dir = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), "..", "data", "full")
|
|
)
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output-dir", default=default_dir)
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
for name in FILES:
|
|
url = f"{BASE_URL}/{name}"
|
|
dest = os.path.join(args.output_dir, name)
|
|
print(f"Downloading {url} -> {dest}", flush=True)
|
|
urllib.request.urlretrieve(url, dest)
|
|
size = os.path.getsize(dest)
|
|
print(f" done ({size / 1_000_000:.1f} MB)", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|