from __future__ import annotations import argparse import hashlib from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any import boto3 from botocore.config import Config PREFIX = "minecraft/" GROUPS = ( "mods", "config", "defaultconfigs", "kubejs", "libraries", "libraries-integratedscripting", "settings", "tools", "world/datapacks", "world/serverconfig", ) def inventory_name(group: str) -> str: return group.replace("/", "-") + ".sha256" def files_for(source: Path, tools: Path, exporter: Path) -> dict[str, dict[str, Path]]: files: dict[str, dict[str, Path]] = {group: {} for group in GROUPS} for group in ("config", "defaultconfigs", "kubejs", "libraries", "libraries-integratedscripting"): root = source / group files[group] = {p.relative_to(root).as_posix(): p for p in root.rglob("*") if p.is_file()} files["mods"] = {p.name: p for p in (source / "mods").glob("*.jar")} files["mods"][exporter.name] = exporter files["settings"]["user_jvm_args.txt"] = source / "user_jvm_args.txt" files["tools"] = {p.relative_to(tools).as_posix(): p for p in tools.rglob("*") if p.is_file()} ranks = source / "world/serverconfig/ftbranks/ranks.snbt" files["world/serverconfig"]["ftbranks/ranks.snbt"] = ranks return files def digest(path: Path) -> str: with path.open("rb") as source: return hashlib.file_digest(source, "sha256").hexdigest() def publish(client: Any, files: dict[str, dict[str, Path]]) -> None: def upload(item: tuple[str, str, Path]) -> tuple[str, str, str]: group, name, path = item expected = digest(path) key = PREFIX + group + "/" + name with path.open("rb") as source: client.put_object(Bucket="artifacts", Key=key, Body=source) body = client.get_object(Bucket="artifacts", Key=key)["Body"] try: actual = hashlib.file_digest(body, "sha256").hexdigest() finally: body.close() if actual != expected: raise ValueError(f"checksum differs: {key}") return group, name, expected entries = [(group, name, path) for group, items in files.items() for name, path in sorted(items.items())] with ThreadPoolExecutor(max_workers=8) as executor: hashes = [] for result in executor.map(upload, entries): hashes.append(result) if len(hashes) % 100 == 0: print(f"verified {len(hashes)}/{len(entries)} files", flush=True) for group in GROUPS: lines = [f"{sha} {name}\n" for current, name, sha in hashes if current == group] body = "".join(lines).encode() key = PREFIX + inventory_name(group) client.put_object(Bucket="artifacts", Key=key, Body=body) remote = client.get_object(Bucket="artifacts", Key=key)["Body"] try: if remote.read() != body: raise ValueError(f"inventory differs: {key}") finally: remote.close() print(f"published {group}: {len(lines)} files", flush=True) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("source", type=Path) parser.add_argument("tools", type=Path) parser.add_argument("exporter", type=Path) args = parser.parse_args() client = boto3.client( "s3", endpoint_url="https://s3.quorate.internal", region_name="us-east-1", config=Config(s3={"addressing_style": "path"}, max_pool_connections=8), ) publish(client, files_for(args.source, args.tools, args.exporter)) if __name__ == "__main__": main()