ce7cb702218693d08ea6d83464d2457c9250027c

services/minecraft/publish.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import boto3
+from botocore.config import Config
+
+PREFIX = "minecraft/"
+GROUPS = (
+    "mods",
+    "config",
+    "defaultconfigs",
+    "kubejs",
+    "libraries",
+    "libraries-integratedscripting",
+    "settings",
+    "tools",
+    "world/datapacks",
+    "world/serverconfig",
+)
+
+
+def inventory_name(group: str) -> str:
+    return group.replace("/", "-") + ".sha256"
+
+
+def files_for(source: Path, tools: Path, exporter: Path) -> dict[str, dict[str, Path]]:
+    files: dict[str, dict[str, Path]] = {group: {} for group in GROUPS}
+    for group in ("config", "defaultconfigs", "kubejs", "libraries", "libraries-integratedscripting"):
+        root = source / group
+        files[group] = {p.relative_to(root).as_posix(): p for p in root.rglob("*") if p.is_file()}
+    files["mods"] = {p.name: p for p in (source / "mods").glob("*.jar")}
+    files["mods"][exporter.name] = exporter
+    files["settings"]["user_jvm_args.txt"] = source / "user_jvm_args.txt"
+    files["tools"] = {p.relative_to(tools).as_posix(): p for p in tools.rglob("*") if p.is_file()}
+    ranks = source / "world/serverconfig/ftbranks/ranks.snbt"
+    files["world/serverconfig"]["ftbranks/ranks.snbt"] = ranks
+    return files
+
+
+def digest(path: Path) -> str:
+    with path.open("rb") as source:
+        return hashlib.file_digest(source, "sha256").hexdigest()
+
+
+def publish(client: Any, files: dict[str, dict[str, Path]]) -> None:
+    def upload(item: tuple[str, str, Path]) -> tuple[str, str, str]:
+        group, name, path = item
+        expected = digest(path)
+        key = PREFIX + group + "/" + name
+        with path.open("rb") as source:
+            client.put_object(Bucket="artifacts", Key=key, Body=source)
+        body = client.get_object(Bucket="artifacts", Key=key)["Body"]
+        try:
+            actual = hashlib.file_digest(body, "sha256").hexdigest()
+        finally:
+            body.close()
+        if actual != expected:
+            raise ValueError(f"checksum differs: {key}")
+        return group, name, expected
+
+    entries = [(group, name, path) for group, items in files.items() for name, path in sorted(items.items())]
+    with ThreadPoolExecutor(max_workers=8) as executor:
+        hashes = []
+        for result in executor.map(upload, entries):
+            hashes.append(result)
+            if len(hashes) % 100 == 0:
+                print(f"verified {len(hashes)}/{len(entries)} files", flush=True)
+    for group in GROUPS:
+        lines = [f"{sha}  {name}\n" for current, name, sha in hashes if current == group]
+        body = "".join(lines).encode()
+        key = PREFIX + inventory_name(group)
+        client.put_object(Bucket="artifacts", Key=key, Body=body)
+        remote = client.get_object(Bucket="artifacts", Key=key)["Body"]
+        try:
+            if remote.read() != body:
+                raise ValueError(f"inventory differs: {key}")
+        finally:
+            remote.close()
+        print(f"published {group}: {len(lines)} files", flush=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("source", type=Path)
+    parser.add_argument("tools", type=Path)
+    parser.add_argument("exporter", type=Path)
+    args = parser.parse_args()
+    client = boto3.client(
+        "s3",
+        endpoint_url="https://s3.quorate.internal",
+        region_name="us-east-1",
+        config=Config(s3={"addressing_style": "path"}, max_pool_connections=8),
+    )
+    publish(client, files_for(args.source, args.tools, args.exporter))
+
+
+if __name__ == "__main__":
+    main()