setup.nim -> add-mirrors

.gitignore

@@ -7,5 +7,5 @@ forgejo/gitea/*
 
 
 soft/*
-!soft/setup.nim
+!soft/add-mirrors
 !soft/repos.txt

soft/add-mirrors

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List
+
+
+@dataclass(frozen=True)
+class Repo:
+    name: str
+    html_url: str
+    description: str = ""
+
+    @classmethod
+    def from_response(cls, data: Dict[str, str]) -> "Repo":
+        return cls(
+            **{
+                k: v
+                for k, v in data.items()
+                if k in cls.__dataclass_fields__.keys() and v is not None
+            }
+        )
+
+
+def get_soft_repos() -> List[str]:
+    return (
+        subprocess.check_output(
+            shlex.split("ssh -p 23231 localhost repos list"), text=True
+        )
+        .strip()
+        .splitlines()
+    )
+
+
+def get_link_url(links: str) -> str:
+    """parse a link from a github api header response
+
+    will look like this:
+    '<https://api.github.com/user/47667941/repos?per_page=100&page=2>; rel="next" <https://api.github.com/user/47667941/repos?per_page=100&page=2>; rel="last"'
+    """
+    return links.split(",")[0].split(";")[0].strip("<>")
+
+
+def get_gh_repos() -> List[Repo]:
+    """get a list of all repos on GH with pagination"""
+    token = os.getenv("GITHUB_TOKEN")
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if token != "":
+        headers["Authorization"] = f"Bearer {token}"
+    url = "https://api.github.com/user/47667941/repos?per_page=100"
+    pages_left = True
+    data = []
+    while pages_left:
+        req = urllib.request.Request(url, headers=headers)
+        response = urllib.request.urlopen(req)
+        data.extend(json.loads(response.read()))
+        pages_left = 'rel="next"' in (link_header := response.getheader("link"))
+        if pages_left:
+            url = get_link_url(link_header)
+
+    return [Repo.from_response(repo_data) for repo_data in data]
+
+
+def mirror_repo(repo: Repo, dry_run=False) -> None:
+    print(f"mirroring: {repo.name}")
+    cmd = [
+        *shlex.split("ssh -p 23231 localhost repos import --mirror"),
+
+        repo.name,
+        repo.html_url,
+    ]
+    if repo.description != "":
+        cmd.append("-d")
+        cmd.append(f"'{repo.description}'")
+    if dry_run:
+        print(" ".join(str(s) for s in cmd))
+    else:
+        subprocess.check_output(cmd)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        help="set dry-run (don't actually mirror with soft-serve yet)",
+        action="store_true",
+    )
+    args = parser.parse_args()
+    dry_run = False
+    repos = (Path(__file__).parent / "repos.txt").read_text().splitlines()
+    gh_repos = get_gh_repos()
+    soft_repos = get_soft_repos()
+
+    non_existent = {
+        repo for repo in repos if repo not in (repo.name for repo in gh_repos)
+    }
+    if non_existent:
+        sys.exit("some repos don't exist on github: " + ";".join(non_existent))
+
+    to_mirror = {
+        repo for repo in gh_repos if repo.name in repos and repo.name not in soft_repos
+    }
+    if not to_mirror:
+        sys.exit("no repos need to be mirrored")
+
+    print(f"mirroring {len(to_mirror)} repo(s)")
+    for repo in to_mirror:
+        mirror_repo(repo, dry_run=args.dry_run)

soft/repos.txt

@@ -1,14 +1,15 @@
-oizys
+advent-of-code-2023
 bbansi
 dotfiles
-git-server
-monolisa-nerdfont-patch
-yartsu
-task.mk
-viv
-logo
 forge
-advent-of-code-2023
+git-server
-nnl
 hwylterm
+logo
+monolisa-nerdfont-patch
+nim2nix
+nnl
+oizys
+task.mk
 tsm
+viv
+yartsu

soft/setup.nim

@@ -1,96 +0,0 @@
-#[
-# fetch all repos from user -> first 100
-curl -L \
-  -H "Accept: application/vnd.github+json" \
-  -H "Authorization: Bearer $GITHUB_TOKEN" \
-  -H "X-GitHub-Api-Version: 2022-11-28" \
-  'https://api.github.com/users/daylinmorgan/repos?per_page=100' > repos.json
-]#
-{.define: ssl.}
-import std/[
-  httpclient, json, os, osproc,
-  options, strformat, strutils,
-  sugar
-]
-
-type
-  Repo = object
-    name: string
-    description: string
-    html_url: string
-
-template use(client: HttpClient, body: untyped) =
-  try:
-    body
-  finally:
-    client.close()
-
-proc getGhRepos(): seq[Repo] =
-  let
-    token = getEnv("GITHUB_TOKEN")
-    url = "https://api.github.com/users/daylinmorgan/repos?per_page=100"
-  var
-    response: string
-    headers = @[
-      ("Accept", "application/vnd.github+json"),
-      ("X-GitHub-Api-Version", "2022-11-28")
-    ]
-  if token != "": headers.add ("Authorization", "Bearer " & token)
-  var client = newHttpClient(headers = newHttpHeaders(headers))
-  use client:
-    response = client.getContent(url)
-  parseJson(response).to(seq[Repo])
-
-
-proc getSoftRepos(): seq[string] =
-  let (output, errCode) = execCmdEx("ssh -p 23231 localhost repos list")
-  if errCode != 0:
-    echo "error fetching repos"
-    echo "result:"
-    echo output
-    quit(QuitFailure)
-  return output.strip().split "\n"
-
-proc mirrorToSoft(repo: Repo, dryRun: bool) =
-  var cmd = "ssh -p 23231 localhost repos import "
-  cmd.add fmt"{repo.name} {repo.html_url} -m "
-  if repo.description != "":
-    # I've never had such problems with quotes before
-    cmd.add fmt("""-d \"{quoteshell(repo.description)}\"""")
-  echo cmd
-  if not dryrun:
-    let (output, errCode) = execCmdEx(cmd)
-    if errCode != 0:
-      echo "ERROR:"
-      echo output
-
-when isMainModule:
-  import std/parseopt
-  # const reposList = slurp("repos.txt").strip().split("\n")
-  let reposList = readFile("repos.txt").strip().split('\n')
-  var dryrun = false
-  var p = initOptParser()
-  for kind, key, val in p.getopt():
-    case kind:
-    of cmdEnd: break
-    of cmdShortOption, cmdLongOption:
-      case key:
-        of "n", "--dryrun": dryrun = true
-        else:
-          echo "unexpected option/value -> ", key, ", ", val
-    of cmdArgument:
-      echo "unknown argument: ", key
-
-  let
-    ghRepos = getGhRepos()
-    softRepos = getSoftRepos()
-    toBeMirrored = collect:
-      for repo in ghRepos:
-        if repo.name in reposList and
-          repo.name notin softRepos:
-          repo
-
-  if toBeMirrored.len > 0:
-    for repo in toBeMirrored:
-      mirrorToSoft(repo, dryrun)
-