diff --git a/.gitignore b/.gitignore index 9d9b1a5..5f0cfd4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,5 @@ forgejo/gitea/* soft/* -!soft/setup.nim +!soft/add-mirrors !soft/repos.txt diff --git a/soft/add-mirrors b/soft/add-mirrors new file mode 100755 index 0000000..846a3f7 --- /dev/null +++ b/soft/add-mirrors @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shlex +import subprocess +import sys +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List + + +@dataclass(frozen=True) +class Repo: + name: str + html_url: str + description: str = "" + + @classmethod + def from_response(cls, data: Dict[str, str]) -> "Repo": + return cls( + **{ + k: v + for k, v in data.items() + if k in cls.__dataclass_fields__.keys() and v is not None + } + ) + + +def get_soft_repos() -> List[str]: + return ( + subprocess.check_output( + shlex.split("ssh -p 23231 localhost repos list"), text=True + ) + .strip() + .splitlines() + ) + + +def get_link_url(links: str) -> str: + """parse a link from a github api header response + + will look like this: + '; rel="next" ; rel="last"' + """ + return links.split(",")[0].split(";")[0].strip("<>") + + +def get_gh_repos() -> List[Repo]: + """get a list of all repos on GH with pagination""" + token = os.getenv("GITHUB_TOKEN") + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token != "": + headers["Authorization"] = f"Bearer {token}" + url = "https://api.github.com/user/47667941/repos?per_page=100" + pages_left = True + data = [] + while pages_left: + req = urllib.request.Request(url, headers=headers) + response = urllib.request.urlopen(req) + data.extend(json.loads(response.read())) + pages_left = 'rel="next"' in (link_header := response.getheader("link")) + if pages_left: + url = get_link_url(link_header) + + return [Repo.from_response(repo_data) for repo_data in data] + + +def mirror_repo(repo: Repo, dry_run=False) -> None: + print(f"mirroring: {repo.name}") + cmd = [ + *shlex.split("ssh -p 23231 localhost repos import --mirror"), + + repo.name, + repo.html_url, + ] + if repo.description != "": + cmd.append("-d") + cmd.append(f"'{repo.description}'") + if dry_run: + print(" ".join(str(s) for s in cmd)) + else: + subprocess.check_output(cmd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", + "--dry-run", + help="set dry-run (don't actually mirror with soft-serve yet)", + action="store_true", + ) + args = parser.parse_args() + dry_run = False + repos = (Path(__file__).parent / "repos.txt").read_text().splitlines() + gh_repos = get_gh_repos() + soft_repos = get_soft_repos() + + non_existent = { + repo for repo in repos if repo not in (repo.name for repo in gh_repos) + } + if non_existent: + sys.exit("some repos don't exist on github: " + ";".join(non_existent)) + + to_mirror = { + repo for repo in gh_repos if repo.name in repos and repo.name not in soft_repos + } + if not to_mirror: + sys.exit("no repos need to be mirrored") + + print(f"mirroring {len(to_mirror)} repo(s)") + for repo in to_mirror: + mirror_repo(repo, dry_run=args.dry_run) diff --git a/soft/repos.txt b/soft/repos.txt index 76e4a81..f6fea7c 100644 --- a/soft/repos.txt +++ b/soft/repos.txt @@ -1,14 +1,15 @@ -oizys +advent-of-code-2023 bbansi dotfiles -git-server -monolisa-nerdfont-patch -yartsu -task.mk -viv -logo forge -advent-of-code-2023 -nnl +git-server hwylterm +logo +monolisa-nerdfont-patch +nim2nix +nnl +oizys +task.mk tsm +viv +yartsu diff --git a/soft/setup.nim b/soft/setup.nim deleted file mode 100644 index 4d27c08..0000000 --- a/soft/setup.nim +++ /dev/null @@ -1,96 +0,0 @@ -#[ -# fetch all repos from user -> first 100 -curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - 'https://api.github.com/users/daylinmorgan/repos?per_page=100' > repos.json -]# -{.define: ssl.} -import std/[ - httpclient, json, os, osproc, - options, strformat, strutils, - sugar -] - -type - Repo = object - name: string - description: string - html_url: string - -template use(client: HttpClient, body: untyped) = - try: - body - finally: - client.close() - -proc getGhRepos(): seq[Repo] = - let - token = getEnv("GITHUB_TOKEN") - url = "https://api.github.com/users/daylinmorgan/repos?per_page=100" - var - response: string - headers = @[ - ("Accept", "application/vnd.github+json"), - ("X-GitHub-Api-Version", "2022-11-28") - ] - if token != "": headers.add ("Authorization", "Bearer " & token) - var client = newHttpClient(headers = newHttpHeaders(headers)) - use client: - response = client.getContent(url) - parseJson(response).to(seq[Repo]) - - -proc getSoftRepos(): seq[string] = - let (output, errCode) = execCmdEx("ssh -p 23231 localhost repos list") - if errCode != 0: - echo "error fetching repos" - echo "result:" - echo output - quit(QuitFailure) - return output.strip().split "\n" - -proc mirrorToSoft(repo: Repo, dryRun: bool) = - var cmd = "ssh -p 23231 localhost repos import " - cmd.add fmt"{repo.name} {repo.html_url} -m " - if repo.description != "": - # I've never had such problems with quotes before - cmd.add fmt("""-d \"{quoteshell(repo.description)}\"""") - echo cmd - if not dryrun: - let (output, errCode) = execCmdEx(cmd) - if errCode != 0: - echo "ERROR:" - echo output - -when isMainModule: - import std/parseopt - # const reposList = slurp("repos.txt").strip().split("\n") - let reposList = readFile("repos.txt").strip().split('\n') - var dryrun = false - var p = initOptParser() - for kind, key, val in p.getopt(): - case kind: - of cmdEnd: break - of cmdShortOption, cmdLongOption: - case key: - of "n", "--dryrun": dryrun = true - else: - echo "unexpected option/value -> ", key, ", ", val - of cmdArgument: - echo "unknown argument: ", key - - let - ghRepos = getGhRepos() - softRepos = getSoftRepos() - toBeMirrored = collect: - for repo in ghRepos: - if repo.name in reposList and - repo.name notin softRepos: - repo - - if toBeMirrored.len > 0: - for repo in toBeMirrored: - mirrorToSoft(repo, dryrun) -