From 720d12d455bcbbe54a93a8b10b06d88adbd4b24d Mon Sep 17 00:00:00 2001 From: Daylin Morgan Date: Mon, 24 Jun 2024 12:53:07 -0500 Subject: [PATCH] improved cache population --- .github/workflows/build.yml | 2 +- pkgs/oizys/oizys-go/cmd/build.go | 7 +- pkgs/oizys/oizys-go/internal/oizys/main.go | 151 +++++++++++++++++++-- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1002b3b..b555c71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: for host in ${{ inputs.hosts }}; do nix run . \ -- \ - build --system-path \ + build --minimal \ --host "$host" \ --flake . \ --verbose \ diff --git a/pkgs/oizys/oizys-go/cmd/build.go b/pkgs/oizys/oizys-go/cmd/build.go index b2f79d6..bd325c8 100644 --- a/pkgs/oizys/oizys-go/cmd/build.go +++ b/pkgs/oizys/oizys-go/cmd/build.go @@ -8,12 +8,15 @@ var buildCmd = &cobra.Command{ Use: "build", Short: "nix build", Run: func(cmd *cobra.Command, args []string) { - oizys.NixBuild(nom, args...) + oizys.NixBuild(nom, minimal, args...) }, } +var minimal bool + func init() { rootCmd.AddCommand(buildCmd) buildCmd.Flags().BoolVar(&nom, "nom", false, "display result with nom") - buildCmd.Flags().BoolVar(&systemPath, "system-path", false, "build system path derivation") + // buildCmd.Flags().BoolVar(&systemPath, "system-path", false, "build system path derivation") + buildCmd.Flags().BoolVar(&minimal, "minimal", false, "use system dry-run to make build args") } diff --git a/pkgs/oizys/oizys-go/internal/oizys/main.go b/pkgs/oizys/oizys-go/internal/oizys/main.go index 2cf4b0a..1cffd12 100644 --- a/pkgs/oizys/oizys-go/internal/oizys/main.go +++ b/pkgs/oizys/oizys-go/internal/oizys/main.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "os/exec" + "sort" "strings" "time" @@ -70,13 +71,11 @@ func parseSystemPath(derivation map[string]Derivation) (string, error) { func (o *Oizys) getSystemPath() string { cmd := exec.Command("nix", "derivation", "show", o.nixosConfigAttr()) logCmd(cmd) - // TODO: add spinner? - // cmd.Stderr = os.Stderr s := nixSpinner(o.host) out, err := cmd.Output() s.Stop() if err != nil { - log.Fatal(err) + log.Fatal("failed to evalute nixosConfiguration for system-path.drv", "err", err) } var derivation map[string]Derivation @@ -128,8 +127,8 @@ func (o *Oizys) Set( func terminalSize() (int, int) { fd := os.Stdout.Fd() if !term.IsTerminal(int(fd)) { - log.Error("failed to get terminal size") - return 80, 0 + log.Error("failed to get terminal size") + return 80, 0 // log.Fatal("failed to get terminal size") } w, h, err := term.GetSize(int(fd)) @@ -272,6 +271,30 @@ func parseDryRun(buf string) (*packages, *packages) { parsePackages(parts[1], "packages to fetch") } +// TODO: Refactor this and above +func parseDryRun2(buf string) ([]string, []string) { + lines := strings.Split(strings.TrimSpace(buf), "\n") + var parts [2][]string + i := 0 + for _, line := range lines { + if strings.Contains(line, "fetch") && strings.HasSuffix(line, ":") { + i++ + } + if i == 2 { + log.Fatal("failed to parse output", "output", buf) + } + if strings.HasPrefix(line, " ") { + parts[i] = append(parts[i], strings.TrimSpace(line)) + } + } + + if len(parts[0])+len(parts[1]) == 0 { + log.Info("no changes...") + os.Exit(0) + } + return parts[0], parts[1] +} + func showDryRunResult(nixOutput string, verbose bool) { toBuild, toFetch := parseDryRun(nixOutput) toBuild.show(verbose) @@ -318,7 +341,7 @@ func runCommand(cmd *exec.Cmd) { } } -func (o *Oizys) NixBuild(nom bool, rest ...string) { +func (o *Oizys) NixBuild(nom bool, minimal bool, rest ...string) { var cmdName string if nom { cmdName = "nom" @@ -326,19 +349,123 @@ func (o *Oizys) NixBuild(nom bool, rest ...string) { cmdName = "nix" } cmd := exec.Command(cmdName, "build") - if o.systemPath { - cmd.Args = append(cmd.Args, fmt.Sprintf("%s^*", o.getSystemPath())) - } + cmd.Args = append(cmd.Args, rest...) if o.resetCache { cmd.Args = append(cmd.Args, "--narinfo-cache-positive-ttl", "0") } - if o.inCI { - o.ciPreBuild(cmd) + // if o.inCI { + // o.ciPreBuild(cmd) + // } + if minimal { + log.Debug("populating args with derivations not already built") + drvs := o.systemPathDrvsToBuild() + if len(drvs) == 0 { + log.Info("nothing to build. exiting...") + os.Exit(0) + } + cmd.Args = append(cmd.Args, drvs...) } + runCommand(cmd) } +var ignoredMap = stringSliceToMap( + []string{ + "builder.pl", + "profile", + "system-path", + + "nixos-install", + "nixos-version", + "nixos-manual-html", + "nixos-configuration-reference-manpage", + "nixos-rebuild", + "nixos-help", + "nixos-generate-config", + + "nixos-wsl-version", + "nixos-wsl-welcome-message", + "nixos-wsl-welcome", + + // trivial packages + "restic-gdrive", + "gitea", + "lock", + }, +) + +func drvIsIgnored(drv string) bool { + s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2) + _, ok := ignoredMap[s[len(s)-1]] + return ok +} + +func stringSliceToMap(slice []string) map[string]struct{} { + hashMap := make(map[string]struct{}, len(slice)) + for _, s := range slice { + hashMap[s] = struct{}{} + } + return hashMap +} + +func drvsToInputs(derivation map[string]Derivation) []string { + var drvs []string + for _, drv := range derivation { + for name := range drv.InputDrvs { + drvs = append(drvs, name) + } + } + return drvs +} + +// v2 +func (o *Oizys) systemPathDrvsToBuild() []string { + log.Debug("getting system-path build-only deps") + systemPathDrv := fmt.Sprintf("%s^*", o.getSystemPath()) + derivationCmd := exec.Command("nix", "derivation", "show", systemPathDrv) + logCmd(derivationCmd) + output, err := derivationCmd.Output() + if err != nil { + log.Fatal("failed to evaluate", "drv", systemPathDrv) + } + + // TODO: abstract this? + var derivation map[string]Derivation + if err := json.Unmarshal(output, &derivation); err != nil { + log.Fatal(err) + } + systemPathInputDrvs := stringSliceToMap(drvsToInputs(derivation)) + + // get to build packages for full system + systemCmd := exec.Command("nix", "build", o.nixosConfigAttr(), "--dry-run") + logCmd(systemCmd) + result, err := systemCmd.CombinedOutput() + if err != nil { + log.Fatal("failed to dry-run build full system", "err", err) + } + toBuild, _ := parseDryRun2(string(result)) + + var toActuallyBuild []string + for _, drv := range toBuild { + _, ok := systemPathInputDrvs[drv] + if ok && !drvIsIgnored(drv) { + toActuallyBuild = append(toActuallyBuild, drv) + } + } + + drvs := make([]string, len(toActuallyBuild)) + for i, pkg := range toActuallyBuild { + drvs[i] = fmt.Sprintf("%s^*", strings.TrimSpace(pkg)) + } + return drvs +} + +func contains(s []string, search string) bool { + i := sort.SearchStrings(s, search) + return i < len(s) && s[i] == search +} + func (o *Oizys) writeToGithubStepSummary(txt string) { f, err := os.OpenFile(o.githubSummary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -389,7 +516,7 @@ func (o *Oizys) checkPath(name string) string { func (o *Oizys) Checks(nom bool, rest ...string) { checks := o.getChecks() for _, check := range checks { - o.NixBuild(nom, o.checkPath(check)) + o.NixBuild(nom, false, o.checkPath(check)) } }