improved cache population

This commit is contained in:
Daylin Morgan 2024-06-24 12:53:07 -05:00
parent aebda80ff2
commit 720d12d455
Signed by: daylin
GPG key ID: 950D13E9719334AD
3 changed files with 145 additions and 15 deletions

View file

@ -54,7 +54,7 @@ jobs:
for host in ${{ inputs.hosts }}; do
nix run . \
-- \
build --system-path \
build --minimal \
--host "$host" \
--flake . \
--verbose \

View file

@ -8,12 +8,15 @@ var buildCmd = &cobra.Command{
Use: "build",
Short: "nix build",
Run: func(cmd *cobra.Command, args []string) {
oizys.NixBuild(nom, args...)
oizys.NixBuild(nom, minimal, args...)
},
}
var minimal bool
func init() {
rootCmd.AddCommand(buildCmd)
buildCmd.Flags().BoolVar(&nom, "nom", false, "display result with nom")
buildCmd.Flags().BoolVar(&systemPath, "system-path", false, "build system path derivation")
// buildCmd.Flags().BoolVar(&systemPath, "system-path", false, "build system path derivation")
buildCmd.Flags().BoolVar(&minimal, "minimal", false, "use system dry-run to make build args")
}

View file

@ -7,6 +7,7 @@ import (
"io/fs"
"os"
"os/exec"
"sort"
"strings"
"time"
@ -70,13 +71,11 @@ func parseSystemPath(derivation map[string]Derivation) (string, error) {
func (o *Oizys) getSystemPath() string {
cmd := exec.Command("nix", "derivation", "show", o.nixosConfigAttr())
logCmd(cmd)
// TODO: add spinner?
// cmd.Stderr = os.Stderr
s := nixSpinner(o.host)
out, err := cmd.Output()
s.Stop()
if err != nil {
log.Fatal(err)
log.Fatal("failed to evalute nixosConfiguration for system-path.drv", "err", err)
}
var derivation map[string]Derivation
@ -272,6 +271,30 @@ func parseDryRun(buf string) (*packages, *packages) {
parsePackages(parts[1], "packages to fetch")
}
// TODO: Refactor this and above
func parseDryRun2(buf string) ([]string, []string) {
lines := strings.Split(strings.TrimSpace(buf), "\n")
var parts [2][]string
i := 0
for _, line := range lines {
if strings.Contains(line, "fetch") && strings.HasSuffix(line, ":") {
i++
}
if i == 2 {
log.Fatal("failed to parse output", "output", buf)
}
if strings.HasPrefix(line, " ") {
parts[i] = append(parts[i], strings.TrimSpace(line))
}
}
if len(parts[0])+len(parts[1]) == 0 {
log.Info("no changes...")
os.Exit(0)
}
return parts[0], parts[1]
}
func showDryRunResult(nixOutput string, verbose bool) {
toBuild, toFetch := parseDryRun(nixOutput)
toBuild.show(verbose)
@ -318,7 +341,7 @@ func runCommand(cmd *exec.Cmd) {
}
}
func (o *Oizys) NixBuild(nom bool, rest ...string) {
func (o *Oizys) NixBuild(nom bool, minimal bool, rest ...string) {
var cmdName string
if nom {
cmdName = "nom"
@ -326,19 +349,123 @@ func (o *Oizys) NixBuild(nom bool, rest ...string) {
cmdName = "nix"
}
cmd := exec.Command(cmdName, "build")
if o.systemPath {
cmd.Args = append(cmd.Args, fmt.Sprintf("%s^*", o.getSystemPath()))
}
cmd.Args = append(cmd.Args, rest...)
if o.resetCache {
cmd.Args = append(cmd.Args, "--narinfo-cache-positive-ttl", "0")
}
if o.inCI {
o.ciPreBuild(cmd)
// if o.inCI {
// o.ciPreBuild(cmd)
// }
if minimal {
log.Debug("populating args with derivations not already built")
drvs := o.systemPathDrvsToBuild()
if len(drvs) == 0 {
log.Info("nothing to build. exiting...")
os.Exit(0)
}
cmd.Args = append(cmd.Args, drvs...)
}
runCommand(cmd)
}
var ignoredMap = stringSliceToMap(
[]string{
"builder.pl",
"profile",
"system-path",
"nixos-install",
"nixos-version",
"nixos-manual-html",
"nixos-configuration-reference-manpage",
"nixos-rebuild",
"nixos-help",
"nixos-generate-config",
"nixos-wsl-version",
"nixos-wsl-welcome-message",
"nixos-wsl-welcome",
// trivial packages
"restic-gdrive",
"gitea",
"lock",
},
)
func drvIsIgnored(drv string) bool {
s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2)
_, ok := ignoredMap[s[len(s)-1]]
return ok
}
func stringSliceToMap(slice []string) map[string]struct{} {
hashMap := make(map[string]struct{}, len(slice))
for _, s := range slice {
hashMap[s] = struct{}{}
}
return hashMap
}
func drvsToInputs(derivation map[string]Derivation) []string {
var drvs []string
for _, drv := range derivation {
for name := range drv.InputDrvs {
drvs = append(drvs, name)
}
}
return drvs
}
// v2
func (o *Oizys) systemPathDrvsToBuild() []string {
log.Debug("getting system-path build-only deps")
systemPathDrv := fmt.Sprintf("%s^*", o.getSystemPath())
derivationCmd := exec.Command("nix", "derivation", "show", systemPathDrv)
logCmd(derivationCmd)
output, err := derivationCmd.Output()
if err != nil {
log.Fatal("failed to evaluate", "drv", systemPathDrv)
}
// TODO: abstract this?
var derivation map[string]Derivation
if err := json.Unmarshal(output, &derivation); err != nil {
log.Fatal(err)
}
systemPathInputDrvs := stringSliceToMap(drvsToInputs(derivation))
// get to build packages for full system
systemCmd := exec.Command("nix", "build", o.nixosConfigAttr(), "--dry-run")
logCmd(systemCmd)
result, err := systemCmd.CombinedOutput()
if err != nil {
log.Fatal("failed to dry-run build full system", "err", err)
}
toBuild, _ := parseDryRun2(string(result))
var toActuallyBuild []string
for _, drv := range toBuild {
_, ok := systemPathInputDrvs[drv]
if ok && !drvIsIgnored(drv) {
toActuallyBuild = append(toActuallyBuild, drv)
}
}
drvs := make([]string, len(toActuallyBuild))
for i, pkg := range toActuallyBuild {
drvs[i] = fmt.Sprintf("%s^*", strings.TrimSpace(pkg))
}
return drvs
}
func contains(s []string, search string) bool {
i := sort.SearchStrings(s, search)
return i < len(s) && s[i] == search
}
func (o *Oizys) writeToGithubStepSummary(txt string) {
f, err := os.OpenFile(o.githubSummary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@ -389,7 +516,7 @@ func (o *Oizys) checkPath(name string) string {
func (o *Oizys) Checks(nom bool, rest ...string) {
checks := o.getChecks()
for _, check := range checks {
o.NixBuild(nom, o.checkPath(check))
o.NixBuild(nom, false, o.checkPath(check))
}
}