feat: build multiple systems at once

This commit is contained in:
Daylin Morgan 2024-07-10 11:37:23 -05:00
parent 49c288f3d9
commit 10d1800f04
Signed by: daylin
GPG key ID: 950D13E9719334AD
3 changed files with 219 additions and 200 deletions

View file

@ -1,8 +1,6 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"oizys/internal/oizys"
@ -12,7 +10,7 @@ var outputCmd = &cobra.Command{
Use: "output",
Short: "show nixosConfiguration attr",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println(oizys.Output())
oizys.Output()
},
}

View file

@ -92,54 +92,36 @@ func SetResetCache(reset bool) {
o.resetCache = reset
}
type Derivation struct {
InputDrvs map[string]interface{}
func NixosConfigAttrs() (attrs []string) {
for _, host := range strings.Split(o.host, " ") {
attrs = append(attrs, o.nixosConfigAttr(host))
}
return attrs
}
func parseSystemPath(derivation map[string]Derivation) (string, error) {
for _, nixosDrv := range derivation {
for drv := range nixosDrv.InputDrvs {
if strings.HasSuffix(drv, "system-path.drv") {
return drv, nil
}
}
}
return "", errors.New("failed to find path for system-path.drv")
}
// recreating this command
// nix derivation show `oizys output` | jq -r '.[].inputDrvs | with_entries(select(.key|match("system-path";"i"))) | keys | .[]'
func getSystemPath() string {
cmd := exec.Command("nix", "derivation", "show", o.nixosConfigAttr())
out, err := cmdOutputWithSpinner(cmd, "running nix derivation show for full system", false)
if err != nil {
log.Fatal("failed to evalute nixosConfiguration for system-path.drv", "err", err)
}
var derivation map[string]Derivation
if err := json.Unmarshal(out, &derivation); err != nil {
log.Fatal(err)
}
systemPath, err := parseSystemPath(derivation)
if err != nil {
log.Fatal(err)
}
return systemPath
}
func (o *Oizys) nixosConfigAttr() string {
func (o *Oizys) nixosConfigAttr(host string) string {
return fmt.Sprintf(
"%s#nixosConfigurations.%s.config.system.build.toplevel",
o.flake,
o.host,
host,
)
}
func Output() string {
func Output() {
if o.systemPath {
return getSystemPath()
drv := evaluateDerivations(NixosConfigAttrs()...)
systemPaths, err := findSystemPaths(drv)
// systemPath, err := findSystemPath(drv)
if err != nil {
log.Fatal("error collecting system paths", "err", err)
}
for _, drv := range systemPaths {
fmt.Println(drv)
}
} else {
return o.nixosConfigAttr()
for _, drv := range NixosConfigAttrs() {
fmt.Println(drv)
}
}
}
@ -242,7 +224,7 @@ func Dry(verbose bool, minimal bool, rest ...string) {
spinnerMsg = "evaluting for minimal build needs"
} else {
log.Debug("evalutating full nixosConfiguration")
cmd.Args = append(cmd.Args, o.nixosConfigAttr())
cmd.Args = append(cmd.Args, NixosConfigAttrs()...)
spinnerMsg = fmt.Sprintf("%s %s", "evaluating derivation for:",
lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("6")).Render(o.host),
)
@ -258,7 +240,7 @@ func Dry(verbose bool, minimal bool, rest ...string) {
}
}
// / Setup command completely differently here
// Setup command completely differently here
func NixosRebuild(subcmd string, rest ...string) {
cmd := exec.Command("sudo",
"nixos-rebuild",
@ -303,135 +285,6 @@ func NixBuild(nom bool, minimal bool, rest ...string) {
exitWithCommand(cmd)
}
var ignoredMap = stringSliceToMap(
[]string{
"builder.pl",
"profile",
"system-path",
"nixos-install",
"nixos-version",
"nixos-manual-html",
"nixos-configuration-reference-manpage",
"nixos-rebuild",
"nixos-help",
"nixos-generate-config",
"nixos-enter",
"nixos-container",
"nixos-build-vms",
"ld-library-path",
"nixos-wsl-version",
"nixos-wsl-welcome-message",
"nixos-wsl-welcome",
// trivial packages
"restic-gdrive",
"gitea",
"lock",
},
)
func drvNotIgnored(drv string) bool {
s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2)
_, ok := ignoredMap[s[len(s)-1]]
return !ok
}
func stringSliceToMap(slice []string) map[string]struct{} {
hashMap := make(map[string]struct{}, len(slice))
for _, s := range slice {
hashMap[s] = struct{}{}
}
return hashMap
}
func drvsToInputs(derivation map[string]Derivation) []string {
var drvs []string
for _, drv := range derivation {
for name := range drv.InputDrvs {
drvs = append(drvs, name)
}
}
return drvs
}
// compute the overlap between two slices of strings
func overlapStrings(a []string, b []string) []string {
var overlap []string
set := stringSliceToMap(a)
for _, s := range b {
_, ok := set[s]
if ok {
overlap = append(overlap, s)
}
}
return overlap
}
func nixDerivationShowToInputs(output []byte) []string {
var derivation map[string]Derivation
if err := json.Unmarshal(output, &derivation); err != nil {
log.Fatal(err)
}
return drvsToInputs(derivation)
}
func filter[T any](ss []T, test func(T) bool) (ret []T) {
for _, s := range ss {
if test(s) {
ret = append(ret, s)
}
}
return
}
func toBuildNixosConfiguration() []string {
systemCmd := exec.Command("nix", "build", o.nixosConfigAttr(), "--dry-run")
if o.resetCache {
systemCmd.Args = append(systemCmd.Args, "--narinfo-cache-negative-ttl", "0")
}
result, err := cmdOutputWithSpinner(
systemCmd,
fmt.Sprintf("running dry build for: %s", o.nixosConfigAttr()),
true,
)
if err != nil {
log.Fatal("failed to dry-run build system", "err", err)
}
toBuild, _ := parseDryRun2(string(result))
return toBuild
}
func systemPathDerivationShow() []string {
systemPathDrv := fmt.Sprintf("%s^*", getSystemPath())
derivationCmd := exec.Command("nix", "derivation", "show", systemPathDrv)
output, err := cmdOutputWithSpinner(
derivationCmd,
fmt.Sprintf("evaluating system path: %s", systemPathDrv),
false)
if err != nil {
log.Fatal("failed to evaluate", "drv", systemPathDrv)
}
return nixDerivationShowToInputs(output)
}
func systemPathDrvsToBuild() []string {
toBuild := toBuildNixosConfiguration()
systemPathInputDrvs := systemPathDerivationShow()
toActuallyBuild := filter(
overlapStrings(systemPathInputDrvs, toBuild),
drvNotIgnored,
)
drvs := make([]string, len(toActuallyBuild))
for i, pkg := range toActuallyBuild {
drvs[i] = fmt.Sprintf("%s^*", strings.TrimSpace(pkg))
}
return drvs
}
func (o *Oizys) writeToGithubStepSummary(txt string) {
f, err := os.OpenFile(o.githubSummary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@ -445,22 +298,6 @@ func (o *Oizys) writeToGithubStepSummary(txt string) {
}
}
func (o *Oizys) ciPreBuild(cmd *exec.Cmd) {
// TODO: is this exec.Command call necessary?
ciCmd := exec.Command(cmd.Args[0], cmd.Args[1:]...)
ciCmd.Args = append(ciCmd.Args, "--dry-run")
logCmd(ciCmd)
output, err := ciCmd.CombinedOutput()
if err != nil {
showFailedOutput(output)
log.Fatal(err)
}
toBuild, _ := parseDryRun(string(output))
o.writeToGithubStepSummary(
fmt.Sprintf("# %s\n\n%s", o.host, strings.Join(toBuild.names, "\n")),
)
}
func (o *Oizys) getChecks() []string {
attrName := fmt.Sprintf("%s#%s", o.flake, "checks.x86_64-linux")
cmd := exec.Command("nix", "eval", attrName, "--apply", "builtins.attrNames", "--json")
@ -489,28 +326,18 @@ func Checks(nom bool, rest ...string) {
func CacheBuild(rest ...string) {
args := []string{
"watch-exec", o.cache, "--", "nix",
"build", o.nixosConfigAttr(), "--print-build-logs",
"build", "--print-build-logs",
"--accept-flake-config",
}
args = append(args, NixosConfigAttrs()...)
args = append(args, rest...)
cmd := exec.Command("cachix", args...)
exitWithCommand(cmd)
}
func CheckFlake() {
}
func CI(rest ...string) {
args := []string{"workflow", "run", "build.yml", "-F", fmt.Sprintf("hosts=%s", o.host)}
args = append(args, rest...)
cmd := exec.Command("gh", args...)
exitWithCommand(cmd)
}
// // TODO: deprecate
// func nixSpinner(host string) *spinner.Spinner {
// msg := fmt.Sprintf("%s %s", " evaluating derivation for:",
// lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("6")).Render(host),
// )
// return startSpinner(msg)
// }

View file

@ -0,0 +1,194 @@
package oizys
import (
"encoding/json"
"errors"
"fmt"
"os/exec"
"strings"
"github.com/charmbracelet/log"
)
var ignoredMap = stringSliceToMap(
[]string{
// nix
"ld-library-path", "builder.pl", "profile", "system-path",
// nixos
"nixos-install",
"nixos-version",
"nixos-manual-html",
"nixos-configuration-reference-manpage",
"nixos-rebuild",
"nixos-help",
"nixos-generate-config",
"nixos-enter",
"nixos-container",
"nixos-build-vms",
"nixos-wsl-version", "nixos-wsl-welcome-message", "nixos-wsl-welcome",
// trivial packages
"restic-gdrive", "gitea", "lock", "code",
},
)
type Derivation struct {
InputDrvs map[string]interface{}
Name string
}
func findSystemPath(nixosDrv Derivation) (string, error) {
for drv := range nixosDrv.InputDrvs {
if strings.HasSuffix(drv, "system-path.drv") {
return drv, nil
}
}
return "", errors.New("failed to find path for system-path.drv")
}
func findSystemPaths(drv map[string]Derivation) ([]string, error) {
hosts := strings.Split(o.host, " ")
systemDrvs := make([]string, 0, len(hosts))
for _, p := range Keys(drv) {
if strings.HasPrefix(strings.SplitN(p, "-", 2)[1], "nixos-system-") {
systemDrvs = append(systemDrvs, p)
}
}
if len(hosts) != len(systemDrvs) {
return nil, errors.New("didn't find appropriate number of nixos-system derivations")
}
systemPaths := make([]string, 0, len(hosts))
for _, name := range systemDrvs {
systemPath, err := findSystemPath(drv[name])
if err != nil {
return nil, fmt.Errorf("error finding system-path for %s: %w", name, err)
}
systemPaths = append(systemPaths, systemPath)
}
return systemPaths, nil
}
func drvNotIgnored(drv string) bool {
s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2)
_, ok := ignoredMap[s[len(s)-1]]
return !ok
}
func stringSliceToMap(slice []string) map[string]struct{} {
hashMap := make(map[string]struct{}, len(slice))
for _, s := range slice {
hashMap[s] = struct{}{}
}
return hashMap
}
func drvsToInputs(derivation map[string]Derivation) []string {
var drvs []string
for _, drv := range derivation {
for name := range drv.InputDrvs {
drvs = append(drvs, name)
}
}
return drvs
}
// compute the overlap between two slices of strings
func overlapStrings(a []string, b []string) []string {
var overlap []string
set := stringSliceToMap(a)
for _, s := range b {
_, ok := set[s]
if ok {
overlap = append(overlap, s)
}
}
return overlap
}
func nixDerivationShowToInputs(output []byte) []string {
var derivation map[string]Derivation
if err := json.Unmarshal(output, &derivation); err != nil {
log.Fatal(err)
}
return drvsToInputs(derivation)
}
func filter[T any](ss []T, test func(T) bool) (ret []T) {
for _, s := range ss {
if test(s) {
ret = append(ret, s)
}
}
return
}
func toBuildNixosConfiguration() []string {
systemCmd := exec.Command("nix", "build", "--dry-run")
systemCmd.Args = append(systemCmd.Args, NixosConfigAttrs()...)
if o.resetCache {
systemCmd.Args = append(systemCmd.Args, "--narinfo-cache-negative-ttl", "0")
}
result, err := cmdOutputWithSpinner(
systemCmd,
fmt.Sprintf("running dry build for: %s", strings.Join(NixosConfigAttrs(), " ")),
true,
)
if err != nil {
log.Fatal("failed to dry-run build system", "err", err)
}
toBuild, _ := parseDryRun2(string(result))
return toBuild
}
func evaluateDerivations(drvs ...string) map[string]Derivation {
cmd := exec.Command("nix", "derivation", "show", "-r")
cmd.Args = append(cmd.Args, drvs...)
out, err := cmdOutputWithSpinner(cmd,
fmt.Sprintf("evaluating derivations %s", strings.Join(drvs, " ")),
false)
if err != nil {
log.Fatal("failed to evalute derivation for", "drvs", drvs, "err", err)
}
var derivation map[string]Derivation
if err := json.Unmarshal(out, &derivation); err != nil {
log.Fatal("failed to decode json", "err", err)
}
return derivation
}
// Keys returns the keys of the map m.
// The keys will be an indeterminate order.
func Keys[M ~map[K]V, K comparable, V any](m M) []K {
r := make([]K, 0, len(m))
for k := range m {
r = append(r, k)
}
return r
}
func systemPathDrvsToBuild() []string {
toBuild := toBuildNixosConfiguration()
drv := evaluateDerivations(NixosConfigAttrs()...)
systemPaths, err := findSystemPaths(drv)
// systemPath, err := findSystemPath(drv)
if err != nil {
log.Fatal("error collecting system paths", "err", err)
}
var inputDrvs []string
for _, path := range systemPaths {
inputDrvs = append(inputDrvs, Keys(drv[path].InputDrvs)...)
}
toActuallyBuild := filter(
overlapStrings(inputDrvs, toBuild),
drvNotIgnored,
)
drvs := make([]string, 0, len(toActuallyBuild))
for _, pkg := range toActuallyBuild {
fmt.Println(pkg)
drvs = append(drvs, fmt.Sprintf("%s^*", strings.TrimSpace(pkg)))
}
return drvs
}