Compare commits

..

6 commits

9 changed files with 249 additions and 224 deletions

View file

@ -42,9 +42,9 @@ runs:
--allow-remove-essential \ --allow-remove-essential \
-qq \ -qq \
$(non_manifest_packages) > /dev/null & $(non_manifest_packages) > /dev/null &
wait
fi fi
wait
sudo rm -rf /var/lib/docker & sudo rm -rf /var/lib/docker &
sudo rm -rf /usr/{local,share} & sudo rm -rf /usr/{local,share} &

View file

@ -52,14 +52,12 @@ jobs:
- name: Build - name: Build
run: | run: |
for host in ${{ inputs.hosts }}; do nix run . \
nix run . \ -- \
build --minimal \
--host "${{ inputs.hosts }}" \
--flake . \
--verbose \
--debug \
-- \ -- \
build --minimal \ --print-build-logs
--host "$host" \
--flake . \
--verbose \
--debug \
-- \
--print-build-logs
done

View file

@ -3,7 +3,7 @@
services.resolved = enabled; services.resolved = enabled;
services.fail2ban = { services.fail2ban = {
package = pkgs.callPackage ../../pkgs/fail2ban {}; package = pkgs.callPackage ../../pkgs/fail2ban { };
enable = true; enable = true;
maxretry = 5; maxretry = 5;
bantime = "24h"; bantime = "24h";

View file

@ -22,10 +22,15 @@ let
self.nixosModules.nix self.nixosModules.nix
self.nixosModules.essentials self.nixosModules.essentials
( (
{ pkgs, modulesPath, ... }: {
self,
pkgs,
modulesPath,
...
}:
{ {
imports = [ (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") ]; imports = [ (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") ];
environment.systemPackages = with pkgs; [ neovim ]; environment.systemPackages = (with pkgs; [ neovim ]) ++ [ self.packages.${pkgs.system}.default ];
} }
) )
]; ];
@ -38,6 +43,7 @@ let
; ;
}; };
}; };
mkSystem = mkSystem =
hostName: hostName:
nixosSystem { nixosSystem {

View file

@ -2,6 +2,7 @@
pkgs, pkgs,
lib, lib,
config, config,
enabled,
... ...
}: }:
let let
@ -101,13 +102,14 @@ in
}; };
config = config =
mkIf cfg.enable let
libs =
{ defaultLibraries ++ cfg.extra-libraries ++ (if cfg.overkill.enable then overkillLibraries else [ ]);
programs.nix-ld.enable = true; in
programs.nix-ld.libraries = mkIf cfg.enable {
programs.nix-ld = enabled // {
defaultLibraries ++ cfg.extra-libraries ++ (if cfg.overkill.enable then overkillLibraries else [ ]); package = pkgs.nix-ld-rs;
libraries = libs;
}; };
};
} }

View file

@ -1,8 +1,6 @@
package cmd package cmd
import ( import (
"fmt"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"oizys/internal/oizys" "oizys/internal/oizys"
@ -12,7 +10,7 @@ var outputCmd = &cobra.Command{
Use: "output", Use: "output",
Short: "show nixosConfiguration attr", Short: "show nixosConfiguration attr",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
fmt.Println(oizys.Output()) oizys.Output()
}, },
} }

View file

@ -44,7 +44,7 @@ var rootCmd = &cobra.Command{
Short: "nix begat oizys", Short: "nix begat oizys",
PersistentPreRun: func(cmd *cobra.Command, args []string) { PersistentPreRun: func(cmd *cobra.Command, args []string) {
if debug { if debug {
log.Info("running with verbose mode") log.Info("running in debug mode")
log.SetLevel(log.DebugLevel) log.SetLevel(log.DebugLevel)
} }
oizys.SetFlake(flake) oizys.SetFlake(flake)
@ -76,7 +76,7 @@ func init() {
rootCmd.CompletionOptions.HiddenDefaultCmd = true rootCmd.CompletionOptions.HiddenDefaultCmd = true
rootCmd.PersistentFlags().StringVar(&flake, "flake", "", "path to flake ($OIZYS_DIR or $HOME/oizys)") rootCmd.PersistentFlags().StringVar(&flake, "flake", "", "path to flake ($OIZYS_DIR or $HOME/oizys)")
rootCmd.PersistentFlags().StringVar(&host, "host", "", "host to build (current host)") rootCmd.PersistentFlags().StringVar(&host, "host", "", "host(s) to build (current host)")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "show verbose output") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "show verbose output")
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "show debug output") rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "show debug output")
rootCmd.PersistentFlags().BoolVar(&resetCache, "reset-cache", false, "set narinfo-cache-negative-ttl to 0") rootCmd.PersistentFlags().BoolVar(&resetCache, "reset-cache", false, "set narinfo-cache-negative-ttl to 0")

View file

@ -92,54 +92,36 @@ func SetResetCache(reset bool) {
o.resetCache = reset o.resetCache = reset
} }
type Derivation struct { func NixosConfigAttrs() (attrs []string) {
InputDrvs map[string]interface{} for _, host := range strings.Split(o.host, " ") {
attrs = append(attrs, o.nixosConfigAttr(host))
}
return attrs
} }
func parseSystemPath(derivation map[string]Derivation) (string, error) { func (o *Oizys) nixosConfigAttr(host string) string {
for _, nixosDrv := range derivation {
for drv := range nixosDrv.InputDrvs {
if strings.HasSuffix(drv, "system-path.drv") {
return drv, nil
}
}
}
return "", errors.New("failed to find path for system-path.drv")
}
// recreating this command
// nix derivation show `oizys output` | jq -r '.[].inputDrvs | with_entries(select(.key|match("system-path";"i"))) | keys | .[]'
func getSystemPath() string {
cmd := exec.Command("nix", "derivation", "show", o.nixosConfigAttr())
out, err := cmdOutputWithSpinner(cmd, "running nix derivation show for full system", false)
if err != nil {
log.Fatal("failed to evalute nixosConfiguration for system-path.drv", "err", err)
}
var derivation map[string]Derivation
if err := json.Unmarshal(out, &derivation); err != nil {
log.Fatal(err)
}
systemPath, err := parseSystemPath(derivation)
if err != nil {
log.Fatal(err)
}
return systemPath
}
func (o *Oizys) nixosConfigAttr() string {
return fmt.Sprintf( return fmt.Sprintf(
"%s#nixosConfigurations.%s.config.system.build.toplevel", "%s#nixosConfigurations.%s.config.system.build.toplevel",
o.flake, o.flake,
o.host, host,
) )
} }
func Output() string { func Output() {
if o.systemPath { if o.systemPath {
return getSystemPath() drv := evaluateDerivations(NixosConfigAttrs()...)
systemPaths, err := findSystemPaths(drv)
// systemPath, err := findSystemPath(drv)
if err != nil {
log.Fatal("error collecting system paths", "err", err)
}
for _, drv := range systemPaths {
fmt.Println(drv)
}
} else { } else {
return o.nixosConfigAttr() for _, drv := range NixosConfigAttrs() {
fmt.Println(drv)
}
} }
} }
@ -242,7 +224,7 @@ func Dry(verbose bool, minimal bool, rest ...string) {
spinnerMsg = "evaluting for minimal build needs" spinnerMsg = "evaluting for minimal build needs"
} else { } else {
log.Debug("evalutating full nixosConfiguration") log.Debug("evalutating full nixosConfiguration")
cmd.Args = append(cmd.Args, o.nixosConfigAttr()) cmd.Args = append(cmd.Args, NixosConfigAttrs()...)
spinnerMsg = fmt.Sprintf("%s %s", "evaluating derivation for:", spinnerMsg = fmt.Sprintf("%s %s", "evaluating derivation for:",
lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("6")).Render(o.host), lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("6")).Render(o.host),
) )
@ -258,7 +240,7 @@ func Dry(verbose bool, minimal bool, rest ...string) {
} }
} }
// / Setup command completely differently here // Setup command completely differently here
func NixosRebuild(subcmd string, rest ...string) { func NixosRebuild(subcmd string, rest ...string) {
cmd := exec.Command("sudo", cmd := exec.Command("sudo",
"nixos-rebuild", "nixos-rebuild",
@ -303,135 +285,6 @@ func NixBuild(nom bool, minimal bool, rest ...string) {
exitWithCommand(cmd) exitWithCommand(cmd)
} }
var ignoredMap = stringSliceToMap(
[]string{
"builder.pl",
"profile",
"system-path",
"nixos-install",
"nixos-version",
"nixos-manual-html",
"nixos-configuration-reference-manpage",
"nixos-rebuild",
"nixos-help",
"nixos-generate-config",
"nixos-enter",
"nixos-container",
"nixos-build-vms",
"ld-library-path",
"nixos-wsl-version",
"nixos-wsl-welcome-message",
"nixos-wsl-welcome",
// trivial packages
"restic-gdrive",
"gitea",
"lock",
},
)
func drvNotIgnored(drv string) bool {
s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2)
_, ok := ignoredMap[s[len(s)-1]]
return !ok
}
func stringSliceToMap(slice []string) map[string]struct{} {
hashMap := make(map[string]struct{}, len(slice))
for _, s := range slice {
hashMap[s] = struct{}{}
}
return hashMap
}
func drvsToInputs(derivation map[string]Derivation) []string {
var drvs []string
for _, drv := range derivation {
for name := range drv.InputDrvs {
drvs = append(drvs, name)
}
}
return drvs
}
// compute the overlap between two slices of strings
func overlapStrings(a []string, b []string) []string {
var overlap []string
set := stringSliceToMap(a)
for _, s := range b {
_, ok := set[s]
if ok {
overlap = append(overlap, s)
}
}
return overlap
}
func nixDerivationShowToInputs(output []byte) []string {
var derivation map[string]Derivation
if err := json.Unmarshal(output, &derivation); err != nil {
log.Fatal(err)
}
return drvsToInputs(derivation)
}
func filter[T any](ss []T, test func(T) bool) (ret []T) {
for _, s := range ss {
if test(s) {
ret = append(ret, s)
}
}
return
}
func toBuildNixosConfiguration() []string {
systemCmd := exec.Command("nix", "build", o.nixosConfigAttr(), "--dry-run")
if o.resetCache {
systemCmd.Args = append(systemCmd.Args, "--narinfo-cache-negative-ttl", "0")
}
result, err := cmdOutputWithSpinner(
systemCmd,
fmt.Sprintf("running dry build for: %s", o.nixosConfigAttr()),
true,
)
if err != nil {
log.Fatal("failed to dry-run build system", "err", err)
}
toBuild, _ := parseDryRun2(string(result))
return toBuild
}
func systemPathDerivationShow() []string {
systemPathDrv := fmt.Sprintf("%s^*", getSystemPath())
derivationCmd := exec.Command("nix", "derivation", "show", systemPathDrv)
output, err := cmdOutputWithSpinner(
derivationCmd,
fmt.Sprintf("evaluating system path: %s", systemPathDrv),
false)
if err != nil {
log.Fatal("failed to evaluate", "drv", systemPathDrv)
}
return nixDerivationShowToInputs(output)
}
func systemPathDrvsToBuild() []string {
toBuild := toBuildNixosConfiguration()
systemPathInputDrvs := systemPathDerivationShow()
toActuallyBuild := filter(
overlapStrings(systemPathInputDrvs, toBuild),
drvNotIgnored,
)
drvs := make([]string, len(toActuallyBuild))
for i, pkg := range toActuallyBuild {
drvs[i] = fmt.Sprintf("%s^*", strings.TrimSpace(pkg))
}
return drvs
}
func (o *Oizys) writeToGithubStepSummary(txt string) { func (o *Oizys) writeToGithubStepSummary(txt string) {
f, err := os.OpenFile(o.githubSummary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) f, err := os.OpenFile(o.githubSummary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil { if err != nil {
@ -445,22 +298,6 @@ func (o *Oizys) writeToGithubStepSummary(txt string) {
} }
} }
func (o *Oizys) ciPreBuild(cmd *exec.Cmd) {
// TODO: is this exec.Command call necessary?
ciCmd := exec.Command(cmd.Args[0], cmd.Args[1:]...)
ciCmd.Args = append(ciCmd.Args, "--dry-run")
logCmd(ciCmd)
output, err := ciCmd.CombinedOutput()
if err != nil {
showFailedOutput(output)
log.Fatal(err)
}
toBuild, _ := parseDryRun(string(output))
o.writeToGithubStepSummary(
fmt.Sprintf("# %s\n\n%s", o.host, strings.Join(toBuild.names, "\n")),
)
}
func (o *Oizys) getChecks() []string { func (o *Oizys) getChecks() []string {
attrName := fmt.Sprintf("%s#%s", o.flake, "checks.x86_64-linux") attrName := fmt.Sprintf("%s#%s", o.flake, "checks.x86_64-linux")
cmd := exec.Command("nix", "eval", attrName, "--apply", "builtins.attrNames", "--json") cmd := exec.Command("nix", "eval", attrName, "--apply", "builtins.attrNames", "--json")
@ -489,28 +326,18 @@ func Checks(nom bool, rest ...string) {
func CacheBuild(rest ...string) { func CacheBuild(rest ...string) {
args := []string{ args := []string{
"watch-exec", o.cache, "--", "nix", "watch-exec", o.cache, "--", "nix",
"build", o.nixosConfigAttr(), "--print-build-logs", "build", "--print-build-logs",
"--accept-flake-config", "--accept-flake-config",
} }
args = append(args, NixosConfigAttrs()...)
args = append(args, rest...) args = append(args, rest...)
cmd := exec.Command("cachix", args...) cmd := exec.Command("cachix", args...)
exitWithCommand(cmd) exitWithCommand(cmd)
} }
func CheckFlake() {
}
func CI(rest ...string) { func CI(rest ...string) {
args := []string{"workflow", "run", "build.yml", "-F", fmt.Sprintf("hosts=%s", o.host)} args := []string{"workflow", "run", "build.yml", "-F", fmt.Sprintf("hosts=%s", o.host)}
args = append(args, rest...) args = append(args, rest...)
cmd := exec.Command("gh", args...) cmd := exec.Command("gh", args...)
exitWithCommand(cmd) exitWithCommand(cmd)
} }
// // TODO: deprecate
// func nixSpinner(host string) *spinner.Spinner {
// msg := fmt.Sprintf("%s %s", " evaluating derivation for:",
// lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("6")).Render(host),
// )
// return startSpinner(msg)
// }

View file

@ -0,0 +1,194 @@
package oizys
import (
"encoding/json"
"errors"
"fmt"
"os/exec"
"strings"
"github.com/charmbracelet/log"
)
var ignoredMap = stringSliceToMap(
[]string{
// nix
"ld-library-path", "builder.pl", "profile", "system-path",
// nixos
"nixos-install",
"nixos-version",
"nixos-manual-html",
"nixos-configuration-reference-manpage",
"nixos-rebuild",
"nixos-help",
"nixos-generate-config",
"nixos-enter",
"nixos-container",
"nixos-build-vms",
"nixos-wsl-version", "nixos-wsl-welcome-message", "nixos-wsl-welcome",
// trivial packages
"restic-gdrive", "gitea", "lock", "code",
},
)
type Derivation struct {
InputDrvs map[string]interface{}
Name string
}
func findSystemPath(nixosDrv Derivation) (string, error) {
for drv := range nixosDrv.InputDrvs {
if strings.HasSuffix(drv, "system-path.drv") {
return drv, nil
}
}
return "", errors.New("failed to find path for system-path.drv")
}
func findSystemPaths(drv map[string]Derivation) ([]string, error) {
hosts := strings.Split(o.host, " ")
systemDrvs := make([]string, 0, len(hosts))
for _, p := range Keys(drv) {
if strings.HasPrefix(strings.SplitN(p, "-", 2)[1], "nixos-system-") {
systemDrvs = append(systemDrvs, p)
}
}
if len(hosts) != len(systemDrvs) {
return nil, errors.New("didn't find appropriate number of nixos-system derivations")
}
systemPaths := make([]string, 0, len(hosts))
for _, name := range systemDrvs {
systemPath, err := findSystemPath(drv[name])
if err != nil {
return nil, fmt.Errorf("error finding system-path for %s: %w", name, err)
}
systemPaths = append(systemPaths, systemPath)
}
return systemPaths, nil
}
func drvNotIgnored(drv string) bool {
s := strings.SplitN(strings.Replace(drv, ".drv", "", 1), "-", 2)
_, ok := ignoredMap[s[len(s)-1]]
return !ok
}
func stringSliceToMap(slice []string) map[string]struct{} {
hashMap := make(map[string]struct{}, len(slice))
for _, s := range slice {
hashMap[s] = struct{}{}
}
return hashMap
}
func drvsToInputs(derivation map[string]Derivation) []string {
var drvs []string
for _, drv := range derivation {
for name := range drv.InputDrvs {
drvs = append(drvs, name)
}
}
return drvs
}
// compute the overlap between two slices of strings
func overlapStrings(a []string, b []string) []string {
var overlap []string
set := stringSliceToMap(a)
for _, s := range b {
_, ok := set[s]
if ok {
overlap = append(overlap, s)
}
}
return overlap
}
func nixDerivationShowToInputs(output []byte) []string {
var derivation map[string]Derivation
if err := json.Unmarshal(output, &derivation); err != nil {
log.Fatal(err)
}
return drvsToInputs(derivation)
}
func filter[T any](ss []T, test func(T) bool) (ret []T) {
for _, s := range ss {
if test(s) {
ret = append(ret, s)
}
}
return
}
func toBuildNixosConfiguration() []string {
systemCmd := exec.Command("nix", "build", "--dry-run")
systemCmd.Args = append(systemCmd.Args, NixosConfigAttrs()...)
if o.resetCache {
systemCmd.Args = append(systemCmd.Args, "--narinfo-cache-negative-ttl", "0")
}
result, err := cmdOutputWithSpinner(
systemCmd,
fmt.Sprintf("running dry build for: %s", strings.Join(NixosConfigAttrs(), " ")),
true,
)
if err != nil {
log.Fatal("failed to dry-run build system", "err", err)
}
toBuild, _ := parseDryRun2(string(result))
return toBuild
}
func evaluateDerivations(drvs ...string) map[string]Derivation {
cmd := exec.Command("nix", "derivation", "show", "-r")
cmd.Args = append(cmd.Args, drvs...)
out, err := cmdOutputWithSpinner(cmd,
fmt.Sprintf("evaluating derivations %s", strings.Join(drvs, " ")),
false)
if err != nil {
log.Fatal("failed to evalute derivation for", "drvs", drvs, "err", err)
}
var derivation map[string]Derivation
if err := json.Unmarshal(out, &derivation); err != nil {
log.Fatal("failed to decode json", "err", err)
}
return derivation
}
// Keys returns the keys of the map m.
// The keys will be an indeterminate order.
func Keys[M ~map[K]V, K comparable, V any](m M) []K {
r := make([]K, 0, len(m))
for k := range m {
r = append(r, k)
}
return r
}
func systemPathDrvsToBuild() []string {
toBuild := toBuildNixosConfiguration()
drv := evaluateDerivations(NixosConfigAttrs()...)
systemPaths, err := findSystemPaths(drv)
// systemPath, err := findSystemPath(drv)
if err != nil {
log.Fatal("error collecting system paths", "err", err)
}
var inputDrvs []string
for _, path := range systemPaths {
inputDrvs = append(inputDrvs, Keys(drv[path].InputDrvs)...)
}
toActuallyBuild := filter(
overlapStrings(inputDrvs, toBuild),
drvNotIgnored,
)
drvs := make([]string, 0, len(toActuallyBuild))
for _, pkg := range toActuallyBuild {
fmt.Println(pkg)
drvs = append(drvs, fmt.Sprintf("%s^*", strings.TrimSpace(pkg)))
}
return drvs
}