Compare commits

...

6 commits

Author SHA1 Message Date
fcb48daade
feat(metadata)!: revamp metadata tracking
This breaks any existing cache but now allows for tracking files that
access vivenv's
2023-05-31 19:17:26 -05:00
fc30b5143b
style: seperate line for question 2023-05-31 19:17:09 -05:00
d0796ac350
fix: user proper site.addsitedir 2023-05-31 19:17:00 -05:00
7d98fd422d
docs: also update the readme with new url 2023-05-31 10:54:53 -05:00
ae77c15827
refactor: replace references with new url 2023-05-31 10:46:37 -05:00
df549555ea
ci: use dedicated sub-domain 2023-05-31 10:36:45 -05:00
4 changed files with 194 additions and 222 deletions

View file

@ -22,5 +22,6 @@ jobs:
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
cname: viv.dayl.in
# publish_dir: ./site
publish_dir: ./docs

View file

@ -11,7 +11,7 @@
Try before you buy!
```sh
python3 <(curl -fsSL gh.dayl.in/viv/viv.py) run pycowsay -- "viv isn't venv\!"
python3 <(curl -fsSL viv.dayl.in/viv.py) run pycowsay -- "viv isn't venv\!"
```
---
@ -32,7 +32,7 @@ For that reason any usage of the `CLI` can be accomplished using a remote copy a
Run the below command to install `viv`.
```sh
python3 <(curl -fsSL gh.dayl.in/viv/viv.py) manage install
python3 <(curl -fsSL viv.dayl.in/viv.py) manage install
```
To access `viv` from within scripts you should add it's location to your `PYTHONPATH`.
@ -74,7 +74,7 @@ viv remove $(viv list -q)
To remove `viv` all together you can use the included `purge` command:
```sh
python3 <(curl -fsSL gh.dayl.in/viv/viv.py) manage purge
python3 <(curl -fsSL viv.dayl.in/viv.py) manage purge
```
## Additional Features
@ -87,7 +87,7 @@ After generating this a standalone `shim` you can freely use this script across
See [examples/black](https://github.com/daylinmorgan/viv/blob/dev/examples/black) for output of below command.
```sh
python3 <(curl -fsSL gh.dayl.in/viv/viv.py) shim black -o ./black --standalone --freeze
python3 <(curl -fsSL viv.dayl.in/viv.py) shim black -o ./black --standalone --freeze
```
## Alternatives

View file

@ -2,4 +2,3 @@
TAG=$(git describe --tags --always --dirty=-dev --exclude 'latest')
VERSION="${TAG#v}"
sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/g" src/viv/viv.py
git add src/viv/viv.py

View file

@ -9,6 +9,7 @@
from __future__ import annotations
import hashlib
import inspect
import itertools
import json
import os
@ -31,19 +32,17 @@ from argparse import (
_SubParsersAction,
)
from argparse import ArgumentParser as StdArgParser
from dataclasses import dataclass
from datetime import datetime
from itertools import zip_longest
from pathlib import Path
from textwrap import dedent, fill, wrap
from textwrap import dedent, fill
from types import TracebackType
from typing import (
Any,
Dict,
Generator,
List,
NoReturn,
Optional,
Sequence,
TextIO,
Tuple,
Type,
@ -51,7 +50,7 @@ from typing import (
from urllib.error import HTTPError
from urllib.request import urlopen
__version__ = "23.5a4-13-ga6bd81d-dev"
__version__ = "23.5a4-15-gdf54955-dev"
class Config:
@ -149,16 +148,6 @@ class Spinner:
class Ansi:
"""control ouptut of ansi(VT100) control codes"""
BOX: Dict[str, str] = {
"v": "",
"h": "",
"tl": "",
"tr": "",
"bl": "",
"br": "",
"sep": "",
}
def __init__(self) -> None:
self.bold: str = "\033[1m"
self.dim: str = "\033[2m"
@ -229,104 +218,9 @@ class Ansi:
new_output = [f"{self.red}->{self.end} {line}" for line in output.splitlines()]
sys.stdout.write("\n".join(new_output) + "\n")
def _get_column_sizes(
self, rows: Tuple[Tuple[str, Sequence[str]], ...]
) -> List[int]:
"""convert list of rows to list of columns sizes
First convert rows into list of columns,
then get max string length for each column.
"""
return list(max(map(len, lst)) for lst in map(list, zip(*rows))) # type: ignore
def _make_row(self, row: Generator[Any, None, None]) -> str:
return (
f" {self.BOX['v']} "
+ f" {self.BOX['sep']} ".join(row)
+ f" {self.BOX['v']}"
)
def _sanitize_row(
self, sizes: List[int], row: Tuple[str, Sequence[str]]
) -> Tuple[Tuple[str, Sequence[str]], ...]:
if len(row[1]) > sizes[1]:
return tuple(
zip_longest(
(row[0],),
wrap(str(row[1]), break_on_hyphens=False, width=sizes[1]),
fillvalue="",
)
)
else:
return (row,)
def viv_preamble(self, style: str = "magenta", sep: str = "::") -> str:
return f"{self.cyan}viv{self.end}{self.__dict__[style]}{sep}{self.end}"
def table(
self, rows: Tuple[Tuple[str, Sequence[str]], ...], header_style: str = "cyan"
) -> None:
"""generate a table with outline and styled header assumes two columns
Args:
rows: sequence of the rows, first item assumed to be header
header_style: color/style for header row
"""
sizes = self._get_column_sizes(rows)
col2_limit = shutil.get_terminal_size().columns - 20
if col2_limit < 20:
error("increase screen size to view table", code=1)
elif sizes[1] > col2_limit:
sizes[1] = col2_limit
header, rows = rows[0], rows[1:]
# this is maybe taking comprehensions too far....
table_rows = (
self._make_row(row)
for row in (
# header row
(
self.__dict__[header_style] + f"{cell:<{sizes[i]}}" + self.end
for i, cell in enumerate(header)
),
# rest of the rows
*(
(f"{cell:<{sizes[i]}}" for i, cell in enumerate(row))
for row in (
newrow
for row in rows
for newrow in self._sanitize_row(sizes, row)
)
),
)
)
sys.stderr.write(
"".join(
(
" ",
self.BOX["tl"],
self.BOX["h"] * (sum(sizes) + 5),
self.BOX["tr"],
"\n",
)
)
)
sys.stderr.write("\n".join(table_rows) + "\n")
sys.stderr.write(
"".join(
(
" ",
self.BOX["bl"],
self.BOX["h"] * (sum(sizes) + 5),
self.BOX["br"],
"\n",
)
)
)
a = Ansi()
@ -339,26 +233,45 @@ to create/activate a vivenv:
- from command line: `{a.style("viv -h","bold")}`
- within python script: {a.style('__import__("viv").use("typer", "rich-click")','bold')}
"""
_standalone_func = r"""def _viv_use(*pkgs, track_exe=False, name=""):
T,F,N=True,False,None;i,s,m,spec=__import__,str,map,[*pkgs]
e,w=lambda x: T if x else F,lambda p,t: p.write_text(t)
if not {*m(type,pkgs)}=={s}: raise ValueError(f"spec: {pkgs} is invalid")
ge,sys,P,ew=i("os").getenv,i("sys"),i("pathlib").Path,i("sys").stderr.write
(cache:=(P(ge("XDG_CACHE_HOME",P.home()/".cache"))/"viv"/"venvs")).mkdir(parents=T,exist_ok=T)
((sha256:=i("hashlib").sha256()).update((s(spec)+
(((exe:=("N/A",s(P(i("sys").executable).resolve()))[e(track_exe)])))).encode()))
if {env:=cache/(((_id:=sha256.hexdigest()),name)[e(name)])}-{*cache.glob("*/")} or ge("VIV_FORCE"):
v=e(ge("VIV_VERBOSE"));ew(f"generating new vivenv -> {env.name}\n")
i("venv").EnvBuilder(with_pip=T,clear=T).create(env)
w(env/"pip.conf","[global]\ndisable-pip-version-check=true")
if (rc:=(p:=i("subprocess").run([env/"bin"/"pip","install","--force-reinstall",*spec],text=T,
stdout=(-1,N)[v],stderr=(-2,N)[v])).returncode)!=0:
if env.is_dir():i("shutil").rmtree(env)
ew(f"pip had non zero exit ({rc})\n{p.stdout}\n");sys.exit(rc)
w(env/"viv-info.json",i("json").dumps(
{"created":s(i("datetime").datetime.today()),"id":_id,"spec":spec,"exe":exe}))
sys.path=[p for p in (*sys.path,s(*(env/"lib").glob("py*/si*")))if p!=i("site").USER_SITE]
return env""" # noqa
import hashlib, json, os, site, shutil, sys, venv # noqa
from pathlib import Path # noqa
from datetime import datetime # noqa
from subprocess import run # noqa
if not {*map(type, pkgs)} == {str}:
raise ValueError(f"spec: {pkgs} is invalid")
meta = dict.fromkeys(("created", "accessed"), (t := str(datetime.today())))
runner = str(Path(__file__).absolute().resolve())
force, verbose, xdg = map(os.getenv, ("VIV_FORCE", "VIV_VERBOSE", "XDG_CACHE_HOME"))
cache = (Path(xdg) if xdg else Path.home() / ".cache") / "viv" / "venvs"
cache.mkdir(parents=True, exist_ok=True)
exe = str(Path(sys.executable).resolve()) if track_exe else "N/A"
(sha256 := hashlib.sha256()).update((str(spec := [*pkgs]) + exe).encode())
_id = sha256.hexdigest()
if (env := cache / (name if name else _id)) not in cache.glob("*/") or force:
sys.stderr.write(f"generating new vivenv -> {env.name}\n")
venv.EnvBuilder(with_pip=True, clear=True).create(env)
(env / "pip.conf").write_text("[global]\ndisable-pip-version-check=true")
run_kw = dict(zip(("stdout", "stderr"), ((None,) * 2 if verbose else (-1, 2))))
p = run([env / "bin" / "pip", "install", "--force-reinstall", *spec], **run_kw)
if (rc := p.returncode) != 0:
if env.is_dir():
shutil.rmtree(env)
sys.stderr.write(f"pip had non zero exit ({rc})\n{p.stdout.decode()}\n")
sys.exit(rc)
meta.update(dict(id=_id, spec=spec, exe=exe, name=name, files=[runner]))
else:
meta = json.loads((env / "vivmeta.json").read_text())
meta.update(dict(accessed=t, files=sorted({*meta["files"],runner})))
(env / "vivmeta.json").write_text(json.dumps(meta))
sys.path = [p for p in sys.path if not p != site.USER_SITE]
site.addsitedir(str(*(env / "lib").glob("py*/si*")))
return env
"""
def noqa(self, txt: str) -> str:
max_length = max(map(len, txt.splitlines()))
@ -372,16 +285,13 @@ to create/activate a vivenv:
return f"""__import__("viv").use({spec_str})"""
def standalone(self, spec: List[str]) -> str:
func_use = self.noqa(
"\n".join((self._standalone_func, self._use_str(spec, standalone=True)))
func_use = "\n".join(
(self._standalone_func, self.noqa(self._use_str(spec, standalone=True)))
)
return f"""
# see `python3 <(curl -fsSL gh.dayl.in/viv/viv.py) --help`
# <<<<< auto-generated by viv (v{__version__})
# fmt: off
# see `python3 <(curl -fsSL viv.dayl.in/viv.py) --help`
# AUTOGENERATED by viv (v{__version__})
{func_use}
# fmt: on
# >>>>> code golfed with <3
"""
def _rel_import(self, local_source: Optional[Path]) -> str:
@ -426,9 +336,7 @@ to create/activate a vivenv:
bin: str,
) -> str:
if standalone:
imports = "\n".join(
("# fmt: off", self.noqa(self._standalone_func), "# fmt: on")
)
imports = self._standalone_func
elif path == "abs":
imports = self._absolute_import(local_source)
elif path == "rel":
@ -724,37 +632,78 @@ def get_hash(spec: Tuple[str, ...] | List[str], track_exe: bool = False) -> str:
return sha256.hexdigest()
@dataclass
class Meta:
name: str
id: str
spec: List[str]
files: List[str]
exe: str
created: str = ""
accessed: str = ""
@classmethod
def load(cls, name: str) -> "Meta":
if not (c.venvcache / name / "vivmeta.json").exists():
warn(f"possibly corrupted vivenv: {name}")
# add empty values for corrupted vivenvs so it will still load
return cls(name=name, spec=[""], files=[""], exe="", id="")
else:
meta = json.loads((c.venvcache / name / "vivmeta.json").read_text())
return cls(**meta)
def write(self, p: Path | None = None) -> None:
if not p:
p = (c.venvcache) / self.name / "vivmeta.json"
p.write_text(json.dumps(self.__dict__))
def addfile(self, f: Path) -> None:
self.accessed = str(datetime.today())
self.files = sorted({*self.files, str(f.absolute().resolve())})
class ViVenv:
def __init__(
self,
spec: List[str],
spec: List[str] = [""],
track_exe: bool = False,
id: str | None = None,
name: str = "",
path: Path | None = None,
skip_load: bool = False,
metadata: Meta | None = None,
) -> None:
self.spec = self._validate_spec(spec)
self.exe = str(Path(sys.executable).resolve()) if track_exe else "N/A"
self.id = id if id else get_hash(spec, track_exe)
self.name = name if name else self.id
self.loaded = False
spec = self._validate_spec(spec)
id = id if id else get_hash(spec, track_exe)
self.name = name if name else id
self.path = path if path else c.venvcache / self.name
self.exists = self.name in [d.name for d in c.venvcache.iterdir()]
if not metadata:
if self.name in (d.name for d in c.venvcache.iterdir()):
self.loaded = True
self.meta = Meta.load(self.name)
else:
self.meta = Meta(
spec=spec,
name=self.name,
id=id,
files=[],
exe=str(Path(sys.executable).resolve()) if track_exe else "N/A",
)
else:
self.meta = metadata
@classmethod
def load(cls, name: str) -> "ViVenv":
"""generate a vivenv from a viv-info.json file
"""generate a vivenv from a vivmeta.json
Args:
name: used as lookup in the vivenv cache
"""
if not (c.venvcache / name / "viv-info.json").is_file():
warn(f"possibly corrupted vivenv: {name}")
return cls(name=name, spec=[""])
else:
with (c.venvcache / name / "viv-info.json").open("r") as f:
venvconfig = json.load(f)
vivenv = cls(name=name, spec=venvconfig["spec"], id=venvconfig["id"])
vivenv.exe = venvconfig["exe"]
vivenv = cls(name=name, metadata=Meta.load(name))
return vivenv
@ -781,12 +730,14 @@ class ViVenv:
with (self.path / "pip.conf").open("w") as f:
f.write("[global]\ndisable-pip-version-check = true")
self.meta.created = str(datetime.today())
def install_pkgs(self) -> None:
cmd: List[str] = [
str(self.path / "bin" / "pip"),
"install",
"--force-reinstall",
] + self.spec
] + self.meta.spec
run(
cmd,
@ -795,24 +746,57 @@ class ViVenv:
verbose=bool(os.getenv("VIV_VERBOSE")),
)
def dump_info(self, write: bool = False) -> None:
# TODO: include associated files in 'info'
# means it needs to be loaded first
# or keep a seperate file hash in c.share?
info = {
"created": str(datetime.today()),
"id": self.id,
"spec": self.spec,
"exe": self.exe,
}
def show(self, verbose: bool = False) -> None:
if not verbose:
_id = (
self.meta.id[:8]
if self.meta.id == self.name
else (self.name[:5] + "..." if len(self.name) > 8 else self.name)
)
# save metadata to json file
if write:
with (self.path / "viv-info.json").open("w") as f:
json.dump(info, f)
sys.stdout.write(
f"""{a.bold}{a.cyan}{_id}{a.end} """
f"""{a.style(", ".join(self.meta.spec),'dim')}\n"""
)
else:
info["spec"] = ", ".join(self.spec)
a.table((("key", "value"), *((k, v) for k, v in info.items())))
self.tree()
def _tree_leaves(self, items: List[str], indent: str = "") -> str:
tree_chars = [""] * (len(items) - 1) + [""]
return "\n".join(
(f"{indent}{a.yellow}{c}{a.end} {i}" for c, i in zip(tree_chars, items))
)
def tree(self) -> None:
_id = self.meta.id if self.meta.id == self.name else self.name
# TODO: generarlize and loop this or make a template..
items = [
f"{a.magenta}{k}{a.end}: {v}"
for k, v in {
**{
"spec": ", ".join(self.meta.spec),
"created": self.meta.created,
"accessed": self.meta.accessed,
},
**({"exe": self.meta.exe} if self.meta.exe != "N/A" else {}),
**({"files": ""} if self.meta.files else {}),
}.items()
]
rows = [f"\n{a.bold}{a.cyan}{_id}{a.end}", self._tree_leaves(items)]
if self.meta.files:
rows += (self._tree_leaves(self.meta.files, indent=" "),)
sys.stdout.write("\n".join(rows) + "\n")
def get_caller_path() -> Path:
"""get callers callers file path"""
# viv.py is fist in stack since function is used in `viv.use()`
frame_info = inspect.stack()[2]
filepath = frame_info.filename # in python 3.5+, you can use frame_info.filename
del frame_info # drop the reference to the stack frame to avoid reference cycles
return Path(filepath).absolute()
def use(*packages: str, track_exe: bool = False, name: str = "") -> Path:
@ -823,26 +807,22 @@ def use(*packages: str, track_exe: bool = False, name: str = "") -> Path:
track_exe: if true make env python exe specific
name: use as vivenv name, if not provided id is used
"""
vivenv = ViVenv(list(packages), track_exe=track_exe, name=name)
if not vivenv.exists or os.getenv("VIV_FORCE"):
vivenv = ViVenv(list(packages), track_exe=track_exe, name=name)
if not vivenv.loaded or os.getenv("VIV_FORCE"):
vivenv.create()
vivenv.install_pkgs()
vivenv.dump_info(write=True)
vivenv.meta.addfile(get_caller_path())
vivenv.meta.write()
modify_sys_path(vivenv.path)
return vivenv.path
def modify_sys_path(new_path: Path) -> None:
# remove user-site
for i, path in enumerate(sys.path):
if path == site.USER_SITE:
sys.path.pop(i)
sys.path.append(
str([p for p in (new_path / "lib").glob("python*/site-packages")][0])
)
sys.path = [p for p in sys.path if p is not site.USER_SITE]
site.addsitedir(str(*(new_path / "lib").glob("python*/site-packages")))
def get_venvs() -> Dict[str, ViVenv]:
@ -923,9 +903,7 @@ class Viv:
def __init__(self) -> None:
self.vivenvs = get_venvs()
self._get_sources()
self.name = (
"viv" if self.local else "python3 <(curl -fsSL gh.dayl.in/viv/viv.py)"
)
self.name = "viv" if self.local else "python3 <(curl -fsSL viv.dayl.in/viv.py)"
def _get_sources(self) -> None:
self.local_source: Optional[Path] = None
@ -954,7 +932,9 @@ class Viv:
for k, v in self.vivenvs.items():
if name_id == k or v.name == name_id:
matches.append(v)
elif k.startswith(name_id) or (v.id.startswith(name_id) and v.id == v.name):
elif k.startswith(name_id) or (
v.meta.id.startswith(name_id) and v.meta.id == v.name
):
matches.append(v)
elif v.name.startswith(name_id):
matches.append(v)
@ -994,11 +974,10 @@ class Viv:
# re-create env again since path's are hard-coded
vivenv = ViVenv(spec)
if not vivenv.exists or os.getenv("VIV_FORCE"):
if not vivenv.loaded or os.getenv("VIV_FORCE"):
vivenv.create()
vivenv.install_pkgs()
vivenv.dump_info(write=True)
vivenv.meta.write()
else:
echo("re-using existing vivenv")
@ -1021,19 +1000,8 @@ class Viv:
elif len(self.vivenvs) == 0:
echo("no vivenvs setup")
else:
rows = (
("vivenv", "spec"),
*(
(
f"{vivenv.name[:6]}..."
if len(vivenv.name) > 9
else vivenv.name,
", ".join(vivenv.spec),
)
for vivenv in self.vivenvs.values()
),
)
a.table(rows)
for _, vivenv in self.vivenvs.items():
vivenv.show(args.verbose)
def exe(self, args: Namespace) -> None:
"""run python/pip in existing vivenv"""
@ -1056,14 +1024,12 @@ class Viv:
def info(self, args: Namespace) -> None:
"""get metadata about a vivenv"""
vivenv = self._match_vivenv(args.vivenv)
metadata_file = vivenv.path / "viv-info.json"
metadata_file = vivenv.path / "vivmeta.json"
if not metadata_file.is_file():
error(f"Unable to find metadata for vivenv: {args.vivenv}", code=1)
echo(f"more info about {vivenv.name}:")
vivenv.dump_info()
vivenv.show(verbose=True)
def _install_local_src(self, sha256: str, src: Path, cli: Path) -> None:
echo("updating local source copy of viv")
@ -1071,10 +1037,9 @@ class Viv:
make_executable(src)
echo("symlinking cli")
if cli.is_file() and confirm(
f"Existing file at {a.style(str(cli),'bold')}, "
"would you like to overwrite it?"
):
if cli.is_file():
echo(f"Existing file at {a.style(str(cli),'bold')}")
if confirm("Would you like to overwrite it?"):
cli.unlink()
cli.symlink_to(src)
else:
@ -1172,7 +1137,7 @@ class Viv:
echo(
"to re-install use: "
"`python3 <(curl -fsSL gh.dayl.in/viv/viv.py) manage install`"
"`python3 <(curl -fsSL viv.dayl.in/viv.py) manage install`"
)
def _pick_bin(self, args: Namespace) -> Tuple[str, str]:
@ -1227,7 +1192,7 @@ class Viv:
# ephemeral (default), semi-ephemeral (persist inside /tmp), or
# persist (use c.cache)
if not vivenv.exists or os.getenv("VIV_FORCE"):
if not vivenv.loaded or os.getenv("VIV_FORCE"):
if not args.keep:
with tempfile.TemporaryDirectory(prefix="viv-") as tmpdir:
vivenv.path = Path(tmpdir)
@ -1241,7 +1206,7 @@ class Viv:
else:
vivenv.create()
vivenv.install_pkgs()
vivenv.dump_info(write=True)
vivenv.meta.write()
sys.exit(subprocess.run([vivenv.path / "bin" / bin, *args.rest]).returncode)
@ -1265,7 +1230,7 @@ class Viv:
return parser
def _validate_args(self, args):
def _validate_args(self, args: Namespace) -> None:
if args.func.__name__ in ("freeze", "shim", "run"):
if not args.reqs:
error("must specify a requirement", code=1)
@ -1326,6 +1291,13 @@ class Viv:
p_vivenv_arg.add_argument("vivenv", help="name/hash of vivenv")
p_list = self._get_subcmd_parser(subparsers, "list")
p_list.add_argument(
"-v",
"--verbose",
help="show full metadata for vivenvs",
default=False,
action="store_true",
)
p_list.add_argument(
"-q",
"--quiet",