Promote cram out

trunk
Reid 'arrdem' McKenzie 8 months ago
parent 438df41f91
commit 8e19f22640

@ -8,6 +8,7 @@ And so I'm going the other way; Bazel in a monorepo with subprojects so I'm able
- [Datalog](projects/datalog) and the matching [shell](projects/datalog-shell)
- [YAML Schema](projects/yamlschema) (JSON schema with knowledge of PyYAML's syntax structure & nice errors)
- [Zapp! (now with a new home and releases)](https://github.com/arrdem/rules_zapp)
- [Cram (now with a new home)](https://github.com/arrdem/cram)
- [Flowmetal](projects/flowmetal)
- [Lilith](projects/lilith)

@ -1,28 +0,0 @@
py_library(
name = "lib",
srcs = glob(["src/python/**/*.py"]),
deps = [
"//projects/vfs",
py_requirement("click"),
py_requirement("toposort"),
py_requirement("toml"),
]
)
zapp_binary(
name = "cram",
main = "src/python/cram/__main__.py",
shebang = "/usr/bin/env python3",
imports = [
"src/python"
],
deps = [
":lib",
],
)
sh_test(
name = "test_cram",
srcs = glob(["test.sh"]),
data = glob(["test/**/*"]) + [":cram"],
)

@ -1,5 +1,11 @@
# Cram
Cram has graduated [to its own repo](https://github.com/arrdem/cram)!
------
> To force (people or things) into a place or container that is or appears to be too small to contain them.
An alternative to GNU Stow, more some notion of packages with dependencies and install scripts.

@ -1,4 +0,0 @@
__version__ = "0.1.0"
__author__ = "Reid D. 'arrdem' McKenzie <me@arrdem.com>"
__copyright__ = "Copyright 2020"
__license__ = "https://anticapitalist.software/"

@ -1,335 +0,0 @@
"""Cram's entry point."""
from itertools import chain
import logging
import os
from pathlib import Path
import pickle
from typing import List
from . import (
__author__,
__copyright__,
__license__,
__version__,
)
from .v0 import PackageV0, ProfileV0
from .v1 import PackageV1, ProfileV1
import click
import toml
from toposort import toposort_flatten
from vfs import Vfs
log = logging.getLogger(__name__)
def _exit(val):
logging.shutdown()
exit(val)
def load(root: Path, name: str, clss):
for c in clss:
i = c(root, name)
if i.test():
return i
def load_package(root, name):
log.debug(f"Attempting to load package {name} from {root}")
return load(root, name, [PackageV1, PackageV0])
def load_profile(root, name):
log.debug(f"Attempting to load profile {name} from {root}")
return load(root, name, [ProfileV1, ProfileV0])
def load_packages(root: Path) -> dict:
"""Load the configured packages."""
packages = {}
log.debug(f"Trying to load packages from {root}...")
for p in (root / "packages.d").glob("*"):
name = str(p.relative_to(root))
packages[name] = load_package(p, name)
# Add profiles, hosts which contain subpackages.
for mp_root in chain((root / "profiles.d").glob("*"), (root / "hosts.d").glob("*")):
# First find all subpackages
for p in mp_root.glob("*"):
if p.is_dir():
name = str(p.relative_to(root))
packages[name] = load_package(p, name)
# Register the metapackages themselves using the profile type
mp_name = str(mp_root.relative_to(root))
packages[mp_name] = load_profile(mp_root, mp_name)
return packages
def build_fs(root: Path, dest: Path, prelude: List[str]) -> Vfs:
"""Build a VFS by configuring dest from the given config root."""
packages = load_packages(root)
requirements = []
requirements.extend(prelude)
if packages:
for p in packages:
log.debug(f"Loaded package {p}")
else:
log.warning("Loaded no packages!")
for r in requirements:
try:
for d in packages[r].requires():
if d not in requirements:
requirements.append(d)
except KeyError:
log.fatal(f"Error: Unable to load package {r}")
_exit(1)
# Compute the topsort graph
requirements = {r: packages[r].requires() for r in requirements}
fs = Vfs()
# Abstractly execute the current packages
for r in toposort_flatten(requirements):
r = packages[r]
r.install(fs, dest)
return fs
def load_state(statefile: Path) -> Vfs:
"""Load a persisted VFS state from disk. Sort of."""
oldfs = Vfs([])
if statefile.exists():
log.debug("Loading statefile %s", statefile)
with open(statefile, "rb") as fp:
oldfs._log = pickle.load(fp)
else:
log.warning("No previous statefile %s", statefile)
return oldfs
def simplify(old_fs: Vfs, new_fs: Vfs, /, exec_idempotent=True) -> Vfs:
"""Try to reduce a new VFS using diff from the original VFS."""
old_fs = old_fs.copy()
new_fs = new_fs.copy()
# Scrub anything in the new log that's in the old log
for txn in list(old_fs._log):
# Except for execs which are stateful
if txn[0] == "exec" and not exec_idempotent:
continue
try:
new_fs._log.remove(txn)
except ValueError:
pass
# Dedupe the new log while preserving order.
keys = set()
deduped = []
for op in new_fs._log:
key = str(op)
if key not in keys:
keys.add(key)
deduped.append(op)
new_fs._log = deduped
return new_fs
def scrub(old_fs: Vfs, new_fs: Vfs) -> Vfs:
"""Try to eliminate files which were previously installed but are no longer used."""
old_fs = old_fs.copy()
new_fs = new_fs.copy()
cleanup_fs = Vfs([])
# Look for files in the old log which are no longer present in the new log
for txn in old_fs._log:
if txn[0] == "link" and txn not in new_fs._log:
cleanup_fs.unlink(txn[2])
elif txn[0] == "mkdir" and txn not in new_fs._log:
cleanup_fs.unlink(txn[1])
# Do unlink operations before we do install operations.
# This works around being unable to finely straify uninstall operations over their source packages.
cleanup_fs.merge(new_fs)
return cleanup_fs
@click.group()
@click.version_option(version=1, message=f"""Cram {__version__}
Documentation
https://github.com/arrdem/source/tree/trunk/projects/cram/
Features
- 0.0.0 legacy config format
- 0.1.0 TOML config format
- 0.1.0 log based optimizer
- 0.1.0 idempotent default for scripts
About
{__copyright__}, {__author__}.
Published under the terms of the {__license__} license.
""")
def cli():
pass
@cli.command("apply")
@click.option("--execute/--dry-run", default=False)
@click.option("--force/--no-force", default=False)
@click.option("--state-file", default=".cram.log", type=Path)
@click.option("--optimize/--no-optimize", default=True)
@click.option("--require", type=str, multiple=True, default=[f"hosts.d/{os.uname()[1].split('.')[0]}", "profiles.d/default"])
@click.option("--exec-idempotent/--exec-always", "exec_idempotent", default=True)
@click.argument("confdir", type=Path)
@click.argument("destdir", type=Path)
def do_apply(confdir, destdir, state_file, execute, optimize, force, require, exec_idempotent):
"""The entry point of cram."""
# Resolve the two input paths to absolutes
root = confdir.resolve()
dest = destdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
if not state_file.is_absolute():
state_file = root / state_file
if not force:
old_fs = load_state(state_file)
log.debug(f"Loaded old state consisting of {len(old_fs._log)} steps")
else:
# Force an empty state
old_fs = Vfs([])
new_fs = build_fs(root, dest, require)
log.debug(f"Built new state consisting of {len(new_fs._log)} steps")
# Middleware processing of the resulting filesystem(s)
executable_fs = scrub(old_fs, new_fs)
if optimize:
executable_fs = simplify(old_fs, executable_fs,
exec_idempotent=exec_idempotent)
# Dump the new state.
# Note that we dump the UNOPTIMIZED state, because we want to simplify relative complete states.
def cb(e):
print("-", *e)
if execute:
executable_fs.execute(callback=cb)
with open(state_file, "wb") as fp:
pickle.dump(new_fs._log, fp)
else:
for e in executable_fs._log:
cb(e)
@cli.command("list")
@click.argument("confdir", type=Path)
@click.argument("list_packages", nargs=-1)
def do_list(confdir, list_packages):
"""List out packages, profiles, hosts and subpackages in the <confdir>."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
packages = load_packages(root)
if list_packages:
dest = Path("~/")
for pname in list_packages:
fs = Vfs()
p = packages[pname]
p.install(fs, dest)
print(f"{pname}: ({type(p).__name__})")
print("requires:")
for e in p.requires():
print(" -", e)
print("log:")
for e in fs._log:
print(" -", *e)
else:
for pname in sorted(packages.keys()):
p = packages[pname]
print(f"{pname}: ({type(p).__name__})")
for d in p.requires():
print(f"- {d}")
@cli.command("state")
@click.option("--state-file", default=".cram.log", type=Path)
@click.argument("confdir", type=Path)
def do_state(confdir, state_file):
"""List out the last `apply` state in the <confdir>/.cram.log or --state-file."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
if not state_file.is_absolute():
state_file = root / state_file
fs = load_state(state_file)
for e in fs._log:
print(*e)
@cli.command("fmt")
@click.argument("confdir", type=Path)
@click.argument("requirement", type=str)
def do_migrate(confdir, requirement):
"""Format the specified requirement to a canonical-ish representation."""
root = confdir.resolve()
if not root.is_dir():
log.fatal(f"{confdir} does not exist!")
_exit(1)
packages = load_packages(root)
pkg = packages[requirement]
json = pkg.json()
for suffix in pkg.SPECIAL_FILES:
f = (root / requirement / suffix)
if f.exists():
f.unlink()
with open(root / requirement / "pkg.toml", "w") as fp:
toml.dump(json, fp)
if __name__ == "__main__" or 1:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
cli()

@ -1,86 +0,0 @@
#!/usr/bin/env python3
import os
from pathlib import Path
from shlex import quote as sh_quote
import sys
from typing import List, Optional
from vfs import Vfs
# FIXME: This should be a config somewhere
SHELL = "/bin/sh"
# Light monkeypatching because macos ships a "stable" a py
if sys.version_info <= (3, 9, 0):
Path.readlink = lambda p: Path(os.readlink(str(p)))
def sh(cmd: List[str], /,
env: Optional[dict] = None):
prefix = []
if env:
prefix.append("/usr/bin/env")
for k, v in env.items():
v = sh_quote(str(v))
prefix.append(f"{k}={v}")
return tuple(prefix + [SHELL, *cmd])
def stow(fs: Vfs, src_dir: Path, dest_dir: Path, skip=[]):
"""Recursively 'stow' (link) the contents of the source into the destination."""
dest_root = Path(dest_dir)
src_root = Path(src_dir)
skip = [src_root / n for n in skip]
for src in src_root.glob("**/*"):
if src in skip:
continue
elif src.name.endswith(".gitkeep"):
continue
dest = dest_root / src.relative_to(src_root)
if src.is_symlink():
fs.link(src.readlink().resolve(), dest)
elif src.is_dir():
fs.mkdir(dest)
fs.chmod(dest, src.stat().st_mode)
elif src.is_file():
fs.link(src, dest)
class Package(object):
def __init__(self, root: Path, name: str):
self.root = root
self.name = name
def test(self):
return True
def requires(self):
return []
def install(self, fs: Vfs, dest: Path):
self.do_build(fs, dest)
self.pre_install(fs, dest)
self.do_install(fs, dest)
self.post_install(fs, dest)
def do_build(self, fs: Vfs, dest: Path):
pass
def pre_install(self, fs: Vfs, dest: Path):
pass
def do_install(self, fs: Vfs, dest: Path):
pass
def post_install(self, fs: Vfs, dest: Path):
pass

@ -1,109 +0,0 @@
"""Cram's original (v0) configs.
An ill-considered pseudo-format.
"""
from pathlib import Path
import re
from .common import Package, sh, stow
from vfs import Vfs
class PackageV0(Package):
"""The original package format from install.sh."""
SPECIAL_FILES = ["BUILD", "PRE_INSTALL", "INSTALL", "POST_INSTALL", "REQUIRES"]
def requires(self):
"""Get the dependencies of this package."""
requiresf = self.root / "REQUIRES"
requires = []
# Listed dependencies
if requiresf.exists():
with open(requiresf) as fp:
for l in fp:
l = l.strip()
l = re.sub(r"\s*#.*\n", "", l)
if l:
requires.append(l)
return requires
def install(self, fs: Vfs, dest: Path):
"""Install this package."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
else:
stow(fs, self.root, dest, self.SPECIAL_FILES)
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))
def _read(self, p: Path):
if p.exists():
with open(p) as fp:
return fp.read()
else:
return None
def json(self):
buildt = self._read(self.root / "BUILD")
pret = self._read(self.root / "PRE_INSTALL")
installt = self._read(self.root / "INSTALL")
postt = self._read(self.root / "POST_INSTALL")
o = {"cram": {"version": 1}, "package": {"require": []}}
if buildt:
o["package"]["build"] = [{"run": buildt}]
if pret:
o["package"]["pre_install"] = [{"run": pret}]
if installt:
o["package"]["install"] = [{"run": installt}]
if postt:
o["package"]["install"] = [{"run": postt}]
o["package"]["require"] = [{"name": it} for it in sorted(self.requires())]
return o
class ProfileV0(PackageV0):
def requires(self):
requires = super().requires()
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires
def install(self, fs: Vfs, dest: Path):
"""Profiles differ from Packages in that they don't support literal files."""
buildf = self.root / "BUILD"
if buildf.exists():
fs.exec(self.root, sh([str(buildf)]))
pref = self.root / "PRE_INSTALL"
if pref.exists():
fs.exec(self.root, sh([str(pref)]))
installf = self.root / "INSTALL"
if installf.exists():
fs.exec(self.root, sh([str(installf)]))
postf = self.root / "POST_INSTALL"
if postf.exists():
fs.exec(self.root, sh([str(postf)]))

@ -1,116 +0,0 @@
"""Cram's v1 configs.
Based on well* defined TOML manifests, rather than many files.
*Okay. Better.
"""
from hashlib import sha256
from pathlib import Path
from typing import List, Optional, Union
from .common import Package, sh, stow
import toml
from vfs import Vfs
def tempf(name):
root = Path("/tmp/stow")
root.mkdir(exist_ok=True, parents=True)
return root / name
class PackageV1(Package):
"""The v1 package format."""
SPECIAL_FILES = ["pkg.toml"]
_config = None
def config(self):
if not self._config:
with open(self.root / self.SPECIAL_FILES[0], "r") as fp:
self._config = toml.load(fp)
return self._config
def test(self):
return (self.root / self.SPECIAL_FILES[0]).exists() and self.config().get("cram", {}).get("version") == 1
def requires(self):
"""Get the dependencies of this package."""
def _name(it):
if isinstance(it, str):
return it
elif isinstance(it, dict):
return it["name"]
return [
_name(it) for it in self.config().get("package", {}).get("require", [])
]
def do_sh_or_script(self, content: Optional[Union[List[str], str]], fs: Vfs, dest: Path, cwd: Path = "/tmp"):
if content is None:
pass
elif isinstance(content, list):
for c in content:
self.do_sh_or_script(c, fs, dest)
elif isinstance(content, dict):
self.do_sh_or_script(
content["run"],
fs,
dest,
{"cwd": self.root}.get(content.get("root"), "/tmp")
)
elif isinstance(content, str):
sum = sha256()
sum.update(content.encode("utf-8"))
sum = sum.hexdigest()
installf = self.root / content
if installf.exists():
with open(installf, "r") as fp:
self.do_sh_or_script(fp.read(), fs, dest)
elif content:
f = tempf(f"{sum}.sh")
with open(f, "w") as fp:
fp.write(content)
fs.exec(cwd, sh([f]))
def do_build(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("build"), fs, dest)
def pre_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("pre_install"), fs, dest)
def do_install(self, fs: Vfs, dest: Path):
if not self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest):
stow(fs, self.root, dest, self.SPECIAL_FILES)
def post_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("post_install"), fs, dest)
def json(self):
return self.config()
class ProfileV1(PackageV1):
"""Unline packages, profiles don't support recursive stow of contents."""
def do_install(self, fs: Vfs, dest: Path):
self.do_sh_or_script(self.config().get("package", {}).get("install"), fs, dest)
def requires(self):
requires = super().requires()
# Implicitly depended subpackages
for p in self.root.glob("*"):
if p.is_dir():
requires.append(self.name + "/" + p.name)
return requires

@ -1,96 +0,0 @@
#!/usr/bin/env bash
set -ex
cd projects/cram
dest=$(mktemp -d)
./cram --help
# Should be able to list all packages
./cram list test/ | grep "packages.d/p1"
# P3 depends on P1, should show up in the listing
./cram list test/ packages.d/p3 | grep "packages.d/p1"
# P4 depends on P3, should show up in the listing
./cram list test/ packages.d/p4 | grep "packages.d/p3"
# The default profile should depend on its subpackage
./cram list test/ profiles.d/default | grep "profiles.d/default/subpackage"
# And the subpackage has a dep
./cram list test/ profiles.d/default/subpackage | grep "packages.d/p3"
# Install one package
./cram apply --no-optimize --require packages.d/p1 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
./cram state test/ | grep "${dest}/foo"
rm -r "${dest}"/*
# Install two transitively (legacy)
./cram apply --no-optimize --require packages.d/p3 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
./cram state test/ | grep "${dest}/foo"
./cram state test/ | grep "${dest}/bar"
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# Install two transitively (current)
./cram apply --no-optimize --require hosts.d/test --require profiles.d/default --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
rm -r "${dest}"/*
# INSTALL scripts get run as-is
./cram list test/ packages.d/p5 | grep "packages.d/p5/INSTALL"
# Inline scripts get pulled out repeatably
./cram list test/ packages.d/p6 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Inline scripts get pulled out repeatably, even from the list format
./cram list test/ packages.d/p7 | grep "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"
# Test log-based optimization
./cram apply --no-optimize --require packages.d/p4 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
# These paths were already linked, they shouldn't be re-linked when optimizing.
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/foo"
! ./cram apply --require packages.d/p4 --optimize --execute test/ "${dest}" | grep "${dest}/bar"
rm -r "${dest}"/*
# Likewise, if we've exec'd this once we shouldn't do it again
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
! ./cram apply --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# ... unless the user tells us to
./cram apply --no-optimize --require packages.d/p5 --execute test/ "${dest}"
./cram apply --exec-always --require packages.d/p5 --execute test/ "${dest}" | grep "exec"
# If multiple packages provide the same _effective_ script, do it once
./cram apply --require packages.d/p6 --require packages.d/p7 --execute test/ "${dest}" | sort | uniq -c | grep "/tmp/stow/b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b.sh" | grep "1 - exec"
# Test log-based cleanup
./cram apply --require packages.d/p1 --require packages.d/p2 --execute test/ "${dest}"
[ -L "${dest}"/foo ]
[ -L "${dest}"/bar ]
# And how bar shouldn't be installed...
./cram state test/
./cram apply --require packages.d/p1 --execute test/ "${dest}"
./cram state test/
[ -L "${dest}"/foo ]
[ ! -L "${dest}"/bar ]

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p1"

@ -1,2 +0,0 @@
packages.d/p1
packages.d/p2

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"

@ -1,3 +0,0 @@
#!/bin/bash
# A legacy custom install script
true

@ -1,5 +0,0 @@
[cram]
version = 1
[package]
install = "true"

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.install]]
run = "true"

@ -1,4 +0,0 @@
[cram]
version = 1
[package]

@ -1,6 +0,0 @@
[cram]
version = 1
[package]
[[package.require]]
name = "packages.d/p3"
Loading…
Cancel
Save