Repository navigation
Expand file tree
/
Copy pathinstall.py
More file actions
409 lines (361 loc) · 18.4 KB
/
Copy pathinstall.py
File metadata and controls
409 lines (361 loc) · 18.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
"""Run the Ansible play, streaming and logging.
Carried over from run.py rather than rewritten, because the behaviour there was
arrived at by measurement and re-deriving it would be a waste of a live host:
streaming the play is slow and a silent terminal cannot be told apart
from a hung one
heartbeat reports how long the current task has been silent, which is
the number that distinguishes slow from wedged
commands.txt one line per command with its task; log.txt is 150 KB of
which one line alone is 52 KB of gathered facts
redaction a password passed as an extra var comes back in the log at
-vvv, twice: once from our own argv, once from Ansible's
extra_vars echo
The one new thing is where secrets come from. They are read from .env and
written to a temporary file passed as `--extra-vars @file`, so no secret
appears in argv, in `ps`, or in the run logs.
"""
from __future__ import annotations
import json
import os
import re
import subprocess
import threading
import time
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from .config import Config
from .errors import InstallError
TASK_RE = re.compile(r"^TASK \[(.+?)\]\s*\*+\s*$")
RESULT_RE = re.compile(r"^(ok|changed|skipping|failed|fatal|rescued|ignored|unreachable):")
FATAL_RE = re.compile(r"^(fatal|failed):", re.I)
# Ansible writes "[ERROR]:" with brackets. Missing them means a play that dies
# before its first task reports a clean run.
ERROR_RE = re.compile(r"^\[?ERROR\]?:", re.I)
COMMAND_RE = re.compile(r'^\s*"cmd":\s*(\[.*\]|".*")')
EXEC_RE = re.compile(r"^<[^>]+>\s+EXEC ")
REMOTE_EXEC_RE = re.compile(r"SSH:\s*EXEC\s+ssh\b.*?'(.*)'\s*$")
LIST_OPENER_RE = re.compile(r'^\s*"(cmd|argv|extra_args)":\s*\[\s*$')
NULL_ARG_RE = re.compile(r'^\s*"(?:path|src|dest|name)":\s*null,?$')
EXTRA_VARS_RE = re.compile(r"^(\s*)extra_vars:\s*(.*)$")
# The role's own report task. An application ends its role with a debug task
# named "Report the checks that passed", whose `msg` list holds one line per
# check. This is what makes the summary say "the endpoint answered HTTP 200"
# instead of "41 tasks, 0 failed" — the first is evidence, the second is only
# a count.
CHECKS_TASK_RE = re.compile(r"^TASK \[.*?Report the checks that passed\]")
CHECK_LINE_RE = re.compile(r'^\s*"(.*)",?\s*$')
# The list opens with a `"msg": [` line and closes with a `]` line. The
# closing line has no comma, so it cannot be confused with an entry.
CHECK_LIST_OPEN_RE = re.compile(r'^\s*"msg":\s*\[\s*$')
CHECK_LIST_CLOSE_RE = re.compile(r"^\s*\]\s*$")
ARG_KEYS = ("cmd", "argv", "_raw_params", "src", "dest", "path", "name",
"requirements", "extra_args", "virtualenv", "content", "url", "state", "mode")
ARG_RE = re.compile(r'^\s*"(' + "|".join(ARG_KEYS) + r')":\s*(.+?),?$')
ARG_MAX_LEN = 120
SECRETISH_RE = re.compile(r"(pass|secret|token|api_?key|credential)", re.I)
SECRET_NAME_RE = re.compile(r"^(--?e\s+|-e)?\s*(?P<name>[A-Za-z_][A-Za-z0-9_]*)=(?P<value>.*)$")
def redact_argv(cmd: list[str]) -> list[str]:
"""Blank the value of any password-shaped -e name=value."""
out = []
for arg in cmd:
m = SECRET_NAME_RE.match(arg.strip())
if m and SECRETISH_RE.search(m.group("name")):
prefix = m.group(1) or ""
out.append(f"{prefix}{m.group('name')}=***REDACTED***".strip())
else:
out.append(arg)
return out
def source_vars(app: Any) -> dict[str, str]:
"""The manifest's own source pin, as variables the role can read.
The manifest names where an application comes from: a package and a
version, a repository and a ref. A role usually pins the same thing
independently, in its own defaults, because a role has to be able to say
what it installs without reading the manifest.
That gives two places that can disagree. This passes the manifest's pin
into the play so a role can refuse to install something the manifest did
not name, instead of quietly installing a different version than the one
that was reviewed.
Only a package source has a version to compare. A repository source has a
ref, and a toolchain source has neither — a role that builds from a
checkout compares the commit, and one that assembles an environment has
nothing to compare.
"""
if app.source.kind != "package" or not app.source.package:
return {}
pin = str(app.source.package)
# "someapp[browser]==1.2.3" -> name "someapp", version "1.2.3".
name, _, version = pin.partition("==")
if not version:
# A package with no version pin cannot be compared, and pretending
# otherwise would make the assertion compare a name against itself.
return {}
# The extras are part of the requirement, not of the name.
name = name.split("[")[0].strip()
return {"appctl_source_package": name, "appctl_source_version": version.strip()}
@dataclass
class Result:
label: str
returncode: int = 0
seconds: float = 0.0
tasks_seen: int = 0
commands_seen: int = 0
tasks_changed: int = 0
tasks_failed: int = 0
directory: Path = field(default_factory=lambda: Path("."))
problems: list[str] = field(default_factory=list)
#: The evidence the application's own checks produced. Populated from the
#: role's own report task, so the summary the operator reads is the
#: application's statement about itself rather than a count of tasks.
checks: list[str] = field(default_factory=list)
def summary(self) -> str:
return (
f"{self.label}: {'OK' if self.returncode == 0 else 'FAILED'} in {self.seconds:.0f}s "
f"({self.tasks_seen} tasks, {self.tasks_changed} changed, "
f"{self.tasks_failed} failed, {self.commands_seen} commands)"
)
def _echo(msg: str = "") -> None:
print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}" if msg else "", flush=True)
class Installer:
"""Runs ansible-playbook and turns its output into both a terminal view and
a set of files worth keeping afterwards."""
def __init__(self, config: Config, *, verbosity: int = 3, forks: int = 10,
heartbeat: int = 30) -> None:
self.config = config
self.verbosity = verbosity
self.forks = forks
self.heartbeat = heartbeat
self._shared = {"last_output": [time.monotonic()], "last_task": ["(starting)"]}
self._list_open: str | None = None
self._list_items: list[str] = []
self._in_checks = False
self._collecting = False
def command(self, extra: list[str], secret_vars: dict[str, str],
rendered: dict[str, str] | None = None,
source_vars: dict[str, str] | None = None) -> list[str]:
cmd = [
"ansible-playbook",
"-i", str(self.config.root / "inventory" / "hosts.yml"),
"playbooks/site.yml",
"--forks", str(self.forks),
]
if self.verbosity:
cmd.append("-" + "v" * self.verbosity)
# Secrets go in a file, never argv: an extra var on the command line is
# visible in ps on the controller for the whole run. The rendered config
# paths travel in the same file: they are controller paths, harmless, and
# it keeps the play's whole variable set in one place.
#
# The manifest's own source pin travels with them, for the same reason
# and with the same care: it is what the operator reviewed, and a role
# that installs a different version than the manifest names is a role
# that disagreed with the manifest. See _source_vars.
handle = self._vars_file(secret_vars, rendered, source_vars)
if handle is not None:
cmd += ["--extra-vars", f"@{handle}"]
cmd += extra
return cmd
def _vars_file(self, values: dict[str, str],
rendered: dict[str, str] | None = None,
source_vars: dict[str, str] | None = None) -> Path | None:
"""Write the extra-vars file, mode 0600, removed when the run ends.
It carries the secrets and, when any application declares one, the paths
of the config files rendered on the controller. Nothing else.
The file holds a FLAT mapping. It must not be wrapped in an `ansible`
key: that namespace is a config-file convention and does not apply to
`--extra-vars @file`. A wrapped file is accepted without complaint and
silently defines nothing, so every secret arrives empty and the role
fails with "all three secrets are required" while the controller holds
perfectly good values. Measured both forms against ansible-playbook:
wrapped gives an undefined variable, flat gives the value.
"""
rendered = rendered or {}
source_vars = source_vars or {}
if not values and not rendered and not source_vars:
return None
import tempfile
fd, name = tempfile.mkstemp(prefix="appctl-vars-", suffix=".json")
with os.fdopen(fd, "w") as handle:
payload = dict(values)
if rendered:
payload["appctl_rendered_configs"] = rendered
if source_vars:
payload.update(source_vars)
json.dump(payload, handle)
os.chmod(name, 0o600)
self._cleanup_files = getattr(self, "_cleanup_files", [])
self._cleanup_files.append(Path(name))
return Path(name)
def cleanup(self) -> None:
for path in getattr(self, "_cleanup_files", []):
path.unlink(missing_ok=True)
self._cleanup_files = []
def run(self, extra: list[str], *, label: str = "play",
outdir: Path | None = None, secret_vars: dict[str, str] | None = None,
rendered: dict[str, str] | None = None,
source_vars: dict[str, str] | None = None) -> Result:
stamp = outdir or (self.config.root / "tmp" / "runs" /
datetime.now().strftime("%Y%m%d-%H%M%S"))
directory = stamp / label
directory.mkdir(parents=True, exist_ok=True)
result = Result(label=label, directory=directory)
cmd = self.command(extra, secret_vars or {}, rendered, source_vars)
_echo("run: " + " ".join(redact_argv(cmd)))
_echo(f"--> {directory / 'commands.txt'}")
started = time.monotonic()
self._shared["last_output"][0] = started
self._shared["last_task"][0] = "(starting)"
log_path = directory / "log.txt"
cmd_path = directory / "commands.txt"
# Both opened together so neither leaks a descriptor if the run dies.
with log_path.open("w", buffering=1) as log, cmd_path.open("w", buffering=1) as cmds:
cmds.write("# commands for this run, one per line\n\n")
log.write(f"# started {datetime.now().isoformat()}\n")
log.write(f"# {' '.join(redact_argv(cmd))}\n")
try:
proc = subprocess.Popen(
cmd, cwd=self.config.root, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, bufsize=1,
)
except FileNotFoundError as exc:
raise InstallError(
"ansible-playbook is not on PATH",
hint="install it with: uv tool install ansible-core "
"or: pipx install ansible-core",
) from exc
reader = threading.Thread(target=self._pump, args=(proc, log, cmds, result), daemon=True)
reader.start()
while reader.is_alive():
reader.join(timeout=self.heartbeat if self.heartbeat else 1.0)
if reader.is_alive() and self.heartbeat:
quiet = time.monotonic() - self._shared["last_output"][0]
_echo(f" still on: {self._shared['last_task'][0]} "
f"({quiet:.0f}s without output, {time.monotonic() - started:.0f}s elapsed)")
reader.join()
proc.wait()
result.returncode = proc.returncode
result.seconds = time.monotonic() - started
self._write_reports(result, cmd)
return result
def _pump(self, proc, log, cmds, result: Result) -> None:
assert proc.stdout is not None
shared = self._shared
for raw in proc.stdout:
line = raw.rstrip("\n")
# Ansible echoes extra vars back at -vvv and above. A secret passed
# with -e returns here in clear text, so the line is rewritten before
# it reaches the file.
ev = EXTRA_VARS_RE.match(line)
if ev:
# Split on the quotes that delimit each tuple element, not on
# commas: a value can itself contain one.
parts = re.findall(r"'([^']*)'|\"([^\"]*)\"", ev.group(2))
values = [a or b for a, b in parts] or [ev.group(2).strip()]
red = ", ".join(redact_argv([v])[0] for v in values)
line = f"{ev.group(1)}extra_vars: ({red},)"
log.write(line + "\n")
shared["last_output"][0] = time.monotonic()
m = TASK_RE.match(line)
if m:
shared["last_task"][0] = m.group(1)
result.tasks_seen += 1
_echo(f"TASK {m.group(1)}")
# The application's own check report. It is collected when the
# list opens, not here: the task line is followed by the task
# path and the result header before the list itself, so
# collecting from this line would collect the metadata.
self._in_checks = m.group(1).endswith("Report the checks that passed")
continue
if self._in_checks:
if CHECK_LIST_OPEN_RE.match(line):
self._collecting = True
continue
if self._collecting:
if CHECK_LIST_CLOSE_RE.match(line):
self._collecting = False
self._in_checks = False
continue
cm = CHECK_LINE_RE.match(line)
if cm:
result.checks.append(cm.group(1))
continue
# Not the list yet. The task line is followed by the task path
# and the result header before the list opens, so the state
# survives them. A new TASK line ends the block, and that is
# handled where the task is matched.
continue
r = RESULT_RE.match(line)
if r:
kind = r.group(1)
if kind == "changed":
result.tasks_changed += 1
elif kind in ("failed", "fatal", "unreachable"):
result.tasks_failed += 1
result.problems.append(f"{shared['last_task'][0]}: {kind}")
_echo(f" !! {kind} :: {shared['last_task'][0]}")
continue
if REMOTE_EXEC_RE.match(line) and self.verbosity >= 4:
remote = REMOTE_EXEC_RE.match(line).group(1)
if remote:
cmds.write(f"{shared['last_task'][0]}\n {remote.strip()}\n\n")
result.commands_seen += 1
continue
if self.verbosity >= 3:
if self._list_open is None and LIST_OPENER_RE.match(line):
am = ARG_RE.match(line.strip().rstrip(":"))
self._list_open = am.group(1) if am else "cmd"
continue
if self._list_open is not None:
if re.match(r"^\s*\],?\s*$", line):
joined = " ".join(self._list_items)
_echo(f" {self._list_open}: {self._list_items}")
cmds.write(f"{shared['last_task'][0]}\n {joined}\n\n")
result.commands_seen += 1
self._list_open, self._list_items = None, []
continue
item = line.strip().rstrip(",").strip('"')
if item:
self._list_items.append(item)
if NULL_ARG_RE.match(line):
continue
am = ARG_RE.match(line)
if am and len(am.group(2)) <= ARG_MAX_LEN:
key, value = am.group(1), am.group(2)
result.commands_seen += 1
shown = "***REDACTED***" if SECRETISH_RE.search(key) or SECRETISH_RE.search(value[:80]) else value
_echo(f" {key}: {shown}")
cmds.write(f"{shared['last_task'][0]}\n {key}: {shown}\n\n")
continue
if self.verbosity >= 4 and EXEC_RE.match(line):
_echo(f" {line.strip()}")
if FATAL_RE.match(line) or ERROR_RE.match(line):
result.tasks_failed += 1
result.problems.append(f"{shared['last_task'][0]}: {line}")
_echo(f" !! {line}")
def _write_reports(self, result: Result, cmd: list[str]) -> None:
text = (result.directory / "log.txt").read_text(errors="replace")
recap = [l for l in text.splitlines()
if l.startswith("PLAY RECAP") or re.match(r"^\S+\s+: ok=", l)]
(result.directory / "recap.txt").write_text("\n".join(recap) + "\n")
problems, task = [], "(unknown)"
for line in text.splitlines():
m = TASK_RE.match(line)
if m:
task = m.group(1)
elif FATAL_RE.match(line) or ERROR_RE.match(line):
problems.append(f"{task}\n {line.strip()}")
if result.returncode != 0 and not problems:
problems.append(f"the play exited {result.returncode} but no error line was "
f"recognised. Read {result.directory / 'log.txt'} for the raw output.")
(result.directory / "errors.txt").write_text("\n".join(problems) + "\n" if problems else "")
(result.directory / "meta.txt").write_text(json.dumps({
"label": result.label,
"exit_code": result.returncode,
"seconds": round(result.seconds, 1),
"tasks_seen": result.tasks_seen,
"commands_seen": result.commands_seen,
"tasks_changed": result.tasks_changed,
"tasks_failed": result.tasks_failed,
"finished_at": datetime.now().isoformat(),
"command": redact_argv(cmd),
}, indent=2) + "\n")