ProvSQL C/C++ API
Adding support for provenance and uncertainty management to PostgreSQL databases
Loading...
Searching...
No Matches
external_tool.cpp
Go to the documentation of this file.
1/**
2 * @file external_tool.cpp
3 * @brief Implementation of the external-tool helpers.
4 *
5 * Reads the @c provsql.tool_search_path GUC (exposed as
6 * @c provsql_tool_search_path) and uses it both to extend @c $PATH around
7 * @c system() and to drive the pre-flight @c find_external_tool() lookup.
8 */
9extern "C" {
10#include "postgres.h"
11#include "provsql_utils.h"
12#include "miscadmin.h"
13
14#include <signal.h>
15#include <stdlib.h>
16#include <stdio.h>
17#include <errno.h>
18#include <unistd.h>
19#include <sys/wait.h>
20}
21
22#include "external_tool.h"
23#include "Circuit.h"
24#include "ToolRegistry.h"
25#include "process_memory.h"
26#include "provsql_interrupt.h"
27#include "provsql_config.h"
28
29#include <chrono>
30#include <string>
31#include <unordered_map>
32
33// PATH that /bin/sh resolves binaries against when the environment has no
34// PATH set. PostgreSQL backends inherit no PATH from systemd, so
35// getenv("PATH") is NULL inside the server; without an explicit fallback,
36// setting PATH to "<GUC>" alone would mask /usr/local/bin and friends
37// (dash's compiled-in default), making the GUC-set case strictly narrower
38// than the GUC-empty case. Matches dash's _PATH_STDPATH on Debian/Ubuntu
39// and bash's default on macOS.
40static const char *DEFAULT_PATH =
41 "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
42
43/*
44 * Run @p cmdline via @c /bin/sh @c -c in its OWN process group, polling for
45 * a query cancel / backend termination while it runs. Returns a wait(2)
46 * status (so @c WIFEXITED / @c WIFSIGNALED / @c WEXITSTATUS decode exactly
47 * as the old @c system() return did), or -1 if @c fork failed.
48 *
49 * Why not @c system(): it runs the child in the BACKEND's process group,
50 * and statement_timeout / pg_cancel_backend deliver SIGINT to that group
51 * (PostgreSQL backends @c setsid, and @c StatementTimeoutHandler does
52 * @c kill(-MyProcPid, SIGINT)). A well-behaved tool dies on it, but a tool
53 * that catches/ignores SIGINT -- or that forks a worker into another
54 * process group, as KCBox/Panini does -- survives, so the timeout is
55 * silently not honoured (a single long OBDD compile runs unbounded). Here
56 * the child leads its own process group, and when a cancel/terminate is
57 * pending we @c SIGKILL that whole group: uncatchable, and it reaches any
58 * forked workers. The pending interrupt is then raised by the
59 * @c CHECK_FOR_INTERRUPTS() in @c run_external_tool, with the child reaped.
60 */
61/** Set by @c run_in_own_pgroup when it killed a tool that took the
62 * statement past @c provsql.max_memory. */
63static bool tool_memory_exceeded = false;
64
65/** Time limit, in ms, of the innermost @c ExternalToolTimeLimit scope (0:
66 * none). */
67static double tool_time_limit_ms = 0.;
68
69/** Set by @c run_in_own_pgroup when it killed a tool that ran past
70 * @c tool_time_limit_ms. */
71static bool tool_time_exceeded = false;
72
78
83
84static int run_in_own_pgroup(const std::string &cmdline)
85{
86#ifdef PROVSQL_NO_SUBPROCESS
87 /* No subprocesses in the WASM sandbox. Mirror fork()'s failure return
88 so find_external_tool() reports "not found" and callers fall back to
89 the in-process compiler. */
90 (void) cmdline;
91 return -1;
92#else
93 fflush(NULL); /* flush stdio before fork, like system() */
94
95 pid_t child = fork();
96 if (child < 0)
97 return -1; /* mirror system()'s failure return */
98
99 if (child == 0) {
100 /* Child: lead a new process group so a later killpg targets this whole
101 * subtree (including tools that fork their own workers) and never the
102 * backend. exec resets caught signals to default, so SIGINT / SIGTERM
103 * / SIGKILL terminate the tool normally. */
104 setpgid(0, 0);
105 execl("/bin/sh", "sh", "-c", cmdline.c_str(), (char *) NULL);
106 _exit(127); /* exec failed: shell-style "not found" */
107 }
108
109 /* Parent: close the setpgid race (whichever process runs first wins). */
110 setpgid(child, child);
111
112 int status = 0;
113 unsigned polls = 0;
114 const auto started = std::chrono::steady_clock::now();
115 for (;;) {
116 pid_t w = waitpid(child, &status, WNOHANG);
117 if (w == child)
118 break; /* tool finished; status is set */
119 if (w < 0 && errno != EINTR) {
120 status = -1; /* unexpected; surface as failure */
121 break;
122 }
123 /* A pending query cancel (statement_timeout, pg_cancel_backend) or
124 * backend termination (pg_terminate_backend / SIGTERM) must stop the
125 * tool now. Kill its whole process group hard and reap it. We do NOT
126 * call CHECK_FOR_INTERRUPTS() while the child is alive: a throw there
127 * would leak the running child as an orphan -- the very bug being
128 * fixed. The caller's CHECK_FOR_INTERRUPTS() (below) raises the
129 * pending interrupt once the child is reaped. */
130 if (QueryCancelPending || ProcDiePending) {
131 killpg(child, SIGKILL);
132 pid_t r;
133 do { r = waitpid(child, &status, 0); } while (r < 0 && errno == EINTR);
134 break;
135 }
136 /* provsql.max_memory: every 100 ms, the resident memory of the tool and
137 * of the processes it forked (its process group), with what the
138 * statement's evaluations have added to the backend; past the budget,
139 * the group is killed as on a cancel. */
140 if (provsql_max_memory > 0 && ++polls % 10 == 0 &&
142 (std::size_t)provsql_max_memory * 1024 * 1024) {
143 killpg(child, SIGKILL);
144 pid_t r;
145 do { r = waitpid(child, &status, 0); } while (r < 0 && errno == EINTR);
147 break;
148 }
149 /* The time limit of an ExternalToolTimeLimit scope: past it, the group
150 * is killed as on a cancel, and run_external_tool reports it. */
151 if (tool_time_limit_ms > 0. &&
152 std::chrono::duration<double, std::milli>(
153 std::chrono::steady_clock::now() - started).count()
155 killpg(child, SIGKILL);
156 pid_t r;
157 do { r = waitpid(child, &status, 0); } while (r < 0 && errno == EINTR);
158 tool_time_exceeded = true;
159 break;
160 }
161 pg_usleep(10000); /* 10 ms; an arriving signal wakes us via EINTR */
162 }
163 return status;
164#endif /* PROVSQL_NO_SUBPROCESS */
165}
166
167int run_external_tool(const std::string &cmdline) {
168 bool override_path = (provsql_tool_search_path != NULL
169 && provsql_tool_search_path[0] != '\0');
170 std::string saved_path;
171 bool had_path = false;
172
173 if (override_path) {
174 const char *cur = getenv("PATH");
175 if (cur != NULL) {
176 saved_path = cur;
177 had_path = true;
178 }
179 std::string new_path(provsql_tool_search_path);
180 new_path += ':';
181 new_path += had_path ? saved_path : DEFAULT_PATH;
182 setenv("PATH", new_path.c_str(), 1);
183 }
184
185 tool_memory_exceeded = false;
186 tool_time_exceeded = false;
187 int rv = run_in_own_pgroup(cmdline);
188
189 if (override_path) {
190 if (had_path)
191 setenv("PATH", saved_path.c_str(), 1);
192 else
193 unsetenv("PATH");
194 }
195
196 /* If a cancel/terminate fired while the tool ran, run_in_own_pgroup has
197 * already killed and reaped it; raise the interrupt now (cleanly, with no
198 * child left running) so it surfaces as query-cancelled rather than being
199 * masked by a downstream "tool killed by signal" error. A no-op when no
200 * interrupt is pending. */
201 CHECK_FOR_INTERRUPTS();
202
204 tool_memory_exceeded = false;
206 "memory-limit",
207 ("an external tool took the evaluation past provsql.max_memory (" +
208 std::to_string(provsql_max_memory) + " MB) and was stopped").c_str(),
209 "Raise provsql.max_memory (e.g., SET provsql.max_memory = '2GB'), or "
210 "set it to 0 for no limit.");
211 }
212
213 if (tool_time_exceeded) {
214 tool_time_exceeded = false;
216 "an external tool was stopped after " +
217 std::to_string(static_cast<long>(tool_time_limit_ms)) +
218 " ms, the time the cheapest alternative was expected to take");
219 }
220
221 return rv;
222}
223
224std::string find_external_tool(const std::string &name) {
225 // Path-like names (containing '/') are tested directly without any
226 // search-path walk; this matches POSIX execvp semantics.
227 if (name.find('/') != std::string::npos)
228 return access(name.c_str(), X_OK) == 0 ? name : "";
229
230 // Per-session positive-result cache, keyed on (tool_search_path, name).
231 // find_external_tool runs on every compilation() / Ganak() / ... call and
232 // each miss forks /bin/sh for `command -v`; memoizing the (common)
233 // successful lookups removes that fork on the hot path. Keying on the
234 // current provsql.tool_search_path means a runtime change to that
235 // PGC_USERSET GUC simply misses and re-probes -- no explicit invalidation.
236 // We cache positives only and re-probe on miss, mirroring the deliberate
237 // "don't cache failed lookups so pooled backends self-heal" choice for
238 // get_constants(): a tool installed mid-session is picked up on the next
239 // call rather than being remembered as absent. The embedded '\0' separates
240 // the two fields unambiguously (a C-string GUC value cannot contain it).
241 static std::unordered_map<std::string, std::string> tool_cache;
242 std::string key =
244 + '\0' + name;
245 auto cached = tool_cache.find(key);
246 if (cached != tool_cache.end())
247 return cached->second;
248
249 // Delegate the search to /bin/sh via `command -v`, routed through
250 // run_external_tool() so the GUC override is honoured. This reuses
251 // exactly the PATH resolution that the eventual tool invocation will
252 // see, including the shell's compiled-in default when the environment
253 // has no PATH (typical inside a PostgreSQL backend).
254 //
255 // Single-quoting `name` defends against shell metacharacters; the
256 // five tool names provsql actually uses ("d4", "c2d", "minic2d",
257 // "dsharp", "weightmc", "graph-easy") contain none.
258 std::string check = "command -v '" + name + "' >/dev/null 2>&1";
259 // run_external_tool runs the probe in its own process group and raises
260 // any pending statement_timeout / cancel itself (so it does not surface
261 // as a spurious "tool not found").
262 int rv = run_external_tool(check);
263
264 if (rv == 0) {
265 tool_cache[key] = name;
266 return name;
267 }
268 return "";
269}
270
271std::string format_external_tool_status(int rv, const std::string &tool) {
272 if (rv == 0)
273 return "";
274 if (rv == -1)
275 return tool + " could not be invoked (system() returned -1)";
276 if (WIFSIGNALED(rv))
277 return tool + " terminated by signal "
278 + std::to_string(WTERMSIG(rv));
279 if (WIFEXITED(rv)) {
280 int code = WEXITSTATUS(rv);
281 if (code == 127)
282 return tool + " was not found at runtime (shell exit 127); "
283 "install it or add its directory to provsql.tool_search_path";
284 if (code == 126)
285 return tool + " is not executable (shell exit 126); "
286 "check permissions on the binary";
287 return tool + " exited with status " + std::to_string(code);
288 }
289 return tool + " failed with raw status " + std::to_string(rv);
290}
291
293 if (rec.kind == "kcmcp")
294 return !rec.endpoint.empty();
295 if (!rec.binary.empty() && find_external_tool(rec.binary).empty())
296 return false;
297 for (const std::string &dep : rec.dependencies)
298 if (find_external_tool(dep).empty())
299 return false;
300 return true;
301}
Generic directed-acyclic-graph circuit template and gate identifier.
In-memory catalog of the external tools ProvSQL can invoke.
ExternalToolTimeLimit(double ms)
double saved_
Limit of the enclosing scope, restored on exit.
Raised by run_external_tool when it stopped the tool for running past the time limit of the enclosing...
int run_external_tool(const std::string &cmdline)
Run a shell command line in its own process group, optionally extending PATH, interruptible by query ...
static bool tool_memory_exceeded
Set by run_in_own_pgroup when it killed a tool that took the statement past provsql....
static double tool_time_limit_ms
Time limit, in ms, of the innermost ExternalToolTimeLimit scope (0: none).
std::string format_external_tool_status(int rv, const std::string &tool)
Decode a system() return value into a human-readable message.
static bool tool_time_exceeded
Set by run_in_own_pgroup when it killed a tool that ran past tool_time_limit_ms.
static int run_in_own_pgroup(const std::string &cmdline)
bool toolAvailable(const provsql::ToolRecord &rec)
True iff a registry tool can currently be used.
std::string find_external_tool(const std::string &name)
Locate an external tool by name.
static const char * DEFAULT_PATH
Helpers for invoking external command-line tools.
std::size_t provsql_process_group_rss(pid_t pgid)
The resident memory of the processes of group pgid, in bytes, or 0 where the platform does not say.
Resident memory of a process, or of a process group.
char * provsql_tool_search_path
Colon-separated directory list prepended to PATH when invoking external tools (d4,...
Definition provsql.c:122
int provsql_max_memory
Memory, in MB, that the evaluations of a statement may add to the backend's resident set,...
Definition provsql.c:131
Build-configuration switches shared across the C and C++ sources.
void provsql_limit_exceeded(const char *tag, const char *message, const char *hint)
Stop the evaluation at a resource limit (provsql.max_memory, provsql.max_worlds): throws,...
size_t provsql_memory_used(void)
How much the backend's resident memory has grown since the first evaluation of the statement,...
Cancellation of the long C++ evaluation loops.
Core types, constants, and utilities shared across ProvSQL.
One registered external tool.
std::vector< std::string > dependencies
std::string kind
"cli" (spawn a binary) or "kcmcp" (talk to a socket server at endpoint).
std::string endpoint
KCMCP server address for kind "kcmcp": "unix:/path" or "host:port".