ProvSQL C/C++ API
Adding support for provenance and uncertainty management to PostgreSQL databases
Loading...
Searching...
No Matches
provsql_mmap.c
Go to the documentation of this file.
1/**
2 * @file provsql_mmap.c
3 * @brief Background worker registration and IPC primitives for mmap-backed storage.
4 *
5 * Implements the PostgreSQL background worker lifecycle functions declared
6 * in @c provsql_mmap.h:
7 * - @c RegisterProvSQLMMapWorker(): registers the worker with the postmaster
8 * during @c _PG_init().
9 * - @c provsql_mmap_worker(): worker entry point; sets up signal handlers
10 * and enters @c provsql_mmap_main_loop().
11 *
12 * The IPC between normal backends and the background worker is handled in
13 * @c MMappedCircuit.cpp. This file provides the PostgreSQL-specific glue
14 * (background worker API, signal handling).
15 *
16 * Also declares the shared write buffer @c buffer[] and position counter
17 * @c bufferpos used by the @c STARTWRITEM / @c ADDWRITEM / @c SENDWRITEM
18 * macros in @c provsql_mmap.h.
19 *
20 * The gate-creation SQL functions (e.g. @c create_gate()) that backends
21 * call are also implemented here; they acquire the IPC lock, write a
22 * message to the background worker, and wait for an acknowledgment.
23 */
24#include "provsql_mmap.h"
25#include "provsql_rmgr.h"
26#include "provsql_shmem.h"
27#include "provsql_utils.h"
28
29#include <errno.h>
30#include <fcntl.h>
31#include <sys/stat.h>
32#include <unistd.h>
33#include <poll.h>
34#include <math.h>
35#include <assert.h>
36
37#include "postgres.h"
38#include "access/xact.h"
39#include "postmaster/bgworker.h"
40#include "storage/fd.h"
41#include "fmgr.h"
42#include "funcapi.h"
43#include "utils/array.h"
44#include "access/htup_details.h"
45#include "utils/builtins.h"
46
47#include "circuit_cache.h"
48
49#ifdef PROVSQL_INPROCESS_STORE
50
51char *buffer = NULL; // flawfinder: ignore
52unsigned bufferpos = 0;
53size_t buffercap = 0;
54
55void provsql_buffer_ensure(size_t need)
56{
57 if(need > buffercap) {
58 size_t newcap = buffercap ? buffercap * 2 : 4096;
59 while(newcap < need)
60 newcap *= 2;
61 buffer = realloc(buffer, newcap);
62 if(!buffer)
63 provsql_error("ProvSQL: out of memory growing the IPC buffer");
64 buffercap = newcap;
65 }
66}
67
69{
70 return QueryCancelPending || ProcDiePending;
71}
72
73/* No background worker in the single-process build. */
74
75#else
76
77char buffer[PIPE_BUF]={}; // flawfinder: ignore
78unsigned bufferpos=0;
79
80/* The worker's read buffer. A message is a dozen fields, and a read() per
81 field made the worker slower than the backends that feed it: they then
82 wait on the full pipe. The pipe holds 64 KiB, which is what one read()
83 can bring back. */
84#define WORKER_READ_BUFFER (64 * 1024)
85static char worker_buffer[WORKER_READ_BUFFER]; // flawfinder: ignore
86static size_t worker_buffer_pos = 0, worker_buffer_len = 0;
87
88/* Store messages WAL replay could not hand to the worker, which was not
89 running yet: read from the spool file when the worker starts, and served
90 before anything from the pipe (see provsql_replay_store_message). */
91static char *spool_data = NULL;
92static size_t spool_pos = 0, spool_len = 0;
94
96{
97 return spool_pos < spool_len;
98}
99
104
105/* Take the spool file into memory and remove it; called by the worker under
106 the store lock, before it announces itself ready. */
108{
109 int fd = open(PROVSQL_REPLAY_SPOOL, O_RDONLY | PG_BINARY, 0);
110 struct stat st;
111
112 if(fd < 0)
113 return;
114 if(fstat(fd, &st) == 0 && st.st_size > 0) {
115 spool_data = malloc((size_t) st.st_size);
116 if(spool_data == NULL || !provsql_read_all(fd, spool_data, (size_t) st.st_size)) {
117 close(fd);
118 provsql_error("Cannot read the replay spool %s", PROVSQL_REPLAY_SPOOL);
119 }
120 spool_len = (size_t) st.st_size;
121 spool_pos = 0;
122 provsql_log("replaying %zu bytes of store messages spooled during "
123 "crash recovery", spool_len);
124 }
125 close(fd);
126}
127
128bool provsql_worker_read(void *dst, size_t n)
129{
130 char *p = dst;
131
132 while(n > 0 && spool_pos < spool_len) {
133 size_t take = spool_len - spool_pos < n ? spool_len - spool_pos : n;
134 memcpy(p, spool_data + spool_pos, take);
135 spool_pos += take;
136 p += take;
137 n -= take;
138 if(spool_pos == spool_len) {
139 /* Applied in full (a message is never split between the spool and
140 the pipe): the file can go. Should the worker die before, its
141 next start applies the spool again, which the store's writes
142 allow (a gate, an annotation or a probability written twice with
143 the same value). */
144 free(spool_data);
145 spool_data = NULL;
146 spool_pos = spool_len = 0;
147 unlink(PROVSQL_REPLAY_SPOOL);
148 }
149 }
150
151 while(n > 0) {
152 size_t avail = worker_buffer_len - worker_buffer_pos;
153 if(avail > 0) {
154 size_t take = avail < n ? avail : n;
155 memcpy(p, worker_buffer + worker_buffer_pos, take);
156 worker_buffer_pos += take;
157 p += take;
158 n -= take;
159 continue;
160 }
161 {
162 ssize_t r = read(provsql_shared_state->pipebmr, worker_buffer, // flawfinder: ignore
164 if(r < 0 && errno == EINTR)
165 continue;
166 if(r <= 0)
167 return false;
169 worker_buffer_len = (size_t) r;
170 }
171 }
172 return true;
173}
174
175bool provsql_read_all(int fd, void *dst, size_t n)
176{
177 /* A read interrupted by a signal must be tried again: given up, it
178 * leaves the rest of the message in the pipe, for the next reader. */
179 char *p = dst;
180 size_t remaining = n;
181 while(remaining > 0) {
182 ssize_t r = read(fd, p, remaining); // flawfinder: ignore
183 if(r < 0 && errno == EINTR)
184 continue;
185 if(r <= 0)
186 return false;
187 remaining -= r;
188 p += r;
189 }
190 return true;
191}
192
193bool provsql_write_all(int fd, const void *src, size_t n)
194{
195 /* A write to a pipe interrupted by a signal returns what it wrote so
196 * far: the rest must be written too, or the reader waits for it forever. */
197 const char *p = src;
198 size_t remaining = n;
199 while(remaining > 0) {
200 ssize_t r = write(fd, p, remaining);
201 if(r < 0) {
202 if(errno == EINTR)
203 continue;
204 return false;
205 }
206 remaining -= r;
207 p += r;
208 }
209 return true;
210}
211
213{
214 pg_atomic_write_u32(&provsql_shared_state->cancel_request, 0);
215}
216
218{
219 struct pollfd pfd;
220 bool cancelled = false;
221
222 pfd.fd = provsql_shared_state->pipembr;
223 pfd.events = POLLIN;
224 for(;;) {
225 int r;
226 if(!cancelled && (QueryCancelPending || ProcDiePending)) {
227 cancelled = true;
228 pg_atomic_write_u32(&provsql_shared_state->cancel_request, 1);
229 }
230 /* A signal may not interrupt poll(): looked at again every 100 ms */
231 r = poll(&pfd, 1, cancelled ? -1 : 100);
232 /* Readable, or an error the read reports */
233 if(r > 0 || (r < 0 && errno != EINTR))
234 return cancelled;
235 }
236}
237
239{
240 return pg_atomic_read_u32(&provsql_shared_state->cancel_request) != 0;
241}
242
243#if PG_VERSION_NUM >= 190000
244/* PostgreSQL 19 changed the default background-worker SIGTERM handler
245 * from bgworker_die() (immediate FATAL from the signal handler) to the
246 * flag-based die(), which only acts at the next CHECK_FOR_INTERRUPTS().
247 * This worker blocks in read() on the IPC pipe (restarted by
248 * SA_RESTART), so it would never observe the flag and a fast shutdown
249 * would hang on it. Restore the pre-19 semantics: the worker holds no
250 * transaction state, and being interrupted between messages leaves the
251 * store consistent, so exiting mid-read is fine. (Being interrupted
252 * *during* a write is what the write ordering in MMappedCircuit.cpp is
253 * there for; provsql.check_store() reports what an interruption left
254 * behind.) */
255static void provsql_worker_die(SIGNAL_ARGS)
256{
257 ereport(FATAL,
258 (errcode(ERRCODE_ADMIN_SHUTDOWN),
259 errmsg("terminating background worker \"%s\" due to administrator command",
260 MyBgworkerEntry->bgw_type)));
261}
262#endif
263
264PGDLLEXPORT void provsql_mmap_worker(Datum ignored)
265{
266#if PG_VERSION_NUM >= 190000
267 pqsignal(SIGTERM, provsql_worker_die);
268#endif
269 BackgroundWorkerUnblockSignals();
271 close(provsql_shared_state->pipebmw);
272 close(provsql_shared_state->pipembr);
273 /* From here on, WAL replay writes to the pipe; what it wrote before is in
274 the spool, which is applied first. */
277 provsql_shared_state->worker_ready = true;
279 provsql_log("%s initialized", MyBgworkerEntry->bgw_name);
280
282
284}
285
287{
288 BackgroundWorker worker;
289
290 snprintf(worker.bgw_name, BGW_MAXLEN, "ProvSQL MMap Worker");
291 snprintf(worker.bgw_type, BGW_MAXLEN, "ProvSQL MMap");
292
293 worker.bgw_flags = BGWORKER_SHMEM_ACCESS;
294 worker.bgw_start_time = BgWorkerStart_PostmasterStart;
295 worker.bgw_restart_time = 1;
296
297 snprintf(worker.bgw_library_name, BGW_MAXLEN, "provsql");
298 snprintf(worker.bgw_function_name, BGW_MAXLEN, "provsql_mmap_worker");
299 worker.bgw_main_arg = (Datum) 0;
300 worker.bgw_notify_pid = 0;
301
302 RegisterBackgroundWorker(&worker);
303}
304
305#endif /* PROVSQL_INPROCESS_STORE */
306
307/* -------------------------------------------------------------------------
308 * Durability of the store across a machine crash
309 *
310 * A backend's writes reach the worker over a FIFO pipe and the worker
311 * applies them to the mmap files, but nothing forces those files to disk
312 * at the moment the writing transaction commits: the heap's WAL record is
313 * fsynced, the circuit's bytes are not. After a crash of the machine
314 * (PostgreSQL crashing is harmless -- the page cache outlives it) a
315 * committed row can reference a gate that never reached the disk, and an
316 * unknown token reads back as an input gate, so the loss is silent.
317 *
318 * Two things narrow that window. The worker forces the files out shortly
319 * after the last write (PROVSQL_STORE_FLUSH_INTERVAL_MS), which bounds the
320 * loss. And, when provsql.store_synchronous_commit is on, a transaction that
321 * wrote to the store sends a sync request before it commits and waits for
322 * the reply, which closes the window entirely: the reply comes after every
323 * earlier message of this backend has been applied and forced.
324 * ------------------------------------------------------------------------- */
325
327
328/** Whether the current transaction has written anything to the store. */
329static bool store_written = false;
330static bool store_callbacks_registered = false;
331
332/** @brief Send the sync barrier and wait for the worker's acknowledgement. */
334{
335 char ack;
336
337 STARTWRITEM();
338 ADDWRITEM("S", char);
339 ADDWRITEDB();
340
342 if(!SENDWRITEM() || !READB(ack, char)) {
344 provsql_error("Cannot communicate with pipe (message type S)");
345 }
347}
348
349static void provsql_store_xact_callback(XactEvent event, void *arg)
350{
351 (void) arg;
352 switch(event) {
353 case XACT_EVENT_PRE_COMMIT:
354 case XACT_EVENT_PRE_PREPARE:
355 /* Still inside the transaction, so raising here aborts the commit
356 rather than leaving it half-durable. */
359 break;
360 case XACT_EVENT_COMMIT:
361 case XACT_EVENT_ABORT:
362 case XACT_EVENT_PREPARE:
363 case XACT_EVENT_PARALLEL_COMMIT:
364 case XACT_EVENT_PARALLEL_ABORT:
365 store_written = false;
366 break;
367 default:
368 break;
369 }
370}
371
372/** @brief What every store mutation does before it reaches the pipe:
373 * refuse it on a standby, and write it to the WAL. @p data is the
374 * complete message, opcode first. */
375static void provsql_log_store_write(const char *data, size_t len)
376{
378 ereport(ERROR,
379 (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION),
380 errmsg("cannot write to the ProvSQL circuit store during recovery"),
381 errdetail("The store is maintained on a standby by replaying the "
382 "primary's records; a backend writing to it would make "
383 "the two diverge."),
384 errhint("Provenance queries create gates as they run, reads "
385 "included, so they only work on the primary.")));
387}
388
389/** @brief The same, for a mutation made by a live transaction: it also
390 * arms the at-commit sync barrier. */
391static void provsql_before_store_write(const char *data, size_t len)
392{
393 provsql_log_store_write(data, len);
395}
396
398{
400 RegisterXactCallback(provsql_store_xact_callback, NULL);
402 }
403 store_written = true;
404}
405
407{
408 return store_written;
409}
410
411void provsql_circuit_cleanup_request(bool dry_run, const pg_uuid_t *roots,
412 int64 nb_roots,
414{
415 char flag = dry_run ? 1 : 0;
416 unsigned long n = (unsigned long) nb_roots;
417
418 /* The root set is as large as the number of distinct tokens stored in
419 the database, so it does not fit one atomic pipe write. The lock is
420 held across the whole exchange, which is what makes the sequence of
421 writes one message; nothing else can be talking to the worker anyway,
422 since the caller holds the database exclusively. */
424
425 STARTWRITEM();
426 ADDWRITEM("X", char);
427 ADDWRITEDB();
428 ADDWRITEM(&flag, char);
429 ADDWRITEM(&n, unsigned long);
430 if(!SENDWRITEM()) {
432 provsql_error("Cannot write to pipe (message type X)");
433 }
434
435#ifdef PROVSQL_INPROCESS_STORE
436 /* The in-memory FIFO has no atomicity limit and the dispatch runs
437 inside SENDWRITEM, so the roots must accompany the header. */
439 provsql_error("circuit_cleanup is not available in the single-process build");
440#else
441 {
442 unsigned long per_batch = PIPE_BUF / sizeof(pg_uuid_t);
443 for(unsigned long i = 0; i < n; ) {
444 unsigned long j;
445 STARTWRITEM();
446 for(j = 0; j < per_batch && i < n; ++j, ++i)
447 ADDWRITEM(&roots[i], pg_uuid_t);
448 if(!SENDWRITEM()) {
450 provsql_error("Cannot write to pipe (message type X)");
451 }
452 }
453 }
454
455 if(!READB(out->gates_before, uint64) || !READB(out->gates_after, uint64)
456 || !READB(out->wires_before, uint64) || !READB(out->wires_after, uint64)
457 || !READB(out->extra_before, uint64) || !READB(out->extra_after, uint64)) {
459 provsql_error("Cannot read response from pipe (message type X)");
460 }
462#endif
463}
464
465#ifndef PROVSQL_INPROCESS_STORE
466/* The replay spool, open in the startup process while it writes to it */
467static int replay_spool_fd = -1;
468#endif
469
471{
472#ifndef PROVSQL_INPROCESS_STORE
473 /* Synced before the end-of-recovery checkpoint, after which the WAL it
474 comes from is not replayed again: the spool is all that is left of it
475 until the worker applies it. */
476 if(replay_spool_fd >= 0) {
477 if(pg_fsync(replay_spool_fd) != 0)
478 provsql_error("Cannot sync the replay spool %s", PROVSQL_REPLAY_SPOOL);
479 close(replay_spool_fd);
480 replay_spool_fd = -1;
481 }
482#endif
483}
484
485void provsql_replay_store_message(const char *data, size_t len)
486{
487#ifdef PROVSQL_INPROCESS_STORE
488 (void) data; (void) len;
489#else
490 const char *p = data;
491 size_t left = len;
492
493 if(len == 0)
494 return;
495
496 /* The worker is the single writer, in recovery as in normal running:
497 the message goes back down the same pipe a backend would use. The
498 lock is held across the whole write so the (possibly chunked)
499 message stays one message. */
501
502 /* After a crash, PostgreSQL starts the background workers only once
503 recovery is over: nothing reads the pipe, which fills, and a message
504 that is answered waits for ever, so recovery never ends. Until the
505 worker announces itself, the message goes to a spool file the worker
506 applies first when it starts. */
507 if(!provsql_shared_state->worker_ready) {
508 if(replay_spool_fd < 0) {
510 O_WRONLY | O_CREAT | O_APPEND | PG_BINARY,
511 S_IRUSR | S_IWUSR);
512 if(replay_spool_fd < 0) {
514 provsql_error("Cannot open the replay spool %s", PROVSQL_REPLAY_SPOOL);
515 }
516 }
517 while(left > 0) {
518 ssize_t w = write(replay_spool_fd, p, left);
519 if(w <= 0) {
521 provsql_error("Cannot write to the replay spool %s",
523 }
524 p += w;
525 left -= (size_t) w;
526 }
528 return;
529 }
530
531 while(left > 0) {
532 size_t chunk = left > PIPE_BUF ? PIPE_BUF : left;
533 if(!provsql_write_all(provsql_shared_state->pipebmw, p, chunk)) {
535 provsql_error("Cannot replay a store message to the pipe");
536 }
537 p += chunk;
538 left -= chunk;
539 }
540
541 /* Opcodes that answer must be drained, or the reply would be read as
542 the answer to somebody else's later question. */
543 if(data[0] == 'P') {
544 char result;
545 double stored;
546 if(!READB(result, char) || !READB(stored, double)) {
548 provsql_error("Cannot read the reply to a replayed store message");
549 }
550 }
551
553#endif
554}
555
556PG_FUNCTION_INFO_V1(check_store);
557/**
558 * @brief Report what does not add up in this database's circuit store.
559 *
560 * A store nothing has damaged answers zero to every count. A non-zero
561 * one means a write was interrupted at a point the ordering rules do not
562 * cover, or that a set of files was copied at different instants -- a
563 * file-level backup of a running server, or a base backup.
564 * @c provsql.circuit_cleanup() rebuilds the store from what is still
565 * reachable.
566 */
567Datum check_store(PG_FUNCTION_ARGS)
568{
569 char unclean;
570 unsigned long nb_gates, nb_mapping, next_value,
571 dangling, unreferenced, bad_wires, bad_extra;
572 TupleDesc tupdesc;
573 Datum values[8];
574 bool nulls[8] = {false, false, false, false, false, false, false, false};
575
576 STARTWRITEM();
577 ADDWRITEM("k", char);
578 ADDWRITEDB();
579
581 if(!SENDWRITEM()
582 || !READB(unclean, char)
583 || !READB(nb_gates, unsigned long)
584 || !READB(nb_mapping, unsigned long)
585 || !READB(next_value, unsigned long)
586 || !READB(dangling, unsigned long)
587 || !READB(unreferenced, unsigned long)
588 || !READB(bad_wires, unsigned long)
589 || !READB(bad_extra, unsigned long)) {
591 provsql_error("Cannot communicate with pipe (message type k)");
592 }
594
595 if(get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
596 provsql_error("check_store: expected composite return type");
597 tupdesc = BlessTupleDesc(tupdesc);
598
599 values[0] = BoolGetDatum(unclean != 0);
600 values[1] = Int64GetDatum((int64) nb_gates);
601 values[2] = Int64GetDatum((int64) nb_mapping);
602 values[3] = Int64GetDatum((int64) next_value);
603 values[4] = Int64GetDatum((int64) dangling);
604 values[5] = Int64GetDatum((int64) unreferenced);
605 values[6] = Int64GetDatum((int64) bad_wires);
606 values[7] = Int64GetDatum((int64) bad_extra);
607
608 PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
609}
610
611PG_FUNCTION_INFO_V1(get_gate_type);
612/** @brief PostgreSQL-callable wrapper for get_gate_type().
613 *
614 * On cache miss this fetches BOTH the gate type and its children from
615 * the worker, in one critical section, then caches them together. If
616 * we cached only the type (with an empty children list), a subsequent
617 * get_children() call for the same token would consult the cache, find
618 * the entry, and return 0 children : never querying the worker for the
619 * real children. provsql.provenance_evaluate hits exactly that pattern
620 * (it calls get_gate_type first, then unnest(get_children(...))) and
621 * silently folds plus/times gates over an empty set.
622 */
623/** @brief Fetch a gate's type and children, cache-first with a worker
624 * round-trip (and cache fill) on a miss. Factored out of the
625 * get_gate_type() wrapper for in-extension callers that walk the circuit
626 * from C (e.g. the annotation-transparent set_prob()). On return
627 * @p *children_out is a @c calloc'd array to be freed by the caller, or
628 * @c NULL when the gate has no children. */
630 unsigned *nb_children_out,
631 pg_uuid_t **children_out)
632{
633 gate_type type;
634 unsigned nb_children = 0;
635 pg_uuid_t *children = NULL;
636
637 type = circuit_cache_get_type(*token);
638 if(type!=gate_invalid) {
639 *nb_children_out = circuit_cache_get_children(*token, children_out);
640 return type;
641 }
642
643 /* Type fetch (message 't'). */
644 STARTWRITEM();
645 ADDWRITEM("t", char);
646 ADDWRITEDB();
647 ADDWRITEM(token, pg_uuid_t);
648
650
651 if(!SENDWRITEM() || !READB(type, gate_type)) {
653 provsql_error("Cannot communicate on pipe (message type t)");
654 }
655
656 /* Children fetch (message 'c'), batched in the same critical
657 * section so the cache entry below is complete. Skipped when the
658 * token is unknown (worker reports gate_invalid). */
659 if(type != gate_invalid) {
660 STARTWRITEM();
661 ADDWRITEM("c", char);
662 ADDWRITEDB();
663 ADDWRITEM(token, pg_uuid_t);
664
665 if(!SENDWRITEM() || !READB(nb_children, unsigned)) {
667 provsql_error("Cannot communicate on pipe (message type c during get_gate_type)");
668 }
669
670 if(nb_children > 0) {
671 children = calloc(nb_children, sizeof(pg_uuid_t));
672 if(!READB_BYTES(children, nb_children * sizeof(pg_uuid_t))) {
674 provsql_error("Cannot read children from pipe (during get_gate_type)");
675 }
676 }
677 }
678
680
681 /* Skip caching the gate_input lazy default: MMappedCircuit::getGateType
682 * returns gate_input both for real input gates and for tokens that are
683 * not yet in the mapping. Caching the latter would poison subsequent
684 * create_gate() calls in this session (the cache hit would short-circuit
685 * the worker IPC, dropping the gate). The cost is one extra IPC per
686 * lookup of a real input gate -- acceptable. */
687 if(!(type == gate_input && nb_children == 0))
688 circuit_cache_create_gate(*token, type, nb_children, children);
689 *nb_children_out = nb_children;
690 *children_out = children;
691 return type;
692}
693
694Datum get_gate_type(PG_FUNCTION_ARGS)
695{
696 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
697 gate_type type;
698 constants_t constants=get_constants(true);
699 unsigned nb_children = 0;
700 pg_uuid_t *children = NULL;
701
702 if(PG_ARGISNULL(0))
703 PG_RETURN_NULL();
704
705 type = provsql_fetch_gate(token, &nb_children, &children);
706 if(children) free(children);
707 PG_RETURN_INT32(constants.GATE_TYPE_TO_OID[type]);
708}
709
710/** @brief Internal entry point behind create_gate(): cache + worker IPC.
711 *
712 * Factored out of the SQL-callable wrapper so in-extension C/C++ code
713 * (e.g. the decomposition-aligned reachability materialiser) can create
714 * gates without Datum marshalling or gate-type-OID lookups. Same
715 * semantics: write-through to the per-session cache, then the C message
716 * to the background worker; MMappedCircuit::createGate is idempotent on
717 * already-mapped tokens. */
719 unsigned nb_children,
720 const pg_uuid_t *children_data)
721{
722 /* Populate the per-session cache, but unconditionally fall through to
723 * the worker IPC: a cache hit only proves "this token has been seen
724 * in this session before" (e.g. by get_gate_type returning the
725 * gate_input lazy default for an unknown token) -- not "the worker
726 * already has a gate for it". Skipping the IPC on a cache hit caused
727 * silently-dropped create_gate calls under concurrent backends.
728 * MMappedCircuit::createGate is idempotent on already-mapped tokens. */
729 circuit_cache_create_gate(*token, type, nb_children, children_data);
730
731 /* The WAL record is the whole logical message, children included, even
732 though the pipe may need several writes for it. */
733 {
734 size_t header = sizeof(char) + 2 * sizeof(Oid) + sizeof(pg_uuid_t)
735 + sizeof(gate_type) + sizeof(unsigned);
736 size_t len = header + nb_children * sizeof(pg_uuid_t);
737 char *msg = palloc(len);
738 char *p = msg;
739 *p++ = 'C';
740 memcpy(p, &MyDatabaseId, sizeof(Oid)); p += sizeof(Oid);
741 memcpy(p, &MyDatabaseTableSpace, sizeof(Oid)); p += sizeof(Oid);
742 memcpy(p, token, sizeof(pg_uuid_t)); p += sizeof(pg_uuid_t);
743 memcpy(p, &type, sizeof(gate_type)); p += sizeof(gate_type);
744 memcpy(p, &nb_children, sizeof(unsigned)); p += sizeof(unsigned);
745 for(unsigned i=0; i<nb_children; ++i) {
746 memcpy(p, &children_data[i], sizeof(pg_uuid_t));
747 p += sizeof(pg_uuid_t);
748 }
750 pfree(msg);
751 }
752
753 STARTWRITEM();
754 ADDWRITEM("C", char);
755 ADDWRITEDB();
756 ADDWRITEM(token, pg_uuid_t);
757 ADDWRITEM(&type, gate_type);
758 ADDWRITEM(&nb_children, unsigned);
759
760#ifdef PROVSQL_INPROCESS_STORE
761 /* The in-memory FIFO has no PIPE_BUF atomicity limit: always send the
762 gate and all its children as a single message. */
763 if(1) {
764#else
765 if(PIPE_BUF-bufferpos>nb_children*sizeof(pg_uuid_t)) {
766#endif
767 // Enough space in the buffer for an atomic write, no need of
768 // exclusive locks
769
770 for(unsigned i=0; i<nb_children; ++i)
771 ADDWRITEM(&children_data[i], pg_uuid_t);
772
774 if(!SENDWRITEM()) {
776 provsql_error("Cannot write to pipe (message type C)");
777 }
779 }
780#ifndef PROVSQL_INPROCESS_STORE
781 else {
782 // Not enough space in buffer, pipe write won't be atomic, we need to
783 // make several writes and use locks
784 unsigned children_per_batch = PIPE_BUF/sizeof(pg_uuid_t);
785
787
788 if(!SENDWRITEM()) {
790 provsql_error("Cannot write to pipe (message type C)");
791 }
792
793 for(unsigned j=0; j<1+(nb_children-1)/children_per_batch; ++j) {
794 STARTWRITEM();
795
796 for(unsigned i=j*children_per_batch; i<(j+1)*children_per_batch && i<nb_children; ++i) {
797 ADDWRITEM(&children_data[i], pg_uuid_t);
798 }
799
800 if(!SENDWRITEM()) {
802 provsql_error("Cannot write to pipe (message type C)");
803 }
804 }
805
807 }
808#endif
809}
810
811/** @brief Send a probability write and read back what the store made of
812 * it. @p tracked arms the at-commit sync barrier; the one caller that
813 * passes false is the rollback path, which runs when the transaction
814 * that would have committed is already gone. */
816 double prob,
817 double *existing,
818 bool tracked)
819{
820 char result;
821 double stored;
822
823 STARTWRITEM();
824 ADDWRITEM("P", char);
825 ADDWRITEDB();
826 ADDWRITEM(token, pg_uuid_t);
827 ADDWRITEM(&prob, double);
828 if(tracked)
830 else
832
834 if(!SENDWRITEM() || !READB(result, char) || !READB(stored, double)) {
836 provsql_error("Cannot communicate with pipe (message type P)");
837 }
839
840 if(existing)
841 *existing = stored;
842 return (provsql_set_prob_result) result;
843}
844
846 double prob,
847 double *existing)
848{
849 return provsql_send_set_prob(token, prob, existing, true);
850}
851
853{
854 provsql_send_set_prob(token, NAN, NULL, false);
855}
856
857bool provsql_internal_get_prob_written(const pg_uuid_t *token, double *prob)
858{
859 char has;
860 double stored;
861
862 STARTWRITEM();
863 ADDWRITEM("q", char);
864 ADDWRITEDB();
865 ADDWRITEM(token, pg_uuid_t);
866
868 if(!SENDWRITEM() || !READB(has, char) || !READB(stored, double)) {
870 provsql_error("Cannot communicate with pipe (message type q)");
871 }
873
874 if(prob)
875 *prob = stored;
876 return has != 0;
877}
878
880 unsigned nb_children,
881 const pg_uuid_t *children,
882 bool has_infos, unsigned info1,
883 unsigned info2, const char *extra)
884{
885 char flag = has_infos ? 1 : 0;
886 unsigned len = extra != NULL ? (unsigned) strlen(extra) : 0;
887 size_t total = sizeof(char) + 2 * sizeof(Oid) + sizeof(pg_uuid_t)
888 + sizeof(gate_type) + sizeof(unsigned)
889 + nb_children * sizeof(pg_uuid_t)
890 + sizeof(char) + 3 * sizeof(unsigned) + len;
891 char *msg = palloc(total);
892 char *p = msg;
893 unsigned i;
894
895 *p++ = 'G';
896 memcpy(p, &MyDatabaseId, sizeof(Oid)); p += sizeof(Oid);
897 memcpy(p, &MyDatabaseTableSpace, sizeof(Oid)); p += sizeof(Oid);
898 memcpy(p, token, sizeof(pg_uuid_t)); p += sizeof(pg_uuid_t);
899 memcpy(p, &type, sizeof(gate_type)); p += sizeof(gate_type);
900 memcpy(p, &nb_children, sizeof(unsigned)); p += sizeof(unsigned);
901 for(i=0; i<nb_children; ++i) {
902 memcpy(p, &children[i], sizeof(pg_uuid_t));
903 p += sizeof(pg_uuid_t);
904 }
905 *p++ = flag;
906 memcpy(p, &info1, sizeof(unsigned)); p += sizeof(unsigned);
907 memcpy(p, &info2, sizeof(unsigned)); p += sizeof(unsigned);
908 memcpy(p, &len, sizeof(unsigned)); p += sizeof(unsigned);
909 if(len > 0)
910 memcpy(p, extra, len);
911
912 circuit_cache_create_gate(*token, type, nb_children, children);
913 provsql_before_store_write(msg, total);
914
915 /* The message goes down the pipe as it is: in one write when it fits, so
916 that it is atomic and the shared lock is enough; in PIPE_BUF pieces
917 under the exclusive lock otherwise. */
918#ifdef PROVSQL_INPROCESS_STORE
920 if(!provsql_inproc_send(msg, total)) {
922 provsql_error("Cannot write to pipe (message type G)");
923 }
925#else
926 if(total <= PIPE_BUF) {
928 if(!provsql_write_all(provsql_shared_state->pipebmw, msg, total)) {
930 provsql_error("Cannot write to pipe (message type G)");
931 }
933 } else {
934 size_t left = total;
935 p = msg;
937 while(left > 0) {
938 size_t chunk = left > PIPE_BUF ? PIPE_BUF : left;
939 if(!provsql_write_all(provsql_shared_state->pipebmw, p, chunk)) {
941 provsql_error("Cannot write to pipe (message type G)");
942 }
943 p += chunk;
944 left -= chunk;
945 }
947 }
948#endif
949 pfree(msg);
950}
951
952
953
954PG_FUNCTION_INFO_V1(create_gate);
955/** @brief PostgreSQL-callable wrapper for create_gate(). */
956Datum create_gate(PG_FUNCTION_ARGS)
957{
958 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
959 Oid oid_type = PG_GETARG_INT32(1);
960 ArrayType *children = PG_ARGISNULL(2)?NULL:PG_GETARG_ARRAYTYPE_P(2);
961 unsigned nb_children = 0;
962 gate_type type = gate_invalid;
963 constants_t constants;
964 pg_uuid_t *children_data;
965
966 if(PG_ARGISNULL(0) || PG_ARGISNULL(1))
967 provsql_error("Invalid NULL value passed to create_gate");
968
969 if(children) {
970 if(ARR_NDIM(children) > 1)
971 provsql_error("Invalid multi-dimensional array passed to create_gate");
972 if(array_contains_nulls(children))
973 provsql_error("create_gate: children array must not contain NULL "
974 "elements (filter them out before calling)");
975 if(ARR_NDIM(children) == 1)
976 nb_children = *ARR_DIMS(children);
977 }
978
979 constants=get_constants(true);
980
981 for(int i=0; i<nb_gate_types; ++i) {
982 if(constants.GATE_TYPE_TO_OID[i]==oid_type) {
983 type = i;
984 break;
985 }
986 }
987 if(type == gate_invalid) {
988 provsql_error("Invalid gate type");
989 }
990
991 if(nb_children>0)
992 children_data = (pg_uuid_t*) ARR_DATA_PTR(children);
993 else
994 children_data = NULL;
995
996 if(PG_NARGS() > 3) {
997 /* create_gate(token, type, children, info1, info2, extra): the gate
998 and what it records in one unanswered message. */
999 bool has_infos = !PG_ARGISNULL(3) || !PG_ARGISNULL(4);
1000 unsigned info1 = PG_ARGISNULL(3) ? 0 : (unsigned) PG_GETARG_INT32(3);
1001 unsigned info2 = PG_ARGISNULL(4) ? 0 : (unsigned) PG_GETARG_INT32(4);
1002 char *extra = PG_ARGISNULL(5) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(5));
1003
1004 provsql_internal_create_gate_with(token, type, nb_children, children_data,
1005 has_infos, info1, info2, extra);
1006 PG_RETURN_VOID();
1007 }
1008
1009 provsql_internal_create_gate(token, type, nb_children, children_data);
1010
1011 PG_RETURN_VOID();
1012}
1013
1014/* set_infos and set_extra: the SQL functions are gone (a gate is created
1015 with what it records), but the install scripts of earlier versions bind
1016 these symbols, and an upgrade runs them. What they send is applied
1017 write-once and not answered, like everything a backend writes. */
1018static void send_unanswered(const char *msg, size_t len)
1019{
1020#ifdef PROVSQL_INPROCESS_STORE
1021 if(!provsql_inproc_send(msg, len))
1022 provsql_error("Cannot write to pipe (message type %c)", msg[0]);
1023#else
1024 if(len > PIPE_BUF)
1025 provsql_error("message type %c too long", msg[0]);
1027 if(!provsql_write_all(provsql_shared_state->pipebmw, msg, len)) {
1029 provsql_error("Cannot write to pipe (message type %c)", msg[0]);
1030 }
1032#endif
1033}
1034
1035PG_FUNCTION_INFO_V1(set_infos);
1036/** @brief Entry point of the @c set_infos of earlier versions' scripts. */
1037Datum set_infos(PG_FUNCTION_ARGS)
1038{
1039 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1040 unsigned info1 = PG_ARGISNULL(1) ? 0 : (unsigned) PG_GETARG_INT32(1);
1041 unsigned info2 = PG_ARGISNULL(2) ? 0 : (unsigned) PG_GETARG_INT32(2);
1042 char msg[sizeof(char) + 2 * sizeof(Oid) + sizeof(pg_uuid_t) + 2 * sizeof(unsigned)];
1043 char *p = msg;
1044
1045 if(PG_ARGISNULL(0))
1046 PG_RETURN_VOID();
1047 *p++ = 'I';
1048 memcpy(p, &MyDatabaseId, sizeof(Oid)); p += sizeof(Oid);
1049 memcpy(p, &MyDatabaseTableSpace, sizeof(Oid)); p += sizeof(Oid);
1050 memcpy(p, token, sizeof(pg_uuid_t)); p += sizeof(pg_uuid_t);
1051 memcpy(p, &info1, sizeof(unsigned)); p += sizeof(unsigned);
1052 memcpy(p, &info2, sizeof(unsigned));
1053 provsql_before_store_write(msg, sizeof(msg));
1054 send_unanswered(msg, sizeof(msg));
1055 PG_RETURN_VOID();
1056}
1057
1058PG_FUNCTION_INFO_V1(set_extra);
1059/** @brief Entry point of the @c set_extra of earlier versions' scripts. */
1060Datum set_extra(PG_FUNCTION_ARGS)
1061{
1062 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1063 char *str = text_to_cstring(PG_GETARG_TEXT_PP(1));
1064 unsigned len = (unsigned) strlen(str);
1065 size_t total = sizeof(char) + 2 * sizeof(Oid) + sizeof(pg_uuid_t) + sizeof(unsigned) + len;
1066 char *msg = palloc(total);
1067 char *p = msg;
1068
1069 *p++ = 'E';
1070 memcpy(p, &MyDatabaseId, sizeof(Oid)); p += sizeof(Oid);
1071 memcpy(p, &MyDatabaseTableSpace, sizeof(Oid)); p += sizeof(Oid);
1072 memcpy(p, token, sizeof(pg_uuid_t)); p += sizeof(pg_uuid_t);
1073 memcpy(p, &len, sizeof(unsigned)); p += sizeof(unsigned);
1074 memcpy(p, str, len);
1075 provsql_before_store_write(msg, total);
1076 send_unanswered(msg, total);
1077 pfree(msg);
1078 pfree(str);
1079 PG_RETURN_VOID();
1080}
1081
1082PG_FUNCTION_INFO_V1(get_extra);
1083/** @brief PostgreSQL-callable wrapper for get_extra(). */
1084Datum get_extra(PG_FUNCTION_ARGS)
1085{
1086 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1087 text *result;
1088 unsigned len;
1089
1090 if(PG_ARGISNULL(0))
1091 PG_RETURN_NULL();
1092
1093 STARTWRITEM();
1094 ADDWRITEM("e", char);
1095 ADDWRITEDB();
1096 ADDWRITEM(token, pg_uuid_t);
1097
1099
1100 if(!SENDWRITEM() || !READB(len, unsigned)) {
1102 provsql_error("Cannot communicate with pipe (message type e)");
1103 }
1104
1105 result = palloc(len + VARHDRSZ);
1106 SET_VARSIZE(result, VARHDRSZ + len);
1107
1108 if(!READB_BYTES(VARDATA(result), len)) {
1110 provsql_error("Cannot communicate with pipe (message type e)");
1111 }
1112
1114
1115 PG_RETURN_TEXT_P(result);
1116}
1117
1118PG_FUNCTION_INFO_V1(get_nb_gates);
1119/** @brief PostgreSQL-callable wrapper for get_nb_gates(). */
1120Datum get_nb_gates(PG_FUNCTION_ARGS)
1121{
1122 unsigned long nb;
1123
1124 STARTWRITEM();
1125 ADDWRITEM("n", char);
1126 ADDWRITEDB();
1127
1129
1130 if(!SENDWRITEM() || !READB(nb, unsigned long)) {
1132 provsql_error("Cannot communicate with pipe (message type n)");
1133 }
1134
1136
1137 PG_RETURN_INT64((long) nb);
1138}
1139
1140PG_FUNCTION_INFO_V1(get_children);
1141/** @brief PostgreSQL-callable wrapper for get_children(). */
1142Datum get_children(PG_FUNCTION_ARGS)
1143{
1144 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1145 ArrayType *result = NULL;
1146 unsigned nb_children;
1147 pg_uuid_t *children;
1148 Datum *children_ptr;
1149 constants_t constants;
1150
1151 if(PG_ARGISNULL(0))
1152 PG_RETURN_NULL();
1153
1154 nb_children = circuit_cache_get_children(*token, &children);
1155
1156 if(!children) {
1157 STARTWRITEM();
1158 ADDWRITEM("c", char);
1159 ADDWRITEDB();
1160 ADDWRITEM(token, pg_uuid_t);
1161
1163
1164 if(!SENDWRITEM()) {
1166 provsql_error("Cannot write to pipe (message type c)");
1167 }
1168
1169 if(!READB(nb_children, unsigned)) {
1171 provsql_error("Cannot read response from pipe (message type c)");
1172 }
1173
1174 children=calloc(nb_children, sizeof(pg_uuid_t));
1175
1176 if(!READB_BYTES(children, nb_children*sizeof(pg_uuid_t))) {
1178 provsql_error("Cannot read from pipe (message type c)");
1179 }
1181
1182 /* Skip caching when the worker reports zero children: we cannot
1183 * distinguish a real zero-child gate (input/zero/one/...) from a
1184 * token unknown to the worker, and caching the latter poisons
1185 * subsequent create_gate() calls in this session. */
1186 if(nb_children > 0)
1187 circuit_cache_create_gate(*token, gate_invalid, nb_children, children);
1188 }
1189
1190 children_ptr = palloc(nb_children * sizeof(Datum));
1191 for(unsigned i=0; i<nb_children; ++i)
1192 children_ptr[i] = UUIDPGetDatum(&children[i]);
1193
1194 constants=get_constants(true);
1195 result = construct_array(
1196 children_ptr,
1197 nb_children,
1198 constants.OID_TYPE_UUID,
1199 16,
1200 false,
1201 'c');
1202 pfree(children_ptr);
1203 free(children);
1204
1205 PG_RETURN_ARRAYTYPE_P(result);
1206}
1207
1208PG_FUNCTION_INFO_V1(get_prob);
1209/** @brief PostgreSQL-callable wrapper for get_prob(). */
1210Datum get_prob(PG_FUNCTION_ARGS)
1211{
1212 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1213 double result;
1214
1215 if(PG_ARGISNULL(0))
1216 PG_RETURN_NULL();
1217
1218 STARTWRITEM();
1219 ADDWRITEM("p", char);
1220 ADDWRITEDB();
1221 ADDWRITEM(token, pg_uuid_t);
1222
1224
1225 if(!SENDWRITEM() || !READB(result, double)) {
1227 provsql_error("Cannot communicate with pipe (message type p)");
1228 }
1229
1231
1232 if(isnan(result))
1233 PG_RETURN_NULL();
1234 else
1235 PG_RETURN_FLOAT8(result);
1236}
1237
1239 unsigned *info1, unsigned *info2)
1240{
1241 STARTWRITEM();
1242 ADDWRITEM("i", char);
1243 ADDWRITEDB();
1244 ADDWRITEM(token, pg_uuid_t);
1245
1247
1248 if(!SENDWRITEM() || !READB(*info1, int) || !READB(*info2, int)) {
1250 provsql_error("Cannot communicate with pipe (message type i)");
1251 }
1252
1254}
1255
1257{
1258 text *t = DatumGetTextPP(DirectFunctionCall1(get_extra,
1259 UUIDPGetDatum(token)));
1260 return text_to_cstring(t);
1261}
1262
1263PG_FUNCTION_INFO_V1(get_infos);
1264/** @brief PostgreSQL-callable wrapper for get_infos(). */
1265Datum get_infos(PG_FUNCTION_ARGS)
1266{
1267 pg_uuid_t *token = DatumGetUUIDP(PG_GETARG_DATUM(0));
1268 unsigned info1 =0, info2 = 0;
1269
1270 if(PG_ARGISNULL(0))
1271 PG_RETURN_NULL();
1272
1273 provsql_internal_get_infos(token, &info1, &info2);
1274
1275 {
1276 TupleDesc tupdesc;
1277 Datum values[2];
1278 bool nulls[2] = {false, false};
1279
1280 get_call_result_type(fcinfo,NULL,&tupdesc);
1281 tupdesc = BlessTupleDesc(tupdesc);
1282
1283 values[0] = Int32GetDatum(info1);
1284 values[1] = Int32GetDatum(info2);
1285
1286 PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
1287 }
1288}
void destroy_provsql_mmap()
Unmap and close the mmap files.
void provsql_mmap_main_loop()
Main processing loop of the mmap background worker.
void initialize_provsql_mmap()
Initialise the circuit store.
C-linkage interface to the in-process provenance circuit cache.
gate_type circuit_cache_get_type(pg_uuid_t token)
Retrieve the type of a cached gate.
unsigned circuit_cache_get_children(pg_uuid_t token, pg_uuid_t **children)
Retrieve the children of a cached gate.
bool circuit_cache_create_gate(pg_uuid_t token, gate_type type, unsigned nb_children, const pg_uuid_t *children)
Insert a new gate into the circuit cache.
Datum get_gate_type(PG_FUNCTION_ARGS)
#define provsql_error(fmt,...)
Report a fatal ProvSQL error and abort the current transaction.
#define provsql_log(fmt,...)
Write a ProvSQL message to the server log only.
bool provsql_write_all(int fd, const void *src, size_t n)
Write the n bytes of src to fd, however many write calls it takes; false on error.
bool provsql_store_synchronous_commit
Global variable set by the provsql.store_synchronous_commit run-time configuration parameter: when tr...
Datum get_infos(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for get_infos().
void provsql_mmap_worker(Datum ignored)
Entry point for the ProvSQL mmap background worker.
bool provsql_store_written(void)
Whether this transaction has written to the circuit store.
void provsql_internal_create_gate(const pg_uuid_t *token, gate_type type, unsigned nb_children, const pg_uuid_t *children_data)
Internal entry point behind create_gate(): cache + worker IPC.
static char * spool_data
void provsql_replay_store_message(const char *data, size_t len)
Feed a logged store message back to the worker.
void provsql_request_start(void)
Start a request that may keep the worker busy (the building of a circuit): no cancel is pending for i...
#define WORKER_READ_BUFFER
static void provsql_take_replay_spool(void)
Datum check_store(PG_FUNCTION_ARGS)
Report what does not add up in this database's circuit store.
void provsql_internal_create_gate_with(const pg_uuid_t *token, gate_type type, unsigned nb_children, const pg_uuid_t *children, bool has_infos, unsigned info1, unsigned info2, const char *extra)
Create a gate together with its infos and its text, in one message that is not answered.
static int replay_spool_fd
void provsql_store_note_write(void)
Note that this transaction has written to the circuit store.
static void provsql_store_xact_callback(XactEvent event, void *arg)
static size_t spool_len
static void provsql_store_sync_barrier(void)
Send the sync barrier and wait for the worker's acknowledgement.
static void provsql_before_store_write(const char *data, size_t len)
The same, for a mutation made by a live transaction: it also arms the at-commit sync barrier.
static size_t spool_pos
void provsql_replay_spool_close(void)
Sync and close the replay spool at the end of WAL replay (the resource manager's cleanup).
bool provsql_read_all(int fd, void *dst, size_t n)
Read exactly n bytes from fd into dst; false on EOF/error.
static void send_unanswered(const char *msg, size_t len)
static size_t worker_buffer_pos
static bool store_written
Whether the current transaction has written anything to the store.
Datum set_extra(PG_FUNCTION_ARGS)
Entry point of the set_extra of earlier versions' scripts.
Datum get_nb_gates(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for get_nb_gates().
Datum create_gate(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for create_gate().
bool provsql_worker_spooled(void)
Whether store messages spooled during crash recovery remain to be applied (they come before the pipe)...
static bool store_callbacks_registered
gate_type provsql_fetch_gate(const pg_uuid_t *token, unsigned *nb_children_out, pg_uuid_t **children_out)
PostgreSQL-callable wrapper for get_gate_type().
Datum set_infos(PG_FUNCTION_ARGS)
Entry point of the set_infos of earlier versions' scripts.
void provsql_circuit_cleanup_request(bool dry_run, const pg_uuid_t *roots, int64 nb_roots, provsql_cleanup_result *out)
Ask the worker to rebuild this database's store, keeping only what roots reach.
bool provsql_worker_buffered(void)
Whether the worker's read buffer holds unread bytes, which poll() cannot see.
bool provsql_worker_read(void *dst, size_t n)
The worker's buffered read of the request pipe: what the pipe holds is read in one call,...
Datum get_gate_type(PG_FUNCTION_ARGS)
static size_t worker_buffer_len
provsql_set_prob_result provsql_internal_set_prob(const pg_uuid_t *token, double prob, double *existing)
Write a gate's probability from in-extension C/C++ code.
bool provsql_store_cancel_requested(void)
Whether the circuit being built should be given up: in the worker, the backend that asked for it was ...
char buffer[PIPE_BUF]
Shared write buffer used with STARTWRITEM / ADDWRITEM / SENDWRITEM.
bool provsql_internal_get_prob_written(const pg_uuid_t *token, double *prob)
Report whether a probability has been written on a gate.
void provsql_internal_clear_prob(const pg_uuid_t *token)
Drop a gate's probability, leaving it as it was before anyone wrote one.
void provsql_internal_get_infos(const pg_uuid_t *token, unsigned *info1, unsigned *info2)
Read a gate's info1 and info2, through the worker.
Datum get_children(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for get_children().
bool provsql_wait_reply(void)
Wait, under the store lock, until the worker's reply to this backend's request can be read.
static void provsql_log_store_write(const char *data, size_t len)
What every store mutation does before it reaches the pipe: refuse it on a standby,...
Datum get_extra(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for get_extra().
char * provsql_internal_get_extra(const pg_uuid_t *token)
Read a gate's extra text, through the worker; palloc'd.
static provsql_set_prob_result provsql_send_set_prob(const pg_uuid_t *token, double prob, double *existing, bool tracked)
Send a probability write and read back what the store made of it.
static char worker_buffer[WORKER_READ_BUFFER]
Datum get_prob(PG_FUNCTION_ARGS)
PostgreSQL-callable wrapper for get_prob().
unsigned bufferpos
Current write position within buffer.
void RegisterProvSQLMMapWorker(void)
Register the ProvSQL mmap background worker with PostgreSQL.
bool provsql_worker_from_spool
Whether the message being applied comes from the replay spool: it is not answered,...
Background worker and IPC primitives for mmap-backed circuit storage.
#define ADDWRITEDB()
Append the per-message database header to the write buffer.
#define READB_BYTES(ptr, n)
Read exactly n bytes of a reply from the main-to-background pipe.
provsql_set_prob_result
Outcome of a probability write, mirroring MMappedCircuit::SetProbResult across the IPC boundary.
#define READB(var, type)
Read one value of type from the main-to-background pipe.
#define STARTWRITEM()
Reset the shared write buffer for a new batched write.
#define ADDWRITEM(pvar, type)
Append one value of type to the shared write buffer.
#define SENDWRITEM()
Flush the shared write buffer to the background-to-main pipe atomically.
#define PROVSQL_REPLAY_SPOOL
Where WAL replay writes the store messages while the worker is not running, relative to the data dire...
void provsql_wal_log_store_message(const char *data, size_t len)
Write one store message to the WAL, if WAL logging is on.
bool provsql_store_write_allowed(void)
Whether this process may write to the circuit store.
WAL-logging the circuit store: the entry points other files use.
void provsql_shmem_unlock(void)
Release the ProvSQL LWLock.
void provsql_shmem_lock_exclusive(void)
Acquire the ProvSQL LWLock in exclusive mode.
provsqlSharedState * provsql_shared_state
Pointer to the ProvSQL shared-memory segment (set in provsql_shmem_startup).
void provsql_shmem_lock_exclusive_wait(void)
Acquire the ProvSQL LWLock in exclusive mode, waiting whatever happens: for the background worker,...
void provsql_shmem_lock_shared(void)
Acquire the ProvSQL LWLock in shared mode.
Shared-memory segment and inter-process pipe management.
constants_t get_constants(bool failure_if_not_possible)
Retrieve the cached OID constants for the current database.
Core types, constants, and utilities shared across ProvSQL.
Structure to store the value of various constants.
Oid GATE_TYPE_TO_OID[nb_gate_types]
Array of the OID of each provenance_gate ENUM value.
Oid OID_TYPE_UUID
OID of the uuid TYPE.
What provsql.circuit_cleanup() reports.
uint64 wires_before
Child wires before.
uint64 wires_after
Child wires kept.
uint64 extra_before
Annotation bytes before.
uint64 gates_after
Gate records kept.
uint64 extra_after
Annotation bytes kept.
uint64 gates_before
Gate records before.