From 29db2da5fccf42fd255bded58417e41d3cc40688 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 03:09:57 +0000 Subject: [PATCH] Add fix_pikapods_dump.py to the repo; cross-reference it from the migration script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extras/fix_pikapods_dump.py patches two confirmed Adminer PostgreSQL-export bugs that otherwise make a PikaPods Mattermost migration fail outright: unquoted enum-label DEFAULT values (Postgres reads the bare label as a column reference and rejects the CREATE TABLE) and boolean columns serialized as bare 0/1 instead of true/false (Postgres doesn't implicitly cast integers to boolean). Boolean columns are discovered by actually parsing each CREATE TABLE in the dump rather than working from a hand-curated list — Postgres only reports the first bad column per failed row, so a list built from error output alone would likely be incomplete. Already verified earlier this session against a real local Postgres 16 instance; reviewed now for anything needing redaction before committing — it's a generic text-processing tool with no hostnames, credentials, file paths, or personal data in it, so nothing needed changing. Cross-referenced from the generated migrate-from-pikapods.sh's header comment (services/mattermost.sh) so anyone hitting a CREATE TYPE/CREATE TABLE or boolean-column import error is pointed at the fix instead of having to rediscover it. --- extras/fix_pikapods_dump.py | 302 ++++++++++++++++++++++++++++++++++++ services/mattermost.sh | 9 ++ 2 files changed, 311 insertions(+) create mode 100755 extras/fix_pikapods_dump.py diff --git a/extras/fix_pikapods_dump.py b/extras/fix_pikapods_dump.py new file mode 100755 index 0000000..6c90409 --- /dev/null +++ b/extras/fix_pikapods_dump.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +""" +fix_pikapods_dump.py — Patches two confirmed Adminer PostgreSQL-export bugs +in a Mattermost SQL dump, before importing it via migrate-from-pikapods.sh. + +Bug 1: Adminer omits quotes around enum-label DEFAULT values, e.g. + DEFAULT link instead of DEFAULT 'link' + DEFAULT client_credentials instead of DEFAULT 'client_credentials' + This makes Postgres treat the label as a column reference, which is + illegal in a DEFAULT expression, so the whole CREATE TABLE fails. + +Bug 2: Adminer serializes PostgreSQL boolean columns as bare integer + literals (0 / 1) in INSERT statements instead of true/false. + Postgres does not implicitly cast integer literals to boolean, so + every row touching one of those columns is rejected. + +Bug 2's fix parses each CREATE TABLE in the dump to find every column +declared as `boolean` (not a hand-curated list from partial error +messages — Postgres only reports the FIRST bad column per row, so a +list built from error output alone would likely be incomplete). It then +rewrites only the VALUES-tuple positions that correspond to those +specific boolean columns, leaving every other value in the row +(including other literal 0/1 integers) untouched. + +Usage: + python3 fix_pikapods_dump.py input.sql output.sql + +Always writes to a NEW file — never modifies the input in place — so the +original export is preserved if something looks wrong afterward. +""" +import re +import sys + + +def split_top_level(s, sep=","): + """Split s on sep, but only outside single-quoted strings and outside + nested parens. '' inside a quoted string is the SQL escape for a + literal quote and does not end the string.""" + parts = [] + buf = [] + depth = 0 + in_quote = False + i = 0 + n = len(s) + while i < n: + c = s[i] + if in_quote: + if c == "'": + # doubled quote = escaped literal quote, stays in_quote + if i + 1 < n and s[i + 1] == "'": + buf.append("''") + i += 2 + continue + in_quote = False + buf.append(c) + i += 1 + continue + buf.append(c) + i += 1 + continue + if c == "'": + in_quote = True + buf.append(c) + i += 1 + continue + if c == "(": + depth += 1 + buf.append(c) + i += 1 + continue + if c == ")": + depth -= 1 + buf.append(c) + i += 1 + continue + if c == sep and depth == 0: + parts.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(c) + i += 1 + parts.append("".join(buf)) + return parts + + +def split_statements(sql_text): + """Split a whole dump into individual statements at semicolons that + are not inside a quoted string. Returns list of (statement_text, + trailing_terminator) so the exact original text can be reassembled + byte-for-byte from the pieces.""" + stmts = [] + buf = [] + in_quote = False + i = 0 + n = len(sql_text) + while i < n: + c = sql_text[i] + if in_quote: + buf.append(c) + if c == "'": + if i + 1 < n and sql_text[i + 1] == "'": + buf.append(sql_text[i + 1]) + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + buf.append(c) + i += 1 + continue + if c == ";": + buf.append(c) + stmts.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(c) + i += 1 + if buf: + stmts.append("".join(buf)) + return stmts + + +CREATE_TABLE_NAME_RE = re.compile( + r'CREATE TABLE\s+(?:"public"\.)?"([^"]+)"\s*\(', + re.IGNORECASE, +) +INSERT_RE = re.compile( + r'^(INSERT INTO\s+(?:"public"\.)?"([^"]+)"\s*\()([^)]*)\)\s*VALUES\s*(.*);\s*$', + re.IGNORECASE | re.DOTALL, +) + + +def find_matching_paren(s, open_idx): + """Return the index of the ')' matching the '(' at open_idx, skipping + over quoted strings so a paren inside a quoted default value (e.g. a + function call in a DEFAULT expression) doesn't miscount depth.""" + depth = 0 + in_quote = False + i = open_idx + n = len(s) + while i < n: + c = s[i] + if in_quote: + if c == "'": + if i + 1 < n and s[i + 1] == "'": + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + i += 1 + continue + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + +def parse_boolean_columns(create_table_stmt): + """Given a full CREATE TABLE statement (trailing syntax after the + closing paren — WITHOUT OIDS, TABLESPACE, etc — is fine, not assumed + absent), return the set of column names declared as boolean.""" + name_m = CREATE_TABLE_NAME_RE.search(create_table_stmt) + if not name_m: + return set() + open_idx = name_m.end() - 1 # the '(' the regex matched + close_idx = find_matching_paren(create_table_stmt, open_idx) + if close_idx == -1: + return set() + body = create_table_stmt[open_idx + 1 : close_idx] + cols = set() + for coldef in split_top_level(body): + coldef = coldef.strip() + cm = re.match(r'^"([^"]+)"\s+([A-Za-z_][A-Za-z0-9_]*)', coldef) + if cm and cm.group(2).lower() == "boolean": + cols.add(cm.group(1)) + return cols + + +def fix_default_quoting(stmt): + """Bug 1: unquoted enum-label DEFAULTs. Only touches DEFAULT clauses + that name one of the two enum types confirmed broken in this export + (channel_bookmark_type, outgoingoauthconnections_granttype) — narrow + and conservative rather than a blanket 'quote anything after DEFAULT' + rule that could misfire on legitimate unquoted defaults elsewhere + (numbers, now(), etc).""" + stmt, n1 = re.subn( + r"(channel_bookmark_type\s+DEFAULT\s+)([A-Za-z_][A-Za-z0-9_]*)(?=[,\)])", + r"\1'\2'", + stmt, + ) + stmt, n2 = re.subn( + r"(outgoingoauthconnections_granttype\s+DEFAULT\s+)([A-Za-z_][A-Za-z0-9_]*)(?=[,\)])", + r"\1'\2'", + stmt, + ) + return stmt, n1 + n2 + + +def patch_insert_booleans(stmt, table, col_list_raw, values_raw, bool_cols): + """Rewrite bare 0/1 literals to false/true at the positions in + col_list_raw that correspond to bool_cols. Returns (new_values_text, + count_of_values_changed).""" + col_names = [c.strip().strip('"') for c in split_top_level(col_list_raw)] + bool_positions = {i for i, c in enumerate(col_names) if c in bool_cols} + if not bool_positions: + return values_raw, 0 + + tuples = split_top_level(values_raw) + changed = 0 + new_tuples = [] + for tup in tuples: + tup_stripped = tup.strip() + if not (tup_stripped.startswith("(") and tup_stripped.endswith(")")): + new_tuples.append(tup) + continue + inner = tup_stripped[1:-1] + vals = split_top_level(inner) + for pos in bool_positions: + if pos >= len(vals): + continue + v = vals[pos].strip() + if v == "0": + vals[pos] = "false" + changed += 1 + elif v == "1": + vals[pos] = "true" + changed += 1 + prefix = tup[: len(tup) - len(tup.lstrip())] + suffix = tup[len(tup.rstrip()):] + new_tuples.append(prefix + "(" + ",".join(vals) + ")" + suffix) + return ",".join(new_tuples), changed + + +def process(input_path, output_path): + with open(input_path, "r", encoding="utf-8", errors="surrogateescape") as f: + text = f.read() + + statements = split_statements(text) + bool_cols_by_table = {} + out = [] + default_fixes = 0 + total_value_fixes = 0 + tables_patched = set() + + for stmt in statements: + stripped = stmt.strip() + upper = stripped.upper() + + if upper.startswith("CREATE TABLE"): + fixed_stmt, n = fix_default_quoting(stmt) + default_fixes += n + name_m = CREATE_TABLE_NAME_RE.search(fixed_stmt) + if name_m: + bool_cols_by_table[name_m.group(1)] = parse_boolean_columns(fixed_stmt) + out.append(fixed_stmt) + continue + + if upper.startswith("INSERT INTO"): + m = INSERT_RE.match(stripped) + if m: + prefix, table, col_list_raw, values_raw = m.groups() + bool_cols = bool_cols_by_table.get(table, set()) + if bool_cols: + new_values, n = patch_insert_booleans( + stripped, table, col_list_raw, values_raw, bool_cols + ) + if n: + total_value_fixes += n + tables_patched.add(table) + rebuilt = f'{prefix}{col_list_raw}) VALUES {new_values};' + out.append(rebuilt) + continue + out.append(stmt) + continue + + out.append(stmt) + + with open(output_path, "w", encoding="utf-8", errors="surrogateescape") as f: + f.write("".join(out)) + + print(f"DEFAULT-clause quoting fixes: {default_fixes}") + print(f"Boolean literal fixes: {total_value_fixes} across {len(tables_patched)} table(s)") + if tables_patched: + print("Tables patched: " + ", ".join(sorted(tables_patched))) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} input.sql output.sql", file=sys.stderr) + sys.exit(1) + process(sys.argv[1], sys.argv[2]) diff --git a/services/mattermost.sh b/services/mattermost.sh index 1d4c1c3..4a5a3bf 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -781,6 +781,15 @@ MD # default). If you have a custom-format pg_dump instead, use `pg_restore` # in place of the `psql < dump` step below. # +# Adminer's plain-text export has two confirmed bugs of its own — unquoted +# enum-label DEFAULTs, and boolean columns serialized as bare 0/1 instead +# of true/false — either of which makes the import below fail outright. +# extras/fix_pikapods_dump.py patches both; run it on the dump BEFORE this +# script if psql reports errors on CREATE TYPE/CREATE TABLE or boolean +# columns: +# python3 fix_pikapods_dump.py input.sql output.sql +# + # IMPORTANT: point the files argument at the SUBDIRECTORY that holds # Mattermost's own file storage inside whatever you downloaded via SFTP # (commonly named `data`), not the whole SFTP root — PikaPods' exact