feat(W4): daily full reconciliation — user/role drift repair + ReconcileLogs audit
- w4-body.js: ONE-DIRECTIONAL sync (NocoDB status = SOT, never reverse);
drift matrix: missing user/role (provision w/ username lookup to avoid
duplicates), dangling refs (adopt/recreate), disabled<->status, canonical
base_path '/' + permission 16384, scopes vs calculateEffectiveGrants,
username drift detect+log_only; verify-after-repair on every change;
password never pushed on repair (方案 C); dry-run via ?dryRun=true
- gen_w4.js: daily 03:00 cron + webhook alist-w4-reconcile; API-import-safe
(UUID node ids, no pinData, webhookId)
- ReconcileLogs table mktsb7le6fqr2u1 (add_reconcilelogs_table.py, idempotent,
v2 API); FK column nc_jitu___Customers_id for bt-link writes (v2 ignores
link column itself)
- test_w4_reconcile.js: 34 assertions / 12 mock worlds, ALL PASSED
- live-verified against production: dry-run (3 drifts + 1 error found),
real repair (3 repaired, verify all passed, 4 log rows), idempotent rerun
- NocoDB 2026.08.x pitfalls documented (v1 col endpoints gone, row delete
via body [{Id}]); n8n API import/activation pitfalls (activeVersionId)
- docs: W4-DESIGN.md new; DATABASE_SCHEMA 8 tables; N8N_WORKFLOWS, SETUP,
PROJECT_STATE updated
This commit is contained in:
+377
@@ -0,0 +1,377 @@
|
||||
// ============================================================================
|
||||
// W4 runtime body — Daily full reconciliation + drift repair + reconcile log.
|
||||
// Rebuilt by `node n8n/gen_w4.js` (single source of truth = shared/effective-grants.js).
|
||||
//
|
||||
// ONE-DIRECTIONAL sync: NocoDB `Customers.status` is the BUSINESS source of
|
||||
// truth; AList user enabled/disabled is only ENFORCEMENT state. W4 NEVER
|
||||
// writes Customers.status. A manual `disabled` flip in the AList admin UI is
|
||||
// repaired back to match NocoDB — never the other way round.
|
||||
//
|
||||
// Drift matrix (auto-repair):
|
||||
// status active -> user must exist (provision if missing; lookup by
|
||||
// username FIRST to avoid duplicate users when the
|
||||
// NocoDB id was lost), be ENABLED, carry the desired
|
||||
// role scopes (calculateEffectiveGrants), base_path
|
||||
// '/', permission 16384.
|
||||
// status non-active -> if a user exists: must be DISABLED with empty role
|
||||
// scopes (soft state kept). pending/never-provisioned
|
||||
// customers stay WITHOUT an AList user (no auto-create).
|
||||
// dangling refs -> alist_user_id / alist_role_id point at deleted AList
|
||||
// records: lookup by username / role name, adopt if
|
||||
// found, recreate if not, then re-sync.
|
||||
// username drift -> NocoDB username differs from AList username: DETECT
|
||||
// + log only (no safe official rename API assumed; no
|
||||
// destructive migration).
|
||||
//
|
||||
// Every repair is VERIFIED by re-reading AList state afterwards; a failed
|
||||
// verify is logged as verify_failed. ReconcileLogs receives ONLY
|
||||
// drift/repair/error rows — healthy customers produce nothing.
|
||||
//
|
||||
// Triggers: daily cron (03:00) + manual webhook. Dry run: webhook
|
||||
// `?dryRun=true` or env W4_DRY_RUN=true -> plan only, ZERO writes.
|
||||
// ============================================================================
|
||||
// __SHARED_INLINE__ (replaced by shared/effective-grants.js source at build)
|
||||
// ============================================================================
|
||||
|
||||
const ctx = { $env, helpers: this.helpers };
|
||||
|
||||
const query = ($json && $json.query) || {};
|
||||
const DRY_RUN = query.dryRun === 'true' || query.dryRun === true ||
|
||||
String($env.W4_DRY_RUN || '').toLowerCase() === 'true';
|
||||
const T_CUST = $env.NOCODB_TABLE_CUSTOMERS;
|
||||
const T_LOG = $env.NOCODB_TABLE_RECONCILELOGS;
|
||||
// NocoDB v2 (2026.08.x): bt links are NOT created by writing the link column
|
||||
// (scalar/array/object all ignored). Write the FK column directly:
|
||||
// `nc_jitu___Customers_id` (verified live 2026-08-23).
|
||||
const FK_CUST = $env.NOCODB_FK_RECONCILELOG_CUSTOMER || 'nc_jitu___Customers_id';
|
||||
const runId = 'recon-' + new Date().toISOString().replace(/[:.]/g, '-');
|
||||
|
||||
// canonical AList user configuration — must stay identical to W1's (verified
|
||||
// working: base_path '/' + absolute role scopes + permission 16384)
|
||||
const BASE_PATH = '/';
|
||||
const PERMISSION = 16384;
|
||||
|
||||
const logRows = []; // ReconcileLogs payload — drift/repair/error rows ONLY
|
||||
|
||||
const summary = {
|
||||
runId, dryRun: DRY_RUN,
|
||||
customersScanned: 0, driftsDetected: 0, repaired: 0, skipped: 0,
|
||||
errors: [], details: [],
|
||||
};
|
||||
|
||||
function pushLog(cid, username, driftType, details, action, result) {
|
||||
const row = {
|
||||
run_id: runId,
|
||||
drift_type: driftType,
|
||||
details: details == null ? '' : String(details).slice(0, 2000),
|
||||
action,
|
||||
result,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
if (cid != null) row[FK_CUST] = cid; // FK column (link writes are ignored)
|
||||
logRows.push(row);
|
||||
}
|
||||
|
||||
// ---- AList helpers (read) ----
|
||||
async function listUsers(token) {
|
||||
const r = await alistReq(ctx, 'GET', '/api/admin/user/list?page=1&per_page=1000', null, token);
|
||||
return (r && (r.data?.content || r.data?.list)) || [];
|
||||
}
|
||||
async function listRoles(token) {
|
||||
const r = await alistReq(ctx, 'GET', '/api/admin/role/list?page=1&per_page=1000', null, token);
|
||||
return (r && (r.data?.content || r.data?.list)) || [];
|
||||
}
|
||||
async function getUser(token, id) {
|
||||
const r = await alistReq(ctx, 'GET', `/api/admin/user/get?id=${id}`, null, token);
|
||||
return (r && r.data) ? r.data : null;
|
||||
}
|
||||
async function getRole(token, id) {
|
||||
const r = await alistReq(ctx, 'GET', `/api/admin/role/get?id=${id}`, null, token);
|
||||
return (r && r.data) ? r.data : null;
|
||||
}
|
||||
|
||||
// ---- AList helpers (write) — all no-op under DRY_RUN ----
|
||||
async function createRole(token, name, description) {
|
||||
if (DRY_RUN) return { id: null, name, permission_scopes: [] };
|
||||
await alistReq(ctx, 'POST', '/api/admin/role/create', {
|
||||
name, description, default: false, permission_scopes: [],
|
||||
}, token);
|
||||
const roles = await listRoles(token);
|
||||
const r = roles.find(x => x.name === name);
|
||||
if (!r) throw new Error('Created role not found: ' + name);
|
||||
return r;
|
||||
}
|
||||
async function createUser(token, body) {
|
||||
if (DRY_RUN) return { id: null, username: body.username };
|
||||
await alistReq(ctx, 'POST', '/api/admin/user/create', body, token);
|
||||
const users = await listUsers(token);
|
||||
const u = users.find(x => x.username === body.username);
|
||||
if (!u) throw new Error('Created user not found: ' + body.username);
|
||||
return u;
|
||||
}
|
||||
async function updateUser(token, fields) {
|
||||
if (DRY_RUN) return;
|
||||
await alistReq(ctx, 'POST', '/api/admin/user/update', fields, token);
|
||||
}
|
||||
async function updateRoleScopes(token, roleId, name, description, scopes) {
|
||||
if (DRY_RUN) return;
|
||||
await alistReq(ctx, 'POST', '/api/admin/role/update', {
|
||||
id: roleId, name, description, permission_scopes: scopes,
|
||||
}, token);
|
||||
}
|
||||
|
||||
// ---- NocoDB bookkeeping write-back (adopted user/role ids). Echo-safe: only
|
||||
// alist_user_id / alist_role_id are written; W1's old/new-field comparison
|
||||
// treats these as bookkeeping -> echo guard short-circuits. ----
|
||||
async function writeBack(cid, fields) {
|
||||
if (DRY_RUN || !Object.keys(fields).length) return;
|
||||
const live = await ncGet(ctx, `/api/v2/tables/${T_CUST}/records/${cid}`);
|
||||
const changes = {};
|
||||
for (const k of Object.keys(fields)) {
|
||||
if ((live[k] ?? null) !== (fields[k] ?? null)) changes[k] = fields[k];
|
||||
}
|
||||
if (!Object.keys(changes).length) return; // value unchanged -> no PATCH -> no webhook
|
||||
await ctx.helpers.httpRequest({
|
||||
method: 'PATCH',
|
||||
url: `${$env.NOCODB_URL}/api/v2/tables/${T_CUST}/records`,
|
||||
headers: { 'xc-token': $env.NOCODB_TOKEN },
|
||||
body: [Object.assign({ Id: cid }, changes)],
|
||||
json: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ---- verification (post-repair read-back) ----
|
||||
async function verifyUserState(token, userId, expect) {
|
||||
const u = await getUser(token, userId);
|
||||
if (!u) return { ok: false, actual: null };
|
||||
for (const k of Object.keys(expect)) {
|
||||
const want = expect[k];
|
||||
const have = u[k];
|
||||
const wantS = Array.isArray(want) ? JSON.stringify(want) : want;
|
||||
const haveS = Array.isArray(have) ? JSON.stringify(have) : have;
|
||||
if (String(haveS ?? '') !== String(wantS ?? '')) return { ok: false, actual: u };
|
||||
}
|
||||
return { ok: true, actual: u };
|
||||
}
|
||||
async function verifyRoleScopes(token, roleId, expectScopes) {
|
||||
const r = await getRole(token, roleId);
|
||||
if (!r) return { ok: false, actual: null };
|
||||
const cur = (r.permission_scopes || []).map(s => s.path).sort();
|
||||
const exp = expectScopes.map(s => s.path).sort();
|
||||
const ok = cur.length === exp.length && cur.every((p, i) => p === exp[i]);
|
||||
return { ok, actual: r };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// per-customer reconciliation
|
||||
// ============================================================================
|
||||
async function reconcileOne(cust, token) {
|
||||
const cid = cust.Id;
|
||||
const username = cust.username;
|
||||
const status = cust.status || 'active'; // W1 convention: missing = active
|
||||
const isActive = status === 'active';
|
||||
const drifts = []; // {type, action, result}
|
||||
|
||||
const note = (type, details, action, result) => {
|
||||
drifts.push({ type, action, result });
|
||||
pushLog(cid, username, type, details, action, result);
|
||||
};
|
||||
|
||||
// ---------- 1. resolve the AList user ----------
|
||||
let user = null;
|
||||
if (cust.alist_user_id) {
|
||||
user = await getUser(token, cust.alist_user_id);
|
||||
if (user && user.username !== username) {
|
||||
note('username_drift',
|
||||
JSON.stringify({ noco: username, alist: user.username, user_id: user.id }),
|
||||
'log_only', 'log_only');
|
||||
}
|
||||
}
|
||||
if (!user) {
|
||||
// id missing or dangling: lookup by username to avoid duplicate users
|
||||
const byName = (await listUsers(token)).find(u => u.username === username);
|
||||
if (byName) {
|
||||
user = byName;
|
||||
if (cust.alist_user_id !== byName.id) {
|
||||
note('dangling_user_ref',
|
||||
JSON.stringify({ ref: cust.alist_user_id, adopted: byName.id, found_by: 'username' }),
|
||||
'adopted', 'repaired');
|
||||
await writeBack(cid, { alist_user_id: byName.id });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!user && isActive) {
|
||||
// genuinely missing: provision role first, then user (empty role -> 400)
|
||||
let role = null;
|
||||
const roleName = 'customer-' + username;
|
||||
const byName = (await listRoles(token)).find(r => r.name === roleName);
|
||||
if (byName) role = byName;
|
||||
else if (!DRY_RUN) role = await createRole(token, roleName, 'Auto role for ' + username);
|
||||
else role = { id: null, name: roleName, permission_scopes: [] };
|
||||
user = await createUser(token, {
|
||||
username,
|
||||
password: cust.password || 'change_me',
|
||||
base_path: BASE_PATH,
|
||||
role: role.id ? [role.id] : [],
|
||||
permission: PERMISSION,
|
||||
});
|
||||
note('user_missing',
|
||||
JSON.stringify({ alist_user_id: cust.alist_user_id ?? null, alist_role_id: cust.alist_role_id ?? null }),
|
||||
DRY_RUN ? 'provision_planned' : 'provisioned', 'repaired');
|
||||
if (role.id) await writeBack(cid, { alist_role_id: role.id });
|
||||
}
|
||||
if (!user) {
|
||||
// non-active and never provisioned -> pure label, keep NO AList user.
|
||||
// (Healthy state: no drift, no log row.)
|
||||
return drifts;
|
||||
}
|
||||
if (user.id) await writeBack(cid, { alist_user_id: user.id });
|
||||
|
||||
// ---------- 2. resolve the role (the user's role binding) ----------
|
||||
const roleFor = async (wantName) => {
|
||||
const roleName = wantName || ('customer-' + username);
|
||||
const bound = Array.isArray(user.role) ? user.role[0] : (user.role || null);
|
||||
if (bound) {
|
||||
const r = await getRole(token, bound);
|
||||
if (r) return r;
|
||||
const byName = (await listRoles(token)).find(x => x.name === roleName);
|
||||
note('dangling_role_ref',
|
||||
JSON.stringify({ ref: bound, adopted: byName ? byName.id : null, want: roleName }),
|
||||
byName ? 'adopted' : 'recreate_required', 'repaired');
|
||||
if (byName) return byName;
|
||||
}
|
||||
const byName = (await listRoles(token)).find(x => x.name === roleName);
|
||||
if (byName) return byName;
|
||||
if (isActive) {
|
||||
const created = await createRole(token, roleName, 'Auto role for ' + username);
|
||||
note('role_missing', JSON.stringify({ roleName }), 'created', 'repaired');
|
||||
return created;
|
||||
}
|
||||
return null; // non-active with no role: nothing to clear
|
||||
};
|
||||
let role = await roleFor();
|
||||
|
||||
// ---------- 3. desired scopes (shared algorithm; also reads manual grants) ----------
|
||||
const g = await calculateEffectiveGrants(ctx, cid);
|
||||
const desiredScopes = isActive ? g.newScopes : [];
|
||||
|
||||
// ---------- 4. apply fixes (canonical state = W1's verified config) ----------
|
||||
const userPatches = [];
|
||||
const expectUser = {};
|
||||
if (!isActive) {
|
||||
if (user.disabled !== true) { userPatches.push('disabled'); note('user_disabled_drift', JSON.stringify({ have: user.disabled, want: true }), 'disabled', 'repaired'); }
|
||||
expectUser.disabled = true;
|
||||
} else {
|
||||
if (user.disabled === true) { userPatches.push('re-enabled'); note('user_disabled_drift', JSON.stringify({ have: true, want: false }), 'enabled', 'repaired'); }
|
||||
expectUser.disabled = false;
|
||||
}
|
||||
if (user.base_path !== BASE_PATH) {
|
||||
userPatches.push('base_path'); note('base_path_drift', JSON.stringify({ have: user.base_path, want: BASE_PATH }), 'fixed', 'repaired');
|
||||
}
|
||||
expectUser.base_path = BASE_PATH;
|
||||
if (user.permission !== PERMISSION) {
|
||||
userPatches.push('permission'); note('permission_drift', JSON.stringify({ have: user.permission, want: PERMISSION }), 'fixed', 'repaired');
|
||||
}
|
||||
expectUser.permission = PERMISSION;
|
||||
const boundRoleId = role && role.id;
|
||||
if (boundRoleId && !(Array.isArray(user.role) && user.role.includes(boundRoleId))) {
|
||||
userPatches.push('role_binding'); note('role_binding_drift', JSON.stringify({ have: user.role, want: [boundRoleId] }), 'rebound', 'repaired');
|
||||
}
|
||||
expectUser.role = boundRoleId ? [boundRoleId] : user.role || [];
|
||||
|
||||
if (userPatches.length) {
|
||||
// NOTE: password intentionally NOT included in repairs — NocoDB plaintext
|
||||
// may be stale (方案 C: customer may have changed their own AList password;
|
||||
// pushing it here would clobber it). Password only used at provision.
|
||||
await updateUser(token, Object.assign({ id: user.id }, {
|
||||
username, base_path: BASE_PATH, permission: PERMISSION, disabled: !isActive,
|
||||
role: boundRoleId ? [boundRoleId] : [],
|
||||
}));
|
||||
}
|
||||
|
||||
// scopes drift (role may be null for non-active with no role -> nothing to clear)
|
||||
if (role && role.id) {
|
||||
const cur = (role.permission_scopes || []).map(s => s.path).sort();
|
||||
const want = desiredScopes.map(s => s.path).sort();
|
||||
if (cur.length !== want.length || cur.some((p, i) => p !== want[i])) {
|
||||
note('scopes_drift',
|
||||
JSON.stringify({ have: cur, want }),
|
||||
DRY_RUN ? 'rewrite_planned' : 'rewritten', 'repaired');
|
||||
await updateRoleScopes(token, role.id, role.name, role.description, desiredScopes);
|
||||
}
|
||||
} else if (isActive && !DRY_RUN) {
|
||||
// active but could not bind a role (should not happen; roleFor creates one)
|
||||
throw new Error('Active customer has no role to write scopes into');
|
||||
}
|
||||
|
||||
// ---------- 5. verify actual AList state after every repair ----------
|
||||
if (!DRY_RUN && user.id) {
|
||||
const uu = await verifyUserState(token, user.id, expectUser);
|
||||
if (!uu.ok) {
|
||||
note('verify_failed', JSON.stringify({ expect: expectUser, actual: uu.actual }), 'verify', 'failed');
|
||||
}
|
||||
if (role && role.id) {
|
||||
const rr = await verifyRoleScopes(token, role.id, desiredScopes);
|
||||
if (!rr.ok) {
|
||||
note('verify_failed', JSON.stringify({ expect_scopes: desiredScopes, actual: rr.actual }), 'verify', 'failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return drifts;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// main sweep
|
||||
// ============================================================================
|
||||
const login = await alistReq(ctx, 'POST', '/api/auth/login', {
|
||||
username: $env.ALIST_ADMIN_USERNAME,
|
||||
password: $env.ALIST_ADMIN_PASSWORD,
|
||||
}, null);
|
||||
const token = login?.data?.token;
|
||||
if (!token) throw new Error('AList login failed');
|
||||
|
||||
const customers = (await ncGet(ctx, `/api/v2/tables/${T_CUST}/records?limit=10000`)).list || [];
|
||||
|
||||
for (const cust of customers) {
|
||||
if (!cust.username) {
|
||||
summary.errors.push({ customerId: cust.Id, username: null, error: 'missing username' });
|
||||
pushLog(cust.Id, null, 'reconcile_error', JSON.stringify({ row: cust.Id }), 'log_only', 'failed');
|
||||
summary.customersScanned++;
|
||||
continue;
|
||||
}
|
||||
summary.customersScanned++;
|
||||
const entry = { customerId: cust.Id, username: cust.username, status: cust.status || 'active', drifts: [] };
|
||||
try {
|
||||
entry.drifts = await reconcileOne(cust, token);
|
||||
if (entry.drifts.length === 0) summary.skipped++;
|
||||
else summary.details.push(entry);
|
||||
} catch (e) {
|
||||
summary.errors.push({ customerId: cust.Id, username: cust.username, error: e.message });
|
||||
pushLog(cust.Id, cust.username, 'reconcile_error', JSON.stringify({ error: e.message }), 'log_only', 'failed');
|
||||
}
|
||||
}
|
||||
|
||||
summary.driftsDetected = logRows.filter(r => r.result !== 'failed').length;
|
||||
summary.repaired = logRows.filter(r => r.result === 'repaired').length;
|
||||
|
||||
// ---- write ReconcileLogs (drift/repair/error rows only) ----
|
||||
if (logRows.length && !DRY_RUN) {
|
||||
if (!T_LOG) {
|
||||
summary.logWriteSkipped = 'NOCODB_TABLE_RECONCILELOGS env missing — rows not persisted';
|
||||
} else {
|
||||
await ctx.helpers.httpRequest({
|
||||
method: 'POST',
|
||||
url: `${$env.NOCODB_URL}/api/v2/tables/${T_LOG}/records`,
|
||||
headers: { 'xc-token': $env.NOCODB_TOKEN },
|
||||
body: logRows,
|
||||
json: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { json: Object.assign({}, summary, {
|
||||
at: new Date().toISOString(),
|
||||
logRowsWritten: DRY_RUN ? 0 : logRows.length,
|
||||
plannedLogRows: DRY_RUN ? logRows.length : 0,
|
||||
}) };
|
||||
Reference in New Issue
Block a user