Skip to content

Commit bda82ee

Browse files
tea-artistgithub-actions[bot]caoxing9boris-wJocky-Teable
authored
[sync] feat: v2 stability improve (#3097)
Synced from teableio/teable-ee@e761db1 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Aries X <caoxing9@gmail.com> Co-authored-by: Boris <boris2code@outlook.com> Co-authored-by: Jocky-Teable <jocky@teable.ai> Co-authored-by: Jun Lu <hammond@teable.io> Co-authored-by: Pengap <penganpingprivte@gmail.com> Co-authored-by: SkyHuang <sky.huang.fe@gmail.com> Co-authored-by: Uno <uno@teable.ai> Co-authored-by: nichenqin <nichenqin@hotmail.com>
1 parent ceba9a7 commit bda82ee

527 files changed

Lines changed: 29897 additions & 4886 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,8 +238,11 @@ db-migration: ## Reruns the existing migration history in the shadow database i
238238

239239
postgres.mode: ## postgres.mode
240240
@cd ./packages/db-main-prisma; \
241-
pnpm prisma-generate --schema ./prisma/postgres/schema.prisma; \
241+
pnpm prisma-generate; \
242242
pnpm prisma-migrate deploy --schema ./prisma/postgres/schema.prisma
243+
@cd ./packages/db-data-prisma; \
244+
pnpm prisma-generate; \
245+
pnpm prisma-migrate deploy --schema ./prisma/schema.prisma
243246
# Override environment variable files based on variables
244247
RUN_DB_MODE ?= postgres
245248
FILE_ENV_PATHS = $(ENV_PATH)/.env.development* $(ENV_PATH)/.env.test*

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,4 +232,4 @@ In essence, Teable isn't just another no-code solution, it's a comprehensive ans
232232

233233
Teable Community Edition (CE) is free for self-hosting under the AGPL license. See [./LICENSE](./LICENSE) for details.
234234

235-
Teable Enterprise Edition (EE) includes advanced features such as AI, authority matrix, automation and advanced admin. For detailed information and pricing, please visit [pricing](https://app.teable.ai/public/pricing?host=self-hosted&billing=year).
235+
Teable Enterprise Edition (EE) includes advanced features such as AI, authority matrix, automation and advanced admin. For detailed information and pricing, please visit [pricing](https://teable.ai/pricing).

‎apps/nestjs-backend/package.json‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
"test-unit:watch": "vitest --watch",
4444
"test-unit": "vitest run --silent --bail 1",
4545
"test-unit-cover": "pnpm test-unit --coverage ${VITEST_SHARD:+--shard=$VITEST_SHARD}",
46+
"dual-db-cutover-validate": "node ./scripts/validate-dual-db-cutover.mjs",
4647
"pre-test-e2e": "cross-env NODE_ENV=test pnpm -F @teable/db-main-prisma prisma-db-seed -- --e2e",
4748
"test-e2e": "pnpm pre-test-e2e && vitest run --config ./vitest-e2e.config.ts --silent",
4849
"test-e2e-cover": "pnpm test-e2e --coverage --bail 1 ${VITEST_SHARD:+--shard=$VITEST_SHARD}",
@@ -123,17 +124,18 @@
123124
"webpack": "5.91.0"
124125
},
125126
"dependencies": {
126-
"@ai-sdk/amazon-bedrock": "4.0.69",
127-
"@ai-sdk/anthropic": "3.0.50",
128-
"@ai-sdk/azure": "3.0.38",
129-
"@ai-sdk/cohere": "3.0.22",
130-
"@ai-sdk/deepseek": "2.0.21",
131-
"@ai-sdk/google": "3.0.34",
132-
"@ai-sdk/mistral": "3.0.21",
133-
"@ai-sdk/openai": "3.0.37",
134-
"@ai-sdk/openai-compatible": "2.0.31",
135-
"@ai-sdk/togetherai": "2.0.35",
136-
"@ai-sdk/xai": "3.0.60",
127+
"@ai-sdk/amazon-bedrock": "4.0.97",
128+
"@ai-sdk/anthropic": "3.0.72",
129+
"@ai-sdk/azure": "3.0.55",
130+
"@ai-sdk/cohere": "3.0.31",
131+
"@ai-sdk/deepseek": "2.0.30",
132+
"@ai-sdk/google": "3.0.65",
133+
"@ai-sdk/mistral": "3.0.31",
134+
"@ai-sdk/openai": "3.0.54",
135+
"@ai-sdk/openai-compatible": "2.0.42",
136+
"@ai-sdk/provider": "3.0.9",
137+
"@ai-sdk/togetherai": "2.0.46",
138+
"@ai-sdk/xai": "3.0.84",
137139
"@an-epiphany/websocket-json-stream": "1.2.0",
138140
"@aws-sdk/client-s3": "3.609.0",
139141
"@aws-sdk/lib-storage": "3.609.0",
@@ -154,7 +156,7 @@
154156
"@nestjs/swagger": "7.3.0",
155157
"@nestjs/terminus": "10.2.3",
156158
"@nestjs/websockets": "10.3.5",
157-
"@openrouter/ai-sdk-provider": "2.2.3",
159+
"@openrouter/ai-sdk-provider": "2.8.1",
158160
"@opentelemetry/api": "1.9.0",
159161
"@opentelemetry/context-async-hooks": "2.5.0",
160162
"@opentelemetry/exporter-logs-otlp-http": "0.201.1",
@@ -180,6 +182,7 @@
180182
"@smithy/node-http-handler": "^3.1.1",
181183
"@teable/common-i18n": "workspace:^",
182184
"@teable/core": "workspace:^",
185+
"@teable/db-data-prisma": "workspace:^",
183186
"@teable/db-main-prisma": "workspace:^",
184187
"@teable/openapi": "workspace:^",
185188
"@teable/v2-adapter-db-postgres-pg": "workspace:*",
@@ -193,7 +196,7 @@
193196
"@teable/v2-di": "workspace:*",
194197
"@teable/v2-import": "workspace:*",
195198
"@valibot/to-json-schema": "1.3.0",
196-
"ai": "6.0.105",
199+
"ai": "6.0.169",
197200
"ajv": "8.12.0",
198201
"archiver": "7.0.1",
199202
"axios": "1.7.7",
@@ -241,7 +244,7 @@
241244
"oauth2orize": "1.12.0",
242245
"oauth2orize-pkce": "0.1.2",
243246
"object-sizeof": "2.6.4",
244-
"ollama-ai-provider-v2": "3.0.2",
247+
"ollama-ai-provider-v2": "3.5.0",
245248
"p-limit": "3.1.0",
246249
"papaparse": "5.4.1",
247250
"passport": "0.7.0",

‎apps/nestjs-backend/scripts/validate-dual-db-cutover.mjs‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
#!/usr/bin/env node
2+
import pg from 'pg';
3+
4+
const { Client } = pg;
5+
6+
const DATA_PLANE_TABLES = [
7+
'computed_update_outbox',
8+
'computed_update_outbox_seed',
9+
'computed_update_dead_letter',
10+
'computed_update_pause_scope',
11+
'record_history',
12+
'table_trash',
13+
'record_trash',
14+
'__undo_log',
15+
];
16+
17+
const META_PLANE_TABLES = ['base', 'table_meta', 'field'];
18+
19+
const usage = `Usage:
20+
node ./scripts/validate-dual-db-cutover.mjs --source <single-db-url> --meta <meta-db-url> --data <data-db-url> [--schema-prefix bse]
21+
22+
Environment fallback:
23+
DUAL_DB_SOURCE_URL
24+
DUAL_DB_TARGET_META_URL
25+
DUAL_DB_TARGET_DATA_URL
26+
DUAL_DB_SCHEMA_PREFIX
27+
`;
28+
29+
const parseArgs = (argv) => {
30+
const args = {};
31+
for (let index = 0; index < argv.length; index += 1) {
32+
const arg = argv[index];
33+
if (!arg.startsWith('--')) {
34+
continue;
35+
}
36+
const key = arg.slice(2);
37+
const value = argv[index + 1];
38+
if (!value || value.startsWith('--')) {
39+
throw new Error(`Missing value for --${key}`);
40+
}
41+
args[key] = value;
42+
index += 1;
43+
}
44+
return args;
45+
};
46+
47+
const formatList = (items) => (items.length ? items.join(', ') : '(none)');
48+
49+
const diffSets = (source, target) => {
50+
const sourceSet = new Set(source);
51+
const targetSet = new Set(target);
52+
return {
53+
missingInTarget: source.filter((item) => !targetSet.has(item)),
54+
extraInTarget: target.filter((item) => !sourceSet.has(item)),
55+
};
56+
};
57+
58+
const getSchemas = async (client, schemaPrefix) => {
59+
const result = await client.query(
60+
`
61+
SELECT schema_name
62+
FROM information_schema.schemata
63+
WHERE schema_name LIKE $1
64+
ORDER BY schema_name
65+
`,
66+
[`${schemaPrefix}%`]
67+
);
68+
return result.rows.map((row) => row.schema_name);
69+
};
70+
71+
const getSchemaTableCounts = async (client, schemaPrefix) => {
72+
const result = await client.query(
73+
`
74+
SELECT table_schema AS schema_name, COUNT(*)::int AS table_count
75+
FROM information_schema.tables
76+
WHERE table_type = 'BASE TABLE'
77+
AND table_schema LIKE $1
78+
GROUP BY table_schema
79+
ORDER BY table_schema
80+
`,
81+
[`${schemaPrefix}%`]
82+
);
83+
return new Map(result.rows.map((row) => [row.schema_name, Number(row.table_count)]));
84+
};
85+
86+
const getTableCount = async (client, tableName) => {
87+
const existsResult = await client.query(`SELECT to_regclass($1) AS relation_name`, [
88+
`public.${tableName}`,
89+
]);
90+
if (!existsResult.rows[0]?.relation_name) {
91+
return null;
92+
}
93+
const result = await client.query(`SELECT COUNT(*)::bigint AS count FROM public."${tableName}"`);
94+
return Number(result.rows[0]?.count ?? 0);
95+
};
96+
97+
const getTableCounts = async (client, tableNames) => {
98+
const entries = await Promise.all(
99+
tableNames.map(async (tableName) => [tableName, await getTableCount(client, tableName)])
100+
);
101+
return new Map(entries);
102+
};
103+
104+
const getFunctionExists = async (client, functionName) => {
105+
const result = await client.query(
106+
`
107+
SELECT EXISTS (
108+
SELECT 1
109+
FROM pg_proc p
110+
JOIN pg_namespace n ON n.oid = p.pronamespace
111+
WHERE n.nspname = 'public'
112+
AND p.proname = $1
113+
) AS exists
114+
`,
115+
[functionName]
116+
);
117+
return Boolean(result.rows[0]?.exists);
118+
};
119+
120+
const compareCountMaps = (source, target) => {
121+
const mismatches = [];
122+
const keys = new Set([...source.keys(), ...target.keys()]);
123+
for (const key of [...keys].sort()) {
124+
const sourceCount = source.get(key) ?? null;
125+
const targetCount = target.get(key) ?? null;
126+
if (sourceCount !== targetCount) {
127+
mismatches.push({ key, sourceCount, targetCount });
128+
}
129+
}
130+
return mismatches;
131+
};
132+
133+
const logSection = (title) => {
134+
console.log(`\n[${title}]`);
135+
};
136+
137+
const main = async () => {
138+
const args = parseArgs(process.argv.slice(2));
139+
const sourceUrl = args.source ?? process.env.DUAL_DB_SOURCE_URL;
140+
const metaUrl = args.meta ?? process.env.DUAL_DB_TARGET_META_URL;
141+
const dataUrl = args.data ?? process.env.DUAL_DB_TARGET_DATA_URL;
142+
const schemaPrefix = args['schema-prefix'] ?? process.env.DUAL_DB_SCHEMA_PREFIX ?? 'bse';
143+
144+
if (!sourceUrl || !metaUrl || !dataUrl) {
145+
console.error(usage);
146+
throw new Error('Missing one or more required database urls');
147+
}
148+
149+
const sourceClient = new Client({ connectionString: sourceUrl });
150+
const metaClient = new Client({ connectionString: metaUrl });
151+
const dataClient = new Client({ connectionString: dataUrl });
152+
153+
await Promise.all([sourceClient.connect(), metaClient.connect(), dataClient.connect()]);
154+
155+
try {
156+
const [sourceSchemas, targetSchemas, sourceSchemaTableCounts, targetSchemaTableCounts] =
157+
await Promise.all([
158+
getSchemas(sourceClient, schemaPrefix),
159+
getSchemas(dataClient, schemaPrefix),
160+
getSchemaTableCounts(sourceClient, schemaPrefix),
161+
getSchemaTableCounts(dataClient, schemaPrefix),
162+
]);
163+
164+
const schemaDiff = diffSets(sourceSchemas, targetSchemas);
165+
const schemaTableCountDiffs = compareCountMaps(sourceSchemaTableCounts, targetSchemaTableCounts);
166+
167+
const [sourceDataCounts, targetDataCounts, sourceMetaCounts, targetMetaCounts, undoFunctionExists] =
168+
await Promise.all([
169+
getTableCounts(sourceClient, DATA_PLANE_TABLES),
170+
getTableCounts(dataClient, DATA_PLANE_TABLES),
171+
getTableCounts(sourceClient, META_PLANE_TABLES),
172+
getTableCounts(metaClient, META_PLANE_TABLES),
173+
getFunctionExists(dataClient, '__teable_capture_undo_row'),
174+
]);
175+
176+
const dataCountDiffs = compareCountMaps(sourceDataCounts, targetDataCounts);
177+
const metaCountDiffs = compareCountMaps(sourceMetaCounts, targetMetaCounts);
178+
179+
logSection('Schema Summary');
180+
console.log(`source ${schemaPrefix}* schemas: ${sourceSchemas.length}`);
181+
console.log(`target ${schemaPrefix}* schemas: ${targetSchemas.length}`);
182+
console.log(`missing in target: ${formatList(schemaDiff.missingInTarget)}`);
183+
console.log(`extra in target: ${formatList(schemaDiff.extraInTarget)}`);
184+
185+
logSection('Per-Schema Table Count');
186+
if (!schemaTableCountDiffs.length) {
187+
console.log('all matched');
188+
} else {
189+
for (const diff of schemaTableCountDiffs.slice(0, 20)) {
190+
console.log(
191+
`${diff.key}: source=${String(diff.sourceCount ?? 'missing')} target=${String(diff.targetCount ?? 'missing')}`
192+
);
193+
}
194+
if (schemaTableCountDiffs.length > 20) {
195+
console.log(`... ${schemaTableCountDiffs.length - 20} more mismatches`);
196+
}
197+
}
198+
199+
logSection('Meta Table Row Count');
200+
if (!metaCountDiffs.length) {
201+
console.log('all matched');
202+
} else {
203+
for (const diff of metaCountDiffs) {
204+
console.log(
205+
`${diff.key}: source=${String(diff.sourceCount ?? 'missing')} target=${String(diff.targetCount ?? 'missing')}`
206+
);
207+
}
208+
}
209+
210+
logSection('Data-Plane Table Row Count');
211+
if (!dataCountDiffs.length) {
212+
console.log('all matched');
213+
} else {
214+
for (const diff of dataCountDiffs) {
215+
console.log(
216+
`${diff.key}: source=${String(diff.sourceCount ?? 'missing')} target=${String(diff.targetCount ?? 'missing')}`
217+
);
218+
}
219+
}
220+
221+
logSection('Data-Plane Function');
222+
console.log(
223+
`public.__teable_capture_undo_row: ${undoFunctionExists ? 'present on target data db' : 'missing on target data db'}`
224+
);
225+
226+
const hasMismatch =
227+
schemaDiff.missingInTarget.length > 0 ||
228+
schemaDiff.extraInTarget.length > 0 ||
229+
schemaTableCountDiffs.length > 0 ||
230+
metaCountDiffs.length > 0 ||
231+
dataCountDiffs.length > 0 ||
232+
!undoFunctionExists;
233+
234+
if (hasMismatch) {
235+
process.exitCode = 1;
236+
console.error('\nDual-db cutover validation failed.');
237+
return;
238+
}
239+
240+
console.log('\nDual-db cutover validation passed.');
241+
} finally {
242+
await Promise.allSettled([sourceClient.end(), metaClient.end(), dataClient.end()]);
243+
}
244+
};
245+
246+
main().catch((error) => {
247+
console.error(error instanceof Error ? error.message : error);
248+
process.exit(1);
249+
});

‎apps/nestjs-backend/src/cache/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export interface ICacheStore {
3636
[key: `oauth:token-rate:${string}:${string}`]: number;
3737
[key: `automation:email:rate:${string}:${number}`]: number;
3838
[key: `automation:email-att:${string}`]: string[];
39+
[key: `automation:fail-notify-count:${string}`]: number;
3940
// Distributed lock keys
4041
[key: `lock:${string}`]: string;
4142
[key: `import:result:manifest:${string}`]: {

0 commit comments

Comments
 (0)