Skip to content

Commit ebfb6e5

Browse files
elevatebartfarnabaz
elevatebart
and
farnabaz
authored
fix(collection): avoid double update of some record by using the hash column as index (#3304)
--------- Co-authored-by: Farnabaz <farnabaz@gmail.com>
1 parent e1a98d4 commit ebfb6e5

File tree

6 files changed

+72
-36
lines changed

6 files changed

+72
-36
lines changed

‎src/module.ts

+3-9
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,12 @@ import { findPreset } from './presets'
3232
import type { Manifest } from './types/manifest'
3333
import { setupPreview, shouldEnablePreview } from './utils/preview/module'
3434
import { parseSourceBase } from './utils/source'
35-
import { getLocalDatabase, refineDatabaseConfig, resolveDatabaseAdapter } from './utils/database'
35+
import { databaseVersion, getLocalDatabase, refineDatabaseConfig, resolveDatabaseAdapter } from './utils/database'
3636

3737
// Export public utils
3838
export * from './utils'
3939
export type * from './types'
4040

41-
/**
42-
* Database version is used to identify schema changes
43-
* and drop the info table when the version is not supported
44-
*/
45-
const databaseVersion = 'v3.3.0'
46-
4741
export default defineNuxtModule<ModuleOptions>({
4842
meta: {
4943
name: '@nuxt/content',
@@ -307,7 +301,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
307301
let parsedContent
308302
if (cache && cache.checksum === checksum) {
309303
cachedFilesCount += 1
310-
parsedContent = JSON.parse(cache.parsedContent)
304+
parsedContent = JSON.parse(cache.value)
311305
}
312306
else {
313307
parsedFilesCount += 1
@@ -317,7 +311,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
317311
path: fullPath,
318312
})
319313
if (parsedContent) {
320-
db.insertDevelopmentCache(keyInCollection, checksum, JSON.stringify(parsedContent))
314+
db.insertDevelopmentCache(keyInCollection, JSON.stringify(parsedContent), checksum)
321315
}
322316
}
323317

‎src/types/database.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { Primitive, Connector } from 'db0'
22

3-
export type CacheEntry = { id: string, checksum: string, parsedContent: string }
3+
export type CacheEntry = { id: string, value: string, checksum: string }
44

55
export type DatabaseBindParams = Primitive[]
66

‎src/utils/collection.ts

+21-14
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ export const MAX_SQL_QUERY_SIZE = 100000
151151
export const SLICE_SIZE = 70000
152152

153153
// Convert collection data to SQL insert statement
154-
export function generateCollectionInsert(collection: ResolvedCollection, data: ParsedContentFile, opts: { hashColumn?: boolean } = {}): { queries: string[], hash: string } {
154+
export function generateCollectionInsert(collection: ResolvedCollection, data: ParsedContentFile): { queries: string[], hash: string } {
155155
const fields: string[] = []
156156
const values: Array<string | number | boolean> = []
157157
const sortedKeys = getOrderedSchemaKeys((collection.extendedSchema).shape)
@@ -191,9 +191,7 @@ export function generateCollectionInsert(collection: ResolvedCollection, data: P
191191
})
192192

193193
const valuesHash = hash(values)
194-
if (opts.hashColumn !== false) {
195-
values.push(`'${valuesHash}'`)
196-
}
194+
values.push(`'${valuesHash}'`)
197195

198196
let index = 0
199197
const sql = `INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(values.length).slice(0, -2)});`
@@ -217,15 +215,26 @@ export function generateCollectionInsert(collection: ResolvedCollection, data: P
217215
let sliceIndex = SLICE_SIZE
218216
values[bigColumnIndex] = `${biggestColumn.slice(0, sliceIndex)}'`
219217
index = 0
218+
219+
const bigValueSliceWithHash = [...values.slice(0, -1), `'${valuesHash}-${sliceIndex}'`]
220+
220221
const SQLQueries = [
221-
`INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(values.length).slice(0, -2)});`.replace(/\?/g, () => values[index++] as string),
222+
`INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(bigValueSliceWithHash.length).slice(0, -2)});`.replace(/\?/g, () => bigValueSliceWithHash[index++] as string),
222223
]
223224
while (sliceIndex < biggestColumn.length) {
224-
const newSlice = `'${biggestColumn.slice(sliceIndex, sliceIndex + SLICE_SIZE)}` + (sliceIndex + SLICE_SIZE < biggestColumn.length ? '\'' : '')
225-
SQLQueries.push(
226-
`UPDATE ${collection.tableName} SET ${bigColumnName} = CONCAT(${bigColumnName}, ${newSlice}) WHERE id = ${values[0]};`,
227-
)
225+
const prevSliceIndex = sliceIndex
228226
sliceIndex += SLICE_SIZE
227+
228+
const isLastSlice = sliceIndex > biggestColumn.length
229+
const newSlice = `'${biggestColumn.slice(prevSliceIndex, sliceIndex)}` + (!isLastSlice ? '\'' : '')
230+
const sliceHash = isLastSlice ? valuesHash : `${valuesHash}-${sliceIndex}`
231+
SQLQueries.push([
232+
'UPDATE',
233+
collection.tableName,
234+
`SET ${bigColumnName} = CONCAT(${bigColumnName}, ${newSlice}), "__hash__" = '${sliceHash}'`,
235+
'WHERE',
236+
`id = ${values[0]} AND "__hash__" = '${valuesHash}-${prevSliceIndex}';`,
237+
].join(' '))
229238
}
230239
return { queries: SQLQueries, hash: valuesHash }
231240
}
@@ -237,7 +246,7 @@ export function generateCollectionInsert(collection: ResolvedCollection, data: P
237246
}
238247

239248
// Convert a collection with Zod schema to SQL table definition
240-
export function generateCollectionTableDefinition(collection: ResolvedCollection, opts: { drop?: boolean, hashColumn?: boolean } = {}) {
249+
export function generateCollectionTableDefinition(collection: ResolvedCollection, opts: { drop?: boolean } = {}) {
241250
const sortedKeys = getOrderedSchemaKeys((collection.extendedSchema).shape)
242251
const sqlFields = sortedKeys.map((key) => {
243252
const type = (collection.extendedSchema).shape[key]!
@@ -282,10 +291,8 @@ export function generateCollectionTableDefinition(collection: ResolvedCollection
282291
return `"${key}" ${sqlType}${constraints.join(' ')}`
283292
})
284293

285-
if (opts.hashColumn !== false) {
286-
// add __hash__ field for inserts
287-
sqlFields.push('"__hash__" TEXT UNIQUE')
288-
}
294+
// add __hash__ field for inserts
295+
sqlFields.push('"__hash__" TEXT UNIQUE')
289296

290297
let definition = `CREATE TABLE IF NOT EXISTS ${collection.tableName} (${sqlFields.join(', ')});`
291298

‎src/utils/database.ts

+34-8
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ import type { ModuleOptions } from '../types/module'
1010
import { logger } from './dev'
1111
import { generateCollectionInsert, generateCollectionTableDefinition } from './collection'
1212

13+
/**
14+
* Database version is used to identify schema changes
15+
* and drop the info table when the version is not supported
16+
*/
17+
export const databaseVersion = 'v3.5.0'
18+
1319
export async function refineDatabaseConfig(database: ModuleOptions['database'], opts: { rootDir: string, updateSqliteFileName?: boolean }) {
1420
if (database.type === 'd1') {
1521
if (!('bindingName' in database)) {
@@ -69,31 +75,51 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa
6975
tableName: '_development_cache',
7076
extendedSchema: z.object({
7177
id: z.string(),
78+
value: z.string(),
7279
checksum: z.string(),
73-
parsedContent: z.string(),
7480
}),
7581
fields: {
7682
id: 'string',
83+
value: 'string',
7784
checksum: 'string',
78-
parsedContent: 'string',
7985
},
8086
} as unknown as ResolvedCollection
8187

82-
_localDatabase[databaseLocation] = db
83-
await db.exec(generateCollectionTableDefinition(cacheCollection, { hashColumn: false }))
88+
// If the database is already initialized, we need to drop the cache table
89+
if (!_localDatabase[databaseLocation]) {
90+
_localDatabase[databaseLocation] = db
91+
92+
let dropCacheTable = false
93+
try {
94+
dropCacheTable = await db.prepare('SELECT * FROM _development_cache WHERE id = ?')
95+
.get('__DATABASE_VERSION__').then(row => (row as unknown as { value: string })?.value !== databaseVersion)
96+
}
97+
catch {
98+
dropCacheTable = true
99+
}
100+
101+
const initQueries = generateCollectionTableDefinition(cacheCollection, { drop: Boolean(dropCacheTable) })
102+
for (const query of initQueries.split('\n')) {
103+
await db.exec(query)
104+
}
105+
// Initialize the database version
106+
if (dropCacheTable) {
107+
await db.exec(generateCollectionInsert(cacheCollection, { id: '__DATABASE_VERSION__', value: databaseVersion, checksum: databaseVersion }).queries[0])
108+
}
109+
}
84110

85111
const fetchDevelopmentCache = async () => {
86112
const result = await db.prepare('SELECT * FROM _development_cache').all() as CacheEntry[]
87113
return result.reduce((acc, cur) => ({ ...acc, [cur.id]: cur }), {} as Record<string, CacheEntry>)
88114
}
89115

90-
const fetchDevelopmentCacheForKey = async (key: string) => {
91-
return await db.prepare('SELECT * FROM _development_cache WHERE id = ?').get(key) as CacheEntry | undefined
116+
const fetchDevelopmentCacheForKey = async (id: string) => {
117+
return await db.prepare('SELECT * FROM _development_cache WHERE id = ?').get(id) as CacheEntry | undefined
92118
}
93119

94-
const insertDevelopmentCache = async (id: string, checksum: string, parsedContent: string) => {
120+
const insertDevelopmentCache = async (id: string, value: string, checksum: string) => {
95121
deleteDevelopmentCache(id)
96-
const insert = generateCollectionInsert(cacheCollection, { id, checksum, parsedContent }, { hashColumn: false })
122+
const insert = generateCollectionInsert(cacheCollection, { id, value, checksum })
97123
for (const query of insert.queries) {
98124
await db.exec(query)
99125
}

‎src/utils/dev.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ export async function watchContents(nuxt: Nuxt, options: ModuleOptions, manifest
136136
const checksum = getContentChecksum(content)
137137
const localCache = await db.fetchDevelopmentCacheForKey(keyInCollection)
138138

139-
let parsedContent = localCache?.parsedContent || ''
139+
let parsedContent = localCache?.value || ''
140140

141141
// If the local cache is not present or the checksum does not match, we need to parse the content
142142
if (!localCache || localCache?.checksum !== checksum) {

‎test/unit/generateCollectionInsert.test.ts

+12-3
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,10 @@ describe('generateCollectionInsert', () => {
8383
})
8484

8585
const querySlices: string[] = [content.slice(0, SLICE_SIZE - 1)]
86+
const sliceIndexes = [SLICE_SIZE]
8687
for (let i = 1; i < (content.length / SLICE_SIZE); i++) {
8788
querySlices.push(content.slice((SLICE_SIZE * i) - 1, (SLICE_SIZE * (i + 1)) - 1))
89+
sliceIndexes.push(SLICE_SIZE * (i + 1))
8890
}
8991

9092
// check that the content will be split into multiple queries
@@ -93,18 +95,25 @@ describe('generateCollectionInsert', () => {
9395
// check that concatenated all the values are equal to the original content
9496
expect(content).toEqual(querySlices.join(''))
9597

98+
const hash = 'QMyFxMru9gVfaNx0fzjs5is7SvAZMEy3tNDANjkdogg'
99+
96100
expect(sql[0]).toBe([
97101
`INSERT INTO ${getTableName('content')}`,
98102
' VALUES',
99-
` ('foo.md', '${querySlices[0]}', 'md', '{}', 'foo', 'QMyFxMru9gVfaNx0fzjs5is7SvAZMEy3tNDANjkdogg');`,
103+
` ('foo.md', '${querySlices[0]}', 'md', '{}', 'foo', '${hash}-${sliceIndexes[0]}');`,
100104
].join(''))
101105
let index = 1
102-
while (index < sql.length - 1) {
106+
107+
while (index < sql.length) {
108+
// last statement should update the hash column to the hash itself
109+
const nextHash = index === sql.length - 1 ? hash : `${hash}-${sliceIndexes[index]}`
103110
expect(sql[index]).toBe([
104111
`UPDATE ${getTableName('content')}`,
105112
' SET',
106113
` content = CONCAT(content, '${querySlices[index]}')`,
107-
' WHERE id = \'foo.md\';',
114+
`, "__hash__" = '${nextHash}'`,
115+
' WHERE id = \'foo.md\'',
116+
` AND "__hash__" = '${hash}-${sliceIndexes[index - 1]}';`,
108117
].join(''))
109118
index++
110119
}

0 commit comments

Comments
 (0)