-
Notifications
You must be signed in to change notification settings - Fork 216
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix issue where query get interrupted during prepare. (backport #7592)…
… [release/4.10.x] (#7654) Co-authored-by: affank <khanaffan@gmail.com> Co-authored-by: Affan Khan <khanaffan@users.noreply.github.com> Co-authored-by: imodeljs-admin <38288322+imodeljs-admin@users.noreply.github.com> Co-authored-by: Arun George <11051042+aruniverse@users.noreply.github.com>
- Loading branch information
5 people
authored
Feb 12, 2025
1 parent
dabe07e
commit 33d451b
Showing
13 changed files
with
341 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
common/changes/@itwin/core-backend/affank-ccq-fix_2025-01-22-22-18.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "changes": [ | ||
| { | ||
| "packageName": "@itwin/core-backend", | ||
| "comment": "Fix issue with concurrent query where it interrupt statement during prepare", | ||
| "type": "none" | ||
| } | ||
| ], | ||
| "packageName": "@itwin/core-backend" | ||
| } |
10 changes: 10 additions & 0 deletions
10
common/changes/@itwin/core-common/affank-ccq-fix_2025-01-22-22-18.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "changes": [ | ||
| { | ||
| "packageName": "@itwin/core-common", | ||
| "comment": "Update ECSql reader api to support no stat and error code.", | ||
| "type": "none" | ||
| } | ||
| ], | ||
| "packageName": "@itwin/core-common" | ||
| } |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,186 @@ | ||
| import { Logger, LogLevel, StopWatch } from "@itwin/core-bentley"; | ||
| import { DbQueryConfig, ECSqlReader, QueryStats } from "@itwin/core-common"; | ||
| import { expect } from "chai"; | ||
| import { ConcurrentQuery } from "../../ConcurrentQuery"; | ||
| import { ECDb } from "../../ECDb"; | ||
| import { IModelDb, SnapshotDb } from "../../IModelDb"; | ||
| import { _nativeDb } from "../../core-backend"; | ||
| import { IModelTestUtils } from "../IModelTestUtils"; | ||
|
|
||
| interface ITaskResult { | ||
| stats: QueryStats; | ||
| error?: any; | ||
| } | ||
|
|
||
| interface ISenario { | ||
| name: string; | ||
| config?: DbQueryConfig; | ||
| totalBatches: number; | ||
| taskPerBatch: number; | ||
| createReader: (db: ECDb | IModelDb) => ECSqlReader; | ||
| } | ||
|
|
||
|
|
||
| class LoadSimulator { | ||
| constructor(public db: ECDb | IModelDb, public senario: ISenario) { } | ||
| private async runQueryTask(reader: ECSqlReader): Promise<ITaskResult> { | ||
| try { | ||
| while (await reader.step()) { } | ||
| return { stats: reader.stats }; | ||
| } catch (err) { | ||
| return { stats: reader.stats, error: err }; | ||
| } | ||
| } | ||
|
|
||
| public async run() { | ||
| ConcurrentQuery.shutdown(this.db[_nativeDb]); | ||
| if (this.senario.config) { | ||
| const config = ConcurrentQuery.resetConfig(this.db[_nativeDb], this.senario.config); | ||
| // eslint-disable-next-line no-console | ||
| console.log(config); | ||
| } | ||
| const overalTime = new StopWatch(); | ||
| overalTime.start(); | ||
| const results: ITaskResult[] = []; | ||
| for (let i = 0; i < this.senario.totalBatches; ++i) { | ||
| const promises: Promise<ITaskResult>[] = []; | ||
| const readerTasks = Array(this.senario.taskPerBatch).fill(undefined).map(() => this.senario.createReader(this.db)); | ||
| readerTasks.forEach((reader) => { | ||
| promises.push(this.runQueryTask(reader)); | ||
| }); | ||
| results.push(... await Promise.all(promises)); | ||
|
|
||
| } | ||
| overalTime.stop(); | ||
| const errors = results.filter((x) => x.error !== undefined); | ||
| const errorsMap = new Map<string, number>(); | ||
| errors.forEach((x) => { | ||
| if (x.error instanceof Error) { | ||
| if (!errorsMap.has(x.error.message)) { | ||
| errorsMap.set(x.error.message, 1); | ||
| } else { | ||
| errorsMap.set(x.error.message, errorsMap.get(x.error.message)! + 1); | ||
| } | ||
| } else { | ||
| if (!errorsMap.has("error")) { | ||
| errorsMap.set("error", 1); | ||
| } else { | ||
| errorsMap.set("error", errorsMap.get("error")! + 1); | ||
| } | ||
| } | ||
| }); | ||
| const errorCount = errors.length; | ||
| let backendCpuTime: bigint = BigInt(0); | ||
| let backendTotalTime: bigint = BigInt(0); | ||
| let backendMemUsed: bigint = BigInt(0); | ||
| let backendRowsReturned: bigint = BigInt(0); | ||
| let totalTime: bigint = BigInt(0); | ||
| let retryCount: bigint = BigInt(0); | ||
| let prepareTime: bigint = BigInt(0); | ||
|
|
||
| // Calculate average | ||
| results.forEach((r: ITaskResult) => { | ||
| backendCpuTime += BigInt(r.stats.backendCpuTime); | ||
| backendTotalTime += BigInt(r.stats.backendTotalTime); | ||
| backendMemUsed += BigInt(r.stats.backendMemUsed); | ||
| backendRowsReturned += BigInt(r.stats.backendRowsReturned); | ||
| totalTime += BigInt(r.stats.totalTime); | ||
| retryCount += BigInt(r.stats.retryCount); | ||
| prepareTime += BigInt(r.stats.prepareTime); | ||
| }); | ||
|
|
||
| backendCpuTime /= BigInt(results.length); | ||
| backendTotalTime /= BigInt(results.length); | ||
| backendMemUsed /= BigInt(results.length); | ||
| backendRowsReturned /= BigInt(results.length); | ||
| totalTime /= BigInt(results.length); | ||
| retryCount /= BigInt(results.length); | ||
| // prepareTime /= BigInt(results.length); | ||
|
|
||
| return { | ||
| result: { | ||
| backendCpuTime, | ||
| backendTotalTime, | ||
| backendMemUsed, | ||
| backendRowsReturned, | ||
| totalTime, | ||
| retryCount, | ||
| prepareTime, | ||
| }, | ||
| overalTimeInSec: overalTime.currentSeconds, | ||
| errorCount, | ||
| totalQueries: results.length, | ||
| errorMap: errorsMap | ||
| }; | ||
|
|
||
| } | ||
| } | ||
|
|
||
| describe.skip("ConcurrentQueryLoad", () => { | ||
| it("should run", async () => { | ||
| Logger.initializeToConsole(); | ||
| Logger.setLevel("ECDb.ConcurrentQuery", LogLevel.Trace); | ||
| // { | ||
| // workerThreads: 4, | ||
| // requestQueueSize: 2000, | ||
| // ignorePriority: false, | ||
| // ignoreDelay: true, | ||
| // doNotUsePrimaryConnToPrepare: false, | ||
| // autoShutdowWhenIdlelForSeconds: 300, | ||
| // statementCacheSizePerWorker: 40, | ||
| // monitorPollInterval: 1000, | ||
| // memoryMapFileSize: 0, | ||
| // allowTestingArgs: false, | ||
| // globalQuota: { time: 60, memory: 8388608 } | ||
| // } | ||
|
|
||
| const senario: ISenario = { | ||
| name: "ConcurrentQueryLoad", | ||
| config: { | ||
|
|
||
| }, | ||
| totalBatches: 1, | ||
| taskPerBatch: 1, | ||
| createReader: (dbs: ECDb | IModelDb) => { | ||
| const quries = [ | ||
| { | ||
| sql: ` | ||
| WITH sequence(n) AS ( | ||
| SELECT 1 | ||
| UNION ALL | ||
| SELECT n + 1 FROM sequence WHERE n < 10000 | ||
| ) | ||
| SELECT COUNT(*) | ||
| FROM bis.SpatialIndex i, sequence s | ||
| WHERE i.ECInstanceId MATCH iModel_spatial_overlap_aabb( | ||
| iModel_bbox(random(), random(), random(), random(),random(), random()))` | ||
| }, | ||
| { | ||
| sql: ` | ||
| WITH sequence(n) AS ( | ||
| SELECT 1 | ||
| UNION ALL | ||
| SELECT n + 1 FROM sequence WHERE n < 1000000 | ||
| ) | ||
| SELECT COUNT(*) FROM sequence` | ||
| }, | ||
| { | ||
| sql: "SELECT $ FROM bis.Element LIMIT 10000" | ||
| } | ||
| ]; | ||
| const idx = Math.floor(Math.random() * quries.length); | ||
| return dbs.createQueryReader(quries[idx].sql); | ||
| } | ||
| }; | ||
|
|
||
| const verySmallFile = IModelTestUtils.resolveAssetFile("test.bim"); | ||
| const db = SnapshotDb.openFile(verySmallFile); | ||
| const simulator = new LoadSimulator(db, senario); | ||
| const result = await simulator.run(); | ||
| // eslint-disable-next-line no-console | ||
| console.log(result); | ||
| db.close(); | ||
| expect(result.errorCount).to.be.equal(0); | ||
| }); | ||
|
|
||
| }); |
Oops, something went wrong.