From ed42238a280e95995e621ce95ef81670943b5e05 Mon Sep 17 00:00:00 2001 From: Fabian Kunze Date: Fri, 20 Nov 2020 21:06:47 +0100 Subject: [PATCH 1/3] Sentence parser now recognises adding/removing a single sentence in the document. This speeds up interactiv development in a large document. I refactor this code in a seperate commit, as i want to have the working code with copy-pasted, repeated parts in the git. --- .../src/sentence-model/SentenceCollection.ts | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/server/src/sentence-model/SentenceCollection.ts b/server/src/sentence-model/SentenceCollection.ts index d5ba63e6f..887633b8a 100644 --- a/server/src/sentence-model/SentenceCollection.ts +++ b/server/src/sentence-model/SentenceCollection.ts @@ -375,7 +375,63 @@ export class SentenceCollection implements vscode.TextDocument { // removed.forEach((sent) => sent.dispose()); return {removed: removed, added: reparsed, endOfSentences: false}; - } + } else if(idx >= minCount && start+idx+1 < this.sentences.length + && currentOffset+sent.text.length === this.sentences[start+idx+1].getDocumentEndOffset() + && sent.text === this.sentences[start+idx+1].getText()) { + //we joined two sentences by removing a "." somewhere + // no need to parse further; keep remaining sentences + const removed = this.sentences.splice(start, idx+1, ...reparsed) + // adjust prev/next reference at the last reparsed sentence + if(reparsed.length > 0) { + const lastReparsed = reparsed[reparsed.length-1]; + lastReparsed.next = this.sentences[start+reparsed.length] || null; + if(lastReparsed.next) + lastReparsed.next.prev = lastReparsed; + } else { + if(this.sentences[start-1]){ + console.log("Case A: should " + this.sentences[start-1].toString() +" point to next:" +this.sentences[start] +"?"); + this.sentences[start-1].next = this.sentences[start] || null; + } + if(this.sentences[start]) + console.log("Case B: should " + this.sentences[start].toString() +" point to prev:" +this.sentences[start-1] +"?"); + this.sentences[start].prev = this.sentences[start-1] || null; + } + + // this.sentences[start+reparsed.length-1].next = this.sentences[start+reparsed.length]||null; + // if(start+reparsed.length < this.sentences.length) + // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; + // + removed.forEach((sent) => sent.dispose()); + return {removed: removed, added: reparsed, endOfSentences: false}; + } else if(idx >= minCount && 0 <= start+idx-1 && start+idx-1 < this.sentences.length + && currentOffset+sent.text.length === this.sentences[start+idx-1].getDocumentEndOffset() + && sent.text === this.sentences[start+idx-1].getText()) { + //we joined two sentences by removing a "." somewhere + // no need to parse further; keep remaining sentences + const removed = this.sentences.splice(start, idx-1, ...reparsed) + // adjust prev/next reference at the last reparsed sentence + if(reparsed.length > 0) { + const lastReparsed = reparsed[reparsed.length-1]; + lastReparsed.next = this.sentences[start+reparsed.length] || null; + if(lastReparsed.next) + lastReparsed.next.prev = lastReparsed; + } else { + if(this.sentences[start-1]){ + console.log("Case C: should " + this.sentences[start-1].toString() +" point to next:" +this.sentences[start] +"?"); + this.sentences[start-1].next = this.sentences[start] || null; + } + if(this.sentences[start]) + console.log("Case D: should " + this.sentences[start].toString() +" point to prev:" +this.sentences[start-1] +"?"); + this.sentences[start].prev = this.sentences[start-1] || null; + } + + // this.sentences[start+reparsed.length-1].next = this.sentences[start+reparsed.length]||null; + // if(start+reparsed.length < this.sentences.length) + // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; + // + removed.forEach((sent) => sent.dispose()); + return {removed: removed, added: reparsed, endOfSentences: false}; + } const command = sent.text; const range = Range.create(currentPosition, textUtil.positionAtRelative(currentPosition, command, sent.text.length)); From 004aa92f38c89591a30f1eff3871dabb7277b809 Mon Sep 17 00:00:00 2001 From: Fabian Kunze Date: Fri, 20 Nov 2020 21:27:41 +0100 Subject: [PATCH 2/3] refactored the code that should fix #116 . This fix only handles inserting and removing single sentences. A proper fix would maybe use the distance to the end of the file to determine that we do not want to reparse more, but this fails for example when identifiers are renamed --- .../src/sentence-model/SentenceCollection.ts | 89 +++++++------------ 1 file changed, 30 insertions(+), 59 deletions(-) diff --git a/server/src/sentence-model/SentenceCollection.ts b/server/src/sentence-model/SentenceCollection.ts index 887633b8a..8f1e9e30e 100644 --- a/server/src/sentence-model/SentenceCollection.ts +++ b/server/src/sentence-model/SentenceCollection.ts @@ -353,10 +353,31 @@ export class SentenceCollection implements vscode.TextDocument { this.sentences[start-1].next = null; // removed.forEach((sent) => sent.dispose()); + if (reparsed.length > 10){ + console.log("NOTICE: some editing change lead us to reparse until EOF.") + } return {removed: removed, added: reparsed, endOfSentences: true}; - } if(idx >= minCount && start+idx < this.sentences.length && currentOffset+sent.text.length === this.sentences[start+idx].getDocumentEndOffset() && sent.text === this.sentences[start+idx].getText()) { + } + + var fixByLocalGlueing : undefined | number = undefined; + + if(idx >= minCount && start+idx < this.sentences.length + && currentOffset+sent.text.length === this.sentences[start+idx].getDocumentEndOffset() + && sent.text === this.sentences[start+idx].getText()) + fixByLocalGlueing = 0; //we probably edited inside the sentence before this + else if(idx >= minCount && start+idx+1 < this.sentences.length + && currentOffset+sent.text.length === this.sentences[start+idx+1].getDocumentEndOffset() + && sent.text === this.sentences[start+idx+1].getText()) + fixByLocalGlueing = 1; //we probably joined two sentences by removing a "." + else if(idx >= minCount && 0 <= start+idx-1 && start+idx-1 < this.sentences.length + && currentOffset+sent.text.length === this.sentences[start+idx-1].getDocumentEndOffset() + && sent.text === this.sentences[start+idx-1].getText()) + fixByLocalGlueing = -1;//we probably seperated a sentence into two by adding a "." + + if(fixByLocalGlueing !== undefined) { // no need to parse further; keep remaining sentences - const removed = this.sentences.splice(start, idx, ...reparsed) + const removed = this.sentences.splice(start, idx+fixByLocalGlueing, ...reparsed) + // adjust prev/next reference at the last reparsed sentence if(reparsed.length > 0) { const lastReparsed = reparsed[reparsed.length-1]; @@ -374,64 +395,11 @@ export class SentenceCollection implements vscode.TextDocument { // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; // removed.forEach((sent) => sent.dispose()); - return {removed: removed, added: reparsed, endOfSentences: false}; - } else if(idx >= minCount && start+idx+1 < this.sentences.length - && currentOffset+sent.text.length === this.sentences[start+idx+1].getDocumentEndOffset() - && sent.text === this.sentences[start+idx+1].getText()) { - //we joined two sentences by removing a "." somewhere - // no need to parse further; keep remaining sentences - const removed = this.sentences.splice(start, idx+1, ...reparsed) - // adjust prev/next reference at the last reparsed sentence - if(reparsed.length > 0) { - const lastReparsed = reparsed[reparsed.length-1]; - lastReparsed.next = this.sentences[start+reparsed.length] || null; - if(lastReparsed.next) - lastReparsed.next.prev = lastReparsed; - } else { - if(this.sentences[start-1]){ - console.log("Case A: should " + this.sentences[start-1].toString() +" point to next:" +this.sentences[start] +"?"); - this.sentences[start-1].next = this.sentences[start] || null; - } - if(this.sentences[start]) - console.log("Case B: should " + this.sentences[start].toString() +" point to prev:" +this.sentences[start-1] +"?"); - this.sentences[start].prev = this.sentences[start-1] || null; - } - - // this.sentences[start+reparsed.length-1].next = this.sentences[start+reparsed.length]||null; - // if(start+reparsed.length < this.sentences.length) - // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; - // - removed.forEach((sent) => sent.dispose()); - return {removed: removed, added: reparsed, endOfSentences: false}; - } else if(idx >= minCount && 0 <= start+idx-1 && start+idx-1 < this.sentences.length - && currentOffset+sent.text.length === this.sentences[start+idx-1].getDocumentEndOffset() - && sent.text === this.sentences[start+idx-1].getText()) { - //we joined two sentences by removing a "." somewhere - // no need to parse further; keep remaining sentences - const removed = this.sentences.splice(start, idx-1, ...reparsed) - // adjust prev/next reference at the last reparsed sentence - if(reparsed.length > 0) { - const lastReparsed = reparsed[reparsed.length-1]; - lastReparsed.next = this.sentences[start+reparsed.length] || null; - if(lastReparsed.next) - lastReparsed.next.prev = lastReparsed; - } else { - if(this.sentences[start-1]){ - console.log("Case C: should " + this.sentences[start-1].toString() +" point to next:" +this.sentences[start] +"?"); - this.sentences[start-1].next = this.sentences[start] || null; - } - if(this.sentences[start]) - console.log("Case D: should " + this.sentences[start].toString() +" point to prev:" +this.sentences[start-1] +"?"); - this.sentences[start].prev = this.sentences[start-1] || null; - } - - // this.sentences[start+reparsed.length-1].next = this.sentences[start+reparsed.length]||null; - // if(start+reparsed.length < this.sentences.length) - // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; - // - removed.forEach((sent) => sent.dispose()); - return {removed: removed, added: reparsed, endOfSentences: false}; + if (reparsed.length > 10){ + console.log("NOTICE: some editing change lead us to reparse a lot of code, but we catched ourself, most likely because a lot of subsequent sentences where changed.") } + return {removed: removed, added: reparsed, endOfSentences: false}; + } const command = sent.text; const range = Range.create(currentPosition, textUtil.positionAtRelative(currentPosition, command, sent.text.length)); @@ -450,6 +418,9 @@ export class SentenceCollection implements vscode.TextDocument { // treat the rest of the document as unparsed const removed = this.sentences.splice(start, this.sentences.length - start, ...reparsed) removed.forEach((sent) => sent.dispose()); + if (reparsed.length > 10){ + console.log("NOTICE: some editing change lead us to reparse a lot of code, and we catched an exception.") + } return {removed: removed, added: reparsed, endOfSentences: true}; } else { server.connection.console.warn("unknown parsing error: " + util.inspect(error,false,undefined)) From 1e62c3c2701772e3e1d36b205e424f559a5c3d24 Mon Sep 17 00:00:00 2001 From: Fabian Kunze Date: Sat, 21 Nov 2020 13:55:13 +0100 Subject: [PATCH 3/3] support for copy/pasting more than one sentence in a lot of cases. --- .../src/sentence-model/SentenceCollection.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/server/src/sentence-model/SentenceCollection.ts b/server/src/sentence-model/SentenceCollection.ts index 8f1e9e30e..e9d74a4a5 100644 --- a/server/src/sentence-model/SentenceCollection.ts +++ b/server/src/sentence-model/SentenceCollection.ts @@ -342,6 +342,7 @@ export class SentenceCollection implements vscode.TextDocument { } try { + var oldSentenceCandidate : number = start; for(let idx = 0; /**/; ++idx) { const parseText = this.documentText.substring(currentOffset); const sent = parser.parseSentence(parseText); @@ -352,16 +353,23 @@ export class SentenceCollection implements vscode.TextDocument { if(reparsed.length == 0 && this.sentences[start-1]) this.sentences[start-1].next = null; // + if (removed.length>0 && reparsed.length > 0 && removed[removed.length-1].getText()===reparsed[reparsed.length-1].getText() ) + console.log("Internal inefficiency: detecting unchanged suffix after editing failed, and we parsed the whole document until end, please report."); removed.forEach((sent) => sent.dispose()); - if (reparsed.length > 10){ - console.log("NOTICE: some editing change lead us to reparse until EOF.") - } return {removed: removed, added: reparsed, endOfSentences: true}; } var fixByLocalGlueing : undefined | number = undefined; - if(idx >= minCount && start+idx < this.sentences.length + while(oldSentenceCandidate < this.sentences.length && currentOffset+sent.text.length > this.sentences[oldSentenceCandidate].getDocumentEndOffset()) + ++oldSentenceCandidate; + + if (idx >= minCount && oldSentenceCandidate < this.sentences.length + && currentOffset+sent.text.length === this.sentences[oldSentenceCandidate].getDocumentEndOffset() + && sent.text === this.sentences[oldSentenceCandidate].getText()) + fixByLocalGlueing = oldSentenceCandidate - start; //found the old, parsed document again +/* + if (idx >= minCount && start+idx < this.sentences.length && currentOffset+sent.text.length === this.sentences[start+idx].getDocumentEndOffset() && sent.text === this.sentences[start+idx].getText()) fixByLocalGlueing = 0; //we probably edited inside the sentence before this @@ -372,7 +380,7 @@ export class SentenceCollection implements vscode.TextDocument { else if(idx >= minCount && 0 <= start+idx-1 && start+idx-1 < this.sentences.length && currentOffset+sent.text.length === this.sentences[start+idx-1].getDocumentEndOffset() && sent.text === this.sentences[start+idx-1].getText()) - fixByLocalGlueing = -1;//we probably seperated a sentence into two by adding a "." + fixByLocalGlueing = -1;//we probably seperated a sentence into two by adding a "."*/ if(fixByLocalGlueing !== undefined) { // no need to parse further; keep remaining sentences @@ -394,10 +402,9 @@ export class SentenceCollection implements vscode.TextDocument { // if(start+reparsed.length < this.sentences.length) // this.sentences[start+reparsed.length].prev = this.sentences[start+reparsed.length-1]||null; // + if (removed.length>1 && reparsed.length > 1 && removed[removed.length-2].getText()===reparsed[reparsed.length-2].getText()) + console.log("Internal inefficiency: detecting unchanged suffix after editing and reparsed more than needed ("+reparsed.length+" total), please report."); removed.forEach((sent) => sent.dispose()); - if (reparsed.length > 10){ - console.log("NOTICE: some editing change lead us to reparse a lot of code, but we catched ourself, most likely because a lot of subsequent sentences where changed.") - } return {removed: removed, added: reparsed, endOfSentences: false}; } @@ -418,9 +425,7 @@ export class SentenceCollection implements vscode.TextDocument { // treat the rest of the document as unparsed const removed = this.sentences.splice(start, this.sentences.length - start, ...reparsed) removed.forEach((sent) => sent.dispose()); - if (reparsed.length > 10){ - console.log("NOTICE: some editing change lead us to reparse a lot of code, and we catched an exception.") - } + console.log("Notice: detecting unchanged suffix after editing lead to syntax error.") return {removed: removed, added: reparsed, endOfSentences: true}; } else { server.connection.console.warn("unknown parsing error: " + util.inspect(error,false,undefined))