From b65be817516ec566fc3816242cbc152ac43aeca5 Mon Sep 17 00:00:00 2001 From: ycw Date: Fri, 21 Jan 2022 19:25:28 +0800 Subject: [PATCH 1/9] honor encoding --- src/loaders/FileLoader.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index b42e55bc752ebf..c5107abeb485e7 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -63,6 +63,9 @@ class FileLoader extends Loader { } ); // create request + if ( this.mimeType !== undefined ) { + this.requestHeader[ 'Content-Type' ] = this.mimeType; + } const req = new Request( url, { headers: new Headers( this.requestHeader ), credentials: this.withCredentials ? 'include' : 'same-origin', @@ -173,7 +176,21 @@ class FileLoader extends Loader { default: - return response.text(); + return response.arrayBuffer().then( ab => { + let label = 'utf-8'; + if ( this.mimeType !== undefined ) { + const i = this.mimeType.indexOf( '=' ); + if ( ~i ) { + const charset = this.mimeType.substring( i + 1 ).trim(); + if ( charset.length !== 0 ) { + label = charset; + } + } + } + + const decoder = new TextDecoder( label ); + return decoder.decode( ab ); + } ); } From 6956846a0211c88d12af4808dae19e29ece649d7 Mon Sep 17 00:00:00 2001 From: ycw Date: Fri, 21 Jan 2022 19:38:50 +0800 Subject: [PATCH 2/9] npm run lint-fix --- src/loaders/FileLoader.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index c5107abeb485e7..12c1e3e5a04b34 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -64,8 +64,11 @@ class FileLoader extends Loader { // create request if ( this.mimeType !== undefined ) { + this.requestHeader[ 'Content-Type' ] = this.mimeType; + } + const req = new Request( url, { headers: new Headers( this.requestHeader ), credentials: this.withCredentials ? 'include' : 'same-origin', @@ -177,19 +180,27 @@ class FileLoader extends Loader { default: return response.arrayBuffer().then( ab => { + let label = 'utf-8'; if ( this.mimeType !== undefined ) { + const i = this.mimeType.indexOf( '=' ); - if ( ~i ) { + if ( ~ i ) { + const charset = this.mimeType.substring( i + 1 ).trim(); if ( charset.length !== 0 ) { + label = charset; + } - } + + } + } const decoder = new TextDecoder( label ); return decoder.decode( ab ); + } ); } From 097ce1f4331fc17adddb71d512e8e452091e264e Mon Sep 17 00:00:00 2001 From: ycw Date: Fri, 21 Jan 2022 21:10:10 +0800 Subject: [PATCH 3/9] cmp idx --- src/loaders/FileLoader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index 12c1e3e5a04b34..5e040ebd0e5f00 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -63,7 +63,7 @@ class FileLoader extends Loader { } ); // create request - if ( this.mimeType !== undefined ) { + if ( this.mimeType !== undefined ) { this.requestHeader[ 'Content-Type' ] = this.mimeType; @@ -185,7 +185,7 @@ class FileLoader extends Loader { if ( this.mimeType !== undefined ) { const i = this.mimeType.indexOf( '=' ); - if ( ~ i ) { + if ( i !== -1 ) { const charset = this.mimeType.substring( i + 1 ).trim(); if ( charset.length !== 0 ) { From b0a3d8d31468674847e66236c2cdabde9b0d5e3d Mon Sep 17 00:00:00 2001 From: ycw Date: Fri, 21 Jan 2022 21:12:41 +0800 Subject: [PATCH 4/9] lint --- src/loaders/FileLoader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index 5e040ebd0e5f00..68ea575230da88 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -185,7 +185,7 @@ class FileLoader extends Loader { if ( this.mimeType !== undefined ) { const i = this.mimeType.indexOf( '=' ); - if ( i !== -1 ) { + if ( i !== - 1 ) { const charset = this.mimeType.substring( i + 1 ).trim(); if ( charset.length !== 0 ) { From 7eac2480917eebe45a22a3edcb79fc28762700c5 Mon Sep 17 00:00:00 2001 From: ycw Date: Sat, 22 Jan 2022 05:18:43 +0800 Subject: [PATCH 5/9] textual encoding --- examples/jsm/loaders/MMDLoader.js | 4 ++-- src/loaders/FileLoader.js | 40 +++++++++++-------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/examples/jsm/loaders/MMDLoader.js b/examples/jsm/loaders/MMDLoader.js index eb83354730c970..5841ca6dbeb29f 100644 --- a/examples/jsm/loaders/MMDLoader.js +++ b/examples/jsm/loaders/MMDLoader.js @@ -310,9 +310,9 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader - .setMimeType( isUnicode ? undefined : 'text/plain; charset=shift_jis' ) + //.setMimeType( isUnicode ? undefined : 'text/plain; charset=shift_jis' ) .setPath( this.animationPath ) - .setResponseType( 'text' ) + .setResponseType( 'text', 'shift_jis' ) .setRequestHeader( this.requestHeader ) .setWithCredentials( this.withCredentials ) .load( url, function ( text ) { diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index 68ea575230da88..af3f2487479cd9 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -63,18 +63,16 @@ class FileLoader extends Loader { } ); // create request - if ( this.mimeType !== undefined ) { - - this.requestHeader[ 'Content-Type' ] = this.mimeType; - - } - const req = new Request( url, { headers: new Headers( this.requestHeader ), credentials: this.withCredentials ? 'include' : 'same-origin', // An abort controller could be added within a future PR } ); + // record current states before 'fetch' ( avoid data race ) + const responseType = this.responseType; + const textualEncoding = this.textualEncoding; + // start the fetch fetch( req ) .then( response => { @@ -153,7 +151,7 @@ class FileLoader extends Loader { } ) .then( response => { - switch ( this.responseType ) { + switch ( responseType ) { case 'arraybuffer': @@ -181,24 +179,7 @@ class FileLoader extends Loader { return response.arrayBuffer().then( ab => { - let label = 'utf-8'; - if ( this.mimeType !== undefined ) { - - const i = this.mimeType.indexOf( '=' ); - if ( i !== - 1 ) { - - const charset = this.mimeType.substring( i + 1 ).trim(); - if ( charset.length !== 0 ) { - - label = charset; - - } - - } - - } - - const decoder = new TextDecoder( label ); + const decoder = new TextDecoder( textualEncoding ); return decoder.decode( ab ); } ); @@ -259,9 +240,16 @@ class FileLoader extends Loader { } - setResponseType( value ) { + setResponseType( value, textualEncoding ) { this.responseType = value; + + if ( textualEncoding !== undefined ) { + + this.textualEncoding = textualEncoding; + + } + return this; } From 275a846af1e4d3df0641d7c2a60e46a6a33a18ac Mon Sep 17 00:00:00 2001 From: ycw Date: Sat, 22 Jan 2022 06:01:58 +0800 Subject: [PATCH 6/9] record state --- src/loaders/FileLoader.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index af3f2487479cd9..58fed8dc32caa3 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -70,6 +70,7 @@ class FileLoader extends Loader { } ); // record current states before 'fetch' ( avoid data race ) + const mimeType = this.mimeType; const responseType = this.responseType; const textualEncoding = this.textualEncoding; @@ -167,7 +168,7 @@ class FileLoader extends Loader { .then( text => { const parser = new DOMParser(); - return parser.parseFromString( text, this.mimeType ); + return parser.parseFromString( text, mimeType ); } ); From bb37c453183ce60f550a705b1a41c9d0e2dd9913 Mon Sep 17 00:00:00 2001 From: ycw Date: Sat, 22 Jan 2022 06:53:20 +0800 Subject: [PATCH 7/9] new api `.setResponseType` --- examples/jsm/loaders/MMDLoader.js | 4 ---- src/loaders/FileLoader.js | 26 ++++++++++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/examples/jsm/loaders/MMDLoader.js b/examples/jsm/loaders/MMDLoader.js index 5841ca6dbeb29f..edc620fa697999 100644 --- a/examples/jsm/loaders/MMDLoader.js +++ b/examples/jsm/loaders/MMDLoader.js @@ -218,7 +218,6 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader - .setMimeType( undefined ) .setPath( this.path ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -244,7 +243,6 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader - .setMimeType( undefined ) .setPath( this.path ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -276,7 +274,6 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader - .setMimeType( undefined ) .setPath( this.animationPath ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -310,7 +307,6 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader - //.setMimeType( isUnicode ? undefined : 'text/plain; charset=shift_jis' ) .setPath( this.animationPath ) .setResponseType( 'text', 'shift_jis' ) .setRequestHeader( this.requestHeader ) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index 58fed8dc32caa3..edfe9bcc3bca80 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -70,9 +70,9 @@ class FileLoader extends Loader { } ); // record current states before 'fetch' ( avoid data race ) - const mimeType = this.mimeType; const responseType = this.responseType; - const textualEncoding = this.textualEncoding; + const mimeType = this.mimeType; + const textEncoding = this.textEncoding; // start the fetch fetch( req ) @@ -180,7 +180,7 @@ class FileLoader extends Loader { return response.arrayBuffer().then( ab => { - const decoder = new TextDecoder( textualEncoding ); + const decoder = new TextDecoder( textEncoding ); return decoder.decode( ab ); } ); @@ -241,13 +241,25 @@ class FileLoader extends Loader { } - setResponseType( value, textualEncoding ) { + setResponseType( value, encodingOrMimeType ) { this.responseType = value; - if ( textualEncoding !== undefined ) { + if ( value === 'document' ) { + + if ( encodingOrMimeType !== undefined ) { + + this.mimeType = encodingOrMimeType; + + } - this.textualEncoding = textualEncoding; + } else if ( value === 'text' ) { + + if ( encodingOrMimeType !== undefined ) { + + this.textEncoding = encodingOrMimeType; + + } } @@ -257,6 +269,8 @@ class FileLoader extends Loader { setMimeType( value ) { + console.warn( `deprecated, use .setResponseType( 'document', '${value}' ) instead.` ); + this.mimeType = value; return this; From 1a6ae6b512e5942fec04c6880ad60b7b7f992cd9 Mon Sep 17 00:00:00 2001 From: ycw Date: Sat, 22 Jan 2022 22:05:06 +0800 Subject: [PATCH 8/9] rollback mmdloader --- examples/jsm/loaders/MMDLoader.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/jsm/loaders/MMDLoader.js b/examples/jsm/loaders/MMDLoader.js index edc620fa697999..eb83354730c970 100644 --- a/examples/jsm/loaders/MMDLoader.js +++ b/examples/jsm/loaders/MMDLoader.js @@ -218,6 +218,7 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader + .setMimeType( undefined ) .setPath( this.path ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -243,6 +244,7 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader + .setMimeType( undefined ) .setPath( this.path ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -274,6 +276,7 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader + .setMimeType( undefined ) .setPath( this.animationPath ) .setResponseType( 'arraybuffer' ) .setRequestHeader( this.requestHeader ) @@ -307,8 +310,9 @@ class MMDLoader extends Loader { const parser = this._getParser(); this.loader + .setMimeType( isUnicode ? undefined : 'text/plain; charset=shift_jis' ) .setPath( this.animationPath ) - .setResponseType( 'text', 'shift_jis' ) + .setResponseType( 'text' ) .setRequestHeader( this.requestHeader ) .setWithCredentials( this.withCredentials ) .load( url, function ( text ) { From 40819e75d8e89d5c894cb0aff7d150a437f8629e Mon Sep 17 00:00:00 2001 From: ycw Date: Sat, 22 Jan 2022 22:59:53 +0800 Subject: [PATCH 9/9] sniff encoding for 'text' kind --- src/loaders/FileLoader.js | 44 +++++++++++++-------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/loaders/FileLoader.js b/src/loaders/FileLoader.js index edfe9bcc3bca80..d9ea9bc3e66665 100644 --- a/src/loaders/FileLoader.js +++ b/src/loaders/FileLoader.js @@ -69,10 +69,9 @@ class FileLoader extends Loader { // An abort controller could be added within a future PR } ); - // record current states before 'fetch' ( avoid data race ) - const responseType = this.responseType; + // record states ( avoid data race ) const mimeType = this.mimeType; - const textEncoding = this.textEncoding; + const responseType = this.responseType; // start the fetch fetch( req ) @@ -178,12 +177,20 @@ class FileLoader extends Loader { default: - return response.arrayBuffer().then( ab => { + if ( mimeType === undefined ) { + + return response.text(); - const decoder = new TextDecoder( textEncoding ); - return decoder.decode( ab ); + } else { - } ); + // sniff encoding + const re = /charset="?([^;"\s]*)"?/i; + const exec = re.exec( mimeType ); + const label = exec && exec[ 1 ] ? exec[ 1 ].toLowerCase() : undefined; + const decoder = new TextDecoder( label ); + return response.arrayBuffer().then( ab => decoder.decode( ab ) ); + + } } @@ -241,36 +248,15 @@ class FileLoader extends Loader { } - setResponseType( value, encodingOrMimeType ) { + setResponseType( value ) { this.responseType = value; - - if ( value === 'document' ) { - - if ( encodingOrMimeType !== undefined ) { - - this.mimeType = encodingOrMimeType; - - } - - } else if ( value === 'text' ) { - - if ( encodingOrMimeType !== undefined ) { - - this.textEncoding = encodingOrMimeType; - - } - - } - return this; } setMimeType( value ) { - console.warn( `deprecated, use .setResponseType( 'document', '${value}' ) instead.` ); - this.mimeType = value; return this;