diff --git a/core/download.ts b/core/download.ts index 39e0b02a..e934c955 100644 --- a/core/download.ts +++ b/core/download.ts @@ -40,6 +40,12 @@ function detectType(text: string): string | undefined { } } +function parseEncodeType(response: Partial): string { + let headers = response.headers ?? new Headers() + let contentType = headers.get('content-type')?.toLowerCase() ?? '' + return contentType.match(/;\s*charset=([^\s;]+)/)?.[1] ?? 'utf-8' +} + export function createTextResponse( text: string, other: Partial> = {} @@ -122,7 +128,16 @@ export function createDownloadTask(): DownloadTask { }, async text(url, opts) { let response = await this.request(url, opts) - let text = await response.text() + let text: string + let encodeType = parseEncodeType(response) + if (encodeType !== 'utf-8') { + let buffer = await response.arrayBuffer() + let decoder = new TextDecoder(encodeType) + text = decoder.decode(buffer) + } else { + text = await response.text() + } + if (controller.signal.aborted) { throw new DOMException('', 'AbortError') } diff --git a/core/test/download.test.ts b/core/test/download.test.ts index 8d6ee469..d1c5dab3 100644 --- a/core/test/download.test.ts +++ b/core/test/download.test.ts @@ -124,6 +124,22 @@ test('can download text by keeping eyes on abort signal', async () => { await rejects(response2, (e: Error) => e.name === 'AbortError') }) +test('detect unpopular encode type', async () => { + let task = createDownloadTask() + + expectRequest('https://example.com').andRespond( + 200, + 'Hi', + 'text/plain; charset=shift-jis' + ) + let response1 = await task.text('https://example.com') + + equal(response1.ok, true) + equal(response1.status, 200) + equal(response1.url, 'https://example.com') + equal(response1.text, 'Hi') +}) + test('parses XML content', async () => { let text = createTextResponse('Test') equal(text.parseXml()!.firstChild?.lastChild?.nodeName, 'BODY')