Skip to content

Commit 67a4b65

Browse files
stainless-app[bot]stainless-app[bot]
authored
chore(internal): move LineDecoder to a separate file (#1120)
1 parent f7dc520 commit 67a4b65

File tree

2 files changed

+115
-111
lines changed

2 files changed

+115
-111
lines changed

‎src/internal/decoders/line.ts‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { OpenAIError } from '../../error';
2+
3+
type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;
4+
5+
/**
6+
* A re-implementation of httpx's `LineDecoder` in Python that handles incrementally
7+
* reading lines from text.
8+
*
9+
* https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258
10+
*/
11+
export class LineDecoder {
12+
// prettier-ignore
13+
static NEWLINE_CHARS = new Set(['\n', '\r']);
14+
static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
15+
16+
buffer: string[];
17+
trailingCR: boolean;
18+
textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
19+
20+
constructor() {
21+
this.buffer = [];
22+
this.trailingCR = false;
23+
}
24+
25+
decode(chunk: Bytes): string[] {
26+
let text = this.decodeText(chunk);
27+
28+
if (this.trailingCR) {
29+
text = '\r' + text;
30+
this.trailingCR = false;
31+
}
32+
if (text.endsWith('\r')) {
33+
this.trailingCR = true;
34+
text = text.slice(0, -1);
35+
}
36+
37+
if (!text) {
38+
return [];
39+
}
40+
41+
const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
42+
let lines = text.split(LineDecoder.NEWLINE_REGEXP);
43+
44+
// if there is a trailing new line then the last entry will be an empty
45+
// string which we don't care about
46+
if (trailingNewline) {
47+
lines.pop();
48+
}
49+
50+
if (lines.length === 1 && !trailingNewline) {
51+
this.buffer.push(lines[0]!);
52+
return [];
53+
}
54+
55+
if (this.buffer.length > 0) {
56+
lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
57+
this.buffer = [];
58+
}
59+
60+
if (!trailingNewline) {
61+
this.buffer = [lines.pop() || ''];
62+
}
63+
64+
return lines;
65+
}
66+
67+
decodeText(bytes: Bytes): string {
68+
if (bytes == null) return '';
69+
if (typeof bytes === 'string') return bytes;
70+
71+
// Node:
72+
if (typeof Buffer !== 'undefined') {
73+
if (bytes instanceof Buffer) {
74+
return bytes.toString();
75+
}
76+
if (bytes instanceof Uint8Array) {
77+
return Buffer.from(bytes).toString();
78+
}
79+
80+
throw new OpenAIError(
81+
`Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`,
82+
);
83+
}
84+
85+
// Browser
86+
if (typeof TextDecoder !== 'undefined') {
87+
if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {
88+
this.textDecoder ??= new TextDecoder('utf8');
89+
return this.textDecoder.decode(bytes);
90+
}
91+
92+
throw new OpenAIError(
93+
`Unexpected: received non-Uint8Array/ArrayBuffer (${
94+
(bytes as any).constructor.name
95+
}) in a web platform. Please report this error.`,
96+
);
97+
}
98+
99+
throw new OpenAIError(
100+
`Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,
101+
);
102+
}
103+
104+
flush(): string[] {
105+
if (!this.buffer.length && !this.trailingCR) {
106+
return [];
107+
}
108+
109+
const lines = [this.buffer.join('')];
110+
this.buffer = [];
111+
this.trailingCR = false;
112+
return lines;
113+
}
114+
}

‎src/streaming.ts‎

Lines changed: 1 addition & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { ReadableStream, type Response } from './_shims/index';
22
import { OpenAIError } from './error';
3+
import { LineDecoder } from './internal/decoders/line';
34

45
import { APIError } from 'openai/error';
56

@@ -329,117 +330,6 @@ class SSEDecoder {
329330
}
330331
}
331332

332-
/**
333-
* A re-implementation of httpx's `LineDecoder` in Python that handles incrementally
334-
* reading lines from text.
335-
*
336-
* https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258
337-
*/
338-
class LineDecoder {
339-
// prettier-ignore
340-
static NEWLINE_CHARS = new Set(['\n', '\r']);
341-
static NEWLINE_REGEXP = /\r\n|[\n\r]/g;
342-
343-
buffer: string[];
344-
trailingCR: boolean;
345-
textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types.
346-
347-
constructor() {
348-
this.buffer = [];
349-
this.trailingCR = false;
350-
}
351-
352-
decode(chunk: Bytes): string[] {
353-
let text = this.decodeText(chunk);
354-
355-
if (this.trailingCR) {
356-
text = '\r' + text;
357-
this.trailingCR = false;
358-
}
359-
if (text.endsWith('\r')) {
360-
this.trailingCR = true;
361-
text = text.slice(0, -1);
362-
}
363-
364-
if (!text) {
365-
return [];
366-
}
367-
368-
const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');
369-
let lines = text.split(LineDecoder.NEWLINE_REGEXP);
370-
371-
// if there is a trailing new line then the last entry will be an empty
372-
// string which we don't care about
373-
if (trailingNewline) {
374-
lines.pop();
375-
}
376-
377-
if (lines.length === 1 && !trailingNewline) {
378-
this.buffer.push(lines[0]!);
379-
return [];
380-
}
381-
382-
if (this.buffer.length > 0) {
383-
lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];
384-
this.buffer = [];
385-
}
386-
387-
if (!trailingNewline) {
388-
this.buffer = [lines.pop() || ''];
389-
}
390-
391-
return lines;
392-
}
393-
394-
decodeText(bytes: Bytes): string {
395-
if (bytes == null) return '';
396-
if (typeof bytes === 'string') return bytes;
397-
398-
// Node:
399-
if (typeof Buffer !== 'undefined') {
400-
if (bytes instanceof Buffer) {
401-
return bytes.toString();
402-
}
403-
if (bytes instanceof Uint8Array) {
404-
return Buffer.from(bytes).toString();
405-
}
406-
407-
throw new OpenAIError(
408-
`Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`,
409-
);
410-
}
411-
412-
// Browser
413-
if (typeof TextDecoder !== 'undefined') {
414-
if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {
415-
this.textDecoder ??= new TextDecoder('utf8');
416-
return this.textDecoder.decode(bytes);
417-
}
418-
419-
throw new OpenAIError(
420-
`Unexpected: received non-Uint8Array/ArrayBuffer (${
421-
(bytes as any).constructor.name
422-
}) in a web platform. Please report this error.`,
423-
);
424-
}
425-
426-
throw new OpenAIError(
427-
`Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,
428-
);
429-
}
430-
431-
flush(): string[] {
432-
if (!this.buffer.length && !this.trailingCR) {
433-
return [];
434-
}
435-
436-
const lines = [this.buffer.join('')];
437-
this.buffer = [];
438-
this.trailingCR = false;
439-
return lines;
440-
}
441-
}
442-
443333
/** This is an internal helper function that's just used for testing */
444334
export function _decodeChunks(chunks: string[]): string[] {
445335
const decoder = new LineDecoder();

0 commit comments

Comments
 (0)