Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: heavily simplify multipart formdata snippet generation #224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 8, 2024
Merged

feat: heavily simplify multipart formdata snippet generation #224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aeb4c98
feat: heavily simplify multipart formdata snippet generation
erunion Apr 8, 2024
f144eb5
fix: backporting some formdata-to-string work into this
erunion Apr 8, 2024
6d795f1
Apply suggestions from code review
erunion Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,11 @@ const snippet = new HTTPSnippet({
});

// generate Node.js: Native output
console.log(await snippet.convert('node'));
console.log(snippet.convert('node'));

// generate Node.js: Native output, indent with tabs
console.log(
await snippet.convert('node', {
snippet.convert('node', {
indent: '\t',
}),
);
Expand Down Expand Up @@ -104,13 +104,13 @@ const snippet = new HTTPSnippet({

// generate Shell: cURL output
console.log(
await snippet.convert('shell', 'curl', {
snippet.convert('shell', 'curl', {
indent: '\t',
}),
);

// generate Node.js: Unirest output
console.log(await snippet.convert('node', 'unirest'));
console.log(snippet.convert('node', 'unirest'));
```

### addTarget(target)
Expand Down
9 changes: 0 additions & 9 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@
"test": "vitest run --coverage"
},
"dependencies": {
"formdata-to-string": "^2.0.2",
"qs": "^6.11.2",
"stringify-object": "^3.3.0"
},
Expand Down
2 changes: 1 addition & 1 deletion src/fixtures/runCustomFixtures.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export const runCustomFixtures = ({ targetId, clientId, tests }: CustomFixture)
}

const snippet = new HTTPSnippet(request, opts);
const result = await snippet.convert(targetId, clientId, options);
const result = snippet.convert(targetId, clientId, options);
const filePath = path.join(__dirname, '..', 'targets', targetId, clientId, 'fixtures', fixtureFile);
if (process.env.OVERWRITE_EVERYTHING) {
writeFileSync(filePath, String(result));
Expand Down
60 changes: 30 additions & 30 deletions src/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,32 +10,32 @@ import short from './fixtures/requests/short.cjs';
import { HTTPSnippet } from './index.js';

describe('HTTPSnippet', () => {
it('should return false if no matching target', async () => {
it('should return false if no matching target', () => {
const snippet = new HTTPSnippet(short.log.entries[0].request as Request);

// @ts-expect-error intentionally incorrect
const result = await snippet.convert(null);
const result = snippet.convert(null);

expect(result).toBe(false);
});

describe('repair malformed `postData`', () => {
it('should repair a HAR with an empty `postData` object', async () => {
it('should repair a HAR with an empty `postData` object', () => {
const snippet = new HTTPSnippet({
method: 'POST',
url: 'https://httpbin.org/anything',
postData: {},
} as Request);

await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.postData).toStrictEqual({
mimeType: 'application/octet-stream',
});
});

it('should repair a HAR with a `postData` params object missing `mimeType`', async () => {
it('should repair a HAR with a `postData` params object missing `mimeType`', () => {
// @ts-expect-error Testing a malformed HAR case.
const snippet = new HTTPSnippet({
method: 'POST',
Expand All @@ -44,7 +44,7 @@ describe('HTTPSnippet', () => {
params: [],
},
} as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.postData).toStrictEqual({
Expand All @@ -53,15 +53,15 @@ describe('HTTPSnippet', () => {
});
});

it('should repair a HAR with a `postData` text object missing `mimeType`', async () => {
it('should repair a HAR with a `postData` text object missing `mimeType`', () => {
const snippet = new HTTPSnippet({
method: 'POST',
url: 'https://httpbin.org/anything',
postData: {
text: '',
},
} as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.postData).toStrictEqual({
Expand All @@ -71,7 +71,7 @@ describe('HTTPSnippet', () => {
});
});

it('should parse HAR file with multiple entries', async () => {
it('should parse HAR file with multiple entries', () => {
const snippet = new HTTPSnippet({
log: {
version: '1.2',
Expand All @@ -96,7 +96,7 @@ describe('HTTPSnippet', () => {
},
});

await snippet.convert('node');
snippet.convert('node');

expect(snippet).toHaveProperty('requests');
expect(Array.isArray(snippet.requests)).toBeTruthy();
Expand Down Expand Up @@ -136,28 +136,28 @@ describe('HTTPSnippet', () => {
] as {
expected: string;
input: keyof typeof mimetypes;
}[])('mimetype conversion of $input to $output', async ({ input, expected }) => {
}[])('mimetype conversion of $input to $output', ({ input, expected }) => {
const snippet = new HTTPSnippet(mimetypes[input]);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.postData.mimeType).toStrictEqual(expected);
});
});

it('should set postData.text to empty string when postData.params is undefined in application/x-www-form-urlencoded', async () => {
it('should set postData.text to empty string when postData.params is undefined in application/x-www-form-urlencoded', () => {
const snippet = new HTTPSnippet(mimetypes['application/x-www-form-urlencoded']);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.postData.text).toBe('');
});

describe('requestExtras', () => {
describe('uriObj', () => {
it('should add uriObj', async () => {
it('should add uriObj', () => {
const snippet = new HTTPSnippet(query.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];

Expand All @@ -181,29 +181,29 @@ describe('HTTPSnippet', () => {
});
});

it('should fix the `path` property of uriObj to match queryString', async () => {
it('should fix the `path` property of uriObj to match queryString', () => {
const snippet = new HTTPSnippet(query.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.uriObj.path).toBe('/anything?foo=bar&foo=baz&baz=abc&key=value');
});
});

describe('queryObj', () => {
it('should add queryObj', async () => {
it('should add queryObj', () => {
const snippet = new HTTPSnippet(query.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.queryObj).toMatchObject({ baz: 'abc', key: 'value', foo: ['bar', 'baz'] });
});
});

describe('headersObj', () => {
it('should add headersObj', async () => {
it('should add headersObj', () => {
const snippet = new HTTPSnippet(headers.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.headersObj).toMatchObject({
Expand All @@ -212,7 +212,7 @@ describe('HTTPSnippet', () => {
});
});

it('should add headersObj to source object case insensitive when HTTP/1.0', async () => {
it('should add headersObj to source object case insensitive when HTTP/1.0', () => {
const snippet = new HTTPSnippet({
...headers.log.entries[0].request,
httpVersion: 'HTTP/1.1',
Expand All @@ -225,7 +225,7 @@ describe('HTTPSnippet', () => {
],
} as Request);

await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];

Expand All @@ -236,7 +236,7 @@ describe('HTTPSnippet', () => {
});
});

it('should add headersObj to source object lowercased when HTTP/2.x', async () => {
it('should add headersObj to source object lowercased when HTTP/2.x', () => {
const snippet = new HTTPSnippet({
...headers.log.entries[0].request,
httpVersion: 'HTTP/2',
Expand All @@ -249,7 +249,7 @@ describe('HTTPSnippet', () => {
],
} as Request);

await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];

Expand All @@ -262,19 +262,19 @@ describe('HTTPSnippet', () => {
});

describe('url', () => {
it('should modify the original url to strip query string', async () => {
it('should modify the original url to strip query string', () => {
const snippet = new HTTPSnippet(query.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.url).toBe('https://httpbin.org/anything');
});
});

describe('fullUrl', () => {
it('adds fullURL', async () => {
it('adds fullURL', () => {
const snippet = new HTTPSnippet(query.log.entries[0].request as Request);
await snippet.convert('node');
snippet.convert('node');

const request = snippet.requests[0];
expect(request.fullUrl).toBe('https://httpbin.org/anything?foo=bar&foo=baz&baz=abc&key=value');
Expand Down
49 changes: 31 additions & 18 deletions src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import type { UrlWithParsedQuery } from 'node:url';

import { format as urlFormat, parse as urlParse } from 'node:url';

import formDataToString from 'formdata-to-string';
import { stringify as queryStringify } from 'qs';

import { getHeaderName } from './helpers/headers.js';
Expand Down Expand Up @@ -100,12 +99,10 @@ export class HTTPSnippet {
}
}

async init() {
init() {
this.initCalled = true;

const promises: Promise<Request>[] = [];

this.entries.forEach(({ request }) => {
this.requests = this.entries.map(({ request }) => {
// add optional properties to make validation successful
const req = {
bodySize: 0,
Expand All @@ -125,15 +122,13 @@ export class HTTPSnippet {
req.postData.mimeType = 'application/octet-stream';
}

promises.push(this.prepare(req as HarRequest, this.options));
return this.prepare(req as HarRequest, this.options);
});

this.requests = await Promise.all(promises);

return this;
}

async prepare(harRequest: HarRequest, options: HTTPSnippetOptions) {
prepare(harRequest: HarRequest, options: HTTPSnippetOptions) {
const request: Request = {
...harRequest,
fullUrl: '',
Expand Down Expand Up @@ -195,24 +190,42 @@ export class HTTPSnippet {
request.postData.mimeType = 'multipart/form-data';

if (request.postData?.params) {
const form = new FormData();
const boundary = '---011000010111000001101001'; // this is binary for "api" (easter egg)
const carriage = `${boundary}--`;
const rn = '\r\n';

request.postData?.params.forEach(param => {
/*! formdata-polyfill. MIT License. Jimmy Wärting <https://jimmy.warting.se/opensource> */
const escape = (str: string) => str.replace(/\n/g, '%0A').replace(/\r/g, '%0D').replace(/"/g, '%22');
const normalizeLinefeeds = (value: string) => value.replace(/\r?\n|\r/g, '\r\n');

const payload = [`--${boundary}`];
request.postData?.params.forEach((param, i) => {
const name = param.name;
const value = param.value || '';
const filename = param.fileName || null;
const contentType = param.contentType || '';
const contentType = param.contentType || 'application/octet-stream';

if (filename) {
form.append(name, new Blob([value], { type: contentType }), filename);
payload.push(
`Content-Disposition: form-data; name="${escape(normalizeLinefeeds(name))}"; filename="${filename}"`,
);
payload.push(`Content-Type: ${contentType}`);
} else {
form.append(name, value);
payload.push(`Content-Disposition: form-data; name="${escape(normalizeLinefeeds(name))}"`);
}

payload.push('');
payload.push(normalizeLinefeeds(value));

if (i !== (request.postData.params as Param[]).length - 1) {
payload.push(`--${boundary}`);
}
});

const boundary = '---011000010111000001101001'; // this is binary for "api" (easter egg)
payload.push(`--${carriage}`);

request.postData.boundary = boundary;
request.postData.text = await formDataToString(form, { boundary });
request.postData.text = payload.join(rn);

// Since headers are case-sensitive we need to see if there's an existing `Content-Type` header that we can override.
const contentTypeHeader = getHeaderName(request.headersObj, 'content-type') || 'content-type';
Expand Down Expand Up @@ -305,9 +318,9 @@ export class HTTPSnippet {
};
}

async convert(targetId: TargetId, clientId?: ClientId, options?: any) {
convert(targetId: TargetId, clientId?: ClientId, options?: any) {
if (!this.initCalled) {
await this.init();
this.init();
}

if (!options && clientId) {
Expand Down
Loading