Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b4bcfe0

Browse files
hanshuaihanshuai
authored
Merge pull request lowcoder-org#294 from openblocks-dev/0308-datasources
0308 datasources
2 parents d3d5141 + 22b00f5 commit b4bcfe0

File tree

146 files changed

+690812
-149
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

146 files changed

+690812
-149
lines changed

‎server/node-service/jest.config.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ module.exports = {
44
testEnvironment: "node",
55
testTimeout: 10 * 60000,
66
testPathIgnorePatterns: ["/node_modules/", "/build/"],
7+
setupFiles: ["./jest.setup.ts"],
78
};

‎server/node-service/jest.setup.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
import "./src/common/logger";

‎server/node-service/nodemon.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"restartable": "rs",
3+
"ignore": [".git", "node_modules/**/node_modules"],
4+
"verbose": true,
5+
"execMap": {
6+
"ts": "node --require ts-node/register"
7+
},
8+
"watch": ["src/"],
9+
"env": {
10+
"NODE_ENV": "development"
11+
},
12+
"ext": "js,json,ts"
13+
}

‎server/node-service/package.json

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,22 @@
77
},
88
"main": "src/server.ts",
99
"scripts": {
10-
"dev": "nodemon --files src/server.ts",
10+
"dev": "nodemon src/server.ts",
11+
"debug": "nodemon --inspect src/server.ts",
1112
"start": "node ./build/src/server.js",
1213
"test": "jest",
13-
"build": "rm -rf build/ && tsc && yarn run copy",
14-
"copy": "copyfiles -u 1 src/**/*.!(ts|test.ts) build/src"
14+
"copy": "copyfiles -u 1 src/**/*.!(ts|test.ts) build/src",
15+
"genOpenApiPlugin": "npx ts-node --files ./scripts/openApiDataSourceGen.ts",
16+
"build": "rm -rf build/ && yarn test && tsc && cp -r src/static build/src/static"
1517
},
1618
"devDependencies": {
1719
"@types/jest": "^29.2.4",
20+
"commander": "^10.0.0",
1821
"copyfiles": "^2.4.1",
1922
"jest": "^29.3.1",
2023
"nock": "^13.3.0",
2124
"nodemon": "^2.0.20",
25+
"postman-to-openapi": "^3.0.1",
2226
"svgo": "^3.0.2",
2327
"ts-jest": "^29.0.3",
2428
"ts-node": "^10.9.1"
@@ -30,30 +34,42 @@
3034
"@aws-sdk/client-lambda": "^3.272.0",
3135
"@aws-sdk/client-s3": "^3.238.0",
3236
"@aws-sdk/s3-request-presigner": "^3.241.0",
37+
"@google-cloud/bigquery": "^6.1.0",
3338
"@google-cloud/storage": "^6.9.3",
39+
"@supabase/supabase-js": "^2.10.0",
3440
"@types/axios": "^0.14.0",
3541
"@types/express": "^4.17.14",
3642
"@types/jsonpath": "^0.2.0",
3743
"@types/lodash": "^4.14.190",
3844
"@types/morgan": "^1.9.3",
3945
"@types/node": "^18.11.18",
46+
"@types/node-fetch": "^2.6.2",
4047
"axios": "^1.2.0",
48+
"base64-arraybuffer": "^1.0.2",
4149
"dynamodb-data-types": "^4.0.1",
4250
"express": "^4.18.2",
4351
"express-async-errors": "^3.1.1",
4452
"firebase-admin": "^11.5.0",
4553
"formdata-node": "4",
54+
"graphql": "^16.6.0",
55+
"graphql-request": "^5.1.0",
4656
"jsonpath": "^1.1.1",
4757
"lodash": "^4.17.21",
4858
"loglevel": "^1.8.1",
4959
"morgan": "^1.10.0",
60+
"node-fetch": "2",
5061
"openapi-types": "^12.1.0",
5162
"openblocks-core": "^0.0.7",
52-
"openblocks-sdk": "^0.0.35",
63+
"openblocks-sdk": "0.0.38",
64+
"pino": "^8.11.0",
65+
"proxy-agent": "^5.0.0",
5366
"stylis": "^4.1.3",
5467
"swagger-client": "^3.18.5",
5568
"typescript": "^4.9.3",
5669
"yaml": "^2.2.1"
5770
},
71+
"resolutions": {
72+
"@apidevtools/json-schema-ref-parser": "9.0.7"
73+
},
5874
"packageManager": "[email protected]"
5975
}

‎server/node-service/scripts/openApiDataSourceGen.ts

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { readYaml } from "../src/common/util";
2+
import fs from "fs";
3+
import _ from "lodash";
4+
import path from "path";
5+
import { authParamsConfig, retrieveSpec } from "../src/plugins/openApi/parse";
6+
import postManToOpenApi from "postman-to-openapi";
7+
import { program } from "commander";
8+
9+
interface Options {
10+
force: boolean;
11+
name: string;
12+
url: string;
13+
postMan: boolean;
14+
postManCollectionFile: string;
15+
id?: string;
16+
}
17+
18+
async function gen(options: Options) {
19+
const { postMan, force, postManCollectionFile, name, url: specUrl, id: pluginId } = options;
20+
const id = pluginId ?? _.camelCase(name);
21+
const pluginDir = path.join(path.dirname(__dirname), "src/plugins", id);
22+
const pluginEntryFile = path.join(pluginDir, "index.ts");
23+
const pluginSpecYamlFile = path.join(pluginDir, `${id}.spec.yaml`);
24+
const pluginSpecJsonFile = path.join(pluginDir, `${id}.spec.json`);
25+
const pluginDefaultCollectionFile = path.join(pluginDir, `${id}.collection.json`);
26+
27+
console.info();
28+
console.info("start generate plugin, id:", id, "name:", name);
29+
30+
if (postMan) {
31+
console.info("is PostMan Collection start transforming...");
32+
const collection = postManCollectionFile ?? pluginDefaultCollectionFile;
33+
await postManToOpenApi(collection, pluginSpecYamlFile, { defaultTag: "General" });
34+
}
35+
36+
if (!fs.existsSync(pluginDir)) {
37+
fs.mkdirSync(pluginDir, { recursive: true });
38+
console.info(`plugin dir ${id} created.`);
39+
}
40+
41+
if (!force && fs.existsSync(pluginEntryFile)) {
42+
console.info(`plugin: ${id} is already existed.`);
43+
return;
44+
}
45+
46+
// 1. fetch spec url
47+
let spec: any;
48+
const isYamlSpecExist = fs.existsSync(pluginSpecYamlFile);
49+
const isJsonSpecExist = fs.existsSync(pluginSpecJsonFile);
50+
if (!isYamlSpecExist && !isJsonSpecExist) {
51+
if (!specUrl) {
52+
console.error("specUrl is required to fetch OpenAPI spec.");
53+
return;
54+
}
55+
console.info(`start fetching:`, specUrl);
56+
const { spec } = await retrieveSpec(specUrl);
57+
fs.writeFileSync(pluginSpecJsonFile, JSON.stringify(spec, null, 2));
58+
console.info(`${name}spec.json saved`);
59+
} else {
60+
if (isJsonSpecExist) {
61+
const specJson = fs.readFileSync(pluginSpecJsonFile).toString();
62+
spec = JSON.parse(specJson);
63+
console.info("got spec from json file:", pluginSpecJsonFile);
64+
}
65+
if (isYamlSpecExist) {
66+
spec = readYaml(pluginSpecYamlFile);
67+
console.info("got spec from yaml file:", pluginSpecYamlFile);
68+
}
69+
}
70+
71+
if (!spec) {
72+
console.error("can not get spec");
73+
return;
74+
}
75+
76+
// 2. get data source params
77+
const dataSourceParams = await authParamsConfig(spec);
78+
79+
// 3. gen code
80+
const template = fs
81+
.readFileSync(path.join(__dirname, "./openApiDataSourceTemplate.tpl"))
82+
.toString();
83+
const data = {
84+
id,
85+
name,
86+
isJsonSpec: isJsonSpecExist,
87+
isYamlSpec: isYamlSpecExist,
88+
dataSourceParams: JSON.stringify(dataSourceParams, null, 2),
89+
};
90+
const compiledTemplate = _.template(template);
91+
const code = compiledTemplate(data);
92+
fs.writeFileSync(pluginEntryFile, code);
93+
console.info("success generate plugin:", pluginDir);
94+
console.info();
95+
}
96+
97+
const plugins = [
98+
// ["Jira", "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json"],
99+
[],
100+
];
101+
102+
program
103+
.option("--force")
104+
.option("--post-man")
105+
.option("-f, --post-man-collection-file [postman collection file path]")
106+
.option("-n, --name <char>")
107+
.option("-i, --id [plugin id]")
108+
.option("--url [spec-download-url]");
109+
110+
program.parse();
111+
112+
const options = program.opts<Options>();
113+
114+
gen(options);

‎server/node-service/scripts/openApiDataSourceTemplate.tpl

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { readYaml } from "../../common/util";
2+
import _ from "lodash";
3+
import path from "path";
4+
import { OpenAPIV3, OpenAPI } from "openapi-types";
5+
import { ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
6+
import { runOpenApi } from "../openApi";
7+
import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
8+
<% if (isJsonSpec) {%>
9+
import spec from './<%=id %>.spec.json';
10+
<% } %>
11+
12+
<% if (isYamlSpec) {%>
13+
const spec = readYaml(path.join(__dirname, "./<%=id %>.spec.yaml"));
14+
<% } %>
15+
16+
const dataSourceConfig = {
17+
type: "dataSource",
18+
params: <%=dataSourceParams %>
19+
} as const;
20+
21+
const parseOptions: ParseOpenApiOptions = {
22+
actionLabel: (method: string, path: string, operation: OpenAPI.Operation) => {
23+
return _.upperFirst(operation.operationId || "");
24+
},
25+
};
26+
27+
type DataSourceConfigType = ConfigToType<typeof dataSourceConfig>;
28+
29+
const <%=id %>Plugin: DataSourcePlugin<any, DataSourceConfigType> = {
30+
id: "<%=id %>",
31+
name: "<%=name %>",
32+
icon: "<%=id %>.svg",
33+
category: "api",
34+
dataSourceConfig,
35+
queryConfig: async () => {
36+
const { actions, categories } = await parseOpenApi(spec<% if (isJsonSpec) { %> as unknown<% } %> as OpenAPI.Document, parseOptions);
37+
return {
38+
type: "query",
39+
label: "Action",
40+
categories: {
41+
label: "Resources",
42+
items: categories,
43+
},
44+
actions,
45+
};
46+
},
47+
run: function (actionData, dataSourceConfig): Promise<any> {
48+
const runApiDsConfig = {
49+
url: "",
50+
serverURL: "",
51+
dynamicParamsConfig: dataSourceConfig,
52+
};
53+
return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV3.Document);
54+
},
55+
};
56+
57+
export default <%=id %>Plugin;

‎server/node-service/src/common/fetch.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import nodeFetch, { RequestInit, RequestInfo } from "node-fetch";
2+
import proxyAgent from "proxy-agent";
3+
4+
export function fetch(url: RequestInfo, init?: RequestInit) {
5+
return nodeFetch(url, { agent: proxyAgent(), ...init });
6+
}

‎server/node-service/src/common/logger.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import pino from "pino";
2+
3+
global.logger = pino();

‎server/node-service/src/common/util.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
11
import yaml from "yaml";
22
import fs from "fs";
33

4+
export function kvToRecord(
5+
kvs: { key: string; value: string }[],
6+
trimEmpty: boolean = true
7+
): Record<string, string> {
8+
const ret: Record<string, string> = {};
9+
(kvs || []).forEach(({ key, value }) => {
10+
if (trimEmpty && !value) {
11+
return;
12+
}
13+
ret[key] = value;
14+
});
15+
return ret;
16+
}
17+
418
export function toString(value: any): string {
519
if (value === undefined || value === null) {
620
return "";

‎server/node-service/src/global.d.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { Logger } from "pino";
2+
3+
declare global {
4+
var logger: Logger;
5+
}
6+
7+
export {};

0 commit comments

Comments
 (0)