From b66200419dc15e7866b16f8d01cc62a975554b0c Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Mon, 15 May 2017 23:40:53 +0300 Subject: [PATCH 1/8] Init with old bad code --- customerWorker/1-customer.js | 81 +++++++++++++++++++++++++++++++++ customerWorker/2-worker.js | 16 +++++++ customerWorker/worker.js | 39 ++++++++++++++++ customerWorker/workersMaster.js | 26 +++++++++++ 4 files changed, 162 insertions(+) create mode 100644 customerWorker/1-customer.js create mode 100644 customerWorker/2-worker.js create mode 100644 customerWorker/worker.js create mode 100644 customerWorker/workersMaster.js diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js new file mode 100644 index 0000000..aa6cf1d --- /dev/null +++ b/customerWorker/1-customer.js @@ -0,0 +1,81 @@ +var api = {}; +global.api = api; +api.net = require('net'); +api.os = require('os'); + +var workersCount = 10; +var startPort = 10000; +var defaultElementsByTask = 1; +var host = '127.0.0.1'; +var task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11]; + +var tasks = ChunkArray(task,defaultElementsByTask,workersCount); +SendTasksAndAggregateAnswers(tasks); + +function ChunkArray(arr,elementsByPart,clientsCount) { + var tasks = []; + var needWorkers = arr.length/elementsByPart; + if (needWorkers <= clientsCount) { + + } else { + console.log(needWorkers+'-'+elementsByPart); + while (needWorkers > clientsCount){ + elementsByPart++; + needWorkers = arr.length/elementsByPart; + console.log(needWorkers+'-'+elementsByPart); + } + } + var i = 0; + while (arr.length > 0) { + tasks.push({ + index: i++, + task: arr.splice(0, elementsByPart) + } + ); + } + //console.dir(tasks); + return tasks; +} +function SendTasksAndAggregateAnswers(tasks) { + var answers = []; + var sockets = []; + var answersCount = tasks.length; + for(var i = startPort;i<(answersCount+startPort);i++) { + var socket = new api.net.Socket(); + socket.on('data', function (data) { + console.log('Data received (by client): ' + data); + var res = JSON.parse(data); + answers.push(res); + if (answers.length === answersCount) { + //console.dir(answers); + SortAndMergeAnswers(answers); + } + }); + //socket.on('close', function () { + // console.log('Connection closed'); + //}); + socket.connect({port: i, host: host}); + sockets.push(socket); + } + + tasks.forEach(function (task, i) { + console.log('Data send (by client): ' + JSON.stringify(task)); + // console.dir(task); + var dataObj = task; + sockets[i].write(JSON.stringify(dataObj)); + }); +} + +function SortAndMergeAnswers(answers) { + var res = []; + var temp = []; + answers.forEach(function (value) { + temp[value.index] = value.answer; + }); + for(var i=0;i Date: Tue, 16 May 2017 00:56:04 +0300 Subject: [PATCH 2/8] Fixed linter errors. Some small changes --- customerWorker/1-customer.js | 130 +++++++++++++++----------------- customerWorker/2-worker.js | 5 +- customerWorker/worker.js | 60 +++++++-------- customerWorker/workersMaster.js | 31 ++++---- 4 files changed, 104 insertions(+), 122 deletions(-) diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js index aa6cf1d..5b18c70 100644 --- a/customerWorker/1-customer.js +++ b/customerWorker/1-customer.js @@ -1,81 +1,71 @@ -var api = {}; -global.api = api; +'use strict'; + +global.api = {}; api.net = require('net'); api.os = require('os'); -var workersCount = 10; -var startPort = 10000; -var defaultElementsByTask = 1; -var host = '127.0.0.1'; -var task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11]; - -var tasks = ChunkArray(task,defaultElementsByTask,workersCount); -SendTasksAndAggregateAnswers(tasks); +const workersCount = 10; +const startPort = 10000; +const defaultElementsByTask = 1; +const host = '127.0.0.1'; +const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11]; -function ChunkArray(arr,elementsByPart,clientsCount) { - var tasks = []; - var needWorkers = arr.length/elementsByPart; - if (needWorkers <= clientsCount) { +const tasks = chunkArray(task, defaultElementsByTask, workersCount); +sendTasks(tasks); - } else { - console.log(needWorkers+'-'+elementsByPart); - while (needWorkers > clientsCount){ - elementsByPart++; - needWorkers = arr.length/elementsByPart; - console.log(needWorkers+'-'+elementsByPart); - } - } - var i = 0; - while (arr.length > 0) { - tasks.push({ - index: i++, - task: arr.splice(0, elementsByPart) - } - ); +function chunkArray(arr, elementsByPart, clientsCount) { + const tasks = []; + let needWorkers = arr.length / elementsByPart; + if (needWorkers > clientsCount) { + console.log(needWorkers + '-' + elementsByPart); + while (needWorkers > clientsCount) { + elementsByPart++; + needWorkers = arr.length / elementsByPart; } - //console.dir(tasks); - return tasks; + } + let i = 0; + while (arr.length > 0) { + tasks.push({ + index: i++, + task: arr.splice(0, elementsByPart) + }); + } + return tasks; } -function SendTasksAndAggregateAnswers(tasks) { - var answers = []; - var sockets = []; - var answersCount = tasks.length; - for(var i = startPort;i<(answersCount+startPort);i++) { - var socket = new api.net.Socket(); - socket.on('data', function (data) { - console.log('Data received (by client): ' + data); - var res = JSON.parse(data); - answers.push(res); - if (answers.length === answersCount) { - //console.dir(answers); - SortAndMergeAnswers(answers); - } - }); - //socket.on('close', function () { - // console.log('Connection closed'); - //}); - socket.connect({port: i, host: host}); - sockets.push(socket); - } - - tasks.forEach(function (task, i) { - console.log('Data send (by client): ' + JSON.stringify(task)); - // console.dir(task); - var dataObj = task; - sockets[i].write(JSON.stringify(dataObj)); +function sendTasks(tasks) { + const answers = []; + const sockets = []; + const answersCount = tasks.length; + for (let i = startPort; i < (answersCount + startPort); i++) { + const socket = new api.net.Socket(); + socket.on('data', (data) => { + console.log('Data received (by client): ' + data); + const res = JSON.parse(data); + answers.push(res); + if (answers.length === answersCount) { + mergeAnswers(answers); + } }); + socket.connect({ port: i, host }); + sockets.push(socket); + } + + tasks.forEach((task, i) => { + console.log('Data send (by client): ' + JSON.stringify(task)); + const dataObj = task; + sockets[i].write(JSON.stringify(dataObj)); + }); } -function SortAndMergeAnswers(answers) { - var res = []; - var temp = []; - answers.forEach(function (value) { - temp[value.index] = value.answer; - }); - for(var i=0;i { + temp[value.index] = value.answer; + }); + for (let i = 0; i < temp.length; i++) { + res = res.concat(temp[i]); + } + console.dir(temp); + console.dir(res); } diff --git a/customerWorker/2-worker.js b/customerWorker/2-worker.js index 1761f39..8d57d01 100644 --- a/customerWorker/2-worker.js +++ b/customerWorker/2-worker.js @@ -1,6 +1,5 @@ -/** - * Created by Yura on 11.04.2016. - */ +'use strict'; + global.api = {}; global.port = 10000; api.cluster = require('cluster'); diff --git a/customerWorker/worker.js b/customerWorker/worker.js index 8e43dd8..582b019 100644 --- a/customerWorker/worker.js +++ b/customerWorker/worker.js @@ -1,39 +1,33 @@ -module.exports = function() { - process.on('message', function (message) { - CreateServer(parseInt(message)); - }); -}; +'use strict'; -function CreateServer(port) { - console.log('Listen port # '+port); - var server = api.net.createServer(function(socket) { - console.log('CONNECTED: ' + socket.remoteAddress +':'+ socket.remotePort); - socket.on('error', function(err) { - console.log(err) - }) - socket.on('data', function(data) { - var dataObj = JSON.parse(data); - console.log('Data received (by worker): ' + data); - // console.dir(dataObj); - var result = ProcessData(dataObj.task); - var answer = {index:dataObj.index,answer: result}; - console.log('Counting complete. Sending result - '+JSON.stringify(answer)); - setTimeout(function () { - socket.write(JSON.stringify(answer),function (err) { - socket.end(); - }); - },getRandomInt(1,5)*1000); - }); +module.exports = function() { + process.on('message', (message) => { + const port = parseInt(message); + console.log('Listen port # ' + port); + api.net.createServer((socket) => { + console.log('Conn: ' + socket.remoteAddress + ':' + socket.remotePort); + socket.on('error', (err) => { + console.error(err); + }); + socket.on('data', (data) => { + const dataObj = JSON.parse(data); + console.log('Data received (by worker): ' + data); + const result = multiply(dataObj.task); + const answer = { index: dataObj.index, answer: result }; + console.log('Task complete. Send result - ' + JSON.stringify(answer)); + setTimeout(() => { + socket.end(JSON.stringify(answer)); + }, getRandomInt(1, 5) * 1000); + }); }).listen(port); -} + }); +}; -function ProcessData(data) { - var res = data.map(function(item) { - return item * 2; - }); - return res; +function multiply(data) { + const res = data.map((item) => item * 2); + return res; } function getRandomInt(min, max) { - return Math.floor(Math.random() * (max - min + 1)) + min; -} \ No newline at end of file + return Math.floor(Math.random() * (max - min + 1)) + min; +} diff --git a/customerWorker/workersMaster.js b/customerWorker/workersMaster.js index 9ee4efa..ba4360e 100644 --- a/customerWorker/workersMaster.js +++ b/customerWorker/workersMaster.js @@ -1,26 +1,25 @@ -/** - * Created by Yura on 10.04.2016. - */ -module.exports = function() { +'use strict'; - var workersCount = api.clientsCount; +module.exports = function() { - var workers = []; - for (var i = 0; i < workersCount; i++) { - var worker = api.cluster.fork(); - workers.push(worker); - } + const workersCount = api.clientsCount; - var startPort = global.port; + const workers = []; + for (let i = 0; i < workersCount; i++) { + const worker = api.cluster.fork(); + workers.push(worker); + } - workers.forEach(function(worker) { + let startPort = global.port; - worker.send(startPort++); + workers.forEach((worker) => { - worker.on('exit', function (code) { - console.log('exit ' + worker.process.pid + ' ' + code); - }); + worker.send(startPort++); + worker.on('exit', (code) => { + console.log('exit ' + worker.process.pid + ' ' + code); }); + }); + }; From cef1f521b727e99965c8ec0cb9bcaf0a88a9f4a9 Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Thu, 1 Jun 2017 16:55:31 +0300 Subject: [PATCH 3/8] Rewroted client --- customerWorker/1-customer.js | 104 ++++++++++++++++++----------------- customerWorker/2-worker.js | 2 +- 2 files changed, 55 insertions(+), 51 deletions(-) diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js index 5b18c70..dbf683f 100644 --- a/customerWorker/1-customer.js +++ b/customerWorker/1-customer.js @@ -3,69 +3,73 @@ global.api = {}; api.net = require('net'); api.os = require('os'); +api.metasync = require('metasync'); -const workersCount = 10; -const startPort = 10000; +const host = { workerIp: '127.0.0.1', workerPort: 10000 }; +const workersCount = 15; const defaultElementsByTask = 1; -const host = '127.0.0.1'; -const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11]; +const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11, 2, 17, 3, 2, 5]; -const tasks = chunkArray(task, defaultElementsByTask, workersCount); -sendTasks(tasks); +const createConnection = (task, counter) => { + const conn = { port: host.workerPort + counter, host: host.workerIp }; + return (data, cb) => { + const socket = new api.net.Socket(); + socket.on('data', (readData) => { + console.log('Data received (by client): ' + readData); + const res = JSON.parse(readData); + data[parseInt(res.index)] = res.answer; + cb(data); + }); + socket.connect(conn); + console.log('Data send (by client): ' + JSON.stringify(task)); + socket.write(JSON.stringify(task)); + }; +}; -function chunkArray(arr, elementsByPart, clientsCount) { +const mergeResult = (data) => { + console.log('Merging - '); + console.dir(data); + //merge array of results in one + const res = data.reduce((a, b) => a.concat(b)); + console.log(task); + console.log(res); +}; + +const createTasks = (arr, elementsByPart, clientsCount) => { const tasks = []; - let needWorkers = arr.length / elementsByPart; + const needWorkers = arr.length / elementsByPart; + let elems = elementsByPart; if (needWorkers > clientsCount) { - console.log(needWorkers + '-' + elementsByPart); - while (needWorkers > clientsCount) { - elementsByPart++; - needWorkers = arr.length / elementsByPart; - } + elems = Math.ceil(arr.length / clientsCount); } let i = 0; while (arr.length > 0) { tasks.push({ index: i++, - task: arr.splice(0, elementsByPart) + task: arr.splice(0, elems) }); } return tasks; -} -function sendTasks(tasks) { - const answers = []; - const sockets = []; - const answersCount = tasks.length; - for (let i = startPort; i < (answersCount + startPort); i++) { - const socket = new api.net.Socket(); - socket.on('data', (data) => { - console.log('Data received (by client): ' + data); - const res = JSON.parse(data); - answers.push(res); - if (answers.length === answersCount) { - mergeAnswers(answers); - } - }); - socket.connect({ port: i, host }); - sockets.push(socket); - } +}; - tasks.forEach((task, i) => { - console.log('Data send (by client): ' + JSON.stringify(task)); - const dataObj = task; - sockets[i].write(JSON.stringify(dataObj)); - }); -} +const sendTasks = (tasks) => { + api.metasync.reduce( + tasks, + (prev, curr, cb, counter) => { + prev.push(createConnection(curr, counter)); + cb(null, prev); + }, + (err, res) => { + if (err) console.error(err); + api.metasync.parallel( + res, + (data) => { + mergeResult(data); + }, + [] + ); + }, + []); +}; -function mergeAnswers(answers) { - let res = []; - const temp = []; - answers.forEach((value) => { - temp[value.index] = value.answer; - }); - for (let i = 0; i < temp.length; i++) { - res = res.concat(temp[i]); - } - console.dir(temp); - console.dir(res); -} +sendTasks(createTasks(task.slice(), defaultElementsByTask, workersCount)); diff --git a/customerWorker/2-worker.js b/customerWorker/2-worker.js index 8d57d01..39a93ec 100644 --- a/customerWorker/2-worker.js +++ b/customerWorker/2-worker.js @@ -5,7 +5,7 @@ global.port = 10000; api.cluster = require('cluster'); api.os = require('os'); api.net = require('net'); -api.clientsCount = 10; +api.clientsCount = 15; global.application = {}; application.master = require('./workersMaster.js'); From b67ea64245f7d2aac1b0354647763ce0724aa949 Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Thu, 1 Jun 2017 18:59:03 +0300 Subject: [PATCH 4/8] Rewroted worker --- customerWorker/1-customer.js | 1 + customerWorker/2-worker.js | 2 -- customerWorker/worker.js | 32 +++++++++++++++++--------------- customerWorker/workersMaster.js | 10 ++-------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js index dbf683f..648a950 100644 --- a/customerWorker/1-customer.js +++ b/customerWorker/1-customer.js @@ -46,6 +46,7 @@ const createTasks = (arr, elementsByPart, clientsCount) => { while (arr.length > 0) { tasks.push({ index: i++, + taskId: 1, task: arr.splice(0, elems) }); } diff --git a/customerWorker/2-worker.js b/customerWorker/2-worker.js index 39a93ec..e974c60 100644 --- a/customerWorker/2-worker.js +++ b/customerWorker/2-worker.js @@ -1,11 +1,9 @@ 'use strict'; global.api = {}; -global.port = 10000; api.cluster = require('cluster'); api.os = require('os'); api.net = require('net'); -api.clientsCount = 15; global.application = {}; application.master = require('./workersMaster.js'); diff --git a/customerWorker/worker.js b/customerWorker/worker.js index 582b019..f041c93 100644 --- a/customerWorker/worker.js +++ b/customerWorker/worker.js @@ -2,32 +2,34 @@ module.exports = function() { process.on('message', (message) => { + const funcs = { + 0: (item) => (item * 2), + 1: (item) => (item * item) + }; + const calculate = (funcId, array) => array.map(funcs[funcId]); + const createHandler = (socket) => (data) => { + const dataObj = JSON.parse(data); + const taskId = parseInt(dataObj.taskId); + console.log('Data received (by worker): ' + data); + const result = calculate(taskId, dataObj.task); + const answer = { index: dataObj.index, answer: result }; + console.log('Task complete. Send result - ' + JSON.stringify(answer)); + socket.end(JSON.stringify(answer)); + }; + const port = parseInt(message); console.log('Listen port # ' + port); api.net.createServer((socket) => { console.log('Conn: ' + socket.remoteAddress + ':' + socket.remotePort); + const handler = createHandler(socket); socket.on('error', (err) => { console.error(err); }); - socket.on('data', (data) => { - const dataObj = JSON.parse(data); - console.log('Data received (by worker): ' + data); - const result = multiply(dataObj.task); - const answer = { index: dataObj.index, answer: result }; - console.log('Task complete. Send result - ' + JSON.stringify(answer)); - setTimeout(() => { - socket.end(JSON.stringify(answer)); - }, getRandomInt(1, 5) * 1000); - }); + socket.on('data', handler); }).listen(port); }); }; -function multiply(data) { - const res = data.map((item) => item * 2); - return res; -} - function getRandomInt(min, max) { return Math.floor(Math.random() * (max - min + 1)) + min; } diff --git a/customerWorker/workersMaster.js b/customerWorker/workersMaster.js index ba4360e..dc15a29 100644 --- a/customerWorker/workersMaster.js +++ b/customerWorker/workersMaster.js @@ -1,8 +1,8 @@ 'use strict'; module.exports = function() { - - const workersCount = api.clientsCount; + const workersCount = 15; + let startPort = 10000; const workers = []; for (let i = 0; i < workersCount; i++) { @@ -10,16 +10,10 @@ module.exports = function() { workers.push(worker); } - let startPort = global.port; - workers.forEach((worker) => { - worker.send(startPort++); - worker.on('exit', (code) => { console.log('exit ' + worker.process.pid + ' ' + code); }); - }); - }; From 2fa7e172ac608924c97fc01dc9ddf657f977a6c4 Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Thu, 1 Jun 2017 22:36:53 +0300 Subject: [PATCH 5/8] Added broker that retranslate tasks on workers --- customerWorker/1-customer.js | 6 ++--- customerWorker/3-broker.js | 48 ++++++++++++++++++++++++++++++++++++ customerWorker/worker.js | 8 +++++- 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 customerWorker/3-broker.js diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js index 648a950..748d365 100644 --- a/customerWorker/1-customer.js +++ b/customerWorker/1-customer.js @@ -5,7 +5,7 @@ api.net = require('net'); api.os = require('os'); api.metasync = require('metasync'); -const host = { workerIp: '127.0.0.1', workerPort: 10000 }; +const host = { workerIp: '127.0.0.1', workerPort: 20000 }; const workersCount = 15; const defaultElementsByTask = 1; const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11, 2, 17, 3, 2, 5]; @@ -46,7 +46,7 @@ const createTasks = (arr, elementsByPart, clientsCount) => { while (arr.length > 0) { tasks.push({ index: i++, - taskId: 1, + funcId: 1, task: arr.splice(0, elems) }); } @@ -57,7 +57,7 @@ const sendTasks = (tasks) => { api.metasync.reduce( tasks, (prev, curr, cb, counter) => { - prev.push(createConnection(curr, counter)); + prev.push(createConnection(curr, 0)); cb(null, prev); }, (err, res) => { diff --git a/customerWorker/3-broker.js b/customerWorker/3-broker.js new file mode 100644 index 0000000..1d69c80 --- /dev/null +++ b/customerWorker/3-broker.js @@ -0,0 +1,48 @@ +'use strict'; + +global.api = {}; +api.net = require('net'); +api.os = require('os'); +api.metasync = require('metasync'); +api.fs = require('fs'); + +const workers = []; +let workerId = 0; +let taskId = 0; + +//for clients +api.net.createServer((clientSocket) => { + clientSocket.on('error', (err) => { + console.error(err); + }); + clientSocket.on('data', (data) => { + const obj = JSON.parse(data); + obj.taskId = taskId++; + console.dir(obj); + const workerParams = workers.pop(); + const workerSocket = new api.net.Socket(); + workerSocket.connect({ port: workerParams.port, host: workerParams.host }); + workerSocket.on('data', (data) => { + workers.push(workerParams); + const resp = JSON.parse(data); + delete resp.taskId; + console.dir(resp); + clientSocket.end(JSON.stringify(resp)); + }); + workerSocket.write(JSON.stringify(obj)); + }); +}).listen(20000); + +//for workers +api.net.createServer((socket) => { + socket.on('data', (data) => { + const connParams = JSON.parse(data); + workers.push({ + id: workerId++, + port: connParams.port, + host: socket.remoteAddress + }); + console.dir(workers); + socket.end(); + }); +}).listen(21000); diff --git a/customerWorker/worker.js b/customerWorker/worker.js index f041c93..0daf9c1 100644 --- a/customerWorker/worker.js +++ b/customerWorker/worker.js @@ -9,7 +9,7 @@ module.exports = function() { const calculate = (funcId, array) => array.map(funcs[funcId]); const createHandler = (socket) => (data) => { const dataObj = JSON.parse(data); - const taskId = parseInt(dataObj.taskId); + const taskId = parseInt(dataObj.funcId); console.log('Data received (by worker): ' + data); const result = calculate(taskId, dataObj.task); const answer = { index: dataObj.index, answer: result }; @@ -18,6 +18,12 @@ module.exports = function() { }; const port = parseInt(message); + + const brokerConn = new api.net.Socket(); + brokerConn.connect({ port: 21000, host: '127.0.0.1' }); + const connParam = { port }; + brokerConn.write(JSON.stringify(connParam)); + console.log('Listen port # ' + port); api.net.createServer((socket) => { console.log('Conn: ' + socket.remoteAddress + ':' + socket.remotePort); From 2b85abafc8ab692d64f7620ae55eb1677d8f5cc0 Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Fri, 2 Jun 2017 02:20:50 +0300 Subject: [PATCH 6/8] Added worker queue --- customerWorker/1-customer.js | 15 +++++------ customerWorker/3-broker.js | 36 ++++++++++++++----------- customerWorker/queue.js | 48 +++++++++++++++++++++++++++++++++ customerWorker/worker.js | 14 ++++++---- customerWorker/workersMaster.js | 4 +-- 5 files changed, 86 insertions(+), 31 deletions(-) create mode 100644 customerWorker/queue.js diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js index 748d365..353055b 100644 --- a/customerWorker/1-customer.js +++ b/customerWorker/1-customer.js @@ -5,13 +5,13 @@ api.net = require('net'); api.os = require('os'); api.metasync = require('metasync'); -const host = { workerIp: '127.0.0.1', workerPort: 20000 }; -const workersCount = 15; +const host = { workerIp: '127.0.0.1', workerPort: 50000 }; +const workersCount = 20; const defaultElementsByTask = 1; const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11, 2, 17, 3, 2, 5]; -const createConnection = (task, counter) => { - const conn = { port: host.workerPort + counter, host: host.workerIp }; +const createConnection = (task) => { + const conn = { port: host.workerPort, host: host.workerIp }; return (data, cb) => { const socket = new api.net.Socket(); socket.on('data', (readData) => { @@ -54,11 +54,10 @@ const createTasks = (arr, elementsByPart, clientsCount) => { }; const sendTasks = (tasks) => { - api.metasync.reduce( + api.metasync.map( tasks, - (prev, curr, cb, counter) => { - prev.push(createConnection(curr, 0)); - cb(null, prev); + (curr, cb) => { + cb(null, createConnection(curr)); }, (err, res) => { if (err) console.error(err); diff --git a/customerWorker/3-broker.js b/customerWorker/3-broker.js index 1d69c80..43cd505 100644 --- a/customerWorker/3-broker.js +++ b/customerWorker/3-broker.js @@ -4,9 +4,9 @@ global.api = {}; api.net = require('net'); api.os = require('os'); api.metasync = require('metasync'); -api.fs = require('fs'); +api.queue = require('./queue.js'); -const workers = []; +const workersQueue = new api.queue.Queue(); let workerId = 0; let taskId = 0; @@ -19,30 +19,34 @@ api.net.createServer((clientSocket) => { const obj = JSON.parse(data); obj.taskId = taskId++; console.dir(obj); - const workerParams = workers.pop(); - const workerSocket = new api.net.Socket(); - workerSocket.connect({ port: workerParams.port, host: workerParams.host }); - workerSocket.on('data', (data) => { - workers.push(workerParams); - const resp = JSON.parse(data); - delete resp.taskId; - console.dir(resp); - clientSocket.end(JSON.stringify(resp)); + workersQueue.use((workerParams, cb) => { + const workerSocket = new api.net.Socket(); + workerSocket.connect({ + port: workerParams.port, + host: workerParams.host + }); + workerSocket.on('data', (data) => { + const resp = JSON.parse(data); + delete resp.taskId; + console.dir(resp); + clientSocket.end(JSON.stringify(resp)); + cb(); + }); + workerSocket.write(JSON.stringify(obj)); }); - workerSocket.write(JSON.stringify(obj)); }); -}).listen(20000); +}).listen(50000); //for workers api.net.createServer((socket) => { socket.on('data', (data) => { const connParams = JSON.parse(data); - workers.push({ + workersQueue.put({ id: workerId++, port: connParams.port, host: socket.remoteAddress }); - console.dir(workers); + console.dir(workersQueue.queue); socket.end(); }); -}).listen(21000); +}).listen(51000); diff --git a/customerWorker/queue.js b/customerWorker/queue.js new file mode 100644 index 0000000..141c8e0 --- /dev/null +++ b/customerWorker/queue.js @@ -0,0 +1,48 @@ +'use strict'; + +exports.Queue = Queue +//simple queue for workers +function Queue() { + //free workers + this.queue = []; + //functions waiting for worker + this.process = []; +} + +Queue.prototype.size = function() { + return this.queue.length; +}; + +Queue.prototype.canGet = function() { + return this.queue.length > 0; +}; + +Queue.prototype.put = function(data) { + this.queue.push(data); + if (this.process.length > 0) { + const fn = this.process.shift(); + const item = this.get(); + fn(item, () => { + this.put(item); + }); + } +}; + +Queue.prototype.get = function() { + return this.queue.shift(); +} + +Queue.prototype.use = function(func) { + if (this.queue.length > 0) { + const worker = this.get(); + func(worker, () => { + this.put(worker); + }); + } else { + this.process.push(func); + } +} + +Queue.prototype.clear = function() { + this.queue = []; +}; \ No newline at end of file diff --git a/customerWorker/worker.js b/customerWorker/worker.js index 0daf9c1..776e12a 100644 --- a/customerWorker/worker.js +++ b/customerWorker/worker.js @@ -2,6 +2,7 @@ module.exports = function() { process.on('message', (message) => { + const broker = { port: 51000, host: '127.0.0.1' }; const funcs = { 0: (item) => (item * 2), 1: (item) => (item * item) @@ -16,14 +17,17 @@ module.exports = function() { console.log('Task complete. Send result - ' + JSON.stringify(answer)); socket.end(JSON.stringify(answer)); }; + const registerInBroker = (myPort, connParams) => { + const brokerConn = new api.net.Socket(); + const connParam = { port: myPort }; + brokerConn.connect(connParams, () => { + brokerConn.write(JSON.stringify(connParam)); + }); + }; const port = parseInt(message); - const brokerConn = new api.net.Socket(); - brokerConn.connect({ port: 21000, host: '127.0.0.1' }); - const connParam = { port }; - brokerConn.write(JSON.stringify(connParam)); - + registerInBroker(port, broker); console.log('Listen port # ' + port); api.net.createServer((socket) => { console.log('Conn: ' + socket.remoteAddress + ':' + socket.remotePort); diff --git a/customerWorker/workersMaster.js b/customerWorker/workersMaster.js index dc15a29..fd74ea1 100644 --- a/customerWorker/workersMaster.js +++ b/customerWorker/workersMaster.js @@ -1,8 +1,8 @@ 'use strict'; module.exports = function() { - const workersCount = 15; - let startPort = 10000; + const workersCount = 5; + let startPort = 52000; const workers = []; for (let i = 0; i < workersCount; i++) { From 893bb103ce73a3a4deb599a6b939be847428680e Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Fri, 2 Jun 2017 07:52:12 +0300 Subject: [PATCH 7/8] Renamed files to start order --- customerWorker/1-broker.js | 109 +++++++++++++++++++++++++++++++++++ customerWorker/1-customer.js | 75 ------------------------ customerWorker/3-broker.js | 52 ----------------- customerWorker/3-customer.js | 27 +++++++++ customerWorker/worker.js | 4 -- 5 files changed, 136 insertions(+), 131 deletions(-) create mode 100644 customerWorker/1-broker.js delete mode 100644 customerWorker/1-customer.js delete mode 100644 customerWorker/3-broker.js create mode 100644 customerWorker/3-customer.js diff --git a/customerWorker/1-broker.js b/customerWorker/1-broker.js new file mode 100644 index 0000000..b329ef0 --- /dev/null +++ b/customerWorker/1-broker.js @@ -0,0 +1,109 @@ +'use strict'; + +global.api = {}; +api.net = require('net'); +api.os = require('os'); +api.metasync = require('metasync'); +api.queue = require('./queue.js'); + +const workersQueue = new api.queue.Queue(); +let workerId = 0; +let taskId = 0; + +const newWorker = (socket) => { + socket.on('data', (data) => { + const connParams = JSON.parse(data); + //add new worker + workersQueue.put({ + id: workerId++, + port: connParams.port, + host: socket.remoteAddress + }); + console.dir(workersQueue.queue); + socket.end(); + }); +}; + + +const createConnection = (task) => (data, datacb) => { + workersQueue.use((workerParams, qcb) => { + const workerSocket = new api.net.Socket(); + workerSocket.on('data', (readData) => { + console.log('Data received (by broker): ' + readData); + const res = JSON.parse(readData); + data[parseInt(res.index)] = res.answer; + qcb(); + datacb(data); + }); + workerSocket.connect({ + port: workerParams.port, + host: workerParams.host + }, () => { + console.log('Data send (by broker): ' + JSON.stringify(task)); + workerSocket.write(JSON.stringify(task)); + }); + }); +}; + +const mergeResult = (data) => { + console.log('Merging - '); + console.dir(data); + //merge array of results in one + const res = data.reduce((a, b) => a.concat(b)); + // console.log(task); + console.log(res); + return res; +}; + +const createTasks = (arr, workers) => { + const tasks = []; + const elemsByTask = Math.ceil(arr.length / workers); + let i = 0; + while (arr.length > 0) { + tasks.push({ + index: i++, + funcId: 1, + task: arr.splice(0, elemsByTask), + taskId: taskId++ + }); + } + return tasks; +}; + +const sendTasks = (tasks, clientSocket) => { + api.metasync.map( + tasks, + (curr, cb) => { + cb(null, createConnection(curr)); + }, + (err, res) => { + if (err) console.error(err); + api.metasync.parallel( + res, + (data) => { + const res = mergeResult(data); + const response = { answer: res }; + clientSocket.end(JSON.stringify(response)); + }, + [] + ); + }); +}; + +const newTasks = (data, clientSocket) => { + const obj = JSON.parse(data); + sendTasks(createTasks(obj.task, workersQueue.size()), clientSocket); +}; + +//for clients +api.net.createServer((clientSocket) => { + clientSocket.on('error', (err) => { + console.error(err); + }); + clientSocket.on('data', (data) => { + newTasks(data, clientSocket); + }); +}).listen(50000); + +//for workers +api.net.createServer(newWorker).listen(51000); diff --git a/customerWorker/1-customer.js b/customerWorker/1-customer.js deleted file mode 100644 index 353055b..0000000 --- a/customerWorker/1-customer.js +++ /dev/null @@ -1,75 +0,0 @@ -'use strict'; - -global.api = {}; -api.net = require('net'); -api.os = require('os'); -api.metasync = require('metasync'); - -const host = { workerIp: '127.0.0.1', workerPort: 50000 }; -const workersCount = 20; -const defaultElementsByTask = 1; -const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11, 2, 17, 3, 2, 5]; - -const createConnection = (task) => { - const conn = { port: host.workerPort, host: host.workerIp }; - return (data, cb) => { - const socket = new api.net.Socket(); - socket.on('data', (readData) => { - console.log('Data received (by client): ' + readData); - const res = JSON.parse(readData); - data[parseInt(res.index)] = res.answer; - cb(data); - }); - socket.connect(conn); - console.log('Data send (by client): ' + JSON.stringify(task)); - socket.write(JSON.stringify(task)); - }; -}; - -const mergeResult = (data) => { - console.log('Merging - '); - console.dir(data); - //merge array of results in one - const res = data.reduce((a, b) => a.concat(b)); - console.log(task); - console.log(res); -}; - -const createTasks = (arr, elementsByPart, clientsCount) => { - const tasks = []; - const needWorkers = arr.length / elementsByPart; - let elems = elementsByPart; - if (needWorkers > clientsCount) { - elems = Math.ceil(arr.length / clientsCount); - } - let i = 0; - while (arr.length > 0) { - tasks.push({ - index: i++, - funcId: 1, - task: arr.splice(0, elems) - }); - } - return tasks; -}; - -const sendTasks = (tasks) => { - api.metasync.map( - tasks, - (curr, cb) => { - cb(null, createConnection(curr)); - }, - (err, res) => { - if (err) console.error(err); - api.metasync.parallel( - res, - (data) => { - mergeResult(data); - }, - [] - ); - }, - []); -}; - -sendTasks(createTasks(task.slice(), defaultElementsByTask, workersCount)); diff --git a/customerWorker/3-broker.js b/customerWorker/3-broker.js deleted file mode 100644 index 43cd505..0000000 --- a/customerWorker/3-broker.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict'; - -global.api = {}; -api.net = require('net'); -api.os = require('os'); -api.metasync = require('metasync'); -api.queue = require('./queue.js'); - -const workersQueue = new api.queue.Queue(); -let workerId = 0; -let taskId = 0; - -//for clients -api.net.createServer((clientSocket) => { - clientSocket.on('error', (err) => { - console.error(err); - }); - clientSocket.on('data', (data) => { - const obj = JSON.parse(data); - obj.taskId = taskId++; - console.dir(obj); - workersQueue.use((workerParams, cb) => { - const workerSocket = new api.net.Socket(); - workerSocket.connect({ - port: workerParams.port, - host: workerParams.host - }); - workerSocket.on('data', (data) => { - const resp = JSON.parse(data); - delete resp.taskId; - console.dir(resp); - clientSocket.end(JSON.stringify(resp)); - cb(); - }); - workerSocket.write(JSON.stringify(obj)); - }); - }); -}).listen(50000); - -//for workers -api.net.createServer((socket) => { - socket.on('data', (data) => { - const connParams = JSON.parse(data); - workersQueue.put({ - id: workerId++, - port: connParams.port, - host: socket.remoteAddress - }); - console.dir(workersQueue.queue); - socket.end(); - }); -}).listen(51000); diff --git a/customerWorker/3-customer.js b/customerWorker/3-customer.js new file mode 100644 index 0000000..d68281b --- /dev/null +++ b/customerWorker/3-customer.js @@ -0,0 +1,27 @@ +'use strict'; + +global.api = {}; +api.net = require('net'); +api.os = require('os'); + +const broker = { host: '127.0.0.1', port: 50000 }; +const task = [2, 17, 3, 2, 5, 7, 15, 22, 1, 14, 15, 9, 0, 11, 2, 17, 3, 2, 5]; + +const sendTask = (task) => { + const conn = { port: broker.port, host: broker.host }; + const socket = new api.net.Socket(); + socket.on('data', (readData) => { + console.log('Data received (by client): ' + readData); + const res = JSON.parse(readData); + console.dir(task); + console.dir(res.answer); + }); + socket.connect(conn, () => { + console.log('Data send (by client): ' + JSON.stringify({ task })); + socket.write(JSON.stringify({ task })); + }); +}; + +sendTask(task); +sendTask(task); +sendTask(task); diff --git a/customerWorker/worker.js b/customerWorker/worker.js index 776e12a..57b9ad2 100644 --- a/customerWorker/worker.js +++ b/customerWorker/worker.js @@ -39,7 +39,3 @@ module.exports = function() { }).listen(port); }); }; - -function getRandomInt(min, max) { - return Math.floor(Math.random() * (max - min + 1)) + min; -} From 017fed6cf3ad21bbc74e30f2f7125085584da9fa Mon Sep 17 00:00:00 2001 From: YuraG97 Date: Fri, 2 Jun 2017 07:59:16 +0300 Subject: [PATCH 8/8] Added line at the end file --- customerWorker/queue.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/customerWorker/queue.js b/customerWorker/queue.js index 141c8e0..6a5fbe3 100644 --- a/customerWorker/queue.js +++ b/customerWorker/queue.js @@ -1,6 +1,6 @@ 'use strict'; -exports.Queue = Queue +exports.Queue = Queue; //simple queue for workers function Queue() { //free workers @@ -30,7 +30,7 @@ Queue.prototype.put = function(data) { Queue.prototype.get = function() { return this.queue.shift(); -} +}; Queue.prototype.use = function(func) { if (this.queue.length > 0) { @@ -41,8 +41,8 @@ Queue.prototype.use = function(func) { } else { this.process.push(func); } -} +}; Queue.prototype.clear = function() { this.queue = []; -}; \ No newline at end of file +};