powerloom · xadahiya · May 29, 2023 · Jan 27, 2023 · Jan 30, 2023 · Jan 30, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,13 +1,12 @@
 .env
 .python-version
-settings.json
 __pycache__/
-logs/
 .idea/
 .DS_Store
 screenlog*
 temp/*
 uniswapTokenData
-test.py
-pooler/static/cached_pair_addresses.json
-pooler/auth/settings/auth_settings.json
+**/*projects.json
+**/*aggregator.json
+**/*settings.json
+**/*.backup
diff --git a/Dockerfile b/Dockerfile
@@ -19,7 +19,7 @@ RUN chmod +x ./snapshotter_autofill.sh ./init_processes.sh
 
 # Expose the port that the application will listen on
 EXPOSE 8002
-EXPOSE 8555
+# EXPOSE 9090
 
 # Start the application using PM2
 # CMD pm2 start pm2.config.js && pm2 logs --lines 100
diff --git a/README.md b/README.md
diff --git a/build-docker.sh b/build-docker.sh
@@ -1 +1 @@
-docker build -t powerloom-pooler .
+docker build -t powerloom-pooler .
diff --git a/config/aggregator.example.json b/config/aggregator.example.json
@@ -0,0 +1,114 @@
+{
+  "config": [{
+      "project_type": "aggregate_uniswap_pairContract_24h_trade_volume",
+      "aggregate_on": "SingleProject",
+      "filters": {
+        "projectId": "uniswap_pairContract_trade_volume"
+      },
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_24h",
+        "class_name": "AggreagateTradeVolumeProcessor"
+      }
+    },
+    {
+      "project_type": "aggregate_uniswap_pairContract_7d_trade_volume",
+      "aggregate_on": "SingleProject",
+      "filters": {
+        "projectId": "aggregate_uniswap_pairContract_24h_trade_volume"
+      },
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_7d",
+        "class_name": "AggreagateTradeVolumeProcessor"
+      }
+    },
+    {
+      "project_type": "aggregate_uniswap_24h_top_pairs_lite",
+      "aggregate_on": "MultiProject",
+      "projects_to_wait_for": [
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod"
+      ],
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_24h",
+        "class_name": "AggreagateTopPairsProcessor"
+      }
+    },
+    {
+      "project_type": "aggregate_uniswap_24h_top_tokens_lite",
+      "aggregate_on": "MultiProject",
+      "projects_to_wait_for": [
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod"
+      ],
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_tokens",
+        "class_name": "AggreagateTopTokensProcessor"
+      }
+    },
+    {
+      "project_type": "aggregate_uniswap_7d_top_pairs_lite",
+      "aggregate_on": "MultiProject",
+      "projects_to_wait_for": [
+        "aggregate_uniswap_pairContract_7d_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod"
+      ],
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_7d",
+        "class_name": "AggreagateTopPairsProcessor"
+      }
+    },
+    {
+      "project_type": "aggregate_uniswap_24h_stats_lite",
+      "aggregate_on": "MultiProject",
+      "projects_to_wait_for": [
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod",
+        "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod",
+        "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod"
+      ],
+      "processor": {
+        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_stats",
+        "class_name": "AggreagateStatsProcessor"
+      }
+    }
+  ]
+}
diff --git a/.../auth/settings/auth_settings.example.json → config/auth_settings.example.json b/.../auth/settings/auth_settings.example.json → config/auth_settings.example.json
diff --git a/config/projects.example.json b/config/projects.example.json
@@ -0,0 +1,35 @@
+{
+  "config": [{
+      "project_type": "uniswap_pairContract_pair_total_reserves",
+      "projects":[
+        "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
+        "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
+        "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852",
+        "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f",
+        "0xd3d2e2692501a5c9ca623199d38826e513033a17",
+        "0xbb2b8038a1640196fbe3e38816f3e67cba72d940",
+        "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
+        ],
+      "processor":{
+        "module": "pooler.modules.uniswapv2.pair_total_reserves",
+        "class_name": "PairTotalReservesProcessor"
+      }
+    },
+    {
+      "project_type": "uniswap_pairContract_trade_volume",
+      "projects":[
+        "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
+        "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
+        "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852",
+        "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f",
+        "0xd3d2e2692501a5c9ca623199d38826e513033a17",
+        "0xbb2b8038a1640196fbe3e38816f3e67cba72d940",
+        "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
+        ],
+        "processor":{
+          "module": "pooler.modules.uniswapv2.trade_volume",
+          "class_name": "TradeVolumeProcessor"
+        }
+    }
+  ]
+}
diff --git a/config/settings.example.json b/config/settings.example.json
@@ -0,0 +1,126 @@
+{
+  "namespace": "relevant-namespace",
+  "core_api": {
+    "host": "0.0.0.0",
+    "port": 8002,
+    "auth": {
+      "enabled": false,
+      "header_key": "X-API-KEY"
+    },
+    "public_rate_limit": "10000/day;200/minute;5/second"
+  },
+  "instance_id": "account-address",
+  "rpc": {
+    "full_nodes": [{
+      "url": "https://rpc-url",
+      "rate_limit": "10000000/day;2000/minute;100/second"
+    }],
+    "archive_nodes": [],
+    "force_archive_blocks": 100,
+    "retry": 5,
+    "request_time_out": 5,
+    "skip_epoch_threshold_blocks": 30,
+    "polling_interval": 15,
+    "connection_limits":{
+      "max_connections": 100,
+      "max_keepalive_connections": 50,
+      "keepalive_expiry": 300
+    }
+  },
+  "rlimit": {
+    "file_descriptors": 2048
+  },
+  "rabbitmq": {
+    "user": "guest",
+    "password": "guest",
+    "host": "rabbitmq",
+    "port": 5672,
+    "setup": {
+      "core": {
+        "exchange": "powerloom-backend"
+      },
+      "callbacks": {
+        "exchange": "powerloom-backend-callbacks"
+      },
+      "commit_payload": {
+        "exchange": "powerloom-backend-commit-payload"
+      },
+      "event_detector": {
+        "exchange": "powerloom-backend-event-detector"
+      }
+    }
+  },
+  "reporting": {
+    "slack_url": "https://slack-reporting-url",
+    "service_url": "https://powerloom-reporting-url"
+  },
+  "redis": {
+    "host": "redis",
+    "port": 6379,
+    "db": 0,
+    "password": null,
+    "ssl": false,
+    "cluster_mode": false
+  },
+  "redis_reader": {
+    "host": "redis",
+    "port": 6379,
+    "db": 0,
+    "password": null
+  },
+  "logs": {
+    "trace_enabled": false,
+    "write_to_files": true
+  },
+  "projects_config_path": "config/projects.json",
+  "aggregator_config_path": "config/aggregator.json",
+  "pair_contract_abi": "pooler/static/abis/UniswapV2Pair.json",
+  "protocol_state": {
+    "abi": "pooler/static/abis/ProtocolContract.json",
+    "address": "protocol-state-contract"
+  },
+  "callback_worker_config": {
+    "num_snapshot_workers": 8,
+    "num_aggregation_workers": 16
+  },
+  "ipfs": {
+    "url": "ipfs-writer-url",
+    "url_auth": {
+      "apiKey": "ipfs-writer-key",
+      "apiSecret": "ipfs-writer-secret"
+    },
+    "reader_url": "ipfs-reader-url",
+    "reader_url_auth": {
+      "apiKey": "ipfs-reader-key",
+      "apiSecret": "ipfs-reader-secret"
+    },
+    "write_rate_limit": {
+      "req_per_sec": 10,
+      "burst": 10
+    },
+    "timeout": 60,
+    "local_cache_path": "/home/ubuntu/local_cache/",
+    "connection_limits":{
+      "max_connections": 100,
+      "max_keepalive_connections": 50,
+      "keepalive_expiry": 300
+    }
+  },
+  "web3storage":{
+    "upload_snapshots": true,
+    "upload_aggregates": true
+  },
+  "anchor_chain_rpc": {
+    "full_nodes": [{
+      "url": "https://prost-rpc-url",
+      "rate_limit": "100000000/day;18000/minute;300/second"
+    }],
+    "retry": 5,
+    "request_time_out": 5,
+    "connection_limits":{
+      "max_connections": 100,
+      "max_keepalive_connections": 50,
+      "keepalive_expiry": 300
+    }
+  }
+}
diff --git a/init_processes.sh b/init_processes.sh
@@ -3,18 +3,14 @@
 echo 'starting processes...';
 pm2 start pm2.config.js
 
-echo 'registering projects...';
-poetry run python -m pooler.register_projects || exit 1
-
-
 # Waiting for other processes to start
 echo 'waiting for processes to start..';
 sleep 10
 
-poetry run python -m pooler.processhub_cmd start EpochCallbackManager
+poetry run python -m pooler.processhub_cmd start ProcessorDistributor
 sleep 3
 
-poetry run python -m pooler.processhub_cmd start SystemEpochDetector
+poetry run python -m pooler.processhub_cmd start SystemEventDetector
 
 echo 'started all pooler scripts';
 

diff --git a/logs/.gitignore b/logs/.gitignore
@@ -0,0 +1 @@
+*.log
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		docker build -t powerloom-pooler .
		docker build -t powerloom-pooler .