fix issue with blocking sockets in HTTP2 server; minor refactoring of /graph endpoint (#3091)

whummer · web-flow · commit c24fe07f19f2 · 2020-10-12T08:52:17.000+02:00
diff --git a/localstack/dashboard/api.py b/localstack/dashboard/api.py
@@ -29,6 +29,7 @@ def spec():
 
 @app.route('/graph', methods=['POST'])
 def get_graph():
+    # TODO remove?
     """ Get deployment graph
         ---
         operationId: 'getGraph'
@@ -38,7 +39,7 @@ def get_graph():
     """
     data = get_payload()
     env = Environment.from_string(data.get('awsEnvironment'))
-    graph = infra.get_graph(name_filter=data['nameFilter'], env=env)
+    graph = infra.get_graph(name_filter=data['nameFilter'], env=env, region=data.get('awsRegion'))
     return jsonify(graph)
 
 
diff --git a/localstack/dashboard/infra.py b/localstack/dashboard/infra.py
@@ -121,7 +121,7 @@ def get_kinesis_streams(filter='.*', pool={}, env=None):
                 pool[arn] = stream
                 stream.shards = get_kinesis_shards(stream_details=details, env=env)
                 result.append(stream)
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -153,7 +153,7 @@ def get_sqs_queues(filter='.*', pool={}, env=None):
             if re.match(filter, name):
                 queue = SqsQueue(arn)
                 result.append(queue)
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -237,7 +237,7 @@ def handle(func):
         out = cmd_lambda('list-functions', env)
         out = json.loads(out)
         parallelize(handle, out['Functions'])
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -320,7 +320,7 @@ def handle(domain):
                 result.append(es)
                 pool[arn] = es
         parallelize(handle, out['DomainNames'])
-    except socket.error:
+    except Exception:
         pass
 
     return result
@@ -344,7 +344,7 @@ def handle(table):
                 result.append(db)
                 pool[arn] = db
         parallelize(handle, out['TableNames'])
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -377,7 +377,7 @@ def handle(bucket):
         out = cmd_s3api('list-buckets', env)
         out = json.loads(out)
         parallelize(handle, out['Buckets'])
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -399,7 +399,7 @@ def get_firehose_streams(filter='.*', pool={}, env=None):
                     bucket = EventSource.get(dest_s3, pool=pool)
                     s.destinations.append(bucket)
                 result.append(s)
-    except socket.error:
+    except Exception:
         pass
     return result
 
@@ -413,87 +413,84 @@ def read_kinesis_iterator(shard_iterator, max_results=10, env=None):
 
 
 def get_kinesis_events(stream_name, shard_id, max_results=10, env=None):
-    env = aws_stack.get_environment(env)
-    records = aws_stack.kinesis_get_latest_records(stream_name, shard_id, count=max_results, env=env)
-    for r in records:
-        r['ApproximateArrivalTimestamp'] = mktime(r['ApproximateArrivalTimestamp'])
-    result = {
-        'events': records
-    }
+    records = []
+    try:
+        env = aws_stack.get_environment(env)
+        records = aws_stack.kinesis_get_latest_records(stream_name, shard_id, count=max_results, env=env)
+        for r in records:
+            r['ApproximateArrivalTimestamp'] = mktime(r['ApproximateArrivalTimestamp'])
+    except Exception:
+        pass
+    result = {'events': records}
     return result
 
 
-def get_graph(name_filter='.*', env=None):
+def get_graph(name_filter='.*', env=None, **kwargs):
     result = {
         'nodes': [],
         'edges': []
     }
 
     pool = {}
-
-    if True:
-        result = {
-            'nodes': [],
-            'edges': []
-        }
-        node_ids = {}
-        # Make sure we load components in the right order:
-        # (ES,DynamoDB,S3) -> (Kinesis,Lambda)
-        domains = get_elasticsearch_domains(name_filter, pool=pool, env=env)
-        dbs = get_dynamo_dbs(name_filter, pool=pool, env=env)
-        buckets = get_s3_buckets(name_filter, details=True, pool=pool, env=env)
-        streams = get_kinesis_streams(name_filter, pool=pool, env=env)
-        firehoses = get_firehose_streams(name_filter, pool=pool, env=env)
-        lambdas = get_lambda_functions(name_filter, details=True, pool=pool, env=env)
-        queues = get_sqs_queues(name_filter, pool=pool, env=env)
-
-        for es in domains:
-            uid = short_uid()
-            node_ids[es.id] = uid
-            result['nodes'].append({'id': uid, 'arn': es.id, 'name': es.name(), 'type': 'es'})
-        for b in buckets:
-            uid = short_uid()
-            node_ids[b.id] = uid
-            result['nodes'].append({'id': uid, 'arn': b.id, 'name': b.name(), 'type': 's3'})
-        for db in dbs:
-            uid = short_uid()
-            node_ids[db.id] = uid
-            result['nodes'].append({'id': uid, 'arn': db.id, 'name': db.name(), 'type': 'dynamodb'})
-        for s in streams:
-            uid = short_uid()
-            node_ids[s.id] = uid
-            result['nodes'].append({'id': uid, 'arn': s.id, 'name': s.name(), 'type': 'kinesis'})
-            for shard in s.shards:
-                uid1 = short_uid()
-                name = re.sub(r'shardId-0*', '', shard.id) or '0'
-                result['nodes'].append({'id': uid1, 'arn': shard.id, 'name': name,
-                    'type': 'kinesis_shard', 'streamName': s.name(), 'parent': uid})
-        for f in firehoses:
-            uid = short_uid()
-            node_ids[f.id] = uid
-            result['nodes'].append({'id': uid, 'arn': f.id, 'name': f.name(), 'type': 'firehose'})
-            for d in f.destinations:
-                result['edges'].append({'source': uid, 'target': node_ids[d.id]})
-        for q in queues:
-            uid = short_uid()
-            node_ids[q.id] = uid
-            result['nodes'].append({'id': uid, 'arn': q.id, 'name': q.name(), 'type': 'sqs'})
-        for lda in lambdas:
-            uid = short_uid()
-            node_ids[lda.id] = uid
-            result['nodes'].append({'id': uid, 'arn': lda.id, 'name': lda.name(), 'type': 'lambda'})
-            for s in lda.event_sources:
-                lookup_id = s.id
-                if isinstance(s, DynamoDBStream):
-                    lookup_id = s.table.id
-                result['edges'].append({'source': node_ids.get(lookup_id), 'target': uid})
-            for t in lda.targets:
-                lookup_id = t.id
-                result['edges'].append({'source': uid, 'target': node_ids.get(lookup_id)})
-        for b in buckets:
-            for n in b.notifications:
-                src_uid = node_ids[b.id]
-                tgt_uid = node_ids[n.target.id]
-                result['edges'].append({'source': src_uid, 'target': tgt_uid})
+    node_ids = {}
+
+    # Make sure we load components in the right order:
+    # (ES,DynamoDB,S3) -> (Kinesis,Lambda)
+    domains = get_elasticsearch_domains(name_filter, pool=pool, env=env)
+    dbs = get_dynamo_dbs(name_filter, pool=pool, env=env)
+    buckets = get_s3_buckets(name_filter, details=True, pool=pool, env=env)
+    streams = get_kinesis_streams(name_filter, pool=pool, env=env)
+    firehoses = get_firehose_streams(name_filter, pool=pool, env=env)
+    lambdas = get_lambda_functions(name_filter, details=True, pool=pool, env=env)
+    queues = get_sqs_queues(name_filter, pool=pool, env=env)
+
+    for es in domains:
+        uid = short_uid()
+        node_ids[es.id] = uid
+        result['nodes'].append({'id': uid, 'arn': es.id, 'name': es.name(), 'type': 'es'})
+    for b in buckets:
+        uid = short_uid()
+        node_ids[b.id] = uid
+        result['nodes'].append({'id': uid, 'arn': b.id, 'name': b.name(), 'type': 's3'})
+    for db in dbs:
+        uid = short_uid()
+        node_ids[db.id] = uid
+        result['nodes'].append({'id': uid, 'arn': db.id, 'name': db.name(), 'type': 'dynamodb'})
+    for s in streams:
+        uid = short_uid()
+        node_ids[s.id] = uid
+        result['nodes'].append({'id': uid, 'arn': s.id, 'name': s.name(), 'type': 'kinesis'})
+        for shard in s.shards:
+            uid1 = short_uid()
+            name = re.sub(r'shardId-0*', '', shard.id) or '0'
+            result['nodes'].append({'id': uid1, 'arn': shard.id, 'name': name,
+                'type': 'kinesis_shard', 'streamName': s.name(), 'parent': uid})
+    for f in firehoses:
+        uid = short_uid()
+        node_ids[f.id] = uid
+        result['nodes'].append({'id': uid, 'arn': f.id, 'name': f.name(), 'type': 'firehose'})
+        for d in f.destinations:
+            result['edges'].append({'source': uid, 'target': node_ids[d.id]})
+    for q in queues:
+        uid = short_uid()
+        node_ids[q.id] = uid
+        result['nodes'].append({'id': uid, 'arn': q.id, 'name': q.name(), 'type': 'sqs'})
+    for lda in lambdas:
+        uid = short_uid()
+        node_ids[lda.id] = uid
+        result['nodes'].append({'id': uid, 'arn': lda.id, 'name': lda.name(), 'type': 'lambda'})
+        for s in lda.event_sources:
+            lookup_id = s.id
+            if isinstance(s, DynamoDBStream):
+                lookup_id = s.table.id
+            result['edges'].append({'source': node_ids.get(lookup_id), 'target': uid})
+        for t in lda.targets:
+            lookup_id = t.id
+            result['edges'].append({'source': uid, 'target': node_ids.get(lookup_id)})
+    for b in buckets:
+        for n in b.notifications:
+            src_uid = node_ids[b.id]
+            tgt_uid = node_ids[n.target.id]
+            result['edges'].append({'source': src_uid, 'target': tgt_uid})
 
     return result
diff --git a/localstack/services/edge.py b/localstack/services/edge.py
@@ -191,7 +191,7 @@ def serve_health_endpoint(method, path, data):
 def serve_resource_graph(data):
     data = json.loads(to_str(data or '{}'))
     env = Environment.from_string(data.get('awsEnvironment'))
-    graph = dashboard_infra.get_graph(name_filter=data.get('nameFilter') or '.*', env=env)
+    graph = dashboard_infra.get_graph(name_filter=data.get('nameFilter') or '.*', env=env, region=data.get('awsRegion'))
     return graph
 
 
diff --git a/localstack/services/generic_proxy.py b/localstack/services/generic_proxy.py
@@ -18,10 +18,11 @@
 from six.moves.urllib.parse import urlparse
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from localstack import config
-from localstack.config import USE_SSL, EXTRA_CORS_ALLOWED_HEADERS, EXTRA_CORS_EXPOSE_HEADERS
+from localstack.config import EXTRA_CORS_ALLOWED_HEADERS, EXTRA_CORS_EXPOSE_HEADERS
 from localstack.constants import APPLICATION_JSON
 from localstack.utils.server import http2_server
-from localstack.utils.common import FuncThread, generate_ssl_cert, to_bytes, json_safe, TMP_THREADS, path_from_url
+from localstack.utils.common import (
+    FuncThread, generate_ssl_cert, to_bytes, json_safe, TMP_THREADS, path_from_url, Mock)
 from localstack.utils.testutil import is_local_test_mode
 from localstack.utils.http_utils import uses_chunked_encoding, create_chunked_data
 from localstack.utils.aws.aws_responses import LambdaResponse
@@ -446,9 +447,13 @@ def peek_ssl_header():
             return peek_ssl_header()
         except Exception:
             # Fix for "[Errno 11] Resource temporarily unavailable" - This can
-            # happen if we're using a non-blocking socket in a blocking thread
+            #   happen if we're using a non-blocking socket in a blocking thread.
             newsock.setblocking(1)
-            return peek_ssl_header()
+            newsock.settimeout(1)
+            try:
+                return peek_ssl_header()
+            except Exception:
+                return False
 
 
 # set globally defined SSL socket implementation class
@@ -464,9 +469,10 @@ async def _accept_connection2(self, protocol_factory, conn, extra, sslcontext, *
 
 
 # patch asyncio server to accept SSL and non-SSL traffic over same port
-if hasattr(BaseSelectorEventLoop, '_accept_connection2'):
+if hasattr(BaseSelectorEventLoop, '_accept_connection2') and not hasattr(BaseSelectorEventLoop, '_ls_patched'):
     _accept_connection2_orig = BaseSelectorEventLoop._accept_connection2
     BaseSelectorEventLoop._accept_connection2 = _accept_connection2
+    BaseSelectorEventLoop._ls_patched = True
 
 
 class GenericProxy(FuncThread):
@@ -514,7 +520,7 @@ def create_ssl_cert(cls, serial_number=None):
 
     @classmethod
     def get_flask_ssl_context(cls, serial_number=None):
-        if USE_SSL:
+        if config.USE_SSL:
             _, cert_file_name, key_file_name = cls.create_ssl_cert(serial_number=serial_number)
             return (cert_file_name, key_file_name)
         return None
@@ -548,11 +554,8 @@ def handler(request, data):
         method = request.method
         headers = request.headers
 
-        class T:
-            pass
-
-        request_handler = T()
-        request_handler.proxy = T()
+        request_handler = Mock()
+        request_handler.proxy = Mock()
         request_handler.proxy.port = port
         response = modify_and_forward(method=method, path=path_with_params, data_bytes=data, headers=headers,
             forward_base_url=forward_url, listeners=[listener], request_handler=None,
diff --git a/requirements.txt b/requirements.txt
@@ -17,6 +17,7 @@ boto>=2.49.0
 boto3>=1.14.33                 #basic-lib
 botocore>=1.12.13
 cachetools>=3.1.1,<4.0.0
+cbor2>=5.2.0
 coverage==4.5.4
 crontab>=0.22.6
 dnspython==1.16.0              #basic-lib
@@ -28,6 +29,7 @@ flask>=1.0.2
 flask-cors>=3.0.3,<3.1.0
 flask_swagger==0.2.12
 forbiddenfruit==0.1.3
+jsondiff>=1.2.0
 jsonpatch>=1.24,<2.0
 jsonpath-rw>=1.4.0,<2.0.0
 localstack-ext[full]>=0.11.0
@@ -47,6 +49,3 @@ requests-aws4auth==0.9
 sasl>=0.2.1
 six>=1.12.0                    #basic-lib
 xmltodict>=0.11.0
-cbor2>=5.2.0 
-jsondiff>=1.2.0 
-aws-xray-sdk>=2.6.0
