Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 76f6dae

Browse files
whummerwhummer
committed
enhance logging; extend java KCL credentials provider
1 parent 7efa1b9 commit 76f6dae

12 files changed

+160
-69
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ localstack/infra/
1515
/dist/
1616
*.egg-info/
1717
.eggs/
18+
/target/

‎Makefile

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
VENV_DIR = .venv
22
VENV_RUN = source $(VENV_DIR)/bin/activate
3-
AWS_KCL_URL = http://central.maven.org/maven2/com/amazonaws/amazon-kinesis-client/1.6.3/amazon-kinesis-client-1.6.3.jar
4-
AWS_STS_URL = http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.10.20/aws-java-sdk-sts-1.10.20.jar
5-
# AWS_KCL_URL = http://central.maven.org/maven2/com/amazonaws/amazon-kinesis-client/1.7.0/amazon-kinesis-client-1.7.0.jar
6-
#AWS_STS_URL = http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar
3+
AWS_STS_URL = http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar
74
ES_URL = https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/2.3.3/elasticsearch-2.3.3.zip
85
TMP_ARCHIVE_ES = /tmp/localstack.es.zip
96

@@ -16,10 +13,9 @@ install: ## Install npm/pip dependencies, compile code
1613
($(VENV_RUN) && pip install --upgrade pip)
1714
(test ! -e requirements.txt || ($(VENV_RUN) && pip install -r requirements.txt))
1815
(test -e localstack/infra/elasticsearch || { mkdir -p localstack/infra; cd localstack/infra; test -f $(TMP_ARCHIVE_ES) || (curl -o $(TMP_ARCHIVE_ES) $(ES_URL)); cp $(TMP_ARCHIVE_ES) es.zip; unzip -q es.zip; mv elasticsearch* elasticsearch; rm es.zip; })
19-
(test -e localstack/infra/amazon-kinesis-client/amazon-kinesis-client.jar || { mkdir -p localstack/infra/amazon-kinesis-client; curl -o localstack/infra/amazon-kinesis-client/amazon-kinesis-client.jar $(AWS_KCL_URL); })
2016
(test -e localstack/infra/amazon-kinesis-client/aws-java-sdk-sts.jar || { mkdir -p localstack/infra/amazon-kinesis-client; curl -o localstack/infra/amazon-kinesis-client/aws-java-sdk-sts.jar $(AWS_STS_URL); })
2117
(npm install -g npm || sudo npm install -g npm)
22-
(cd localstack/ && (test ! -e package.json || (npm cache clear && npm install)))
18+
(cd localstack/ && (test ! -e package.json || (npm install)))
2319
make compile
2420
# make install-web
2521

‎localstack/mock/infra.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import logging
99
import requests
1010
import json
11+
import boto3
1112
import __init__
1213
from localstack.utils.aws import aws_stack
1314
from localstack.utils import common
@@ -211,6 +212,18 @@ def check_infra(retries=5, expect_shutdown=False, apis=None, additional_checks=[
211212
check_infra(retries - 1, expect_shutdown=expect_shutdown, apis=apis, additional_checks=additional_checks)
212213

213214

215+
def check_aws_credentials():
216+
session = boto3.Session()
217+
credentials = session.get_credentials()
218+
if not credentials:
219+
# set temporary dummy credentials
220+
os.environ['AWS_ACCESS_KEY_ID'] = 'LocalStackDummyAccessKey'
221+
os.environ['AWS_SECRET_ACCESS_KEY'] = 'LocalStackDummySecretKey'
222+
session = boto3.Session()
223+
credentials = session.get_credentials()
224+
assert credentials
225+
226+
214227
def start_infra(async=False, dynamodb_update_listener=None, kinesis_update_listener=None,
215228
apigateway_update_listener=None,
216229
apis=['s3', 'es', 'apigateway', 'dynamodb', 'kinesis', 'dynamodbstreams', 'firehose', 'lambda']):
@@ -224,6 +237,8 @@ def start_infra(async=False, dynamodb_update_listener=None, kinesis_update_liste
224237
# set environment
225238
os.environ['AWS_REGION'] = DEFAULT_REGION
226239
os.environ['ENV'] = ENV_DEV
240+
# make sure AWS credentials are configured, otherwise boto3 bails on us
241+
check_aws_credentials()
227242
# start services
228243
thread = None
229244
if 'es' in apis:

‎localstack/utils/common.py

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,24 +40,33 @@ def run(self):
4040
except Exception, e:
4141
if not self.quiet:
4242
print("Thread run method %s(%s) failed: %s" %
43-
(self.func, self.params, traceback.format_exc(e)))
43+
(self.func, self.params, traceback.format_exc()))
4444

4545
def stop(self, quiet=False):
4646
if not quiet and not self.quiet:
4747
print("WARN: not implemented: FuncThread.stop(..)")
4848

4949

5050
class ShellCommandThread (FuncThread):
51-
def __init__(self, cmd, params={}, outfile=None, env_vars={}):
51+
def __init__(self, cmd, params={}, outfile=None, env_vars={}, quiet=True):
5252
self.cmd = cmd
5353
self.process = None
5454
self.outfile = outfile
5555
self.env_vars = env_vars
56-
FuncThread.__init__(self, self.run_cmd, params)
56+
FuncThread.__init__(self, self.run_cmd, params, quiet=quiet)
5757

5858
def run_cmd(self, params):
59-
self.process = run(self.cmd, async=True, outfile=self.outfile, env_vars=self.env_vars)
60-
self.process.communicate()
59+
try:
60+
self.process = run(self.cmd, async=True, outfile=self.outfile, env_vars=self.env_vars)
61+
if self.outfile:
62+
self.process.wait()
63+
else:
64+
self.process.communicate()
65+
except Exception, e:
66+
if self.process and not self.quiet:
67+
print('Shell command error "%s": %s' % (e, self.cmd))
68+
if self.process and not self.quiet and self.process.returncode != 0:
69+
print('Shell command exit code "%s": %s' % (self.process.returncode, self.cmd))
6170

6271
def is_killed(self):
6372
if not self.process:

‎localstack/utils/kinesis/java/com/atlassian/DefaultSTSAssumeRoleSessionCredentialsProvider.java

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,30 @@
22

33
import java.util.Map;
44

5-
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
6-
import com.amazonaws.auth.AWSCredentials;
7-
import com.amazonaws.auth.BasicSessionCredentials;
8-
import com.amazonaws.auth.BasicAWSCredentials;
95
import com.amazonaws.auth.AWSCredentialsProvider;
10-
6+
import com.amazonaws.auth.BasicSessionCredentials;
7+
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
8+
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
9+
import com.amazonaws.internal.StaticCredentialsProvider;
10+
11+
/**
12+
* Custom session credentials provider that can be configured to assume a given IAM role.
13+
* Configure the role to assume via the following environment variables:
14+
* - AWS_ASSUME_ROLE_ARN : ARN of the role to assume
15+
* - AWS_ASSUME_ROLE_SESSION_NAME : name of the session to be used when calling assume-role
16+
*
17+
* As long lived credentials, this credentials provider attempts to uses the following:
18+
* - an STS token, via environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
19+
* - instance profile credentials provider (see Google hits for "EC2 instance metadata service")
20+
*
21+
* TODO: Potentially we could simply use the default credentials provider to obtain the long-lived credentials.
22+
*
23+
* @author Waldemar Hummer
24+
*/
1125
public class DefaultSTSAssumeRoleSessionCredentialsProvider extends STSAssumeRoleSessionCredentialsProvider {
1226

1327
public DefaultSTSAssumeRoleSessionCredentialsProvider() {
14-
super(getDefaultCredentials(), getDefaultRoleARN(), getDefaultRoleSessionName());
28+
super(getLongLivedCredentialsProvider(), getDefaultRoleARN(), getDefaultRoleSessionName());
1529
}
1630

1731
private static String getDefaultRoleARN() {
@@ -24,17 +38,20 @@ private static String getDefaultRoleSessionName() {
2438
return env.get("AWS_ASSUME_ROLE_SESSION_NAME");
2539
}
2640

27-
private static AWSCredentials getDefaultCredentials() {
41+
private static AWSCredentialsProvider getLongLivedCredentialsProvider() {
2842
Map<String, String> env = System.getenv();
2943
if(env.containsKey("AWS_SESSION_TOKEN")) {
30-
return new BasicSessionCredentials(
31-
env.get("AWS_ACCESS_KEY_ID"),
32-
env.get("AWS_SECRET_ACCESS_KEY"),
33-
env.get("AWS_SESSION_TOKEN"));
44+
return new StaticCredentialsProvider(
45+
new BasicSessionCredentials(
46+
env.get("AWS_ACCESS_KEY_ID"),
47+
env.get("AWS_SECRET_ACCESS_KEY"),
48+
env.get("AWS_SESSION_TOKEN")));
3449
}
35-
return new BasicAWSCredentials(
36-
env.get("AWS_ACCESS_KEY_ID"),
37-
env.get("AWS_SECRET_ACCESS_KEY"));
50+
return new InstanceProfileCredentialsProvider();
51+
}
52+
53+
public static void main(String args[]) throws Exception {
54+
System.out.println(new DefaultSTSAssumeRoleSessionCredentialsProvider().getCredentials());
3855
}
3956

4057
}

‎localstack/utils/kinesis/java/com/atlassian/KinesisStarter.java

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,15 @@
11
package com.atlassian;
22

33
import java.net.URL;
4-
import java.security.SecureRandom;
5-
import java.security.cert.CertificateException;
6-
import java.security.cert.X509Certificate;
74
import java.util.Properties;
85
import java.util.concurrent.ExecutorService;
96
import java.util.concurrent.Future;
107

11-
import javax.net.ssl.HttpsURLConnection;
12-
import javax.net.ssl.SSLContext;
13-
import javax.net.ssl.TrustManager;
14-
import javax.net.ssl.X509TrustManager;
15-
16-
import com.amazonaws.services.kinesis.multilang.MultiLangDaemon;
17-
import com.amazonaws.services.kinesis.multilang.MultiLangDaemonConfig;
18-
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
198
import com.amazonaws.ClientConfiguration;
209
import com.amazonaws.Protocol;
10+
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
11+
import com.amazonaws.services.kinesis.multilang.MultiLangDaemon;
12+
import com.amazonaws.services.kinesis.multilang.MultiLangDaemonConfig;
2113

2214
/**
2315
* Custom extensions to <code>MultiLangDaemon</code> class from amazon-kinesis-client
@@ -27,6 +19,8 @@
2719
* - dynamodbProtocol: protocol for DynamoDB API (http or https)
2820
* - kinesisProtocol: protocol for Kinesis API (http or https)
2921
* - metricsLevel: level of CloudWatch metrics to report (e.g., SUMMARY or NONE)
22+
*
23+
* @author Waldemar Hummer
3024
*/
3125
public class KinesisStarter {
3226

‎localstack/utils/kinesis/kclipy_helper.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,6 @@ def get_kcl_classpath(properties=None, paths=[]):
5151
# add path of custom java code
5252
dir_name = os.path.dirname(os.path.realpath(__file__))
5353
paths.append(os.path.realpath(os.path.join(dir_name, 'java')))
54-
paths.insert(0, os.path.realpath(os.path.join(dir_name, '..', '..',
55-
'infra', 'amazon-kinesis-client', 'amazon-kinesis-client.jar')))
5654
paths.insert(0, os.path.realpath(os.path.join(dir_name, '..', '..',
5755
'infra', 'amazon-kinesis-client', 'aws-java-sdk-sts.jar')))
5856
return ":".join([p for p in paths if p != ''])

‎localstack/utils/kinesis/kinesis_connector.py

Lines changed: 47 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
LOG_FILE_PATTERN = '/tmp/kclipy.*.log'
2727
DEFAULT_DDB_LEASE_TABLE_SUFFIX = '-app'
2828

29+
# define Java class names
30+
MULTI_LANG_DAEMON_CLASS = 'com.atlassian.KinesisStarter'
31+
2932
# set up log levels
3033
logging.SEVERE = 60
3134
logging.FATAL = 70
@@ -82,15 +85,17 @@ def run_processor(log_file=None, processor_func=None):
8285

8386
class KinesisProcessorThread(ShellCommandThread):
8487
def __init__(self, params):
85-
multi_lang_daemon_class = 'com.atlassian.KinesisStarter'
8688
props_file = params['properties_file']
8789
env_vars = params['env_vars']
8890
cmd = kclipy_helper.get_kcl_app_command('java',
89-
multi_lang_daemon_class, props_file)
91+
MULTI_LANG_DAEMON_CLASS, props_file)
9092
if not params['log_file']:
9193
params['log_file'] = '%s.log' % props_file
9294
TMP_FILES.append(params['log_file'])
93-
ShellCommandThread.__init__(self, cmd, outfile=params['log_file'], env_vars=env_vars)
95+
# print(cmd)
96+
env = aws_stack.get_environment()
97+
quiet = env.region == REGION_LOCAL
98+
ShellCommandThread.__init__(self, cmd, outfile=params['log_file'], env_vars=env_vars, quiet=quiet)
9499

95100
@staticmethod
96101
def start_consumer(kinesis_stream):
@@ -105,42 +110,53 @@ def __init__(self, params):
105110
self.running = True
106111
self.buffer = []
107112
self.params = params
108-
self.prefix = params.get('log_prefix') or 'LOG: '
109113
# number of lines that make up a single log entry
110114
self.buffer_size = 2
111-
self.log_level = params.get('level') or DEFAULT_KCL_LOG_LEVEL
112-
# regular expression to filter the printed output
113-
levels = OutputReaderThread.get_log_level_names(self.log_level)
114-
self.filter_regex = r'.*(%s):.*' % ('|'.join(levels))
115+
# determine log level
116+
self.log_level = params.get('level')
117+
if self.log_level is None:
118+
self.log_level = DEFAULT_KCL_LOG_LEVEL
119+
if self.log_level > 0:
120+
levels = OutputReaderThread.get_log_level_names(self.log_level)
121+
# regular expression to filter the printed output
122+
self.filter_regex = r'.*(%s):.*' % ('|'.join(levels))
123+
# create prefix and logger
124+
self.prefix = params.get('log_prefix') or 'LOG'
125+
self.logger = logging.getLogger(self.prefix)
126+
self.logger.severe = self.logger.critical
127+
self.logger.fatal = self.logger.critical
128+
self.logger.setLevel(self.log_level)
115129

116130
@classmethod
117131
def get_log_level_names(cls, min_level):
118132
return [logging.getLevelName(l) for l in LOG_LEVELS if l >= min_level]
119133

120-
@classmethod
121-
def get_logger_for_level_in_log_line(cls, line, level):
122-
for level in LOG_LEVELS:
123-
level_name = logging.getLevelName(level)
124-
if re.match(r'.*(%s):.*' % level, line):
125-
return LOGGER.__dict__[level.lower()]
134+
def get_logger_for_level_in_log_line(self, line):
135+
level = self.log_level
136+
for l in LOG_LEVELS:
137+
if l >= level:
138+
level_name = logging.getLevelName(l)
139+
if re.match(r'.*(%s):.*' % level_name, line):
140+
return getattr(self.logger, level_name.lower())
126141
return None
127142

128143
def start_reading(self, params):
129144
for line in tail("-n", 0, "-f", params['file'], _iter=True):
130145
if not self.running:
131146
return
132-
line = line.replace('\n', '')
133-
self.buffer.append(line)
134-
if len(self.buffer) >= self.buffer_size:
135-
logger_func = None
136-
for line in self.buffer:
137-
if re.match(self.filter_regex, line):
138-
logger_func = OutputReaderThread.get_logger_for_level_in_log_line(line, self.log_level)
139-
break
140-
if logger_func:
141-
for buffered_line in self.buffer:
142-
logger_func('%s%s' % (self.prefix, buffered_line))
143-
self.buffer = []
147+
if self.log_level > 0:
148+
line = line.replace('\n', '')
149+
self.buffer.append(line)
150+
if len(self.buffer) >= self.buffer_size:
151+
logger_func = None
152+
for line in self.buffer:
153+
if re.match(self.filter_regex, line):
154+
logger_func = self.get_logger_for_level_in_log_line(line)
155+
break
156+
if logger_func:
157+
for buffered_line in self.buffer:
158+
logger_func(buffered_line)
159+
self.buffer = []
144160

145161
def stop(self, quiet=True):
146162
self.running = False
@@ -239,14 +255,18 @@ def start_kcl_client_process(stream_name, listener_script, log_file=None, env=No
239255
'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']:
240256
if var_name in os.environ and var_name not in env_vars:
241257
env_vars[var_name] = os.environ[var_name]
258+
if env.region == REGION_LOCAL:
259+
# need to disable CBOR protocol, enforce use of plain JSON,
260+
# see https://github.com/mhart/kinesalite/issues/31
261+
env_vars['AWS_CBOR_DISABLE'] = 'true'
242262
if kcl_log_level:
243263
if not log_file:
244264
log_file = LOG_FILE_PATTERN.replace('*', short_uid())
245265
TMP_FILES.append(log_file)
246266
run('touch %s' % log_file)
247267
# start log output reader thread which will read the KCL log
248268
# file and print each line to stdout of this process...
249-
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL: '})
269+
reader_thread = OutputReaderThread({'file': log_file, 'level': kcl_log_level, 'log_prefix': 'KCL'})
250270
reader_thread.start()
251271

252272
# construct stream info

‎requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
airspeed==0.5.5.dev20160812
2-
# amazon_kclpy==1.3.1
3-
amazon_kclpy==1.2.0
2+
amazon_kclpy==1.3.1
3+
# amazon_kclpy==1.2.0
44
boto3==1.4.0
55
coverage==4.0.3
66
docopt==0.6.2

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def run(self):
7575

7676
setup(
7777
name='localstack',
78-
version='0.1.2',
78+
version='0.1.4',
7979
description='Provides an easy-to-use test/mocking framework for developing Cloud applications',
8080
author='Waldemar Hummer',
8181
author_email='[email protected]',

‎tests/test_iam_credentials.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env python
2+
3+
import os
4+
import time
5+
import logging
6+
from localstack.utils.kinesis import kinesis_connector
7+
8+
9+
def run_kcl_with_iam_assume_role():
10+
env_vars = {}
11+
if os.environ.get('AWS_ASSUME_ROLE_ARN'):
12+
env_vars['AWS_ASSUME_ROLE_ARN'] = os.environ.get('AWS_ASSUME_ROLE_ARN')
13+
env_vars['AWS_ASSUME_ROLE_SESSION_NAME'] = os.environ.get('AWS_ASSUME_ROLE_SESSION_NAME')
14+
env_vars['ENV'] = os.environ.get('ENV') or 'main'
15+
16+
def process_records(records):
17+
print(records)
18+
19+
# start Kinesis client
20+
stream_name = 'test-foobar'
21+
kinesis_connector.listen_to_kinesis(
22+
stream_name=stream_name,
23+
listener_func=process_records,
24+
env_vars=env_vars,
25+
kcl_log_level=logging.INFO)
26+
time.sleep(60 * 10)
27+
28+
if __name__ == '__main__':
29+
run_kcl_with_iam_assume_role()

0 commit comments

Comments
 (0)