We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Localstack Elasticsearch plugin Ingest User Agent Processor not available
Plugin `Ingest User Agent Processor` is installed by default for Elasticsearch (ELK) on AWS. It is not the case in Localstack and think we basically expect it.

In addition, I was not able to install it manually through command `bin/elasticsearch-plugin install ingest-user-agent` as bin/elasticsearch-plugin is missing.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `localstack/constants.py`
Content:
```
1 import os
2 import localstack_client.config
3 
4 # LocalStack version
5 VERSION = '0.8.10'
6 
7 # default AWS region
8 if 'DEFAULT_REGION' not in os.environ:
9     os.environ['DEFAULT_REGION'] = 'us-east-1'
10 DEFAULT_REGION = os.environ['DEFAULT_REGION']
11 
12 # constant to represent the "local" region, i.e., local machine
13 REGION_LOCAL = 'local'
14 
15 # dev environment
16 ENV_DEV = 'dev'
17 
18 # backend service ports, for services that are behind a proxy (counting down from 4566)
19 DEFAULT_PORT_APIGATEWAY_BACKEND = 4566
20 DEFAULT_PORT_KINESIS_BACKEND = 4565
21 DEFAULT_PORT_DYNAMODB_BACKEND = 4564
22 DEFAULT_PORT_S3_BACKEND = 4563
23 DEFAULT_PORT_SNS_BACKEND = 4562
24 DEFAULT_PORT_SQS_BACKEND = 4561
25 DEFAULT_PORT_ELASTICSEARCH_BACKEND = 4560
26 DEFAULT_PORT_CLOUDFORMATION_BACKEND = 4559
27 
28 DEFAULT_PORT_WEB_UI = 8080
29 
30 LOCALHOST = 'localhost'
31 
32 # version of the Maven dependency with Java utility code
33 LOCALSTACK_MAVEN_VERSION = '0.1.15'
34 
35 # map of default service APIs and ports to be spun up (fetch map from localstack_client)
36 DEFAULT_SERVICE_PORTS = localstack_client.config.get_service_ports()
37 
38 # host to bind to when starting the services
39 BIND_HOST = '0.0.0.0'
40 
41 # AWS user account ID used for tests
42 TEST_AWS_ACCOUNT_ID = '000000000000'
43 os.environ['TEST_AWS_ACCOUNT_ID'] = TEST_AWS_ACCOUNT_ID
44 
45 # root code folder
46 LOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
47 
48 # virtualenv folder
49 LOCALSTACK_VENV_FOLDER = os.path.join(LOCALSTACK_ROOT_FOLDER, '.venv')
50 if not os.path.isdir(LOCALSTACK_VENV_FOLDER):
51     # assuming this package lives here: <python>/lib/pythonX.X/site-packages/localstack/
52     LOCALSTACK_VENV_FOLDER = os.path.realpath(os.path.join(LOCALSTACK_ROOT_FOLDER, '..', '..', '..'))
53 
54 # API Gateway path to indicate a user request sent to the gateway
55 PATH_USER_REQUEST = '_user_request_'
56 
57 # name of LocalStack Docker image
58 DOCKER_IMAGE_NAME = 'localstack/localstack'
59 
60 # environment variable name to tag local test runs
61 ENV_INTERNAL_TEST_RUN = 'LOCALSTACK_INTERNAL_TEST_RUN'
62 
63 # content types
64 APPLICATION_AMZ_JSON_1_0 = 'application/x-amz-json-1.0'
65 APPLICATION_AMZ_JSON_1_1 = 'application/x-amz-json-1.1'
66 APPLICATION_JSON = 'application/json'
67 
68 # Lambda defaults
69 LAMBDA_TEST_ROLE = 'arn:aws:iam::%s:role/lambda-test-role' % TEST_AWS_ACCOUNT_ID
70 
71 # installation constants
72 ELASTICSEARCH_JAR_URL = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.zip'
73 DYNAMODB_JAR_URL = 'https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip'
74 ELASTICMQ_JAR_URL = 'https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.14.2.jar'
75 STS_JAR_URL = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'
76 
77 # API endpoint for analytics events
78 API_ENDPOINT = 'https://api.localstack.cloud/v1'
79
```

Path: `localstack/services/install.py`
Content:
```
1 #!/usr/bin/env python
2 
3 import os
4 import sys
5 import glob
6 import shutil
7 import logging
8 import tempfile
9 from localstack.constants import (DEFAULT_SERVICE_PORTS, ELASTICMQ_JAR_URL, STS_JAR_URL,
10     ELASTICSEARCH_JAR_URL, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)
11 from localstack.utils.common import download, parallelize, run, mkdir, save_file, unzip, rm_rf, chmod_r
12 
13 THIS_PATH = os.path.dirname(os.path.realpath(__file__))
14 ROOT_PATH = os.path.realpath(os.path.join(THIS_PATH, '..'))
15 
16 INSTALL_DIR_INFRA = '%s/infra' % ROOT_PATH
17 INSTALL_DIR_NPM = '%s/node_modules' % ROOT_PATH
18 INSTALL_DIR_ES = '%s/elasticsearch' % INSTALL_DIR_INFRA
19 INSTALL_DIR_DDB = '%s/dynamodb' % INSTALL_DIR_INFRA
20 INSTALL_DIR_KCL = '%s/amazon-kinesis-client' % INSTALL_DIR_INFRA
21 INSTALL_DIR_ELASTICMQ = '%s/elasticmq' % INSTALL_DIR_INFRA
22 INSTALL_PATH_LOCALSTACK_FAT_JAR = '%s/localstack-utils-fat.jar' % INSTALL_DIR_INFRA
23 TMP_ARCHIVE_ES = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')
24 TMP_ARCHIVE_DDB = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')
25 TMP_ARCHIVE_STS = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar')
26 TMP_ARCHIVE_ELASTICMQ = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar')
27 URL_LOCALSTACK_FAT_JAR = ('http://central.maven.org/maven2/' +
28     'cloud/localstack/localstack-utils/{v}/localstack-utils-{v}-fat.jar').format(v=LOCALSTACK_MAVEN_VERSION)
29 
30 # set up logger
31 LOGGER = logging.getLogger(__name__)
32 
33 
34 def install_elasticsearch():
35     if not os.path.exists(INSTALL_DIR_ES):
36         LOGGER.info('Downloading and installing local Elasticsearch server. This may take some time.')
37         mkdir(INSTALL_DIR_INFRA)
38         # download and extract archive
39         download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA)
40         elasticsearch_dir = glob.glob(os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*'))
41         if not elasticsearch_dir:
42             raise Exception('Unable to find Elasticsearch folder in %s' % INSTALL_DIR_INFRA)
43         shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES)
44 
45         for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):
46             dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)
47             mkdir(dir_path)
48             chmod_r(dir_path, 0o777)
49 
50 
51 def install_elasticmq():
52     if not os.path.exists(INSTALL_DIR_ELASTICMQ):
53         LOGGER.info('Downloading and installing local ElasticMQ server. This may take some time.')
54         mkdir(INSTALL_DIR_ELASTICMQ)
55         # download archive
56         if not os.path.exists(TMP_ARCHIVE_ELASTICMQ):
57             download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ)
58         shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)
59 
60 
61 def install_kinesalite():
62     target_dir = '%s/kinesalite' % INSTALL_DIR_NPM
63     if not os.path.exists(target_dir):
64         LOGGER.info('Downloading and installing local Kinesis server. This may take some time.')
65         run('cd "%s" && npm install' % ROOT_PATH)
66 
67 
68 def install_dynamodb_local():
69     if not os.path.exists(INSTALL_DIR_DDB):
70         LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.')
71         mkdir(INSTALL_DIR_DDB)
72         # download and extract archive
73         download_and_extract_with_retry(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB, INSTALL_DIR_DDB)
74 
75     # fix for Alpine, otherwise DynamoDBLocal fails with:
76     # DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found
77     if is_alpine():
78         ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB
79         patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir
80         if not os.path.exists(patched_marker):
81             patched_lib = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +
82                 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so')
83             patched_jar = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +
84                 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')
85             run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" % (ddb_libs_dir, patched_lib))
86             run("curl -L -o %s/sqlite4java.jar '%s'" % (ddb_libs_dir, patched_jar))
87             save_file(patched_marker, '')
88 
89     # fix logging configuration for DynamoDBLocal
90     log4j2_config = """<Configuration status="WARN">
91       <Appenders>
92         <Console name="Console" target="SYSTEM_OUT">
93           <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
94         </Console>
95       </Appenders>
96       <Loggers>
97         <Root level="WARN"><AppenderRef ref="Console"/></Root>
98       </Loggers>
99     </Configuration>"""
100     log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')
101     save_file(log4j2_file, log4j2_config)
102     run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)
103 
104 
105 def install_amazon_kinesis_client_libs():
106     # install KCL/STS JAR files
107     if not os.path.exists(INSTALL_DIR_KCL):
108         mkdir(INSTALL_DIR_KCL)
109         if not os.path.exists(TMP_ARCHIVE_STS):
110             download(STS_JAR_URL, TMP_ARCHIVE_STS)
111         shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)
112     # Compile Java files
113     from localstack.utils.kinesis import kclipy_helper
114     classpath = kclipy_helper.get_kcl_classpath()
115     java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH
116     class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH
117     if not glob.glob(class_files):
118         run('javac -cp "%s" %s' % (classpath, java_files))
119 
120 
121 def install_lambda_java_libs():
122     # install LocalStack "fat" JAR file (contains all dependencies)
123     if not os.path.exists(INSTALL_PATH_LOCALSTACK_FAT_JAR):
124         LOGGER.info('Downloading and installing LocalStack Java libraries. This may take some time.')
125         download(URL_LOCALSTACK_FAT_JAR, INSTALL_PATH_LOCALSTACK_FAT_JAR)
126 
127 
128 def install_component(name):
129     if name == 'kinesis':
130         install_kinesalite()
131     elif name == 'dynamodb':
132         install_dynamodb_local()
133     elif name == 'es':
134         install_elasticsearch()
135     elif name == 'sqs':
136         install_elasticmq()
137 
138 
139 def install_components(names):
140     parallelize(install_component, names)
141     install_lambda_java_libs()
142 
143 
144 def install_all_components():
145     install_components(DEFAULT_SERVICE_PORTS.keys())
146 
147 
148 # -----------------
149 # HELPER FUNCTIONS
150 # -----------------
151 
152 
153 def is_alpine():
154     try:
155         run('cat /etc/issue | grep Alpine', print_error=False)
156         return True
157     except Exception:
158         return False
159 
160 
161 def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
162 
163     def download_and_extract():
164         if not os.path.exists(tmp_archive):
165             download(archive_url, tmp_archive)
166         unzip(tmp_archive, target_dir)
167 
168     try:
169         download_and_extract()
170     except Exception:
171         # try deleting and re-downloading the zip file
172         LOGGER.info('Unable to extract file, re-downloading ZIP archive: %s' % tmp_archive)
173         rm_rf(tmp_archive)
174         download_and_extract()
175 
176 
177 if __name__ == '__main__':
178 
179     if len(sys.argv) > 1:
180         if sys.argv[1] == 'libs':
181             print('Initializing installation.')
182             logging.basicConfig(level=logging.INFO)
183             logging.getLogger('requests').setLevel(logging.WARNING)
184             install_all_components()
185             print('Done.')
186         elif sys.argv[1] == 'testlibs':
187             # Install additional libraries for testing
188             install_amazon_kinesis_client_libs()
189
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/localstack/constants.py b/localstack/constants.py
--- a/localstack/constants.py
+++ b/localstack/constants.py
@@ -70,6 +70,9 @@
 
 # installation constants
 ELASTICSEARCH_JAR_URL = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.zip'
+# https://docs.aws.amazon.com/ja_jp/elasticsearch-service/latest/developerguide/aes-supported-plugins.html
+ELASTICSEARCH_PLUGIN_LIST = ['analysis-icu', 'ingest-attachment', 'ingest-user-agent', 'analysis-kuromoji',
+ 'mapper-murmur3', 'mapper-size', 'analysis-phonetic', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian']
 DYNAMODB_JAR_URL = 'https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip'
 ELASTICMQ_JAR_URL = 'https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.14.2.jar'
 STS_JAR_URL = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'
diff --git a/localstack/services/install.py b/localstack/services/install.py
--- a/localstack/services/install.py
+++ b/localstack/services/install.py
@@ -7,7 +7,7 @@
 import logging
 import tempfile
 from localstack.constants import (DEFAULT_SERVICE_PORTS, ELASTICMQ_JAR_URL, STS_JAR_URL,
-    ELASTICSEARCH_JAR_URL, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)
+    ELASTICSEARCH_JAR_URL, ELASTICSEARCH_PLUGIN_LIST, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)
 from localstack.utils.common import download, parallelize, run, mkdir, save_file, unzip, rm_rf, chmod_r
 
 THIS_PATH = os.path.dirname(os.path.realpath(__file__))
@@ -47,6 +47,14 @@
             mkdir(dir_path)
             chmod_r(dir_path, 0o777)
 
+        # install default plugins
+        for plugin in ELASTICSEARCH_PLUGIN_LIST:
+            if is_alpine():
+                # https://github.com/pires/docker-elasticsearch/issues/56
+                os.environ['ES_TMPDIR'] = '/tmp'
+            plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin', 'elasticsearch-plugin')
+            run('%s install %s' % (plugin_binary, plugin))
+
 
 def install_elasticmq():
     if not os.path.exists(INSTALL_DIR_ELASTICMQ):

{"golden_diff": "diff --git a/localstack/constants.py b/localstack/constants.py\n--- a/localstack/constants.py\n+++ b/localstack/constants.py\n@@ -70,6 +70,9 @@\n \n # installation constants\n ELASTICSEARCH_JAR_URL = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.zip'\n+# https://docs.aws.amazon.com/ja_jp/elasticsearch-service/latest/developerguide/aes-supported-plugins.html\n+ELASTICSEARCH_PLUGIN_LIST = ['analysis-icu', 'ingest-attachment', 'ingest-user-agent', 'analysis-kuromoji',\n+ 'mapper-murmur3', 'mapper-size', 'analysis-phonetic', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian']\n DYNAMODB_JAR_URL = 'https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip'\n ELASTICMQ_JAR_URL = 'https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.14.2.jar'\n STS_JAR_URL = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'\ndiff --git a/localstack/services/install.py b/localstack/services/install.py\n--- a/localstack/services/install.py\n+++ b/localstack/services/install.py\n@@ -7,7 +7,7 @@\n import logging\n import tempfile\n from localstack.constants import (DEFAULT_SERVICE_PORTS, ELASTICMQ_JAR_URL, STS_JAR_URL,\n-    ELASTICSEARCH_JAR_URL, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)\n+    ELASTICSEARCH_JAR_URL, ELASTICSEARCH_PLUGIN_LIST, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)\n from localstack.utils.common import download, parallelize, run, mkdir, save_file, unzip, rm_rf, chmod_r\n \n THIS_PATH = os.path.dirname(os.path.realpath(__file__))\n@@ -47,6 +47,14 @@\n             mkdir(dir_path)\n             chmod_r(dir_path, 0o777)\n \n+        # install default plugins\n+        for plugin in ELASTICSEARCH_PLUGIN_LIST:\n+            if is_alpine():\n+                # https://github.com/pires/docker-elasticsearch/issues/56\n+                os.environ['ES_TMPDIR'] = '/tmp'\n+            plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin', 'elasticsearch-plugin')\n+            run('%s install %s' % (plugin_binary, plugin))\n+\n \n def install_elasticmq():\n     if not os.path.exists(INSTALL_DIR_ELASTICMQ):\n", "issue": "Localstack Elasticsearch plugin Ingest User Agent Processor not available\nPlugin `Ingest User Agent Processor` is installed by default for Elasticsearch (ELK) on AWS. It is not the case in Localstack and think we basically expect it.\r\n\r\nIn addition, I was not able to install it manually through command `bin/elasticsearch-plugin install ingest-user-agent` as bin/elasticsearch-plugin is missing.\n", "before_files": [{"content": "import os\nimport localstack_client.config\n\n# LocalStack version\nVERSION = '0.8.10'\n\n# default AWS region\nif 'DEFAULT_REGION' not in os.environ:\n    os.environ['DEFAULT_REGION'] = 'us-east-1'\nDEFAULT_REGION = os.environ['DEFAULT_REGION']\n\n# constant to represent the \"local\" region, i.e., local machine\nREGION_LOCAL = 'local'\n\n# dev environment\nENV_DEV = 'dev'\n\n# backend service ports, for services that are behind a proxy (counting down from 4566)\nDEFAULT_PORT_APIGATEWAY_BACKEND = 4566\nDEFAULT_PORT_KINESIS_BACKEND = 4565\nDEFAULT_PORT_DYNAMODB_BACKEND = 4564\nDEFAULT_PORT_S3_BACKEND = 4563\nDEFAULT_PORT_SNS_BACKEND = 4562\nDEFAULT_PORT_SQS_BACKEND = 4561\nDEFAULT_PORT_ELASTICSEARCH_BACKEND = 4560\nDEFAULT_PORT_CLOUDFORMATION_BACKEND = 4559\n\nDEFAULT_PORT_WEB_UI = 8080\n\nLOCALHOST = 'localhost'\n\n# version of the Maven dependency with Java utility code\nLOCALSTACK_MAVEN_VERSION = '0.1.15'\n\n# map of default service APIs and ports to be spun up (fetch map from localstack_client)\nDEFAULT_SERVICE_PORTS = localstack_client.config.get_service_ports()\n\n# host to bind to when starting the services\nBIND_HOST = '0.0.0.0'\n\n# AWS user account ID used for tests\nTEST_AWS_ACCOUNT_ID = '000000000000'\nos.environ['TEST_AWS_ACCOUNT_ID'] = TEST_AWS_ACCOUNT_ID\n\n# root code folder\nLOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))\n\n# virtualenv folder\nLOCALSTACK_VENV_FOLDER = os.path.join(LOCALSTACK_ROOT_FOLDER, '.venv')\nif not os.path.isdir(LOCALSTACK_VENV_FOLDER):\n    # assuming this package lives here: <python>/lib/pythonX.X/site-packages/localstack/\n    LOCALSTACK_VENV_FOLDER = os.path.realpath(os.path.join(LOCALSTACK_ROOT_FOLDER, '..', '..', '..'))\n\n# API Gateway path to indicate a user request sent to the gateway\nPATH_USER_REQUEST = '_user_request_'\n\n# name of LocalStack Docker image\nDOCKER_IMAGE_NAME = 'localstack/localstack'\n\n# environment variable name to tag local test runs\nENV_INTERNAL_TEST_RUN = 'LOCALSTACK_INTERNAL_TEST_RUN'\n\n# content types\nAPPLICATION_AMZ_JSON_1_0 = 'application/x-amz-json-1.0'\nAPPLICATION_AMZ_JSON_1_1 = 'application/x-amz-json-1.1'\nAPPLICATION_JSON = 'application/json'\n\n# Lambda defaults\nLAMBDA_TEST_ROLE = 'arn:aws:iam::%s:role/lambda-test-role' % TEST_AWS_ACCOUNT_ID\n\n# installation constants\nELASTICSEARCH_JAR_URL = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.zip'\nDYNAMODB_JAR_URL = 'https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip'\nELASTICMQ_JAR_URL = 'https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.14.2.jar'\nSTS_JAR_URL = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'\n\n# API endpoint for analytics events\nAPI_ENDPOINT = 'https://api.localstack.cloud/v1'\n", "path": "localstack/constants.py"}, {"content": "#!/usr/bin/env python\n\nimport os\nimport sys\nimport glob\nimport shutil\nimport logging\nimport tempfile\nfrom localstack.constants import (DEFAULT_SERVICE_PORTS, ELASTICMQ_JAR_URL, STS_JAR_URL,\n    ELASTICSEARCH_JAR_URL, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)\nfrom localstack.utils.common import download, parallelize, run, mkdir, save_file, unzip, rm_rf, chmod_r\n\nTHIS_PATH = os.path.dirname(os.path.realpath(__file__))\nROOT_PATH = os.path.realpath(os.path.join(THIS_PATH, '..'))\n\nINSTALL_DIR_INFRA = '%s/infra' % ROOT_PATH\nINSTALL_DIR_NPM = '%s/node_modules' % ROOT_PATH\nINSTALL_DIR_ES = '%s/elasticsearch' % INSTALL_DIR_INFRA\nINSTALL_DIR_DDB = '%s/dynamodb' % INSTALL_DIR_INFRA\nINSTALL_DIR_KCL = '%s/amazon-kinesis-client' % INSTALL_DIR_INFRA\nINSTALL_DIR_ELASTICMQ = '%s/elasticmq' % INSTALL_DIR_INFRA\nINSTALL_PATH_LOCALSTACK_FAT_JAR = '%s/localstack-utils-fat.jar' % INSTALL_DIR_INFRA\nTMP_ARCHIVE_ES = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')\nTMP_ARCHIVE_DDB = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')\nTMP_ARCHIVE_STS = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar')\nTMP_ARCHIVE_ELASTICMQ = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar')\nURL_LOCALSTACK_FAT_JAR = ('http://central.maven.org/maven2/' +\n    'cloud/localstack/localstack-utils/{v}/localstack-utils-{v}-fat.jar').format(v=LOCALSTACK_MAVEN_VERSION)\n\n# set up logger\nLOGGER = logging.getLogger(__name__)\n\n\ndef install_elasticsearch():\n    if not os.path.exists(INSTALL_DIR_ES):\n        LOGGER.info('Downloading and installing local Elasticsearch server. This may take some time.')\n        mkdir(INSTALL_DIR_INFRA)\n        # download and extract archive\n        download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA)\n        elasticsearch_dir = glob.glob(os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*'))\n        if not elasticsearch_dir:\n            raise Exception('Unable to find Elasticsearch folder in %s' % INSTALL_DIR_INFRA)\n        shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES)\n\n        for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):\n            dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)\n            mkdir(dir_path)\n            chmod_r(dir_path, 0o777)\n\n\ndef install_elasticmq():\n    if not os.path.exists(INSTALL_DIR_ELASTICMQ):\n        LOGGER.info('Downloading and installing local ElasticMQ server. This may take some time.')\n        mkdir(INSTALL_DIR_ELASTICMQ)\n        # download archive\n        if not os.path.exists(TMP_ARCHIVE_ELASTICMQ):\n            download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ)\n        shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)\n\n\ndef install_kinesalite():\n    target_dir = '%s/kinesalite' % INSTALL_DIR_NPM\n    if not os.path.exists(target_dir):\n        LOGGER.info('Downloading and installing local Kinesis server. This may take some time.')\n        run('cd \"%s\" && npm install' % ROOT_PATH)\n\n\ndef install_dynamodb_local():\n    if not os.path.exists(INSTALL_DIR_DDB):\n        LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.')\n        mkdir(INSTALL_DIR_DDB)\n        # download and extract archive\n        download_and_extract_with_retry(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB, INSTALL_DIR_DDB)\n\n    # fix for Alpine, otherwise DynamoDBLocal fails with:\n    # DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found\n    if is_alpine():\n        ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB\n        patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir\n        if not os.path.exists(patched_marker):\n            patched_lib = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +\n                'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so')\n            patched_jar = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +\n                'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')\n            run(\"curl -L -o %s/libsqlite4java-linux-amd64.so '%s'\" % (ddb_libs_dir, patched_lib))\n            run(\"curl -L -o %s/sqlite4java.jar '%s'\" % (ddb_libs_dir, patched_jar))\n            save_file(patched_marker, '')\n\n    # fix logging configuration for DynamoDBLocal\n    log4j2_config = \"\"\"<Configuration status=\"WARN\">\n      <Appenders>\n        <Console name=\"Console\" target=\"SYSTEM_OUT\">\n          <PatternLayout pattern=\"%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\"/>\n        </Console>\n      </Appenders>\n      <Loggers>\n        <Root level=\"WARN\"><AppenderRef ref=\"Console\"/></Root>\n      </Loggers>\n    </Configuration>\"\"\"\n    log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')\n    save_file(log4j2_file, log4j2_config)\n    run('cd \"%s\" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)\n\n\ndef install_amazon_kinesis_client_libs():\n    # install KCL/STS JAR files\n    if not os.path.exists(INSTALL_DIR_KCL):\n        mkdir(INSTALL_DIR_KCL)\n        if not os.path.exists(TMP_ARCHIVE_STS):\n            download(STS_JAR_URL, TMP_ARCHIVE_STS)\n        shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)\n    # Compile Java files\n    from localstack.utils.kinesis import kclipy_helper\n    classpath = kclipy_helper.get_kcl_classpath()\n    java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH\n    class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH\n    if not glob.glob(class_files):\n        run('javac -cp \"%s\" %s' % (classpath, java_files))\n\n\ndef install_lambda_java_libs():\n    # install LocalStack \"fat\" JAR file (contains all dependencies)\n    if not os.path.exists(INSTALL_PATH_LOCALSTACK_FAT_JAR):\n        LOGGER.info('Downloading and installing LocalStack Java libraries. This may take some time.')\n        download(URL_LOCALSTACK_FAT_JAR, INSTALL_PATH_LOCALSTACK_FAT_JAR)\n\n\ndef install_component(name):\n    if name == 'kinesis':\n        install_kinesalite()\n    elif name == 'dynamodb':\n        install_dynamodb_local()\n    elif name == 'es':\n        install_elasticsearch()\n    elif name == 'sqs':\n        install_elasticmq()\n\n\ndef install_components(names):\n    parallelize(install_component, names)\n    install_lambda_java_libs()\n\n\ndef install_all_components():\n    install_components(DEFAULT_SERVICE_PORTS.keys())\n\n\n# -----------------\n# HELPER FUNCTIONS\n# -----------------\n\n\ndef is_alpine():\n    try:\n        run('cat /etc/issue | grep Alpine', print_error=False)\n        return True\n    except Exception:\n        return False\n\n\ndef download_and_extract_with_retry(archive_url, tmp_archive, target_dir):\n\n    def download_and_extract():\n        if not os.path.exists(tmp_archive):\n            download(archive_url, tmp_archive)\n        unzip(tmp_archive, target_dir)\n\n    try:\n        download_and_extract()\n    except Exception:\n        # try deleting and re-downloading the zip file\n        LOGGER.info('Unable to extract file, re-downloading ZIP archive: %s' % tmp_archive)\n        rm_rf(tmp_archive)\n        download_and_extract()\n\n\nif __name__ == '__main__':\n\n    if len(sys.argv) > 1:\n        if sys.argv[1] == 'libs':\n            print('Initializing installation.')\n            logging.basicConfig(level=logging.INFO)\n            logging.getLogger('requests').setLevel(logging.WARNING)\n            install_all_components()\n            print('Done.')\n        elif sys.argv[1] == 'testlibs':\n            # Install additional libraries for testing\n            install_amazon_kinesis_client_libs()\n", "path": "localstack/services/install.py"}], "after_files": [{"content": "import os\nimport localstack_client.config\n\n# LocalStack version\nVERSION = '0.8.10'\n\n# default AWS region\nif 'DEFAULT_REGION' not in os.environ:\n    os.environ['DEFAULT_REGION'] = 'us-east-1'\nDEFAULT_REGION = os.environ['DEFAULT_REGION']\n\n# constant to represent the \"local\" region, i.e., local machine\nREGION_LOCAL = 'local'\n\n# dev environment\nENV_DEV = 'dev'\n\n# backend service ports, for services that are behind a proxy (counting down from 4566)\nDEFAULT_PORT_APIGATEWAY_BACKEND = 4566\nDEFAULT_PORT_KINESIS_BACKEND = 4565\nDEFAULT_PORT_DYNAMODB_BACKEND = 4564\nDEFAULT_PORT_S3_BACKEND = 4563\nDEFAULT_PORT_SNS_BACKEND = 4562\nDEFAULT_PORT_SQS_BACKEND = 4561\nDEFAULT_PORT_ELASTICSEARCH_BACKEND = 4560\nDEFAULT_PORT_CLOUDFORMATION_BACKEND = 4559\n\nDEFAULT_PORT_WEB_UI = 8080\n\nLOCALHOST = 'localhost'\n\n# version of the Maven dependency with Java utility code\nLOCALSTACK_MAVEN_VERSION = '0.1.15'\n\n# map of default service APIs and ports to be spun up (fetch map from localstack_client)\nDEFAULT_SERVICE_PORTS = localstack_client.config.get_service_ports()\n\n# host to bind to when starting the services\nBIND_HOST = '0.0.0.0'\n\n# AWS user account ID used for tests\nTEST_AWS_ACCOUNT_ID = '000000000000'\nos.environ['TEST_AWS_ACCOUNT_ID'] = TEST_AWS_ACCOUNT_ID\n\n# root code folder\nLOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))\n\n# virtualenv folder\nLOCALSTACK_VENV_FOLDER = os.path.join(LOCALSTACK_ROOT_FOLDER, '.venv')\nif not os.path.isdir(LOCALSTACK_VENV_FOLDER):\n    # assuming this package lives here: <python>/lib/pythonX.X/site-packages/localstack/\n    LOCALSTACK_VENV_FOLDER = os.path.realpath(os.path.join(LOCALSTACK_ROOT_FOLDER, '..', '..', '..'))\n\n# API Gateway path to indicate a user request sent to the gateway\nPATH_USER_REQUEST = '_user_request_'\n\n# name of LocalStack Docker image\nDOCKER_IMAGE_NAME = 'localstack/localstack'\n\n# environment variable name to tag local test runs\nENV_INTERNAL_TEST_RUN = 'LOCALSTACK_INTERNAL_TEST_RUN'\n\n# content types\nAPPLICATION_AMZ_JSON_1_0 = 'application/x-amz-json-1.0'\nAPPLICATION_AMZ_JSON_1_1 = 'application/x-amz-json-1.1'\nAPPLICATION_JSON = 'application/json'\n\n# Lambda defaults\nLAMBDA_TEST_ROLE = 'arn:aws:iam::%s:role/lambda-test-role' % TEST_AWS_ACCOUNT_ID\n\n# installation constants\nELASTICSEARCH_JAR_URL = 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.0.zip'\n# https://docs.aws.amazon.com/ja_jp/elasticsearch-service/latest/developerguide/aes-supported-plugins.html\nELASTICSEARCH_PLUGIN_LIST = ['analysis-icu', 'ingest-attachment', 'ingest-user-agent', 'analysis-kuromoji',\n 'mapper-murmur3', 'mapper-size', 'analysis-phonetic', 'analysis-smartcn', 'analysis-stempel', 'analysis-ukrainian']\nDYNAMODB_JAR_URL = 'https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip'\nELASTICMQ_JAR_URL = 'https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.14.2.jar'\nSTS_JAR_URL = 'http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-sts/1.11.14/aws-java-sdk-sts-1.11.14.jar'\n\n# API endpoint for analytics events\nAPI_ENDPOINT = 'https://api.localstack.cloud/v1'\n", "path": "localstack/constants.py"}, {"content": "#!/usr/bin/env python\n\nimport os\nimport sys\nimport glob\nimport shutil\nimport logging\nimport tempfile\nfrom localstack.constants import (DEFAULT_SERVICE_PORTS, ELASTICMQ_JAR_URL, STS_JAR_URL,\n    ELASTICSEARCH_JAR_URL, ELASTICSEARCH_PLUGIN_LIST, DYNAMODB_JAR_URL, LOCALSTACK_MAVEN_VERSION)\nfrom localstack.utils.common import download, parallelize, run, mkdir, save_file, unzip, rm_rf, chmod_r\n\nTHIS_PATH = os.path.dirname(os.path.realpath(__file__))\nROOT_PATH = os.path.realpath(os.path.join(THIS_PATH, '..'))\n\nINSTALL_DIR_INFRA = '%s/infra' % ROOT_PATH\nINSTALL_DIR_NPM = '%s/node_modules' % ROOT_PATH\nINSTALL_DIR_ES = '%s/elasticsearch' % INSTALL_DIR_INFRA\nINSTALL_DIR_DDB = '%s/dynamodb' % INSTALL_DIR_INFRA\nINSTALL_DIR_KCL = '%s/amazon-kinesis-client' % INSTALL_DIR_INFRA\nINSTALL_DIR_ELASTICMQ = '%s/elasticmq' % INSTALL_DIR_INFRA\nINSTALL_PATH_LOCALSTACK_FAT_JAR = '%s/localstack-utils-fat.jar' % INSTALL_DIR_INFRA\nTMP_ARCHIVE_ES = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')\nTMP_ARCHIVE_DDB = os.path.join(tempfile.gettempdir(), 'localstack.ddb.zip')\nTMP_ARCHIVE_STS = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar')\nTMP_ARCHIVE_ELASTICMQ = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar')\nURL_LOCALSTACK_FAT_JAR = ('http://central.maven.org/maven2/' +\n    'cloud/localstack/localstack-utils/{v}/localstack-utils-{v}-fat.jar').format(v=LOCALSTACK_MAVEN_VERSION)\n\n# set up logger\nLOGGER = logging.getLogger(__name__)\n\n\ndef install_elasticsearch():\n    if not os.path.exists(INSTALL_DIR_ES):\n        LOGGER.info('Downloading and installing local Elasticsearch server. This may take some time.')\n        mkdir(INSTALL_DIR_INFRA)\n        # download and extract archive\n        download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA)\n        elasticsearch_dir = glob.glob(os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*'))\n        if not elasticsearch_dir:\n            raise Exception('Unable to find Elasticsearch folder in %s' % INSTALL_DIR_INFRA)\n        shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES)\n\n        for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):\n            dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)\n            mkdir(dir_path)\n            chmod_r(dir_path, 0o777)\n\n        # install default plugins\n        for plugin in ELASTICSEARCH_PLUGIN_LIST:\n            if is_alpine():\n                # https://github.com/pires/docker-elasticsearch/issues/56\n                os.environ['ES_TMPDIR'] = '/tmp'\n            plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin', 'elasticsearch-plugin')\n            run('%s install %s' % (plugin_binary, plugin))\n\n\ndef install_elasticmq():\n    if not os.path.exists(INSTALL_DIR_ELASTICMQ):\n        LOGGER.info('Downloading and installing local ElasticMQ server. This may take some time.')\n        mkdir(INSTALL_DIR_ELASTICMQ)\n        # download archive\n        if not os.path.exists(TMP_ARCHIVE_ELASTICMQ):\n            download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ)\n        shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)\n\n\ndef install_kinesalite():\n    target_dir = '%s/kinesalite' % INSTALL_DIR_NPM\n    if not os.path.exists(target_dir):\n        LOGGER.info('Downloading and installing local Kinesis server. This may take some time.')\n        run('cd \"%s\" && npm install' % ROOT_PATH)\n\n\ndef install_dynamodb_local():\n    if not os.path.exists(INSTALL_DIR_DDB):\n        LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.')\n        mkdir(INSTALL_DIR_DDB)\n        # download and extract archive\n        download_and_extract_with_retry(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB, INSTALL_DIR_DDB)\n\n    # fix for Alpine, otherwise DynamoDBLocal fails with:\n    # DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found\n    if is_alpine():\n        ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB\n        patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir\n        if not os.path.exists(patched_marker):\n            patched_lib = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +\n                'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so')\n            patched_jar = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +\n                'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')\n            run(\"curl -L -o %s/libsqlite4java-linux-amd64.so '%s'\" % (ddb_libs_dir, patched_lib))\n            run(\"curl -L -o %s/sqlite4java.jar '%s'\" % (ddb_libs_dir, patched_jar))\n            save_file(patched_marker, '')\n\n    # fix logging configuration for DynamoDBLocal\n    log4j2_config = \"\"\"<Configuration status=\"WARN\">\n      <Appenders>\n        <Console name=\"Console\" target=\"SYSTEM_OUT\">\n          <PatternLayout pattern=\"%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\"/>\n        </Console>\n      </Appenders>\n      <Loggers>\n        <Root level=\"WARN\"><AppenderRef ref=\"Console\"/></Root>\n      </Loggers>\n    </Configuration>\"\"\"\n    log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')\n    save_file(log4j2_file, log4j2_config)\n    run('cd \"%s\" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)\n\n\ndef install_amazon_kinesis_client_libs():\n    # install KCL/STS JAR files\n    if not os.path.exists(INSTALL_DIR_KCL):\n        mkdir(INSTALL_DIR_KCL)\n        if not os.path.exists(TMP_ARCHIVE_STS):\n            download(STS_JAR_URL, TMP_ARCHIVE_STS)\n        shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)\n    # Compile Java files\n    from localstack.utils.kinesis import kclipy_helper\n    classpath = kclipy_helper.get_kcl_classpath()\n    java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH\n    class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH\n    if not glob.glob(class_files):\n        run('javac -cp \"%s\" %s' % (classpath, java_files))\n\n\ndef install_lambda_java_libs():\n    # install LocalStack \"fat\" JAR file (contains all dependencies)\n    if not os.path.exists(INSTALL_PATH_LOCALSTACK_FAT_JAR):\n        LOGGER.info('Downloading and installing LocalStack Java libraries. This may take some time.')\n        download(URL_LOCALSTACK_FAT_JAR, INSTALL_PATH_LOCALSTACK_FAT_JAR)\n\n\ndef install_component(name):\n    if name == 'kinesis':\n        install_kinesalite()\n    elif name == 'dynamodb':\n        install_dynamodb_local()\n    elif name == 'es':\n        install_elasticsearch()\n    elif name == 'sqs':\n        install_elasticmq()\n\n\ndef install_components(names):\n    parallelize(install_component, names)\n    install_lambda_java_libs()\n\n\ndef install_all_components():\n    install_components(DEFAULT_SERVICE_PORTS.keys())\n\n\n# -----------------\n# HELPER FUNCTIONS\n# -----------------\n\n\ndef is_alpine():\n    try:\n        run('cat /etc/issue | grep Alpine', print_error=False)\n        return True\n    except Exception:\n        return False\n\n\ndef download_and_extract_with_retry(archive_url, tmp_archive, target_dir):\n\n    def download_and_extract():\n        if not os.path.exists(tmp_archive):\n            download(archive_url, tmp_archive)\n        unzip(tmp_archive, target_dir)\n\n    try:\n        download_and_extract()\n    except Exception:\n        # try deleting and re-downloading the zip file\n        LOGGER.info('Unable to extract file, re-downloading ZIP archive: %s' % tmp_archive)\n        rm_rf(tmp_archive)\n        download_and_extract()\n\n\nif __name__ == '__main__':\n\n    if len(sys.argv) > 1:\n        if sys.argv[1] == 'libs':\n            print('Initializing installation.')\n            logging.basicConfig(level=logging.INFO)\n            logging.getLogger('requests').setLevel(logging.WARNING)\n            install_all_components()\n            print('Done.')\n        elif sys.argv[1] == 'testlibs':\n            # Install additional libraries for testing\n            install_amazon_kinesis_client_libs()\n", "path": "localstack/services/install.py"}]}