使用Python Elasticsearch客户端库,编写Lambda函数实现Python代码到Elasticsearch的迁移。
pip install elasticsearch
import boto3
import json
from elasticsearch import Elasticsearch, RequestsHttpConnection
def lambda_handler(event, context):
es = Elasticsearch(
hosts=[{'host': 'elasticsearch_host', 'port': 443}],
http_auth=('username', 'password'),
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
s3 = boto3.client('s3')
content_object = event['Records'][0]['s3']
bucket = content_object['bucket']['name']
file_key = content_object['object']['key']
file_obj = s3.get_object(Bucket=bucket, Key=file_key)
file_content = file_obj["Body"].read().decode('utf-8').split('\n')
data = []
for line in file_content:
record = json.loads(line)
data.append({
'_index': 'my-index',
'_type': 'my-type',
'_id': record['id'],
'_source': {
'field1': record['field1'],
'field2': record['field2'],
'field3': record['field3']
}
})
es.bulk(index='my-index', body=data, request_timeout=30)
代码示例:
https://github.com/awslabs/amazon-elasticsearch-lambda-samples/blob/master/src/s3_lambda_elasticsearch.py