Send From Edge Delta to Elastic Destinations
Prepare to send logs to Elastic.
7 minute read
Overview
The Elastic destination node sends items to an Elastic destination.
Configuring Elastic
You need to configure Elastic to use it as a data destination in Edge Delta. To do this you create a lifecycle policy and an index template. Then you can update the Edge Delta Pipeline configuration to send data to Elastic.
Step 1: Create a Lifecycle Policy
Index lifecycle policies manage indices based on your performance, resiliency, and retention requirements. You can use the following sample lifecycle policy to start, which creates a new index every day and maintains data from the last 15 days. While you can run the command with the pre-populated settings, you can also change the retention period, as well as other fields.
In your Elastic’s dev console, run this command:
PUT _ilm/policy/ed-agent-log-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1d",
"max_size": "5gb"
},
"set_priority": {
"priority": 100
}
}
},
"delete": {
"min_age": "15d",
"actions": {
"delete": {}
}
}
}
}
}
After you run the command, you should have an index lifecycle policy named ed-agent-log-policy.
Step 2: Create an Index Template
An index template is useful to configure Elastic indices before they are created. While the Edge Delta agent can be configured to stream various types of observations to the Elasticsearch destination, we recommend that you create the target index with the recommend index template. The following sample will create an index template named ed-agent-log with field mappings to the ed-agent-log-policy lifecycle policy.
Too many unique field mappings in Elastic can cause out of memory errors and difficult situations to recover from. Edge Delta supports aggregation of custom fields in Elastic by using a field called “custom_labels_flattened” to index mapping and it utilizes that field for metric and log entries. This field is used by default to send custom labels to Elastic.
In your Elastic’s dev console, run one of the following commands depending on your version of Elastic. For Elasticsearch 8.x, use the following command:
PUT _index_template/ed-agent-log
{
"index_patterns": [
"ed-agent-log-*"
],
"template":{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"lifecycle": {
"name": "ed-agent-log-policy",
"rollover_alias": "ed-agent-log"
},
"analysis": {
"analyzer": {
"custom_label_analyzer": {
"tokenizer": "custom_label_tokenizer"
}
},
"tokenizer": {
"custom_label_tokenizer": {
"type": "pattern",
"pattern": "\\$"
}
}
}
},
"mappings": {
"_meta": {},
"_routing": {
"required": false
},
"dynamic": true,
"numeric_detection": false,
"date_detection": true,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates": [],
"properties": {
"msg": {
"type": "text"
},
"alert_def_id": {
"type": "keyword"
},
"k8s_namespace": {
"type": "keyword"
},
"merge_level": {
"type": "keyword"
},
"ecs_task_family": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"k8s_controller_kind": {
"type": "keyword"
},
"k8s_container_image": {
"type": "keyword"
},
"title": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": false
},
"type": {
"type": "keyword"
},
"src_name": {
"type": "keyword"
},
"k8s_container_name": {
"type": "keyword"
},
"score": {
"type": "double"
},
"sub_type": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"capture_flush_mode": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"tag": {
"type": "keyword"
},
"k8s_controller_logical_name": {
"type": "keyword"
},
"timestamp_end": {
"type": "date"
},
"value": {
"type": "double"
},
"timestamp": {
"index": true,
"ignore_malformed": false,
"store": false,
"type": "date",
"doc_values": true
},
"app": {
"type": "keyword"
},
"capture_size": {
"coerce": true,
"index": false,
"ignore_malformed": false,
"store": false,
"type": "long",
"doc_values": false
},
"ecs_task_version": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"index_options": "docs",
"doc_values": true
},
"stat_type": {
"type": "keyword"
},
"docker_container_name": {
"type": "keyword"
},
"conf_id": {
"type": "keyword"
},
"edac_id": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"k8s_pod_name": {
"type": "keyword"
},
"logical_source": {
"type": "keyword"
},
"environment": {
"type": "keyword"
},
"event_id": {
"type": "keyword"
},
"capture_duration": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"ecs_container": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"capture_bytesize": {
"coerce": true,
"index": false,
"ignore_malformed": false,
"store": false,
"type": "long",
"doc_values": false
},
"group_id": {
"type": "keyword"
},
"org_id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"alert_def_name": {
"type": "keyword"
},
"ecs_cluster": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"threshold_description": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"threshold_type": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"src_type": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"custom_labels": {
"type": "text",
"analyzer": "custom_label_analyzer"
},
"properties": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"docker_image": {
"type": "keyword"
},
"custom_labels_flattened": {
"type": "flattened"
}
}
}
}
}
For Elasticsearch 7.x, use the following command:
PUT _template/ed-agent-log?
{
"order": 0,
"index_patterns": [
"ed-agent-log-*"
],
"settings": {
"index": {
"lifecycle": {
"name": "ed-agent-log-policy",
"rollover_alias": "ed-agent-log"
},
"number_of_shards": "1",
"number_of_replicas": "1"
},
"analysis": {
"analyzer": {
"custom_label_analyzer": {
"tokenizer": "custom_label_tokenizer"
}
},
"tokenizer": {
"custom_label_tokenizer": {
"type": "pattern",
"pattern": "\\$"
}
}
}
},
"mappings": {
"_meta": {},
"_routing": {
"required": false
},
"dynamic": true,
"numeric_detection": false,
"date_detection": true,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates": [],
"properties": {
"msg": {
"type": "text"
},
"alert_def_id": {
"type": "keyword"
},
"k8s_namespace": {
"type": "keyword"
},
"merge_level": {
"type": "keyword"
},
"ecs_task_family": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"k8s_controller_kind": {
"type": "keyword"
},
"k8s_container_image": {
"type": "keyword"
},
"title": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": false
},
"type": {
"type": "keyword"
},
"src_name": {
"type": "keyword"
},
"k8s_container_name": {
"type": "keyword"
},
"score": {
"type": "double"
},
"sub_type": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"capture_flush_mode": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"tag": {
"type": "keyword"
},
"k8s_controller_logical_name": {
"type": "keyword"
},
"timestamp_end": {
"type": "date"
},
"value": {
"type": "double"
},
"timestamp": {
"index": true,
"ignore_malformed": false,
"store": false,
"type": "date",
"doc_values": true
},
"app": {
"type": "keyword"
},
"capture_size": {
"coerce": true,
"index": false,
"ignore_malformed": false,
"store": false,
"type": "long",
"doc_values": false
},
"ecs_task_version": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"index_options": "docs",
"doc_values": true
},
"stat_type": {
"type": "keyword"
},
"docker_container_name": {
"type": "keyword"
},
"conf_id": {
"type": "keyword"
},
"edac_id": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"k8s_pod_name": {
"type": "keyword"
},
"logical_source": {
"type": "keyword"
},
"environment": {
"type": "keyword"
},
"event_id": {
"type": "keyword"
},
"capture_duration": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"ecs_container": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"capture_bytesize": {
"coerce": true,
"index": false,
"ignore_malformed": false,
"store": false,
"type": "long",
"doc_values": false
},
"group_id": {
"type": "keyword"
},
"org_id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"alert_def_name": {
"type": "keyword"
},
"ecs_cluster": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"threshold_description": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"threshold_type": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"src_type": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"custom_labels": {
"type": "text",
"analyzer": "custom_label_analyzer"
},
"properties": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"docker_image": {
"type": "keyword"
},
"custom_labels_flattened": {
"type": "flattened"
}
}
}
}
Step 3: Create the First Index
To generate a daily index, you must create the first index. This first index will inherit field mappings and policies from the template. In your Elastic’s dev console, run one of the following commands depending on your version of Elastic.
- Run one of the following commands depending on your Elastic version:
For Elasticsearch 8.x, use the following command in the Elastic dev console:
PUT /%3Ced-agent-log-%7Bnow%2Fd%7D-000001%3E
{
"aliases": {
"ed-agent-log": {
"is_write_index": true
}
}
}
For Elasticsearch 7.x, use the following command:
PUT /%3Ced-agent-log-%7Bnow%2Fd%7D-000001%3E
{
"aliases": {
"ed-agent-log": {
"is_write_index": true
}
}
}
- Click Index Management > Indices, and then locate a new index, similar to ed-agent-log-2025.10.22-000000 with the current date.
Your Elastic environment is ready for Edge Delta data. Next you configure the Edge Delta agent to flush data to your Elastic index.
Step 4: Configure the Edge Delta Agent
Use Visual Pipelines or the agent YAML to configure the Elastic destination node.