URL

Learn about the URL OTTL converter function.

  2 minute read  

The URL converter parses URLs into common components.

Syntax: URL(string)

  • string: The location of the field containing the URL

In the following example, the body has been decoded and the JSON log parsed into attributes:

set(attributes["decoded_body"], Decode(body, "utf-8"))
set(attributes["parsed_map"], ParseJSON(attributes["decoded_body"]))

Input

{
	"_type": "log",
	"timestamp": 1735883939807,
	"body": "{\"host\": \"192.168.1.100\", \"user-identifier\": \"-\", \"user\": \"john_doe\", \"time\": \"03/Jan/2025:04:54:57 +0000\", \"request\": {\"method\": \"GET\", \"url\": \"http://www.example.com/api/v1/resource?query=example&sort=asc\", \"protocol\": \"HTTP/1.1\"}, \"status\": 200, \"bytes\": 1234, \"referrer\": \"http://www.referer.com\", \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36\"}",
	"resource": {...},
	"attributes": {
		"decoded_body": "{\"host\": \"192.168.1.100\", \"user-identifier\": \"-\", \"user\": \"john_doe\", \"time\": \"03/Jan/2025:04:54:57 +0000\", \"request\": {\"method\": \"GET\", \"url\": \"http://www.example.com/api/v1/resource?query=example&sort=asc\", \"protocol\": \"HTTP/1.1\"}, \"status\": 200, \"bytes\": 1234, \"referrer\": \"http://www.referer.com\", \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36\"}",
		"parsed_map": {
			"bytes": 1234,
			"host": "192.168.1.100",
			"referrer": "http://www.referer.com",
			"request": {
				"method": "GET",
				"protocol": "HTTP/1.1",
				"url": "http://www.example.com/api/v1/resource?query=example&sort=asc"
			},
			"status": 200,
			"time": "03/Jan/2025:04:54:57 +0000",
			"user": "john_doe",
			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",
			"user-identifier": "-"
		}
	}
}

Statement

set(attributes["parsed_url"], URL(attributes["parsed_map"]["request"]["url"]))

Output

{
	"_type": "log",
	"timestamp": 1735883922130,
	"body": "{\"host\": \"192.168.1.100\", \"user-identifier\": \"-\", \"user\": \"john_doe\", \"time\": \"03/Jan/2025:04:54:57 +0000\", \"request\": {\"method\": \"GET\", \"url\": \"http://www.example.com/api/v1/resource?query=example&sort=asc\", \"protocol\": \"HTTP/1.1\"}, \"status\": 200, \"bytes\": 1234, \"referrer\": \"http://www.referer.com\", \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36\"}",
	"resource": {...},
	"attributes": {
		"decoded_body": "{\"host\": \"192.168.1.100\", \"user-identifier\": \"-\", \"user\": \"john_doe\", \"time\": \"03/Jan/2025:04:54:57 +0000\", \"request\": {\"method\": \"GET\", \"url\": \"http://www.example.com/api/v1/resource?query=example&sort=asc\", \"protocol\": \"HTTP/1.1\"}, \"status\": 200, \"bytes\": 1234, \"referrer\": \"http://www.referer.com\", \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36\"}",
		"parsed_map": {
			"bytes": 1234,
			"host": "192.168.1.100",
			"referrer": "http://www.referer.com",
			"request": {
				"method": "GET",
				"protocol": "HTTP/1.1",
				"url": "http://www.example.com/api/v1/resource?query=example&sort=asc"
			},
			"status": 200,
			"time": "03/Jan/2025:04:54:57 +0000",
			"user": "john_doe",
			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",
			"user-identifier": "-"
		},
		"parsed_url": {
			"url.domain": "www.example.com",
			"url.original": "http://www.example.com/api/v1/resource?query=example&sort=asc",
			"url.path": "/api/v1/resource",
			"url.query": "query=example&sort=asc",
			"url.scheme": "http"
		}
	}
}

The URL attribute has been parsed into the domain, path, query and scheme, as well as a copy of the original. Be sure to remove duplicated fields.