Add Splunk logging driver #16207

Allow to send Splunk logs using Http Event Collector

Signed-off-by: Denis Gladkikh <denis@gladkikh.email>

contrib/completion/bash/docker

@@ -321,6 +321,7 @@ __docker_log_drivers() {
 		journald
 		json-file
 		none
+		splunk
 		syslog
 	" -- "$cur" ) )
 }
@@ -333,8 +334,9 @@ __docker_log_driver_options() {
 	local journald_options="env labels"
 	local json_file_options="env labels max-file max-size"
 	local syslog_options="syslog-address syslog-facility tag"
+	local splunk_options="splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url"
 
-	local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options"
+	local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
 
 	case $(__docker_value_of_option --log-driver) in
 		'')
@@ -358,6 +360,9 @@ __docker_log_driver_options() {
 		syslog)
 			COMPREPLY=( $( compgen -W "$syslog_options" -S = -- "$cur" ) )
 			;;
+		splunk)
+			COMPREPLY=( $( compgen -W "$splunk_options" -S = -- "$cur" ) )
+			;;
 		*)
 			return
 			;;
@@ -405,6 +410,17 @@ __docker_complete_log_driver_options() {
 			" -- "${cur#=}" ) )
 			return
 			;;
+		*splunk-url=*)
+			COMPREPLY=( $( compgen -W "http:// https://" -- "${cur#=}" ) )
+			compopt -o nospace
+			__ltrim_colon_completions "${cur}"
+			return
+			;;
+		*splunk-insecureskipverify=*)
+			COMPREPLY=( $( compgen -W "true false" -- "${cur#=}" ) )
+			compopt -o nospace
+			return
+			;;
 	esac
 	return 1
 }

daemon/logdrivers_linux.go

@@ -8,5 +8,6 @@ import (
 	_ "github.com/docker/docker/daemon/logger/gelf"
 	_ "github.com/docker/docker/daemon/logger/journald"
 	_ "github.com/docker/docker/daemon/logger/jsonfilelog"
+	_ "github.com/docker/docker/daemon/logger/splunk"
 	_ "github.com/docker/docker/daemon/logger/syslog"
 )

daemon/logger/splunk/splunk.go

@@ -0,0 +1,256 @@
+// Package splunk provides the log driver for forwarding server logs to
+// Splunk HTTP Event Collector endpoint.
+package splunk
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/pkg/urlutil"
+)
+
+const (
+	driverName                  = "splunk"
+	splunkURLKey                = "splunk-url"
+	splunkTokenKey              = "splunk-token"
+	splunkSourceKey             = "splunk-source"
+	splunkSourceTypeKey         = "splunk-sourcetype"
+	splunkIndexKey              = "splunk-index"
+	splunkCAPathKey             = "splunk-capath"
+	splunkCANameKey             = "splunk-caname"
+	splunkInsecureSkipVerifyKey = "splunk-insecureskipverify"
+)
+
+type splunkLogger struct {
+	client    *http.Client
+	transport *http.Transport
+
+	url         string
+	auth        string
+	nullMessage *splunkMessage
+}
+
+type splunkMessage struct {
+	Event      splunkMessageEvent `json:"event"`
+	Time       string             `json:"time"`
+	Host       string             `json:"host"`
+	Source     string             `json:"source,omitempty"`
+	SourceType string             `json:"sourcetype,omitempty"`
+	Index      string             `json:"index,omitempty"`
+}
+
+type splunkMessageEvent struct {
+	Line        string `json:"line"`
+	ContainerID string `json:"containerId"`
+	Source      string `json:"source"`
+}
+
+func init() {
+	if err := logger.RegisterLogDriver(driverName, New); err != nil {
+		logrus.Fatal(err)
+	}
+	if err := logger.RegisterLogOptValidator(driverName, ValidateLogOpt); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// New creates splunk logger driver using configuration passed in context
+func New(ctx logger.Context) (logger.Logger, error) {
+	hostname, err := ctx.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("%s: cannot access hostname to set source field", driverName)
+	}
+
+	// Parse and validate Splunk URL
+	splunkURL, err := parseURL(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Splunk Token is required parameter
+	splunkToken, ok := ctx.Config[splunkTokenKey]
+	if !ok {
+		return nil, fmt.Errorf("%s: %s is expected", driverName, splunkTokenKey)
+	}
+
+	tlsConfig := &tls.Config{}
+
+	// Splunk is using autogenerated certificates by default,
+	// allow users to trust them with skiping verification
+	if insecureSkipVerifyStr, ok := ctx.Config[splunkInsecureSkipVerifyKey]; ok {
+		insecureSkipVerify, err := strconv.ParseBool(insecureSkipVerifyStr)
+		if err != nil {
+			return nil, err
+		}
+		tlsConfig.InsecureSkipVerify = insecureSkipVerify
+	}
+
+	// If path to the root certificate is provided - load it
+	if caPath, ok := ctx.Config[splunkCAPathKey]; ok {
+		caCert, err := ioutil.ReadFile(caPath)
+		if err != nil {
+			return nil, err
+		}
+		caPool := x509.NewCertPool()
+		caPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caPool
+	}
+
+	if caName, ok := ctx.Config[splunkCANameKey]; ok {
+		tlsConfig.ServerName = caName
+	}
+
+	transport := &http.Transport{
+		TLSClientConfig: tlsConfig,
+	}
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	var nullMessage = &splunkMessage{
+		Host: hostname,
+	}
+
+	// Optional parameters for messages
+	nullMessage.Source = ctx.Config[splunkSourceKey]
+	nullMessage.SourceType = ctx.Config[splunkSourceTypeKey]
+	nullMessage.Index = ctx.Config[splunkIndexKey]
+
+	logger := &splunkLogger{
+		client:      client,
+		transport:   transport,
+		url:         splunkURL.String(),
+		auth:        "Splunk " + splunkToken,
+		nullMessage: nullMessage,
+	}
+
+	err = verifySplunkConnection(logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return logger, nil
+}
+
+func (l *splunkLogger) Log(msg *logger.Message) error {
+	// Construct message as a copy of nullMessage
+	message := *l.nullMessage
+	message.Time = fmt.Sprintf("%f", float64(msg.Timestamp.UnixNano())/1000000000)
+	message.Event = splunkMessageEvent{
+		Line:        string(msg.Line),
+		ContainerID: msg.ContainerID,
+		Source:      msg.Source,
+	}
+
+	jsonEvent, err := json.Marshal(&message)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest("POST", l.url, bytes.NewBuffer(jsonEvent))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", l.auth)
+	res, err := l.client.Do(req)
+	if err != nil {
+		return err
+	}
+	if res.Body != nil {
+		defer res.Body.Close()
+	}
+	if res.StatusCode != http.StatusOK {
+		var body []byte
+		body, err = ioutil.ReadAll(res.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("%s: failed to send event - %s - %s", driverName, res.Status, body)
+	}
+	io.Copy(ioutil.Discard, res.Body)
+	return nil
+}
+
+func (l *splunkLogger) Close() error {
+	l.transport.CloseIdleConnections()
+	return nil
+}
+
+func (l *splunkLogger) Name() string {
+	return driverName
+}
+
+// ValidateLogOpt looks for all supported by splunk driver options
+func ValidateLogOpt(cfg map[string]string) error {
+	for key := range cfg {
+		switch key {
+		case splunkURLKey:
+		case splunkTokenKey:
+		case splunkSourceKey:
+		case splunkSourceTypeKey:
+		case splunkIndexKey:
+		case splunkCAPathKey:
+		case splunkCANameKey:
+		case splunkInsecureSkipVerifyKey:
+		default:
+			return fmt.Errorf("unknown log opt '%s' for %s log driver", key, driverName)
+		}
+	}
+	return nil
+}
+
+func parseURL(ctx logger.Context) (*url.URL, error) {
+	splunkURLStr, ok := ctx.Config[splunkURLKey]
+	if !ok {
+		return nil, fmt.Errorf("%s: %s is expected", driverName, splunkURLKey)
+	}
+
+	splunkURL, err := url.Parse(splunkURLStr)
+	if err != nil {
+		return nil, fmt.Errorf("%s: failed to parse %s as url value in %s", driverName, splunkURLStr, splunkURLKey)
+	}
+
+	if !urlutil.IsURL(splunkURLStr) ||
+		!splunkURL.IsAbs() ||
+		(splunkURL.Path != "" && splunkURL.Path != "/") ||
+		splunkURL.RawQuery != "" ||
+		splunkURL.Fragment != "" {
+		return nil, fmt.Errorf("%s: expected format schema://dns_name_or_ip:port for %s", driverName, splunkURLKey)
+	}
+
+	splunkURL.Path = "/services/collector/event/1.0"
+
+	return splunkURL, nil
+}
+
+func verifySplunkConnection(l *splunkLogger) error {
+	req, err := http.NewRequest("OPTIONS", l.url, nil)
+	if err != nil {
+		return err
+	}
+	res, err := l.client.Do(req)
+	if err != nil {
+		return err
+	}
+	if res.Body != nil {
+		defer res.Body.Close()
+	}
+	if res.StatusCode != http.StatusOK {
+		var body []byte
+		body, err = ioutil.ReadAll(res.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("%s: failed to verify connection - %s - %s", driverName, res.Status, body)
+	}
+	return nil
+}

docs/reference/api/docker_remote_api_v1.22.md

@@ -309,7 +309,7 @@ Json Parameters:
         systems, such as SELinux.
     -   **LogConfig** - Log configuration for the container, specified as a JSON object in the form
           `{ "Type": "<driver_name>", "Config": {"key1": "val1"}}`.
-          Available types: `json-file`, `syslog`, `journald`, `gelf`, `awslogs`, `none`.
+          Available types: `json-file`, `syslog`, `journald`, `gelf`, `awslogs`, `splunk`, `none`.
           `json-file` logging driver.
     -   **CgroupParent** - Path to `cgroups` under which the container's `cgroup` is created. If the path is not absolute, the path is considered to be relative to the `cgroups` path of the init process. Cgroups are created if they do not already exist.
     -   **VolumeDriver** - Driver that this container users to mount volumes.

docs/reference/logging/index.md

@@ -18,3 +18,4 @@ weight=8
 * [Fluentd logging driver](fluentd.md)
 * [Journald logging driver](journald.md)
 * [Amazon CloudWatch Logs logging driver](awslogs.md)
+* [Splunk logging driver](splunk.md)

docs/reference/logging/overview.md

@@ -24,6 +24,7 @@ container's logging driver. The following options are supported:
 | `gelf`      | Graylog Extended Log Format (GELF) logging driver for Docker. Writes log messages to a GELF endpoint likeGraylog or Logstash. |
 | `fluentd`   | Fluentd logging driver for Docker. Writes log messages to `fluentd` (forward input).                                          |
 | `awslogs`   | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs.                              |
+| `splunk`    | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector.                                 |
 
 The `docker logs`command is available only for the `json-file` logging driver.
 
@@ -172,3 +173,13 @@ The Amazon CloudWatch Logs logging driver supports the following options:
 
 
 For detailed information on working with this logging driver, see [the awslogs logging driver](awslogs.md) reference documentation.
+
+## Splunk options
+
+The Splunk logging driver requires the following options:
+
+    --log-opt splunk-token=<splunk_http_event_collector_token>
+    --log-opt splunk-url=https://your_splunk_instance:8088
+
+For detailed information about working with this logging driver, see the [Splunk logging driver](splunk.md)
+reference documentation.

docs/reference/logging/splunk.md

@@ -0,0 +1,56 @@
+<!--[metadata]>
++++
+title = "Splunk logging driver"
+description = "Describes how to use the Splunk logging driver."
+keywords = ["splunk, docker, logging, driver"]
+[menu.main]
+parent = "smn_logging"
+weight = 2
++++
+<![end-metadata]-->
+
+# Splunk logging driver
+
+The `splunk` logging driver sends container logs to
+[HTTP Event Collector](http://dev.splunk.com/view/event-collector/SP-CAAAE6M)
+in Splunk Enterprise and Splunk Cloud.
+
+## Usage
+
+You can configure the default logging driver by passing the `--log-driver`
+option to the Docker daemon:
+
+    docker --log-driver=splunk
+
+You can set the logging driver for a specific container by using the
+`--log-driver` option to `docker run`:
+
+    docker run --log-driver=splunk ...
+
+## Splunk options
+
+You can use the `--log-opt NAME=VALUE` flag to specify these additional Splunk
+logging driver options:
+
+  - `splunk-token` required, Splunk HTTP Event Collector token
+  - `splunk-url` required, path to your Splunk Enterprise or Splunk Cloud instance
+      (including port and schema used by HTTP Event Collector) `https://your_splunk_instance:8088`
+  - `splunk-source` optional, event source
+  - `splunk-sourcetype` optional, event source type
+  - `splunk-index` optional, event index
+  - `splunk-capath` optional, path to root certificate
+  - `splunk-caname` optional, name to use for validating server
+      certificate; by default the hostname of the `splunk-url` will be used
+  - `splunk-insecureskipverify` optional, ignore server certificate validation
+
+Below is an example of the logging option specified for the Splunk Enterprise
+instance. The instance is installed locally on the same machine on which the
+Docker daemon is running. The path to the root certificate and Common Name is
+specified using an HTTPS schema. This is used for verification.
+The `SplunkServerDefaultCert` is automatically generated by Splunk certificates.
+
+    docker run --log-driver=splunk \
+        --log-opt splunk-token=176FCEBF-4CF5-4EDF-91BC-703796522D20 \
+        --log-opt splunk-url=https://localhost:8088 \
+        --log-opt splunk-capath=/opt/splunk/etc/auth/cacert.pem \
+        --log-opt splunk-caname=SplunkServerDefaultCert

docs/reference/run.md

@@ -1071,6 +1071,7 @@ container's logging driver. The following options are supported:
 | `gelf`      | Graylog Extended Log Format (GELF) logging driver for Docker. Writes log messages to a GELF endpoint likeGraylog or Logstash. |
 | `fluentd`   | Fluentd logging driver for Docker. Writes log messages to `fluentd` (forward input).                                          |
 | `awslogs`   | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs                               |
+| `splunk`    | Splunk logging driver for Docker. Writes log messages to `splunk` using Event Http Collector.                                 |
 
 The `docker logs` command is available only for the `json-file` and `journald`
 logging drivers.  For detailed information on working with logging drivers, see

man/docker-create.1.md

@@ -174,7 +174,7 @@ millions of trillions.
    Add link to another container in the form of <name or id>:alias or just
    <name or id> in which case the alias will match the name.
 
-**--log-driver**="|*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*none*"
+**--log-driver**="|*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: the `docker logs` command works only for the `json-file` and
   `journald` logging drivers.

man/docker-run.1.md

@@ -277,7 +277,7 @@ which interface and port to use.
 **--lxc-conf**=[]
    (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
 
-**--log-driver**="|*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*none*"
+**--log-driver**="|*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: the `docker logs` command works only for the `json-file` and
   `journald` logging drivers.