logging.go 34 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007
  1. // Copyright 2016 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // API/gRPC features intentionally missing from this client:
  15. // - You cannot have the server pick the time of the entry. This client
  16. // always sends a time.
  17. // - There is no way to provide a protocol buffer payload.
  18. // - No support for the "partial success" feature when writing log entries.
  19. // TODO(jba): test whether forward-slash characters in the log ID must be URL-encoded.
  20. // These features are missing now, but will likely be added:
  21. // - There is no way to specify CallOptions.
  22. package logging
  23. import (
  24. "bytes"
  25. "context"
  26. "encoding/json"
  27. "errors"
  28. "fmt"
  29. "io"
  30. "log"
  31. "net/http"
  32. "regexp"
  33. "runtime"
  34. "strconv"
  35. "strings"
  36. "sync"
  37. "time"
  38. "unicode/utf8"
  39. vkit "cloud.google.com/go/logging/apiv2"
  40. logpb "cloud.google.com/go/logging/apiv2/loggingpb"
  41. "cloud.google.com/go/logging/internal"
  42. "github.com/golang/protobuf/proto"
  43. "github.com/golang/protobuf/ptypes"
  44. structpb "github.com/golang/protobuf/ptypes/struct"
  45. "google.golang.org/api/option"
  46. "google.golang.org/api/support/bundler"
  47. mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
  48. logtypepb "google.golang.org/genproto/googleapis/logging/type"
  49. "google.golang.org/protobuf/types/known/anypb"
  50. "google.golang.org/protobuf/types/known/timestamppb"
  51. )
  52. const (
  53. // ReadScope is the scope for reading from the logging service.
  54. ReadScope = "https://www.googleapis.com/auth/logging.read"
  55. // WriteScope is the scope for writing to the logging service.
  56. WriteScope = "https://www.googleapis.com/auth/logging.write"
  57. // AdminScope is the scope for administrative actions on the logging service.
  58. AdminScope = "https://www.googleapis.com/auth/logging.admin"
  59. )
  60. const (
  61. // defaultErrorCapacity is the capacity of the channel used to deliver
  62. // errors to the OnError function.
  63. defaultErrorCapacity = 10
  64. // DefaultDelayThreshold is the default value for the DelayThreshold LoggerOption.
  65. DefaultDelayThreshold = time.Second
  66. // DefaultEntryCountThreshold is the default value for the EntryCountThreshold LoggerOption.
  67. DefaultEntryCountThreshold = 1000
  68. // DefaultEntryByteThreshold is the default value for the EntryByteThreshold LoggerOption.
  69. DefaultEntryByteThreshold = 1 << 23 // 8MiB
  70. // DefaultBufferedByteLimit is the default value for the BufferedByteLimit LoggerOption.
  71. DefaultBufferedByteLimit = 1 << 30 // 1GiB
  72. // defaultWriteTimeout is the timeout for the underlying write API calls. As
  73. // write API calls are not idempotent, they are not retried on timeout. This
  74. // timeout is to allow clients to degrade gracefully if underlying logging
  75. // service is temporarily impaired for some reason.
  76. defaultWriteTimeout = 10 * time.Minute
  77. )
  78. var (
  79. // ErrRedirectProtoPayloadNotSupported is returned when Logger is configured to redirect output and
  80. // tries to redirect logs with protobuf payload.
  81. ErrRedirectProtoPayloadNotSupported = errors.New("printEntryToStdout: cannot find valid payload")
  82. // For testing:
  83. now = time.Now
  84. toLogEntryInternal = toLogEntryInternalImpl
  85. // ErrOverflow signals that the number of buffered entries for a Logger
  86. // exceeds its BufferLimit.
  87. ErrOverflow = bundler.ErrOverflow
  88. // ErrOversizedEntry signals that an entry's size exceeds the maximum number of
  89. // bytes that will be sent in a single call to the logging service.
  90. ErrOversizedEntry = bundler.ErrOversizedItem
  91. )
  92. // Client is a Logging client. A Client is associated with a single Cloud project.
  93. type Client struct {
  94. client *vkit.Client // client for the logging service
  95. parent string // e.g. "projects/proj-id"
  96. errc chan error // should be buffered to minimize dropped errors
  97. donec chan struct{} // closed on Client.Close to close Logger bundlers
  98. loggers sync.WaitGroup // so we can wait for loggers to close
  99. closed bool
  100. mu sync.Mutex
  101. nErrs int // number of errors we saw
  102. lastErr error // last error we saw
  103. // OnError is called when an error occurs in a call to Log or Flush. The
  104. // error may be due to an invalid Entry, an overflow because BufferLimit
  105. // was reached (in which case the error will be ErrOverflow) or an error
  106. // communicating with the logging service. OnError is called with errors
  107. // from all Loggers. It is never called concurrently. OnError is expected
  108. // to return quickly; if errors occur while OnError is running, some may
  109. // not be reported. The default behavior is to call log.Printf.
  110. //
  111. // This field should be set only once, before any method of Client is called.
  112. OnError func(err error)
  113. }
  114. // NewClient returns a new logging client associated with the provided parent.
  115. // A parent can take any of the following forms:
  116. //
  117. // projects/PROJECT_ID
  118. // folders/FOLDER_ID
  119. // billingAccounts/ACCOUNT_ID
  120. // organizations/ORG_ID
  121. //
  122. // for backwards compatibility, a string with no '/' is also allowed and is interpreted
  123. // as a project ID.
  124. //
  125. // By default NewClient uses WriteScope. To use a different scope, call
  126. // NewClient using a WithScopes option (see https://godoc.org/google.golang.org/api/option#WithScopes).
  127. func NewClient(ctx context.Context, parent string, opts ...option.ClientOption) (*Client, error) {
  128. parent, err := makeParent(parent)
  129. if err != nil {
  130. return nil, err
  131. }
  132. opts = append([]option.ClientOption{
  133. option.WithScopes(WriteScope),
  134. }, opts...)
  135. c, err := vkit.NewClient(ctx, opts...)
  136. if err != nil {
  137. return nil, err
  138. }
  139. c.SetGoogleClientInfo("gccl", internal.Version)
  140. client := &Client{
  141. client: c,
  142. parent: parent,
  143. errc: make(chan error, defaultErrorCapacity), // create a small buffer for errors
  144. donec: make(chan struct{}),
  145. OnError: func(e error) { log.Printf("logging client: %v", e) },
  146. }
  147. // Call the user's function synchronously, to make life easier for them.
  148. go func() {
  149. for err := range client.errc {
  150. // This reference to OnError is memory-safe if the user sets OnError before
  151. // calling any client methods. The reference happens before the first read from
  152. // client.errc, which happens before the first write to client.errc, which
  153. // happens before any call, which happens before the user sets OnError.
  154. if fn := client.OnError; fn != nil {
  155. fn(err)
  156. } else {
  157. log.Printf("logging (parent %q): %v", parent, err)
  158. }
  159. }
  160. }()
  161. return client, nil
  162. }
  163. func makeParent(parent string) (string, error) {
  164. if !strings.ContainsRune(parent, '/') {
  165. return "projects/" + parent, nil
  166. }
  167. prefix := strings.Split(parent, "/")[0]
  168. if prefix != "projects" && prefix != "folders" && prefix != "billingAccounts" && prefix != "organizations" {
  169. return parent, fmt.Errorf("parent parameter must start with 'projects/' 'folders/' 'billingAccounts/' or 'organizations/'")
  170. }
  171. return parent, nil
  172. }
  173. // Ping reports whether the client's connection to the logging service and the
  174. // authentication configuration are valid. To accomplish this, Ping writes a
  175. // log entry "ping" to a log named "ping".
  176. func (c *Client) Ping(ctx context.Context) error {
  177. unixZeroTimestamp, err := ptypes.TimestampProto(time.Unix(0, 0))
  178. if err != nil {
  179. return err
  180. }
  181. ent := &logpb.LogEntry{
  182. Payload: &logpb.LogEntry_TextPayload{TextPayload: "ping"},
  183. Timestamp: unixZeroTimestamp, // Identical timestamps and insert IDs are both
  184. InsertId: "ping", // necessary for the service to dedup these entries.
  185. }
  186. _, err = c.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
  187. LogName: internal.LogPath(c.parent, "ping"),
  188. Resource: monitoredResource(c.parent),
  189. Entries: []*logpb.LogEntry{ent},
  190. })
  191. return err
  192. }
  193. // error puts the error on the client's error channel
  194. // without blocking, and records summary error info.
  195. func (c *Client) error(err error) {
  196. select {
  197. case c.errc <- err:
  198. default:
  199. }
  200. c.mu.Lock()
  201. c.lastErr = err
  202. c.nErrs++
  203. c.mu.Unlock()
  204. }
  205. func (c *Client) extractErrorInfo() error {
  206. var err error
  207. c.mu.Lock()
  208. if c.lastErr != nil {
  209. err = fmt.Errorf("saw %d errors; last: %w", c.nErrs, c.lastErr)
  210. c.nErrs = 0
  211. c.lastErr = nil
  212. }
  213. c.mu.Unlock()
  214. return err
  215. }
  216. // A Logger is used to write log messages to a single log. It can be configured
  217. // with a log ID, common monitored resource, and a set of common labels.
  218. type Logger struct {
  219. client *Client
  220. logName string // "projects/{projectID}/logs/{logID}"
  221. stdLoggers map[Severity]*log.Logger
  222. bundler *bundler.Bundler
  223. // Options
  224. commonResource *mrpb.MonitoredResource
  225. commonLabels map[string]string
  226. ctxFunc func() (context.Context, func())
  227. populateSourceLocation int
  228. partialSuccess bool
  229. redirectOutputWriter io.Writer
  230. }
  231. // Logger returns a Logger that will write entries with the given log ID, such as
  232. // "syslog". A log ID must be less than 512 characters long and can only
  233. // include the following characters: upper and lower case alphanumeric
  234. // characters: [A-Za-z0-9]; and punctuation characters: forward-slash,
  235. // underscore, hyphen, and period.
  236. func (c *Client) Logger(logID string, opts ...LoggerOption) *Logger {
  237. r := detectResource()
  238. if r == nil {
  239. r = monitoredResource(c.parent)
  240. }
  241. l := &Logger{
  242. client: c,
  243. logName: internal.LogPath(c.parent, logID),
  244. commonResource: r,
  245. ctxFunc: func() (context.Context, func()) { return context.Background(), nil },
  246. populateSourceLocation: DoNotPopulateSourceLocation,
  247. partialSuccess: false,
  248. redirectOutputWriter: nil,
  249. }
  250. l.bundler = bundler.NewBundler(&logpb.LogEntry{}, func(entries interface{}) {
  251. l.writeLogEntries(entries.([]*logpb.LogEntry))
  252. })
  253. l.bundler.DelayThreshold = DefaultDelayThreshold
  254. l.bundler.BundleCountThreshold = DefaultEntryCountThreshold
  255. l.bundler.BundleByteThreshold = DefaultEntryByteThreshold
  256. l.bundler.BufferedByteLimit = DefaultBufferedByteLimit
  257. for _, opt := range opts {
  258. opt.set(l)
  259. }
  260. l.stdLoggers = map[Severity]*log.Logger{}
  261. for s := range severityName {
  262. e := Entry{Severity: s}
  263. l.stdLoggers[s] = log.New(templateEntryWriter{l, &e}, "", 0)
  264. }
  265. c.loggers.Add(1)
  266. // Start a goroutine that cleans up the bundler, its channel
  267. // and the writer goroutines when the client is closed.
  268. go func() {
  269. defer c.loggers.Done()
  270. <-c.donec
  271. l.bundler.Flush()
  272. }()
  273. return l
  274. }
  275. type templateEntryWriter struct {
  276. l *Logger
  277. template *Entry
  278. }
  279. func (w templateEntryWriter) Write(p []byte) (n int, err error) {
  280. e := *w.template
  281. e.Payload = string(p)
  282. // The second argument to logInternal() is how many frames to skip
  283. // from the call stack when determining the source location. In the
  284. // current implementation of log.Logger (i.e. Go's logging library)
  285. // the Write() method is called 2 calls deep so we need to skip 3
  286. // frames to account for the call to logInternal() itself.
  287. w.l.logInternal(e, 3)
  288. return len(p), nil
  289. }
  290. // Close waits for all opened loggers to be flushed and closes the client.
  291. func (c *Client) Close() error {
  292. if c.closed {
  293. return nil
  294. }
  295. close(c.donec) // close Logger bundlers
  296. c.loggers.Wait() // wait for all bundlers to flush and close
  297. // Now there can be no more errors.
  298. close(c.errc) // terminate error goroutine
  299. // Prefer errors arising from logging to the error returned from Close.
  300. err := c.extractErrorInfo()
  301. err2 := c.client.Close()
  302. if err == nil {
  303. err = err2
  304. }
  305. c.closed = true
  306. return err
  307. }
  308. // Severity is the severity of the event described in a log entry. These
  309. // guideline severity levels are ordered, with numerically smaller levels
  310. // treated as less severe than numerically larger levels.
  311. type Severity int
  312. const (
  313. // Default means the log entry has no assigned severity level.
  314. Default = Severity(logtypepb.LogSeverity_DEFAULT)
  315. // Debug means debug or trace information.
  316. Debug = Severity(logtypepb.LogSeverity_DEBUG)
  317. // Info means routine information, such as ongoing status or performance.
  318. Info = Severity(logtypepb.LogSeverity_INFO)
  319. // Notice means normal but significant events, such as start up, shut down, or configuration.
  320. Notice = Severity(logtypepb.LogSeverity_NOTICE)
  321. // Warning means events that might cause problems.
  322. Warning = Severity(logtypepb.LogSeverity_WARNING)
  323. // Error means events that are likely to cause problems.
  324. Error = Severity(logtypepb.LogSeverity_ERROR)
  325. // Critical means events that cause more severe problems or brief outages.
  326. Critical = Severity(logtypepb.LogSeverity_CRITICAL)
  327. // Alert means a person must take an action immediately.
  328. Alert = Severity(logtypepb.LogSeverity_ALERT)
  329. // Emergency means one or more systems are unusable.
  330. Emergency = Severity(logtypepb.LogSeverity_EMERGENCY)
  331. )
  332. var severityName = map[Severity]string{
  333. Default: "Default",
  334. Debug: "Debug",
  335. Info: "Info",
  336. Notice: "Notice",
  337. Warning: "Warning",
  338. Error: "Error",
  339. Critical: "Critical",
  340. Alert: "Alert",
  341. Emergency: "Emergency",
  342. }
  343. // String converts a severity level to a string.
  344. func (v Severity) String() string {
  345. // same as proto.EnumName
  346. s, ok := severityName[v]
  347. if ok {
  348. return s
  349. }
  350. return strconv.Itoa(int(v))
  351. }
  352. // UnmarshalJSON turns a string representation of severity into the type
  353. // Severity.
  354. func (v *Severity) UnmarshalJSON(data []byte) error {
  355. var s string
  356. var i int
  357. if strErr := json.Unmarshal(data, &s); strErr == nil {
  358. *v = ParseSeverity(s)
  359. } else if intErr := json.Unmarshal(data, &i); intErr == nil {
  360. *v = Severity(i)
  361. } else {
  362. return fmt.Errorf("%v; %v", strErr, intErr)
  363. }
  364. return nil
  365. }
  366. // ParseSeverity returns the Severity whose name equals s, ignoring case. It
  367. // returns Default if no Severity matches.
  368. func ParseSeverity(s string) Severity {
  369. sl := strings.ToLower(s)
  370. for sev, name := range severityName {
  371. if strings.ToLower(name) == sl {
  372. return sev
  373. }
  374. }
  375. return Default
  376. }
  377. // Entry is a log entry.
  378. // See https://cloud.google.com/logging/docs/view/logs_index for more about entries.
  379. type Entry struct {
  380. // Timestamp is the time of the entry. If zero, the current time is used.
  381. Timestamp time.Time
  382. // Severity is the entry's severity level.
  383. // The zero value is Default.
  384. Severity Severity
  385. // Payload must be either a string, or something that marshals via the
  386. // encoding/json package to a JSON object (and not any other type of JSON value).
  387. Payload interface{}
  388. // Labels optionally specifies key/value labels for the log entry.
  389. // The Logger.Log method takes ownership of this map. See Logger.CommonLabels
  390. // for more about labels.
  391. Labels map[string]string
  392. // InsertID is a unique ID for the log entry. If you provide this field,
  393. // the logging service considers other log entries in the same log with the
  394. // same ID as duplicates which can be removed. If omitted, the logging
  395. // service will generate a unique ID for this log entry. Note that because
  396. // this client retries RPCs automatically, it is possible (though unlikely)
  397. // that an Entry without an InsertID will be written more than once.
  398. InsertID string
  399. // HTTPRequest optionally specifies metadata about the HTTP request
  400. // associated with this log entry, if applicable. It is optional.
  401. HTTPRequest *HTTPRequest
  402. // Operation optionally provides information about an operation associated
  403. // with the log entry, if applicable.
  404. Operation *logpb.LogEntryOperation
  405. // LogName is the full log name, in the form
  406. // "projects/{ProjectID}/logs/{LogID}". It is set by the client when
  407. // reading entries. It is an error to set it when writing entries.
  408. LogName string
  409. // Resource is the monitored resource associated with the entry.
  410. Resource *mrpb.MonitoredResource
  411. // Trace is the resource name of the trace associated with the log entry,
  412. // if any. If it contains a relative resource name, the name is assumed to
  413. // be relative to //tracing.googleapis.com.
  414. Trace string
  415. // ID of the span within the trace associated with the log entry.
  416. // The ID is a 16-character hexadecimal encoding of an 8-byte array.
  417. SpanID string
  418. // If set, symbolizes that this request was sampled.
  419. TraceSampled bool
  420. // Optional. Source code location information associated with the log entry,
  421. // if any.
  422. SourceLocation *logpb.LogEntrySourceLocation
  423. }
  424. // HTTPRequest contains an http.Request as well as additional
  425. // information about the request and its response.
  426. type HTTPRequest struct {
  427. // Request is the http.Request passed to the handler.
  428. Request *http.Request
  429. // RequestSize is the size of the HTTP request message in bytes, including
  430. // the request headers and the request body.
  431. RequestSize int64
  432. // Status is the response code indicating the status of the response.
  433. // Examples: 200, 404.
  434. Status int
  435. // ResponseSize is the size of the HTTP response message sent back to the client, in bytes,
  436. // including the response headers and the response body.
  437. ResponseSize int64
  438. // Latency is the request processing latency on the server, from the time the request was
  439. // received until the response was sent.
  440. Latency time.Duration
  441. // LocalIP is the IP address (IPv4 or IPv6) of the origin server that the request
  442. // was sent to.
  443. LocalIP string
  444. // RemoteIP is the IP address (IPv4 or IPv6) of the client that issued the
  445. // HTTP request. Examples: "192.168.1.1", "FE80::0202:B3FF:FE1E:8329".
  446. RemoteIP string
  447. // CacheHit reports whether an entity was served from cache (with or without
  448. // validation).
  449. CacheHit bool
  450. // CacheValidatedWithOriginServer reports whether the response was
  451. // validated with the origin server before being served from cache. This
  452. // field is only meaningful if CacheHit is true.
  453. CacheValidatedWithOriginServer bool
  454. // CacheFillBytes is the number of HTTP response bytes inserted into cache. Set only when a cache fill was attempted.
  455. CacheFillBytes int64
  456. // CacheLookup tells whether or not a cache lookup was attempted.
  457. CacheLookup bool
  458. }
  459. func fromHTTPRequest(r *HTTPRequest) (*logtypepb.HttpRequest, error) {
  460. if r == nil {
  461. return nil, nil
  462. }
  463. if r.Request == nil {
  464. return nil, errors.New("logging: HTTPRequest must have a non-nil Request")
  465. }
  466. u := *r.Request.URL
  467. u.Fragment = ""
  468. pb := &logtypepb.HttpRequest{
  469. RequestMethod: r.Request.Method,
  470. RequestUrl: fixUTF8(u.String()),
  471. RequestSize: r.RequestSize,
  472. Status: int32(r.Status),
  473. ResponseSize: r.ResponseSize,
  474. UserAgent: r.Request.UserAgent(),
  475. ServerIp: r.LocalIP,
  476. RemoteIp: r.RemoteIP, // TODO(jba): attempt to parse http.Request.RemoteAddr?
  477. Referer: r.Request.Referer(),
  478. CacheHit: r.CacheHit,
  479. CacheValidatedWithOriginServer: r.CacheValidatedWithOriginServer,
  480. Protocol: r.Request.Proto,
  481. CacheFillBytes: r.CacheFillBytes,
  482. CacheLookup: r.CacheLookup,
  483. }
  484. if r.Latency != 0 {
  485. pb.Latency = ptypes.DurationProto(r.Latency)
  486. }
  487. return pb, nil
  488. }
  489. // fixUTF8 is a helper that fixes an invalid UTF-8 string by replacing
  490. // invalid UTF-8 runes with the Unicode replacement character (U+FFFD).
  491. // See Issue https://github.com/googleapis/google-cloud-go/issues/1383.
  492. func fixUTF8(s string) string {
  493. if utf8.ValidString(s) {
  494. return s
  495. }
  496. // Otherwise time to build the sequence.
  497. buf := new(bytes.Buffer)
  498. buf.Grow(len(s))
  499. for _, r := range s {
  500. if utf8.ValidRune(r) {
  501. buf.WriteRune(r)
  502. } else {
  503. buf.WriteRune('\uFFFD')
  504. }
  505. }
  506. return buf.String()
  507. }
  508. // toProtoStruct converts v, which must marshal into a JSON object,
  509. // into a Google Struct proto.
  510. func toProtoStruct(v interface{}) (*structpb.Struct, error) {
  511. // Fast path: if v is already a *structpb.Struct, nothing to do.
  512. if s, ok := v.(*structpb.Struct); ok {
  513. return s, nil
  514. }
  515. // v is a Go value that supports JSON marshalling. We want a Struct
  516. // protobuf. Some day we may have a more direct way to get there, but right
  517. // now the only way is to marshal the Go value to JSON, unmarshal into a
  518. // map, and then build the Struct proto from the map.
  519. var jb []byte
  520. var err error
  521. if raw, ok := v.(json.RawMessage); ok { // needed for Go 1.7 and below
  522. jb = []byte(raw)
  523. } else {
  524. jb, err = json.Marshal(v)
  525. if err != nil {
  526. return nil, fmt.Errorf("logging: json.Marshal: %w", err)
  527. }
  528. }
  529. var m map[string]interface{}
  530. err = json.Unmarshal(jb, &m)
  531. if err != nil {
  532. return nil, fmt.Errorf("logging: json.Unmarshal: %w", err)
  533. }
  534. return jsonMapToProtoStruct(m), nil
  535. }
  536. func jsonMapToProtoStruct(m map[string]interface{}) *structpb.Struct {
  537. fields := map[string]*structpb.Value{}
  538. for k, v := range m {
  539. fields[k] = jsonValueToStructValue(v)
  540. }
  541. return &structpb.Struct{Fields: fields}
  542. }
  543. func jsonValueToStructValue(v interface{}) *structpb.Value {
  544. switch x := v.(type) {
  545. case bool:
  546. return &structpb.Value{Kind: &structpb.Value_BoolValue{BoolValue: x}}
  547. case float64:
  548. return &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: x}}
  549. case string:
  550. return &structpb.Value{Kind: &structpb.Value_StringValue{StringValue: x}}
  551. case nil:
  552. return &structpb.Value{Kind: &structpb.Value_NullValue{}}
  553. case map[string]interface{}:
  554. return &structpb.Value{Kind: &structpb.Value_StructValue{StructValue: jsonMapToProtoStruct(x)}}
  555. case []interface{}:
  556. var vals []*structpb.Value
  557. for _, e := range x {
  558. vals = append(vals, jsonValueToStructValue(e))
  559. }
  560. return &structpb.Value{Kind: &structpb.Value_ListValue{ListValue: &structpb.ListValue{Values: vals}}}
  561. default:
  562. return &structpb.Value{Kind: &structpb.Value_NullValue{}}
  563. }
  564. }
  565. // LogSync logs the Entry synchronously without any buffering. Because LogSync is slow
  566. // and will block, it is intended primarily for debugging or critical errors.
  567. // Prefer Log for most uses.
  568. func (l *Logger) LogSync(ctx context.Context, e Entry) error {
  569. ent, err := toLogEntryInternal(e, l, l.client.parent, 1)
  570. if err != nil {
  571. return err
  572. }
  573. entries, hasInstrumentation := l.instrumentLogs([]*logpb.LogEntry{ent})
  574. if l.redirectOutputWriter != nil {
  575. for _, ent = range entries {
  576. err = serializeEntryToWriter(ent, l.redirectOutputWriter)
  577. if err != nil {
  578. break
  579. }
  580. }
  581. return err
  582. }
  583. _, err = l.client.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
  584. LogName: l.logName,
  585. Resource: l.commonResource,
  586. Labels: l.commonLabels,
  587. Entries: entries,
  588. PartialSuccess: l.partialSuccess || hasInstrumentation,
  589. })
  590. return err
  591. }
  592. // Log buffers the Entry for output to the logging service. It never blocks.
  593. func (l *Logger) Log(e Entry) {
  594. l.logInternal(e, 1)
  595. }
  596. func (l *Logger) logInternal(e Entry, skipLevels int) {
  597. ent, err := toLogEntryInternal(e, l, l.client.parent, skipLevels+1)
  598. if err != nil {
  599. l.client.error(err)
  600. return
  601. }
  602. entries, _ := l.instrumentLogs([]*logpb.LogEntry{ent})
  603. if l.redirectOutputWriter != nil {
  604. for _, ent = range entries {
  605. err = serializeEntryToWriter(ent, l.redirectOutputWriter)
  606. if err != nil {
  607. l.client.error(err)
  608. }
  609. }
  610. return
  611. }
  612. for _, ent = range entries {
  613. if err := l.bundler.Add(ent, proto.Size(ent)); err != nil {
  614. l.client.error(err)
  615. }
  616. }
  617. }
  618. // Flush blocks until all currently buffered log entries are sent.
  619. //
  620. // If any errors occurred since the last call to Flush from any Logger, or the
  621. // creation of the client if this is the first call, then Flush returns a non-nil
  622. // error with summary information about the errors. This information is unlikely to
  623. // be actionable. For more accurate error reporting, set Client.OnError.
  624. func (l *Logger) Flush() error {
  625. l.bundler.Flush()
  626. return l.client.extractErrorInfo()
  627. }
  628. func (l *Logger) writeLogEntries(entries []*logpb.LogEntry) {
  629. partialSuccess := l.partialSuccess
  630. if len(entries) > 1 {
  631. partialSuccess = partialSuccess || hasInstrumentation(entries)
  632. }
  633. req := &logpb.WriteLogEntriesRequest{
  634. LogName: l.logName,
  635. Resource: l.commonResource,
  636. Labels: l.commonLabels,
  637. Entries: entries,
  638. PartialSuccess: partialSuccess,
  639. }
  640. ctx, afterCall := l.ctxFunc()
  641. ctx, cancel := context.WithTimeout(ctx, defaultWriteTimeout)
  642. defer cancel()
  643. _, err := l.client.client.WriteLogEntries(ctx, req)
  644. if err != nil {
  645. l.client.error(err)
  646. }
  647. if afterCall != nil {
  648. afterCall()
  649. }
  650. }
  651. // StandardLogger returns a *log.Logger for the provided severity.
  652. //
  653. // This method is cheap. A single log.Logger is pre-allocated for each
  654. // severity level in each Logger. Callers may mutate the returned log.Logger
  655. // (for example by calling SetFlags or SetPrefix).
  656. func (l *Logger) StandardLogger(s Severity) *log.Logger { return l.stdLoggers[s] }
  657. // StandardLoggerFromTemplate returns a Go Standard Logging API *log.Logger.
  658. //
  659. // The returned logger emits logs using logging.(*Logger).Log() with an entry
  660. // constructed from the provided template Entry struct.
  661. //
  662. // The caller is responsible for ensuring that the template Entry struct
  663. // does not change during the the lifetime of the returned *log.Logger.
  664. //
  665. // Prefer (*Logger).StandardLogger() which is more efficient if the template
  666. // only sets Severity.
  667. func (l *Logger) StandardLoggerFromTemplate(template *Entry) *log.Logger {
  668. return log.New(templateEntryWriter{l, template}, "", 0)
  669. }
  670. func populateTraceInfo(e *Entry, req *http.Request) bool {
  671. if req == nil {
  672. if e.HTTPRequest != nil && e.HTTPRequest.Request != nil {
  673. req = e.HTTPRequest.Request
  674. } else {
  675. return false
  676. }
  677. }
  678. header := req.Header.Get("Traceparent")
  679. if header != "" {
  680. // do not use traceSampled flag defined by traceparent because
  681. // flag's definition differs from expected by Cloud Tracing
  682. traceID, spanID, _ := deconstructTraceParent(header)
  683. if traceID != "" {
  684. e.Trace = traceID
  685. e.SpanID = spanID
  686. return true
  687. }
  688. }
  689. header = req.Header.Get("X-Cloud-Trace-Context")
  690. if header != "" {
  691. traceID, spanID, traceSampled := deconstructXCloudTraceContext(header)
  692. if traceID != "" {
  693. e.Trace = traceID
  694. e.SpanID = spanID
  695. // enforce sampling if required
  696. e.TraceSampled = e.TraceSampled || traceSampled
  697. return true
  698. }
  699. }
  700. return false
  701. }
  702. // As per format described at https://www.w3.org/TR/trace-context/#traceparent-header-field-values
  703. var validTraceParentExpression = regexp.MustCompile(`^(00)-([a-fA-F\d]{32})-([a-f\d]{16})-([a-fA-F\d]{2})$`)
  704. func deconstructTraceParent(s string) (traceID, spanID string, traceSampled bool) {
  705. matches := validTraceParentExpression.FindStringSubmatch(s)
  706. if matches != nil {
  707. // regexp package does not support negative lookahead preventing all 0 validations
  708. if matches[2] == "00000000000000000000000000000000" || matches[3] == "0000000000000000" {
  709. return
  710. }
  711. flags, err := strconv.ParseInt(matches[4], 16, 16)
  712. if err == nil {
  713. traceSampled = (flags & 0x01) == 1
  714. }
  715. traceID, spanID = matches[2], matches[3]
  716. }
  717. return
  718. }
  719. var validXCloudTraceContext = regexp.MustCompile(
  720. // Matches on "TRACE_ID"
  721. `([a-f\d]+)?` +
  722. // Matches on "/SPAN_ID"
  723. `(?:/([a-f\d]+))?` +
  724. // Matches on ";0=TRACE_TRUE"
  725. `(?:;o=(\d))?`)
  726. func deconstructXCloudTraceContext(s string) (traceID, spanID string, traceSampled bool) {
  727. // As per the format described at https://cloud.google.com/trace/docs/setup#force-trace
  728. // "X-Cloud-Trace-Context: TRACE_ID/SPAN_ID;o=TRACE_TRUE"
  729. // for example:
  730. // "X-Cloud-Trace-Context: 105445aa7843bc8bf206b120001000/1;o=1"
  731. //
  732. // We expect:
  733. // * traceID (optional): "105445aa7843bc8bf206b120001000"
  734. // * spanID (optional): "1"
  735. // * traceSampled (optional): true
  736. matches := validXCloudTraceContext.FindStringSubmatch(s)
  737. if matches != nil {
  738. traceID, spanID, traceSampled = matches[1], matches[2], matches[3] == "1"
  739. }
  740. if spanID == "0" {
  741. spanID = ""
  742. }
  743. return
  744. }
  745. // ToLogEntry takes an Entry structure and converts it to the LogEntry proto.
  746. // A parent can take any of the following forms:
  747. //
  748. // projects/PROJECT_ID
  749. // folders/FOLDER_ID
  750. // billingAccounts/ACCOUNT_ID
  751. // organizations/ORG_ID
  752. //
  753. // for backwards compatibility, a string with no '/' is also allowed and is interpreted
  754. // as a project ID.
  755. //
  756. // ToLogEntry is implied when users invoke Logger.Log or Logger.LogSync,
  757. // but its exported as a pub function here to give users additional flexibility
  758. // when using the library. Don't call this method manually if Logger.Log or
  759. // Logger.LogSync are used, it is intended to be used together with direct call
  760. // to WriteLogEntries method.
  761. func ToLogEntry(e Entry, parent string) (*logpb.LogEntry, error) {
  762. var l Logger
  763. return l.ToLogEntry(e, parent)
  764. }
  765. // ToLogEntry for Logger instance
  766. func (l *Logger) ToLogEntry(e Entry, parent string) (*logpb.LogEntry, error) {
  767. parent, err := makeParent(parent)
  768. if err != nil {
  769. return nil, err
  770. }
  771. return toLogEntryInternal(e, l, parent, 1)
  772. }
  773. func toLogEntryInternalImpl(e Entry, l *Logger, parent string, skipLevels int) (*logpb.LogEntry, error) {
  774. if e.LogName != "" {
  775. return nil, errors.New("logging: Entry.LogName should be not be set when writing")
  776. }
  777. t := e.Timestamp
  778. if t.IsZero() {
  779. t = now()
  780. }
  781. ts := timestamppb.New(t)
  782. if l != nil && l.populateSourceLocation != DoNotPopulateSourceLocation && e.SourceLocation == nil {
  783. if l.populateSourceLocation == AlwaysPopulateSourceLocation ||
  784. l.populateSourceLocation == PopulateSourceLocationForDebugEntries && e.Severity == Severity(Debug) {
  785. // filename and line are captured for source code that calls
  786. // skipLevels up the goroutine calling stack + 1 for this func.
  787. pc, file, line, ok := runtime.Caller(skipLevels + 1)
  788. if ok {
  789. details := runtime.FuncForPC(pc)
  790. e.SourceLocation = &logpb.LogEntrySourceLocation{
  791. File: file,
  792. Function: details.Name(),
  793. Line: int64(line),
  794. }
  795. }
  796. }
  797. }
  798. if e.Trace == "" {
  799. populateTraceInfo(&e, nil)
  800. // format trace
  801. if e.Trace != "" && !strings.Contains(e.Trace, "/traces/") {
  802. e.Trace = fmt.Sprintf("%s/traces/%s", parent, e.Trace)
  803. }
  804. }
  805. req, err := fromHTTPRequest(e.HTTPRequest)
  806. if err != nil {
  807. if l != nil && l.client != nil {
  808. l.client.error(err)
  809. } else {
  810. return nil, err
  811. }
  812. }
  813. ent := &logpb.LogEntry{
  814. Timestamp: ts,
  815. Severity: logtypepb.LogSeverity(e.Severity),
  816. InsertId: e.InsertID,
  817. HttpRequest: req,
  818. Operation: e.Operation,
  819. Labels: e.Labels,
  820. Trace: e.Trace,
  821. SpanId: e.SpanID,
  822. Resource: e.Resource,
  823. SourceLocation: e.SourceLocation,
  824. TraceSampled: e.TraceSampled,
  825. }
  826. switch p := e.Payload.(type) {
  827. case string:
  828. ent.Payload = &logpb.LogEntry_TextPayload{TextPayload: p}
  829. case *anypb.Any:
  830. ent.Payload = &logpb.LogEntry_ProtoPayload{ProtoPayload: p}
  831. default:
  832. s, err := toProtoStruct(p)
  833. if err != nil {
  834. return nil, err
  835. }
  836. ent.Payload = &logpb.LogEntry_JsonPayload{JsonPayload: s}
  837. }
  838. return ent, nil
  839. }
  840. // entry represents the fields of a logging.Entry that can be parsed by Logging agent.
  841. // See the mappings at https://cloud.google.com/logging/docs/structured-logging#special-payload-fields
  842. type structuredLogEntry struct {
  843. // JsonMessage map[string]interface{} `json:"message,omitempty"`
  844. // TextMessage string `json:"message,omitempty"`
  845. Message json.RawMessage `json:"message"`
  846. Severity string `json:"severity,omitempty"`
  847. HTTPRequest *logtypepb.HttpRequest `json:"httpRequest,omitempty"`
  848. Timestamp string `json:"timestamp,omitempty"`
  849. Labels map[string]string `json:"logging.googleapis.com/labels,omitempty"`
  850. InsertID string `json:"logging.googleapis.com/insertId,omitempty"`
  851. Operation *logpb.LogEntryOperation `json:"logging.googleapis.com/operation,omitempty"`
  852. SourceLocation *logpb.LogEntrySourceLocation `json:"logging.googleapis.com/sourceLocation,omitempty"`
  853. SpanID string `json:"logging.googleapis.com/spanId,omitempty"`
  854. Trace string `json:"logging.googleapis.com/trace,omitempty"`
  855. TraceSampled bool `json:"logging.googleapis.com/trace_sampled,omitempty"`
  856. }
  857. func convertSnakeToMixedCase(snakeStr string) string {
  858. words := strings.Split(snakeStr, "_")
  859. mixedStr := words[0]
  860. for _, word := range words[1:] {
  861. mixedStr += strings.Title(word)
  862. }
  863. return mixedStr
  864. }
  865. func (s structuredLogEntry) MarshalJSON() ([]byte, error) {
  866. // extract structuredLogEntry into json map
  867. type Alias structuredLogEntry
  868. var mapData map[string]interface{}
  869. data, err := json.Marshal(Alias(s))
  870. if err == nil {
  871. err = json.Unmarshal(data, &mapData)
  872. }
  873. if err == nil {
  874. // ensure all inner dicts use mixed case instead of snake case
  875. innerDicts := [3]string{"httpRequest", "logging.googleapis.com/operation", "logging.googleapis.com/sourceLocation"}
  876. for _, field := range innerDicts {
  877. if fieldData, ok := mapData[field]; ok {
  878. formattedFieldData := make(map[string]interface{})
  879. for k, v := range fieldData.(map[string]interface{}) {
  880. formattedFieldData[convertSnakeToMixedCase(k)] = v
  881. }
  882. mapData[field] = formattedFieldData
  883. }
  884. }
  885. // serialize json map into raw bytes
  886. return json.Marshal(mapData)
  887. }
  888. return data, err
  889. }
  890. func serializeEntryToWriter(entry *logpb.LogEntry, w io.Writer) error {
  891. jsonifiedEntry := structuredLogEntry{
  892. Severity: entry.Severity.String(),
  893. HTTPRequest: entry.HttpRequest,
  894. Timestamp: entry.Timestamp.String(),
  895. Labels: entry.Labels,
  896. InsertID: entry.InsertId,
  897. Operation: entry.Operation,
  898. SourceLocation: entry.SourceLocation,
  899. SpanID: entry.SpanId,
  900. Trace: entry.Trace,
  901. TraceSampled: entry.TraceSampled,
  902. }
  903. var err error
  904. if entry.GetTextPayload() != "" {
  905. jsonifiedEntry.Message, err = json.Marshal(entry.GetTextPayload())
  906. } else if entry.GetJsonPayload() != nil {
  907. jsonifiedEntry.Message, err = json.Marshal(entry.GetJsonPayload().AsMap())
  908. } else {
  909. return ErrRedirectProtoPayloadNotSupported
  910. }
  911. if err == nil {
  912. err = json.NewEncoder(w).Encode(jsonifiedEntry)
  913. }
  914. return err
  915. }