perf: improves performance when loading large logs

frontend/src/api/nginx_log.ts

@@ -0,0 +1,16 @@
+import http from '@/lib/http'
+
+interface IData {
+    type: string
+    conf_name: string
+    server_idx: number
+    directive_idx: number
+}
+
+const nginx_log = {
+    page(page = 0, data: IData) {
+        return http.post('/nginx_log?page=' + page, data)
+    }
+}
+
+export default nginx_log

frontend/src/views/nginx_log/NginxLog.vue

@@ -5,6 +5,8 @@ import {nextTick, onMounted, onUnmounted, reactive, ref, watch} from 'vue'
 import ReconnectingWebSocket from 'reconnecting-websocket'
 import {useRoute, useRouter} from 'vue-router'
 import FooterToolBar from '@/components/FooterToolbar/FooterToolBar.vue'
+import nginx_log from '@/api/nginx_log'
+import {debounce} from 'lodash'
 
 const {$gettext} = useGettext()
 
@@ -18,7 +20,6 @@ function logType() {
 }
 
 const control = reactive({
-    fetch: 'new',
     type: logType(),
     conf_name: route.query.conf_name,
     server_idx: parseInt(route.query.server_idx as string),
@@ -30,26 +31,51 @@ function openWs() {
 
     websocket.onopen = () => {
         websocket.send(JSON.stringify({
-            ...control,
-            fetch: 'new'
+            ...control
         }))
     }
 
     websocket.onmessage = (m: any) => {
-        const para = document.createElement('p')
-        para.appendChild(document.createTextNode(m.data.trim()));
+        addLog(m.data)
+    }
+}
 
-        (logContainer.value as any as Node).appendChild(para);
+function addLog(data: string, prepend: boolean = false) {
+    const para = document.createElement('p')
+    para.appendChild(document.createTextNode(data.trim()))
 
-        (logContainer.value as any as Element).scroll({
-            top: (logContainer.value as any as Element).scrollHeight,
-            left: 0,
-            behavior: 'smooth'
-        })
+    const node = (logContainer.value as any as Node)
+
+    if (prepend) {
+        node.insertBefore(para, node.firstChild)
+    } else {
+        node.appendChild(para)
     }
+    const elem = (logContainer.value as any as Element)
+    elem.scroll({
+        top: elem.scrollHeight,
+        left: 0,
+    })
+}
+
+const page = ref(0)
+
+function init() {
+    nginx_log.page(0, {
+        conf_name: (route.query.conf_name as string),
+        type: logType(),
+        server_idx: 0,
+        directive_idx: 0
+    }).then(r => {
+        page.value = r.page - 1
+        r.content.split('\n').forEach((v: string) => {
+            addLog(v)
+        })
+    })
 }
 
 onMounted(() => {
+    init()
     openWs()
 })
 
@@ -66,6 +92,8 @@ watch(auto_refresh, (value) => {
 })
 
 watch(route, () => {
+    init()
+
     control.type = logType();
     (logContainer.value as any as Element).innerHTML = ''
 
@@ -88,6 +116,31 @@ onUnmounted(() => {
 })
 
 const router = useRouter()
+const loading = ref(false)
+
+function on_scroll_log() {
+    if (!loading.value && page.value > 0) {
+        loading.value = true
+        const elem = (logContainer.value as any as Element)
+        if (elem.scrollTop / elem.scrollHeight < 0.333) {
+            nginx_log.page(page.value, {
+                conf_name: (route.query.conf_name as string),
+                type: logType(),
+                server_idx: 0,
+                directive_idx: 0
+            }).then(r => {
+                page.value = r.page - 1
+                r.content.split('\n').forEach((v: string) => {
+                    addLog(v, true)
+                })
+            }).finally(() => {
+                loading.value = false
+            })
+        } else {
+            loading.value = false
+        }
+    }
+}
 
 </script>
 
@@ -97,20 +150,11 @@ const router = useRouter()
             <a-form-item :label="$gettext('Auto Refresh')">
                 <a-switch v-model:checked="auto_refresh"/>
             </a-form-item>
-            <a-form-item :label="$gettext('Fetch')">
-                <a-select v-model:value="control.fetch" style="max-width: 200px">
-                    <a-select-option value="all">
-                        <translate>All logs</translate>
-                    </a-select-option>
-                    <a-select-option value="new">
-                        <translate>New logs</translate>
-                    </a-select-option>
-                </a-select>
-            </a-form-item>
         </a-form>
 
         <a-card>
-            <pre class="nginx-log-container" ref="logContainer"></pre>
+            <pre class="nginx-log-container" ref="logContainer"
+                 @scroll="debounce(on_scroll_log,100, null)()"></pre>
         </a-card>
     </a-card>
     <footer-tool-bar v-if="control.type==='site'">
@@ -125,6 +169,7 @@ const router = useRouter()
     height: 60vh;
     overflow: scroll;
     padding: 5px;
+    margin-bottom: 0;
 
     p {
         font-size: 12px;

server/api/nginx_log.go

@@ -8,89 +8,185 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/hpcloud/tail"
 	"github.com/pkg/errors"
+	"github.com/spf13/cast"
 	"io"
 	"log"
 	"net/http"
+	"os"
 	"path/filepath"
 )
 
+const (
+	PageSize = 128 * 1024
+)
+
 type controlStruct struct {
-	Fetch        string `json:"fetch"`
 	Type         string `json:"type"`
 	ConfName     string `json:"conf_name"`
 	ServerIdx    int    `json:"server_idx"`
 	DirectiveIdx int    `json:"directive_idx"`
 }
 
-func tailNginxLog(ws *websocket.Conn, controlChan chan controlStruct, errChan chan error) {
-	defer func() {
-		if err := recover(); err != nil {
-			log.Println("tailNginxLog recovery", err)
-			_ = ws.WriteMessage(websocket.TextMessage, err.([]byte))
+type nginxLogPageResp struct {
+	Content string `json:"content"`
+	Page    int64  `json:"page"`
+}
+
+func GetNginxLogPage(c *gin.Context) {
+	page := cast.ToInt64(c.Query("page"))
+	if page < 0 {
+		page = 0
+	}
+
+	var control controlStruct
+	if !BindAndValid(c, &control) {
+		return
+	}
+
+	logPath, err := getLogPath(&control)
+
+	if err != nil {
+		log.Println("error GetNginxLogPage", err)
+		return
+	}
+
+	f, err := os.Open(logPath)
+
+	if err != nil {
+		c.JSON(http.StatusOK, nginxLogPageResp{})
+		log.Println("error GetNginxLogPage open file", err)
+		return
+	}
+
+	logFileStat, err := os.Stat(logPath)
+
+	if err != nil {
+		c.JSON(http.StatusOK, nginxLogPageResp{})
+		log.Println("error GetNginxLogPage stat", err)
+		return
+	}
+
+	totalPage := logFileStat.Size() / PageSize
+
+	if logFileStat.Size()%PageSize > 0 {
+		totalPage++
+	}
+
+	var buf []byte
+	var offset int64
+	if page == 0 {
+		page = totalPage
+	}
+
+	buf = make([]byte, PageSize)
+	offset = (page - 1) * PageSize
+
+	// seek
+	_, err = f.Seek(offset, io.SeekStart)
+	if err != nil && err != io.EOF {
+		c.JSON(http.StatusOK, nginxLogPageResp{})
+		log.Println("error GetNginxLogPage seek", err)
+		return
+	}
+
+	n, err := f.Read(buf)
+
+	if err != nil && err != io.EOF {
+		c.JSON(http.StatusOK, nginxLogPageResp{})
+		log.Println("error GetNginxLogPage read buf", err)
+		return
+	}
+
+	c.JSON(http.StatusOK, nginxLogPageResp{
+		Page:    page,
+		Content: string(buf[:n]),
+	})
+}
+
+func getLogPath(control *controlStruct) (logPath string, err error) {
+	switch control.Type {
+	case "site":
+		var config *nginx.NgxConfig
+		path := filepath.Join(nginx.GetNginxConfPath("sites-available"), control.ConfName)
+		config, err = nginx.ParseNgxConfig(path)
+		if err != nil {
+			err = errors.Wrap(err, "error parsing ngx config")
 			return
 		}
-	}()
 
-	control := <-controlChan
+		if control.ServerIdx >= len(config.Servers) {
+			err = errors.New("serverIdx out of range")
+			return
+		}
 
-	for {
-		var seek tail.SeekInfo
-		if control.Fetch != "all" {
-			seek.Offset = 0
-			seek.Whence = io.SeekEnd
-		}
-		var logPath string
-		switch control.Type {
-		case "site":
-			path := filepath.Join(nginx.GetNginxConfPath("sites-available"), control.ConfName)
-			config, err := nginx.ParseNgxConfig(path)
-			if err != nil {
-				errChan <- errors.Wrap(err, "error parsing ngx config")
-				return
-			}
+		if control.DirectiveIdx >= len(config.Servers[control.ServerIdx].Directives) {
+			err = errors.New("DirectiveIdx out of range")
+			return
+		}
 
-			if control.ServerIdx >= len(config.Servers) {
-				errChan <- errors.New("serverIdx out of range")
-				return
-			}
+		directive := config.Servers[control.ServerIdx].Directives[control.DirectiveIdx]
 
-			if control.DirectiveIdx >= len(config.Servers[control.ServerIdx].Directives) {
-				errChan <- errors.New("DirectiveIdx out of range")
-				return
-			}
+		switch directive.Directive {
+		case "access_log", "error_log":
+			// ok
+		default:
+			err = errors.New("directive.Params neither access_log nor error_log")
+			return
+		}
 
-			directive := config.Servers[control.ServerIdx].Directives[control.DirectiveIdx]
+		if directive.Params == "" {
+			err = errors.New("directive.Params is empty")
+			return
+		}
 
-			switch directive.Directive {
-			case "access_log", "error_log":
-				// ok
-			default:
-				errChan <- errors.New("directive.Params neither access_log nor error_log")
-				return
-			}
+		logPath = directive.Params
 
-			if directive.Params == "" {
-				errChan <- errors.New("directive.Params is empty")
-				return
-			}
+	case "error":
+		if settings.NginxLogSettings.ErrorLogPath == "" {
+			err = errors.New("settings.NginxLogSettings.ErrorLogPath is empty," +
+				" see https://github.com/0xJacky/nginx-ui/wiki/Nginx-Log-Configuration for more information")
+			return
+		}
+		logPath = settings.NginxLogSettings.ErrorLogPath
 
-			logPath = directive.Params
+	default:
+		if settings.NginxLogSettings.AccessLogPath == "" {
+			err = errors.New("settings.NginxLogSettings.AccessLogPath is empty," +
+				" see https://github.com/0xJacky/nginx-ui/wiki/Nginx-Log-Configuration for more information")
+			return
+		}
+		logPath = settings.NginxLogSettings.AccessLogPath
+	}
 
-		case "error":
-			if settings.NginxLogSettings.ErrorLogPath == "" {
-				errChan <- errors.New("settings.NginxLogSettings.ErrorLogPath is empty," +
-					" see https://github.com/0xJacky/nginx-ui/wiki/Nginx-Log-Configuration for more information")
-				return
-			}
-			logPath = settings.NginxLogSettings.ErrorLogPath
+	return
+}
 
-		default:
-			if settings.NginxLogSettings.AccessLogPath == "" {
-				errChan <- errors.New("settings.NginxLogSettings.AccessLogPath is empty," +
-					" see https://github.com/0xJacky/nginx-ui/wiki/Nginx-Log-Configuration for more information")
+func tailNginxLog(ws *websocket.Conn, controlChan chan controlStruct, errChan chan error) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println("tailNginxLog recovery", err)
+			err = ws.WriteMessage(websocket.TextMessage, err.([]byte))
+			if err != nil {
+				log.Println(err)
 				return
 			}
-			logPath = settings.NginxLogSettings.AccessLogPath
+			return
+		}
+	}()
+
+	control := <-controlChan
+
+	for {
+		logPath, err := getLogPath(&control)
+
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		seek := tail.SeekInfo{
+			Offset: 0,
+			Whence: io.SeekEnd,
 		}
 
 		// Create a tail

server/router/routers.go

@@ -95,6 +95,7 @@ func InitRouter() *gin.Engine {
 
 			// Nginx log
 			g.GET("nginx_log", api.NginxLog)
+			g.POST("nginx_log", api.GetNginxLogPage)
 		}
 	}