Refactor 1/2 + new stuff

* Refactor CLI
* Add config:print command
* Add diagnose command
* Allow including other files in config
* Watch for file changes and automatically restart server
This commit is contained in:
Svilen Markov 2024-11-24 15:39:14 +00:00
parent 7e345dd1f9
commit 2b0dd3ab99
8 changed files with 573 additions and 75 deletions

View file

@ -34,6 +34,7 @@
- [HTML](#html)
## Intro
<!-- TODO: update -->
Configuration is done via a single YAML file and a server restart is required in order for any changes to take effect. Trying to start the server with an invalid config file will result in an error.
## Preconfigured page

2
go.mod
View file

@ -3,6 +3,7 @@ module github.com/glanceapp/glance
go 1.23.1
require (
github.com/fsnotify/fsnotify v1.8.0
github.com/mmcdole/gofeed v1.3.0
github.com/tidwall/gjson v1.18.0
golang.org/x/text v0.18.0
@ -19,4 +20,5 @@ require (
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sys v0.25.0 // indirect
)

4
go.sum
View file

@ -5,6 +5,8 @@ github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@ -52,6 +54,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

View file

@ -2,41 +2,66 @@ package glance
import (
"flag"
"fmt"
"os"
"strings"
)
type CliIntent uint8
type cliIntent uint8
const (
CliIntentServe CliIntent = iota
CliIntentCheckConfig = iota
cliIntentServe cliIntent = iota
cliIntentConfigValidate = iota
cliIntentConfigPrint = iota
cliIntentDiagnose = iota
)
type CliOptions struct {
Intent CliIntent
ConfigPath string
type cliOptions struct {
intent cliIntent
configPath string
}
func ParseCliOptions() (*CliOptions, error) {
func parseCliOptions() (*cliOptions, error) {
flags := flag.NewFlagSet("", flag.ExitOnError)
flags.Usage = func() {
fmt.Println("Usage: glance [options] command")
checkConfig := flags.Bool("check-config", false, "Check whether the config is valid")
fmt.Println("\nOptions:")
flags.PrintDefaults()
fmt.Println("\nCommands:")
fmt.Println(" config:validate Validate the config file")
fmt.Println(" config:print Print the parsed config file with embedded includes")
fmt.Println(" diagnose Run diagnostic checks")
}
configPath := flags.String("config", "glance.yml", "Set config path")
err := flags.Parse(os.Args[1:])
if err != nil {
return nil, err
}
intent := CliIntentServe
var intent cliIntent
var args = flags.Args()
unknownCommandErr := fmt.Errorf("unknown command: %s", strings.Join(args, " "))
if *checkConfig {
intent = CliIntentCheckConfig
if len(args) == 0 {
intent = cliIntentServe
} else if len(args) == 1 {
if args[0] == "config:validate" {
intent = cliIntentConfigValidate
} else if args[0] == "config:print" {
intent = cliIntentConfigPrint
} else if args[0] == "diagnose" {
intent = cliIntentDiagnose
} else {
return nil, unknownCommandErr
}
} else {
return nil, unknownCommandErr
}
return &CliOptions{
Intent: intent,
ConfigPath: *configPath,
return &cliOptions{
intent: intent,
configPath: *configPath,
}, nil
}

View file

@ -1,9 +1,16 @@
package glance
import (
"bytes"
"fmt"
"io"
"log"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/fsnotify/fsnotify"
"gopkg.in/yaml.v3"
)
@ -14,22 +21,16 @@ type Config struct {
Pages []Page `yaml:"pages"`
}
func NewConfigFromYml(contents io.Reader) (*Config, error) {
config := NewConfig()
contentBytes, err := io.ReadAll(contents)
func newConfigFromYAML(contents []byte) (*Config, error) {
config := &Config{}
config.Server.Port = 8080
err := yaml.Unmarshal(contents, config)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(contentBytes, config)
if err != nil {
return nil, err
}
if err = configIsValid(config); err != nil {
if err = isConfigStateValid(config); err != nil {
return nil, err
}
@ -46,16 +47,179 @@ func NewConfigFromYml(contents io.Reader) (*Config, error) {
return config, nil
}
func NewConfig() *Config {
config := &Config{}
var includePattern = regexp.MustCompile(`(?m)^(\s*)!include:\s*(.+)$`)
config.Server.Host = ""
config.Server.Port = 8080
func parseYAMLIncludes(mainFilePath string) ([]byte, map[string]struct{}, error) {
mainFileContents, err := os.ReadFile(mainFilePath)
if err != nil {
return nil, nil, fmt.Errorf("could not read main YAML file: %w", err)
}
return config
mainFileAbsPath, err := filepath.Abs(mainFilePath)
if err != nil {
return nil, nil, fmt.Errorf("could not get absolute path of main YAML file: %w", err)
}
mainFileDir := filepath.Dir(mainFileAbsPath)
includes := make(map[string]struct{})
var includesLastErr error
mainFileContents = includePattern.ReplaceAllFunc(mainFileContents, func(match []byte) []byte {
if includesLastErr != nil {
return nil
}
matches := includePattern.FindSubmatch(match)
if len(matches) != 3 {
includesLastErr = fmt.Errorf("invalid include match: %v", matches)
return nil
}
indent := string(matches[1])
includeFilePath := strings.TrimSpace(string(matches[2]))
if !filepath.IsAbs(includeFilePath) {
includeFilePath = filepath.Join(mainFileDir, includeFilePath)
}
var fileContents []byte
var err error
fileContents, err = os.ReadFile(includeFilePath)
if err != nil {
includesLastErr = fmt.Errorf("could not read included file: %w", err)
return nil
}
includes[includeFilePath] = struct{}{}
return []byte(prefixStringLines(indent, string(fileContents)))
})
if includesLastErr != nil {
return nil, nil, includesLastErr
}
return mainFileContents, includes, nil
}
func configIsValid(config *Config) error {
func prefixStringLines(prefix string, s string) string {
lines := strings.Split(s, "\n")
for i, line := range lines {
lines[i] = prefix + line
}
return strings.Join(lines, "\n")
}
func configFilesWatcher(
mainFilePath string,
lastContents []byte,
lastIncludes map[string]struct{},
onChange func(newContents []byte),
onErr func(error),
) (func() error, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("could not create watcher: %w", err)
}
if err = watcher.Add(mainFilePath); err != nil {
watcher.Close()
return nil, fmt.Errorf("could not add main file to watcher: %w", err)
}
updateWatchedIncludes := func(previousIncludes map[string]struct{}, newIncludes map[string]struct{}) {
for includePath := range previousIncludes {
if _, ok := newIncludes[includePath]; !ok {
watcher.Remove(includePath)
}
}
for includePath := range newIncludes {
if _, ok := previousIncludes[includePath]; !ok {
if err := watcher.Add(includePath); err != nil {
log.Printf(
"Could not add included config file to watcher, changes to this file will not trigger a reload. path: %s, error: %v",
includePath, err,
)
}
}
}
}
updateWatchedIncludes(nil, lastIncludes)
checkForContentChangesBeforeCallback := func() {
currentContents, currentIncludes, err := parseYAMLIncludes(mainFilePath)
if err != nil {
onErr(fmt.Errorf("could not parse main file contents for comparison: %w", err))
return
}
if !bytes.Equal(lastContents, currentContents) {
updateWatchedIncludes(lastIncludes, currentIncludes)
lastContents, lastIncludes = currentContents, currentIncludes
onChange(currentContents)
}
}
const debounceDuration = 500 * time.Millisecond
var debounceTimer *time.Timer
debouncedCallback := func() {
if debounceTimer != nil {
debounceTimer.Stop()
debounceTimer.Reset(debounceDuration)
} else {
debounceTimer = time.AfterFunc(debounceDuration, checkForContentChangesBeforeCallback)
}
}
go func() {
for {
select {
case event, isOpen := <-watcher.Events:
if !isOpen {
return
}
if event.Has(fsnotify.Write) {
debouncedCallback()
}
// maybe also handle .Remove event?
// from testing it appears that a removed file will stop triggering .Write events
// when it gets recreated, in which case we may need to watch the directory for the
// creation of that file and then re-add it to the watcher, though that's
// a lot of effort for a hopefully rare edge case
case err, isOpen := <-watcher.Errors:
if !isOpen {
return
}
onErr(fmt.Errorf("watcher error: %w", err))
}
}
}()
onChange(lastContents)
return func() error {
if debounceTimer != nil {
debounceTimer.Stop()
}
return watcher.Close()
}, nil
}
func isConfigStateValid(config *Config) error {
if len(config.Pages) == 0 {
return fmt.Errorf("no pages configured")
}
if config.Server.AssetsPath != "" {
if _, err := os.Stat(config.Server.AssetsPath); os.IsNotExist(err) {
return fmt.Errorf("assets directory does not exist: %s", config.Server.AssetsPath)
}
}
for i := range config.Pages {
if config.Pages[i].Title == "" {
return fmt.Errorf("Page %d has no title", i+1)

219
internal/glance/diagnose.go Normal file
View file

@ -0,0 +1,219 @@
package glance
import (
"context"
"fmt"
"io"
"net"
"net/http"
"os"
"runtime"
"strings"
"sync"
"time"
)
const httpTestRequestTimeout = 10 * time.Second
var diagnosticSteps = []diagnosticStep{
{
name: "resolve cloudflare.com through Cloudflare DoH",
fn: func() (string, error) {
return testHttpRequestWithHeaders("GET", "https://1.1.1.1/dns-query?name=cloudflare.com", map[string]string{
"accept": "application/dns-json",
}, 200)
},
},
{
name: "resolve cloudflare.com through Google DoH",
fn: func() (string, error) {
return testHttpRequest("GET", "https://8.8.8.8/resolve?name=cloudflare.com", 200)
},
},
{
name: "resolve github.com",
fn: func() (string, error) {
return testDNSResolution("github.com")
},
},
{
name: "resolve reddit.com",
fn: func() (string, error) {
return testDNSResolution("reddit.com")
},
},
{
name: "resolve twitch.tv",
fn: func() (string, error) {
return testDNSResolution("twitch.tv")
},
},
{
name: "fetch data from YouTube RSS feed",
fn: func() (string, error) {
return testHttpRequest("GET", "https://www.youtube.com/feeds/videos.xml?channel_id=UCZU9T1ceaOgwfLRq7OKFU4Q", 200)
},
},
{
name: "fetch data from Twitch.tv GQL",
fn: func() (string, error) {
// this should always return 0 bytes, we're mainly looking for a 200 status code
return testHttpRequest("OPTIONS", "https://gql.twitch.tv/gql", 200)
},
},
{
name: "fetch data from GitHub API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://api.github.com", 200)
},
},
{
name: "fetch data from Open-Meteo API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://geocoding-api.open-meteo.com/v1/search?name=London", 200)
},
},
{
name: "fetch data from Reddit API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://www.reddit.com/search.json", 200)
},
},
{
name: "fetch data from Yahoo finance API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://query1.finance.yahoo.com/v8/finance/chart/NVDA", 200)
},
},
{
name: "fetch data from Hacker News Firebase API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://hacker-news.firebaseio.com/v0/topstories.json", 200)
},
},
{
name: "fetch data from Docker Hub API",
fn: func() (string, error) {
return testHttpRequest("GET", "https://hub.docker.com/v2/namespaces/library/repositories/ubuntu/tags/latest", 200)
},
},
}
func runDiagnostic() {
fmt.Println("```")
fmt.Println("Glance version: " + buildVersion)
fmt.Println("Go version: " + runtime.Version())
fmt.Printf("Platform: %s / %s / %d CPUs\n", runtime.GOOS, runtime.GOARCH, runtime.NumCPU())
fmt.Println("In Docker container: " + boolToString(isRunningInsideDockerContainer(), "yes", "no"))
fmt.Printf("\nChecking network connectivity, this may take up to %d seconds...\n\n", int(httpTestRequestTimeout.Seconds()))
var wg sync.WaitGroup
for i := range diagnosticSteps {
step := &diagnosticSteps[i]
wg.Add(1)
go func() {
defer wg.Done()
start := time.Now()
step.extraInfo, step.err = step.fn()
step.elapsed = time.Since(start)
}()
}
wg.Wait()
for _, step := range diagnosticSteps {
var extraInfo string
if step.extraInfo != "" {
extraInfo = "| " + step.extraInfo + " "
}
fmt.Printf(
"%s %s %s| %dms\n",
boolToString(step.err == nil, "✓ Can", "✗ Can't"),
step.name,
extraInfo,
step.elapsed.Milliseconds(),
)
if step.err != nil {
fmt.Printf("└╴ error: %v\n", step.err)
}
}
fmt.Println("```")
}
type diagnosticStep struct {
name string
fn func() (string, error)
extraInfo string
err error
elapsed time.Duration
}
func boolToString(b bool, trueValue, falseValue string) string {
if b {
return trueValue
}
return falseValue
}
func isRunningInsideDockerContainer() bool {
_, err := os.Stat("/.dockerenv")
return err == nil
}
func testHttpRequest(method, url string, expectedStatusCode int) (string, error) {
return testHttpRequestWithHeaders(method, url, nil, expectedStatusCode)
}
func testHttpRequestWithHeaders(method, url string, headers map[string]string, expectedStatusCode int) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), httpTestRequestTimeout)
defer cancel()
request, _ := http.NewRequestWithContext(ctx, method, url, nil)
for key, value := range headers {
request.Header.Add(key, value)
}
response, err := http.DefaultClient.Do(request)
if err != nil {
return "", err
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
return "", err
}
printableBody := strings.ReplaceAll(string(body), "\n", "")
if len(printableBody) > 50 {
printableBody = printableBody[:50] + "..."
}
if len(printableBody) > 0 {
printableBody = ", " + printableBody
}
extraInfo := fmt.Sprintf("%d bytes%s", len(body), printableBody)
if response.StatusCode != expectedStatusCode {
return extraInfo, fmt.Errorf("expected status code %d, got %d", expectedStatusCode, response.StatusCode)
}
return extraInfo, nil
}
func testDNSResolution(domain string) (string, error) {
ips, err := net.LookupIP(domain)
var ipStrings []string
if err == nil {
for i := range ips {
ipStrings = append(ipStrings, ips[i].String())
}
}
return strings.Join(ipStrings, ", "), err
}

View file

@ -5,7 +5,7 @@ import (
"context"
"fmt"
"html/template"
"log/slog"
"log"
"net/http"
"path/filepath"
"regexp"
@ -122,11 +122,7 @@ func (a *Application) TransformUserDefinedAssetPath(path string) string {
return path
}
func NewApplication(config *Config) (*Application, error) {
if len(config.Pages) == 0 {
return nil, fmt.Errorf("no pages configured")
}
func newApplication(config *Config) *Application {
app := &Application{
Version: buildVersion,
Config: *config,
@ -180,7 +176,7 @@ func NewApplication(config *Config) (*Application, error) {
config.Branding.LogoURL = app.TransformUserDefinedAssetPath(config.Branding.LogoURL)
return app, nil
return app
}
func (a *Application) HandlePageRequest(w http.ResponseWriter, r *http.Request) {
@ -276,7 +272,7 @@ func (a *Application) AssetPath(asset string) string {
return a.Config.Server.BaseURL + "/static/" + a.Config.Server.AssetsHash + "/" + asset
}
func (a *Application) Serve() error {
func (a *Application) Server() (func() error, func() error) {
// TODO: add gzip support, static files must have their gzipped contents cached
// TODO: add HTTPS support
mux := http.NewServeMux()
@ -295,14 +291,9 @@ func (a *Application) Serve() error {
http.StripPrefix("/static/"+a.Config.Server.AssetsHash, FileServerWithCache(http.FS(assets.PublicFS), 24*time.Hour)),
)
var absAssetsPath string
if a.Config.Server.AssetsPath != "" {
absAssetsPath, err := filepath.Abs(a.Config.Server.AssetsPath)
if err != nil {
return fmt.Errorf("invalid assets path: %s", a.Config.Server.AssetsPath)
}
slog.Info("Serving assets", "path", absAssetsPath)
absAssetsPath, _ = filepath.Abs(a.Config.Server.AssetsPath)
assetsFS := FileServerWithCache(http.Dir(a.Config.Server.AssetsPath), 2*time.Hour)
mux.Handle("/assets/{path...}", http.StripPrefix("/assets/", assetsFS))
}
@ -312,8 +303,25 @@ func (a *Application) Serve() error {
Handler: mux,
}
a.Config.Server.StartedAt = time.Now()
slog.Info("Starting server", "host", a.Config.Server.Host, "port", a.Config.Server.Port, "base-url", a.Config.Server.BaseURL)
start := func() error {
a.Config.Server.StartedAt = time.Now()
log.Printf("Starting server on %s:%d (base-url: \"%s\", assets-path: \"%s\")\n",
a.Config.Server.Host,
a.Config.Server.Port,
a.Config.Server.BaseURL,
absAssetsPath,
)
return server.ListenAndServe()
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
return err
}
return nil
}
stop := func() error {
return server.Close()
}
return start, stop
}

View file

@ -2,45 +2,120 @@ package glance
import (
"fmt"
"os"
"log"
)
func Main() int {
options, err := ParseCliOptions()
options, err := parseCliOptions()
if err != nil {
fmt.Println(err)
return 1
}
configFile, err := os.Open(options.ConfigPath)
if err != nil {
fmt.Printf("failed opening config file: %v\n", err)
return 1
}
config, err := NewConfigFromYml(configFile)
configFile.Close()
if err != nil {
fmt.Printf("failed parsing config file: %v\n", err)
return 1
}
if options.Intent == CliIntentServe {
app, err := NewApplication(config)
switch options.intent {
case cliIntentServe:
if err := serveApp(options.configPath); err != nil {
fmt.Println(err)
return 1
}
case cliIntentConfigValidate:
contents, _, err := parseYAMLIncludes(options.configPath)
if err != nil {
fmt.Printf("failed creating application: %v\n", err)
fmt.Printf("failed to parse config file: %v\n", err)
return 1
}
if err := app.Serve(); err != nil {
fmt.Printf("http server error: %v\n", err)
if _, err := newConfigFromYAML(contents); err != nil {
fmt.Printf("config file is invalid: %v\n", err)
return 1
}
case cliIntentConfigPrint:
contents, _, err := parseYAMLIncludes(options.configPath)
if err != nil {
fmt.Printf("failed to parse config file: %v\n", err)
return 1
}
fmt.Println(string(contents))
case cliIntentDiagnose:
runDiagnostic()
}
return 0
}
func serveApp(configPath string) error {
exitChannel := make(chan struct{})
// the onChange method gets called at most once per 500ms due to debouncing so we shouldn't
// need to use atomic.Bool here unless newConfigFromYAML is very slow for some reason
hadValidConfigOnStartup := false
var stopServer func() error
onChange := func(newContents []byte) {
if stopServer != nil {
log.Println("Config file changed, attempting to restart server")
}
config, err := newConfigFromYAML(newContents)
if err != nil {
log.Printf("Config file is invalid: %v", err)
if !hadValidConfigOnStartup {
close(exitChannel)
}
return
} else if !hadValidConfigOnStartup {
hadValidConfigOnStartup = true
}
app := newApplication(config)
if stopServer != nil {
if err := stopServer(); err != nil {
log.Printf("Error while trying to stop server: %v", err)
}
}
go func() {
var startServer func() error
startServer, stopServer = app.Server()
if err := startServer(); err != nil {
log.Printf("Failed to start server: %v", err)
}
}()
}
onErr := func(err error) {
log.Printf("Error watching config files: %v", err)
}
configContents, configIncludes, err := parseYAMLIncludes(configPath)
if err != nil {
return fmt.Errorf("failed to parse config file: %w", err)
}
stopWatching, err := configFilesWatcher(configPath, configContents, configIncludes, onChange, onErr)
if err == nil {
defer stopWatching()
} else {
log.Printf("Error starting file watcher, config file changes will require a manual restart. (%v)", err)
config, err := newConfigFromYAML(configContents)
if err != nil {
return fmt.Errorf("could not parse config file: %w", err)
}
app := newApplication(config)
startServer, _ := app.Server()
if err := startServer(); err != nil {
return fmt.Errorf("failed to start server: %w", err)
}
}
<-exitChannel
return nil
}