AUTH-2596 added new logger package and replaced logrus

logger/create.go

@@ -0,0 +1,150 @@
+package logger
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/alecthomas/units"
+)
+
+// Option is to encaspulate actions that will be called by Parse and run later to build an Options struct
+type Option func(*Options) error
+
+// Options is use to set logging configuration data
+type Options struct {
+	logFileDirectory        string
+	maxFileSize             units.Base2Bytes
+	maxFileCount            uint
+	terminalOutputDisabled  bool
+	supportedFileLevels     []Level
+	supportedTerminalLevels []Level
+}
+
+// DisableTerminal stops terminal output for the logger
+func DisableTerminal(disable bool) Option {
+	return func(c *Options) error {
+		c.terminalOutputDisabled = disable
+		return nil
+	}
+}
+
+// File sets a custom file to log events
+func File(path string, size units.Base2Bytes, count uint) Option {
+	return func(c *Options) error {
+		c.logFileDirectory = path
+		c.maxFileSize = size
+		c.maxFileCount = count
+		return nil
+	}
+}
+
+// DefaultFile configures the log options will the defaults
+func DefaultFile(directoryPath string) Option {
+	return func(c *Options) error {
+		size, err := units.ParseBase2Bytes("1MB")
+		if err != nil {
+			return err
+		}
+
+		c.logFileDirectory = directoryPath
+		c.maxFileSize = size
+		c.maxFileCount = 5
+		return nil
+	}
+}
+
+// SupportedFileLevels sets the supported logging levels for the log file
+func SupportedFileLevels(supported []Level) Option {
+	return func(c *Options) error {
+		c.supportedFileLevels = supported
+		return nil
+	}
+}
+
+// SupportedTerminalevels sets the supported logging levels for the terminal output
+func SupportedTerminalevels(supported []Level) Option {
+	return func(c *Options) error {
+		c.supportedTerminalLevels = supported
+		return nil
+	}
+}
+
+// LogLevelString sets the supported logging levels from a command line flag
+func LogLevelString(level string) Option {
+	return func(c *Options) error {
+		supported, err := ParseLevelString(level)
+		if err != nil {
+			return err
+		}
+		c.supportedFileLevels = supported
+		c.supportedTerminalLevels = supported
+		return nil
+	}
+}
+
+// Parse builds the Options struct so the caller knows what actions should be run
+func Parse(opts ...Option) (*Options, error) {
+	options := &Options{}
+	for _, opt := range opts {
+		if err := opt(options); err != nil {
+			return nil, err
+		}
+	}
+	return options, nil
+}
+
+// New setups a new logger based on the options.
+// The default behavior is to write to standard out
+func New(opts ...Option) (Service, error) {
+	config, err := Parse(opts...)
+
+	if err != nil {
+		return nil, err
+	}
+
+	l := NewOutputWriter(SharedWriteManager)
+	if config.logFileDirectory != "" {
+		l.Add(NewFileRollingWriter(config.logFileDirectory,
+			"cloudflared",
+			int64(config.maxFileSize),
+			config.maxFileCount),
+			NewDefaultFormatter(time.RFC3339Nano), config.supportedFileLevels...)
+	}
+
+	if !config.terminalOutputDisabled {
+		if len(config.supportedTerminalLevels) == 0 {
+			l.Add(os.Stdout, NewTerminalFormatter(""), InfoLevel)
+			l.Add(os.Stderr, NewTerminalFormatter(""), ErrorLevel, FatalLevel)
+		} else {
+			errLevels := []Level{}
+			outLevels := []Level{}
+			for _, level := range config.supportedTerminalLevels {
+				if level == ErrorLevel || level == FatalLevel {
+					errLevels = append(errLevels, level)
+				} else {
+					outLevels = append(outLevels, level)
+				}
+			}
+			l.Add(os.Stdout, NewTerminalFormatter(""), outLevels...)
+			l.Add(os.Stderr, NewTerminalFormatter(""), errLevels...)
+		}
+	}
+	return l, nil
+}
+
+// ParseLevelString returns the expected log levels based on the cmd flag
+func ParseLevelString(lvl string) ([]Level, error) {
+	switch strings.ToLower(lvl) {
+	case "fatal":
+		return []Level{FatalLevel}, nil
+	case "error":
+		return []Level{FatalLevel, ErrorLevel}, nil
+	case "info":
+		return []Level{FatalLevel, ErrorLevel, InfoLevel}, nil
+	case "debug":
+		return []Level{FatalLevel, ErrorLevel, InfoLevel, DebugLevel}, nil
+	}
+	return []Level{}, fmt.Errorf("not a valid log level: %q", lvl)
+}

logger/file_writer.go

@@ -0,0 +1,105 @@
+package logger
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// FileRollingWriter maintains a set of log files numbered in order
+// to keep a subset of log data to ensure it doesn't grow pass defined limits
+type FileRollingWriter struct {
+	baseFileName string
+	directory    string
+	maxFileSize  int64
+	maxFileCount uint
+	fileHandle   *os.File
+}
+
+// NewFileRollingWriter creates a new rolling file writer.
+// directory is the working directory for the files
+// baseFileName is the log file name. This writer appends .log to the name for the file name
+// maxFileSize is the size in bytes of how large each file can be. Not a hard limit, general limit based after each write
+// maxFileCount is the number of rolled files to keep.
+func NewFileRollingWriter(directory, baseFileName string, maxFileSize int64, maxFileCount uint) *FileRollingWriter {
+	return &FileRollingWriter{
+		directory:    directory,
+		baseFileName: baseFileName,
+		maxFileSize:  maxFileSize,
+		maxFileCount: maxFileCount,
+	}
+}
+
+// Write is an implementation of io.writer the rolls the file once it reaches its max size
+// It is expected the caller to Write is doing so in a thread safe manner (as WriteManager does).
+func (w *FileRollingWriter) Write(p []byte) (n int, err error) {
+	logFile := buildPath(w.directory, w.baseFileName)
+	if w.fileHandle == nil {
+		h, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
+		if err != nil {
+			return 0, err
+		}
+		w.fileHandle = h
+	}
+
+	// get size for rolling check
+	info, err := w.fileHandle.Stat()
+	if err != nil {
+		// failed to stat the file. Close the file handle and attempt to open a new handle on the next write
+		w.Close()
+		w.fileHandle = nil
+		return 0, err
+	}
+
+	// write to the file
+	written, err := w.fileHandle.Write(p)
+
+	// check if the file needs to be rolled
+	if err == nil && info.Size()+int64(written) > w.maxFileSize {
+		// close the file handle than do the renaming. A new one will be opened on the next write
+		w.Close()
+		w.rename(logFile, 1)
+	}
+
+	return written, err
+}
+
+// Close closes the file handle if it is open
+func (w *FileRollingWriter) Close() {
+	if w.fileHandle != nil {
+		w.fileHandle.Close()
+		w.fileHandle = nil
+	}
+}
+
+// rename is how the files are rolled. It works recursively to move the base log file to the rolled ones
+// e.g. cloudflared.log -> cloudflared-1.log,
+// but if cloudflared-1.log already exists, it is renamed to cloudflared-2.log,
+// then the other files move in to their postion
+func (w *FileRollingWriter) rename(sourcePath string, index uint) {
+	destinationPath := buildPath(w.directory, fmt.Sprintf("%s-%d", w.baseFileName, index))
+
+	// rolled to the max amount of files allowed on disk
+	if index >= w.maxFileCount {
+		os.Remove(destinationPath)
+	}
+
+	// if the rolled path already exist, rename it to cloudflared-2.log, then do this one.
+	// recursive call since the oldest one needs to be renamed, before the newer ones can be moved
+	if exists(destinationPath) {
+		w.rename(destinationPath, index+1)
+	}
+
+	os.Rename(sourcePath, destinationPath)
+}
+
+func buildPath(directory, fileName string) string {
+	return filepath.Join(directory, fileName+".log")
+}
+
+func exists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}

logger/file_writer_test.go

@@ -0,0 +1,56 @@
+package logger
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileWrite(t *testing.T) {
+	fileName := "test_file"
+	fileLog := fileName + ".log"
+	testData := []byte(string("hello Dalton, how are you doing?"))
+	defer func() {
+		os.Remove(fileLog)
+	}()
+
+	w := NewFileRollingWriter("", fileName, 1000, 2)
+	defer w.Close()
+
+	l, err := w.Write(testData)
+
+	assert.NoError(t, err)
+	assert.Equal(t, l, len(testData), "expected write length and data length to match")
+
+	d, err := ioutil.ReadFile(fileLog)
+	assert.FileExists(t, fileLog, "file doesn't exist at expected path")
+	assert.Equal(t, d, testData, "expected data in file to match test data")
+}
+
+func TestRolling(t *testing.T) {
+	fileName := "test_file"
+	firstFile := fileName + ".log"
+	secondFile := fileName + "-1.log"
+	thirdFile := fileName + "-2.log"
+
+	defer func() {
+		os.Remove(firstFile)
+		os.Remove(secondFile)
+		os.Remove(thirdFile)
+	}()
+
+	w := NewFileRollingWriter("", fileName, 1000, 2)
+	defer w.Close()
+
+	for i := 99; i >= 1; i-- {
+		testData := []byte(fmt.Sprintf("%d bottles of beer on the wall...", i))
+		w.Write(testData)
+	}
+	assert.FileExists(t, firstFile, "first file doesn't exist as expected")
+	assert.FileExists(t, secondFile, "second file doesn't exist as expected")
+	assert.FileExists(t, thirdFile, "third file doesn't exist as expected")
+	assert.False(t, exists(fileName+"-3.log"), "limited to two files and there is more")
+}

logger/formatter.go

@@ -0,0 +1,91 @@
+package logger
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/acmacalister/skittles"
+)
+
+// Level of logging
+type Level int
+
+const (
+	// InfoLevel is for standard log messages
+	InfoLevel Level = iota
+
+	// DebugLevel is for messages that are intended for purposes debugging only
+	DebugLevel
+
+	// ErrorLevel is for error message to indicte something has gone wrong
+	ErrorLevel
+
+	// FatalLevel is for error message that log and kill the program with an os.exit(1)
+	FatalLevel
+)
+
+// Formatter is the base interface for formatting logging messages before writing them out
+type Formatter interface {
+	Timestamp(Level, time.Time) string // format the timestamp string
+	Content(Level, string) string      // format content string (color for terminal, etc)
+}
+
+// DefaultFormatter writes a simple structure timestamp and the message per log line
+type DefaultFormatter struct {
+	format string
+}
+
+// NewDefaultFormatter creates the standard log formatter
+// format is the time format to use for timestamp formatting
+func NewDefaultFormatter(format string) Formatter {
+	return &DefaultFormatter{
+		format: format,
+	}
+}
+
+// Timestamp formats a log line timestamp with a brackets around them
+func (f *DefaultFormatter) Timestamp(l Level, d time.Time) string {
+	if f.format == "" {
+		return ""
+	}
+	return fmt.Sprintf("[%s]: ", d.Format(f.format))
+}
+
+// Content just writes the log line straight to the sources
+func (f *DefaultFormatter) Content(l Level, c string) string {
+	return c
+}
+
+// TerminalFormatter is setup for colored output
+type TerminalFormatter struct {
+	format string
+}
+
+// NewTerminalFormatter creates a Terminal formatter for colored output
+// format is the time format to use for timestamp formatting
+func NewTerminalFormatter(format string) Formatter {
+	return &TerminalFormatter{
+		format: format,
+	}
+}
+
+// Timestamp returns the log level with a matching color to the log type
+func (f *TerminalFormatter) Timestamp(l Level, d time.Time) string {
+	t := ""
+	switch l {
+	case InfoLevel:
+		t = skittles.Cyan("[INFO] ")
+	case ErrorLevel:
+		t = skittles.Red("[ERROR] ")
+	case DebugLevel:
+		t = skittles.Yellow("[DEBUG] ")
+	case FatalLevel:
+		t = skittles.Red("[FATAL] ")
+	}
+	return t
+}
+
+// Content just writes the log line straight to the sources
+func (f *TerminalFormatter) Content(l Level, c string) string {
+	return c
+}

logger/manager.go

@@ -0,0 +1,59 @@
+package logger
+
+import "sync"
+
+// SharedWriteManager is a package level variable to allows multiple loggers to use the same write manager.
+// This is useful when multiple loggers will write to the same file to ensure they don't clobber each other.
+var SharedWriteManager = NewWriteManager()
+
+type writeData struct {
+	writeFunc func([]byte)
+	data      []byte
+}
+
+// WriteManager is a logging service that handles managing multiple writing streams
+type WriteManager struct {
+	shutdown  chan struct{}
+	writeChan chan writeData
+	writers   map[string]Service
+	wg        sync.WaitGroup
+}
+
+// NewWriteManager creates a write manager that implements OutputManager
+func NewWriteManager() OutputManager {
+	m := &WriteManager{
+		shutdown:  make(chan struct{}),
+		writeChan: make(chan writeData, 1000),
+	}
+
+	go m.run()
+	return m
+}
+
+// Append adds a message to the writer runloop
+func (m *WriteManager) Append(data []byte, callback func([]byte)) {
+	m.wg.Add(1)
+	m.writeChan <- writeData{data: data, writeFunc: callback}
+}
+
+// Shutdown stops the sync manager service
+func (m *WriteManager) Shutdown() {
+	m.wg.Wait()
+	close(m.shutdown)
+	close(m.writeChan)
+}
+
+// run is the main runloop that schedules log messages
+func (m *WriteManager) run() {
+	for {
+		select {
+		case event, ok := <-m.writeChan:
+			if ok {
+				event.writeFunc(event.data)
+				m.wg.Done()
+			}
+		case <-m.shutdown:
+			return
+		}
+	}
+}

logger/manager_test.go

@@ -0,0 +1,18 @@
+package logger
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriteManger(t *testing.T) {
+	testData := []byte(string("hello Austin, how are you doing?"))
+	waitChan := make(chan []byte)
+	m := NewWriteManager()
+	m.Append(testData, func(b []byte) {
+		waitChan <- b
+	})
+	resp := <-waitChan
+	assert.Equal(t, testData, resp)
+}

logger/mock_manager.go

@@ -0,0 +1,18 @@
+package logger
+
+// MockWriteManager does nothing and is provided for testing purposes
+type MockWriteManager struct {
+}
+
+// NewMockWriteManager creates an OutputManager that does nothing for testing purposes
+func NewMockWriteManager() OutputManager {
+	return &MockWriteManager{}
+}
+
+// Append is a mock stub
+func (m *MockWriteManager) Append(data []byte, callback func([]byte)) {
+}
+
+// Shutdown is a mock stub
+func (m *MockWriteManager) Shutdown() {
+}

logger/output.go

@@ -0,0 +1,132 @@
+package logger
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+// provided for testing
+var osExit = os.Exit
+
+// OutputManager is used to sync data of Output
+type OutputManager interface {
+	Append([]byte, func([]byte))
+	Shutdown()
+}
+
+// Service is the logging service that is either a group or single log writer
+type Service interface {
+	Error(message string)
+	Info(message string)
+	Debug(message string)
+	Fatal(message string)
+
+	Errorf(format string, args ...interface{})
+	Infof(format string, args ...interface{})
+	Debugf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+
+type sourceGroup struct {
+	writer          io.Writer
+	formatter       Formatter
+	levelsSupported []Level
+}
+
+// OutputWriter is the standard logging implementation
+type OutputWriter struct {
+	groups     []sourceGroup
+	syncWriter OutputManager
+}
+
+// NewOutputWriter create a new logger
+func NewOutputWriter(syncWriter OutputManager) *OutputWriter {
+	return &OutputWriter{
+		syncWriter: syncWriter,
+		groups:     make([]sourceGroup, 0),
+	}
+}
+
+// Add a writer and formatter to output to
+func (s *OutputWriter) Add(writer io.Writer, formatter Formatter, levels ...Level) {
+	s.groups = append(s.groups, sourceGroup{writer: writer, formatter: formatter, levelsSupported: levels})
+}
+
+// Error writes an error to the logging sources
+func (s *OutputWriter) Error(message string) {
+	s.output(ErrorLevel, message)
+}
+
+// Info writes an info string to the logging sources
+func (s *OutputWriter) Info(message string) {
+	s.output(InfoLevel, message)
+}
+
+// Debug writes a debug string to the logging sources
+func (s *OutputWriter) Debug(message string) {
+	s.output(DebugLevel, message)
+}
+
+// Fatal writes a error string to the logging sources and runs does an os.exit()
+func (s *OutputWriter) Fatal(message string) {
+	s.output(FatalLevel, message)
+	s.syncWriter.Shutdown() // waits for the pending logging to finish
+	osExit(1)
+}
+
+// Errorf writes a formatted error to the logging sources
+func (s *OutputWriter) Errorf(format string, args ...interface{}) {
+	s.output(ErrorLevel, fmt.Sprintf(format, args...))
+}
+
+// Infof writes a formatted info statement to the logging sources
+func (s *OutputWriter) Infof(format string, args ...interface{}) {
+	s.output(InfoLevel, fmt.Sprintf(format, args...))
+}
+
+// Debugf writes a formatted debug statement to the logging sources
+func (s *OutputWriter) Debugf(format string, args ...interface{}) {
+	s.output(DebugLevel, fmt.Sprintf(format, args...))
+}
+
+// Fatalf writes a  writes a formatted error statement and runs does an os.exit()
+func (s *OutputWriter) Fatalf(format string, args ...interface{}) {
+	s.output(FatalLevel, fmt.Sprintf(format, args...))
+	s.syncWriter.Shutdown() // waits for the pending logging to finish
+	osExit(1)
+}
+
+// output does the actual write to the sync manager
+func (s *OutputWriter) output(l Level, content string) {
+	for _, group := range s.groups {
+		if isSupported(group, l) {
+			logLine := fmt.Sprintf("%s%s\n", group.formatter.Timestamp(l, time.Now()),
+				group.formatter.Content(l, content))
+			s.append(group, []byte(logLine))
+		}
+	}
+}
+
+func (s *OutputWriter) append(group sourceGroup, logLine []byte) {
+	s.syncWriter.Append(logLine, func(b []byte) {
+		group.writer.Write(b)
+	})
+}
+
+// isSupported checks if the log level is supported
+func isSupported(group sourceGroup, l Level) bool {
+	for _, level := range group.levelsSupported {
+		if l == level {
+			return true
+		}
+	}
+	return false
+}
+
+// Write implements io.Writer to support SetOutput of the log package
+func (s *OutputWriter) Write(p []byte) (n int, err error) {
+	s.Info(string(p))
+	return len(p), nil
+}

logger/output_test.go

@@ -0,0 +1,104 @@
+package logger
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLogLevel(t *testing.T) {
+	timeFormat := "2006-01-02"
+	f := NewDefaultFormatter(timeFormat)
+	m := NewWriteManager()
+
+	var testBuffer bytes.Buffer
+	logger := NewOutputWriter(m)
+	logger.Add(&testBuffer, f, InfoLevel, DebugLevel)
+
+	testTime := f.Timestamp(InfoLevel, time.Now())
+
+	testInfo := "hello Dalton, how are you doing?"
+	logger.Info(testInfo)
+
+	tesErr := "hello Austin, how did it break today?"
+	logger.Error(tesErr)
+
+	testDebug := "hello Bill, who are you?"
+	logger.Debug(testDebug)
+
+	m.Shutdown()
+
+	lines := strings.Split(testBuffer.String(), "\n")
+	assert.Len(t, lines, 3, "only expected two strings in the buffer")
+
+	infoLine := lines[0]
+	debugLine := lines[1]
+
+	compareInfo := fmt.Sprintf("%s%s", testTime, testInfo)
+	assert.Equal(t, compareInfo, infoLine, "expect the strings to match")
+
+	compareDebug := fmt.Sprintf("%s%s", testTime, testDebug)
+	assert.Equal(t, compareDebug, debugLine, "expect the strings to match")
+}
+
+func TestOutputWrite(t *testing.T) {
+	timeFormat := "2006-01-02"
+	f := NewDefaultFormatter(timeFormat)
+	m := NewWriteManager()
+
+	var testBuffer bytes.Buffer
+	logger := NewOutputWriter(m)
+	logger.Add(&testBuffer, f, InfoLevel)
+
+	testData := "hello Bob Bork, how are you doing?"
+	logger.Info(testData)
+	testTime := f.Timestamp(InfoLevel, time.Now())
+
+	m.Shutdown()
+
+	scanner := bufio.NewScanner(&testBuffer)
+	scanner.Scan()
+	line := scanner.Text()
+	assert.NoError(t, scanner.Err())
+
+	compareLine := fmt.Sprintf("%s%s", testTime, testData)
+	assert.Equal(t, compareLine, line, "expect the strings to match")
+}
+
+func TestFatalWrite(t *testing.T) {
+	timeFormat := "2006-01-02"
+	f := NewDefaultFormatter(timeFormat)
+	m := NewWriteManager()
+
+	var testBuffer bytes.Buffer
+	logger := NewOutputWriter(m)
+	logger.Add(&testBuffer, f, FatalLevel)
+
+	oldOsExit := osExit
+	defer func() { osExit = oldOsExit }()
+
+	var got int
+	myExit := func(code int) {
+		got = code
+	}
+
+	osExit = myExit
+
+	testData := "so long y'all"
+	logger.Fatal(testData)
+	testTime := f.Timestamp(FatalLevel, time.Now())
+
+	scanner := bufio.NewScanner(&testBuffer)
+	scanner.Scan()
+	line := scanner.Text()
+	assert.NoError(t, scanner.Err())
+
+	compareLine := fmt.Sprintf("%s%s", testTime, testData)
+	assert.Equal(t, compareLine, line, "expect the strings to match")
+	assert.Equal(t, got, 1, "exit code should be one for a fatal log")
+}