Added logging to file feature

.woodpecker.yml

@@ -1,10 +1,24 @@
 pipeline:
-  build:
+  build_test:
+    image: golang:1.16
+    commands:
+      - go build
+
+  test:
+    image: golang:1.16
+    commands:
+      - apt update
+      - apt install -f lsof
+      - go test -v ./...
+
+  build_release:
     image: golang:1.16
     commands:
       - go mod vendor
       - GOOS=linux GOARCH=amd64 go build -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
       - GOOS=windows GOARCH=amd64 go build -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
+    when:
+      event: tag
 
   gitea_release:
     image: plugins/gitea-release

Makefile

@@ -0,0 +1,37 @@
+BINARY_NAME=adept
+
+all: build test
+
+build:
+	go build -o ${BINARY_NAME} adept.go
+
+test:
+	go test -v ./...
+
+run:
+	go build -o ${BINARY_NAME} adept.go
+	./${BINARY_NAME}
+
+install:
+	useradd ${BINARY_NAME}
+	usermod -aG ${BINARY_NAME} ${BINARY_NAME}
+	cp ./${BINARY_NAME} /usr/bin/${BINARY_NAME}
+	chown root:root /usr/bin/${BINARY_NAME}
+	chmod 755 /usr/bin/${BINARY_NAME}
+	cp ./build/etc/systemd/system/${BINARY_NAME}.service /etc/systemd/system/${BINARY_NAME}.service
+	chown root:root /etc/systemd/system/${BINARY_NAME}.service
+	chmod 644 /etc/systemd/system/${BINARY_NAME}.service
+	mkdir /etc/${BINARY_NAME}
+	cp ./build/etc/${BINARY_NAME}/${BINARY_NAME}.toml /etc/${BINARY_NAME}/
+	chown -R ${BINARY_NAME}:${BINARY_NAME} /etc/${BINARY_NAME}
+	chmod 644 /etc/${BINARY_NAME}/${BINARY_NAME}.toml
+
+uninstall:
+	userdel ${BINARY_NAME}
+	rm /usr/bin/${BINARY_NAME}
+	rm /etc/systemd/system/${BINARY_NAME}.service
+	rm -rf /etc/${BINARY_NAME}
+ 
+clean:
+	go clean
+	rm ${BINARY_NAME}

README.md

@@ -1,3 +1,24 @@
 # adept
 
-Bit Goblin video transcoder service
+Bit Goblin video transcoder service
+
+## Installation
+
+**Note: currently Adept will only run on Linux as `lsof` is a runtime dependency, which doesn't have a Windows equivalent. This will be rectified in the future.**
+
+Build dependencies:
+* go
+* make
+
+Runtime dependencies:
+* ffmpeg
+* lsof
+
+To install dependencies on Ubuntu: `apt install golang make ffmpeg lsof`
+To install dependencies on Red Hat/AlmaLinux: `dnf install go make ffmpeg lsof`
+
+To install Adept as a system service: `make build && sudo make install`
+
+## Uninstallation
+
+To uninstall Adept (if it was installed through make): `sudo make uninstall`

adept.go

@@ -1,24 +1,55 @@
 package main
 
 import (
+  "log"
+  "path/filepath"
+  "time"
+
   "github.com/spf13/viper"
 
   "git.metaunix.net/BitGoblin/adept/config"
   "git.metaunix.net/BitGoblin/adept/transcoder"
+  "git.metaunix.net/BitGoblin/adept/util"
 )
 
 func main() {
+  // load configuration via Viper
   config.LoadConfig()
 
+  // configure our app logging
+  logHandle := config.InitLogging()
+  if logHandle != nil {
+    defer logHandle.Close()
+  }
+
+  // initialize the video repository
   r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
 
-  ingestFiles := r.SearchIngest()
+  // main program loop - runs infinitely until externally terminated
-  for _, i := range ingestFiles {
+  for {
-    // archive file
+    ingestFiles := r.SearchIngest()
-    r.ArchiveFile(i.Name())
+
-    // transcode file
+    if len(ingestFiles) < 1 {
-    transcoder.Transcode(i.Name())
+      log.Println("There were no files found in ingest to transcode; skipping run.")
-    // clean up source file
+    } else {
-    r.CleanupFile(i.Name())
+      for _, i := range ingestFiles {
+        // check if the file is open in another program (e.g. still being written to)
+        if util.IsFileLocked(filepath.Join(viper.GetString("transcoder.repository"), "ingest", i.Name())) {
+          log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
+          continue
+        }
+
+        // archive file
+        r.ArchiveFile(i.Name())
+        // transcode file
+        transcoder.Transcode(i.Name())
+        // clean up source file
+        r.CleanupFile(i.Name())
+      }
+    }
+
+    // sleep for X minutes - specified in the adept.toml config file
+    interval := viper.GetInt("transcoder.interval")
+    time.Sleep(time.Duration(interval) * time.Minute)
   }
 }

build/etc/systemd/system/adept.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Adept video transcoder service
+
+[Service]
+User=adept
+Group=adept
+ExecStart=/usr/bin/adept
+SuccessExitStatus=143
+
+[Install]
+WantedBy=multi-user.target

config/log.go

@@ -0,0 +1,29 @@
+package config
+
+import (
+	"log"
+	"os"
+
+	"github.com/spf13/viper"
+
+	"git.metaunix.net/BitGoblin/adept/util"
+)
+
+func InitLogging() *os.File {
+	var fileHandle *os.File = nil
+
+	if viper.GetBool("log_to_file") {
+		// open a file
+		var err error
+		fileHandle, err = os.OpenFile(util.ResolveTilde(viper.GetString("log_file")), os.O_APPEND | os.O_CREATE | os.O_RDWR, 0644)
+		if err != nil {
+		  log.Fatalf("Error opening log file: %v", err)
+		  os.Exit(1)
+		}
+
+		// set logging to file handle
+		log.SetOutput(fileHandle)
+	}
+
+	return fileHandle
+}

go.mod

@@ -14,10 +14,10 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.12.0
+	github.com/stretchr/testify v1.8.0 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
 )

go.sum

@@ -95,6 +95,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -301,6 +302,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
@@ -344,6 +346,8 @@ github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=
 github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -351,6 +355,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI=
 github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -842,6 +848,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

transcoder/repository.go

@@ -30,6 +30,8 @@ func NewRepository(path string) *Repository {
 func (r *Repository) SearchIngest() []os.FileInfo {
   ingestPath := filepath.Join(r.basePath, "ingest")
 
+  log.Printf("Searching ingest directory for files to transcode...")
+
   ingestDir, err := os.Open(ingestPath)
   if err != nil {
     log.Fatalf("Error opening ingest directory: %s", err)

util/env_test.go

@@ -0,0 +1,29 @@
+package util
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+  "github.com/stretchr/testify/suite"
+)
+
+// define our test suite struct
+type EnvTestSuite struct {
+	suite.Suite
+}
+
+// the tilde should expand to user's home directory
+func (s *EnvTestSuite) TestResolveTilde() {
+	resolvedPath := ResolveTilde("~")
+	assert.NotEqual(s.T(), resolvedPath, "~")
+}
+
+// ensure the tilde + relative path gets expanded fully
+func (s *EnvTestSuite) TestResolveTildePath() {
+	resolvedPath := ResolveTilde("~/test")
+	assert.NotEqual(s.T(), resolvedPath, "~/test")
+}
+
+// this is needed to run the test suite
+func TestEnvTestSuite(t *testing.T) {
+	suite.Run(t, new(EnvTestSuite))
+}

util/file.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+  "os/exec"
   "path"
   "strings"
 )
@@ -8,3 +9,14 @@ import (
 func FilenameWithoutExtension(filename string) string {
   return strings.TrimSuffix(filename, path.Ext(filename))
 }
+
+func IsFileLocked(filepath string) bool {
+  cmd := exec.Command("/usr/bin/lsof", filepath)
+  stdout, _ := cmd.Output()
+
+  if strings.Contains(string(stdout), filepath) {
+    return true
+  }
+
+  return false
+}

util/file_test.go

@@ -0,0 +1,64 @@
+package util
+
+import (
+	"os"
+	"testing"
+	"github.com/stretchr/testify/assert"
+  "github.com/stretchr/testify/suite"
+)
+
+// define our test suite struct
+type FileTestSuite struct {
+	suite.Suite
+	TestFile string
+}
+
+// run before tests to set up
+func (s *FileTestSuite) SetupSuite() {
+	s.TestFile = "testfile.txt"
+	// create the test file for file lock testing
+	file, _ := os.Create(s.TestFile)
+	file.Close() // do this just to make extra sure the file handle is closed
+}
+// run after tests to clean up
+func (s *FileTestSuite) TearDownSuite() {
+	// remove the test file since it's no longer needed
+	os.Remove(s.TestFile)
+}
+
+// test the filename extension removal works
+func (s *FileTestSuite) TestFilenameWithoutExtension() {
+	filename := FilenameWithoutExtension(s.TestFile)
+
+	if filename != "testfile" {
+		s.T().Logf("FilenameWithoutExtension returned '%s'; it should be 'testfile'.", filename)
+	}
+}
+
+// test that IsFileLocked returns true when the file is active
+func (s *FileTestSuite) TestFileShouldBeLocked() {
+	file, err := os.Open(s.TestFile)
+	if err != nil {
+		s.T().Logf("Unable to open file %s: %s", s.TestFile, err)
+	}
+
+	assert.True(s.T(), IsFileLocked(s.TestFile))
+
+	file.Close()
+}
+
+// test that IsFileLocked returns false when the file is not active
+func (s *FileTestSuite) TestFileShouldNotBeLocked() {
+	file, err := os.Open(s.TestFile)
+	if err != nil {
+		s.T().Logf("Unable to open file %s: %s", s.TestFile, err)
+	}
+	file.Close() // we want this closed now so it's NOT open!
+
+	assert.False(s.T(), IsFileLocked(s.TestFile))
+}
+
+// this is needed to run the test suite
+func TestFileTestSuite(t *testing.T) {
+	suite.Run(t, new(FileTestSuite))
+}