Fixed the transcode job to rename the output file to the proper extension

.gitignore

@@ -16,7 +16,7 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 
 # Go workspace file
 go.work

.woodpecker.yml

@@ -1,10 +1,24 @@
 pipeline:
-  build:
+  build_test:
+    image: golang:1.16
+    commands:
+      - go build
+
+  test:
+    image: golang:1.16
+    commands:
+      - apt update
+      - apt install -f lsof
+      - go test -v ./...
+
+  build_release:
     image: golang:1.16
     commands:
       - go mod vendor
       - GOOS=linux GOARCH=amd64 go build -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
       - GOOS=windows GOARCH=amd64 go build -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
+    when:
+      event: tag
 
   gitea_release:
     image: plugins/gitea-release

Makefile

@@ -0,0 +1,37 @@
+BINARY_NAME=adept
+
+all: build test
+
+build:
+	go build -o ${BINARY_NAME} adept.go
+
+test:
+	go test -v ./...
+
+run:
+	go build -o ${BINARY_NAME} adept.go
+	./${BINARY_NAME}
+
+install:
+	useradd ${BINARY_NAME}
+	usermod -aG ${BINARY_NAME} ${BINARY_NAME}
+	cp ./${BINARY_NAME} /usr/bin/${BINARY_NAME}
+	chown root:root /usr/bin/${BINARY_NAME}
+	chmod 755 /usr/bin/${BINARY_NAME}
+	cp ./build/etc/systemd/system/${BINARY_NAME}.service /etc/systemd/system/${BINARY_NAME}.service
+	chown root:root /etc/systemd/system/${BINARY_NAME}.service
+	chmod 644 /etc/systemd/system/${BINARY_NAME}.service
+	mkdir /etc/${BINARY_NAME}
+	cp ./build/etc/${BINARY_NAME}/${BINARY_NAME}.toml /etc/${BINARY_NAME}/
+	chown -R ${BINARY_NAME}:${BINARY_NAME} /etc/${BINARY_NAME}
+	chmod 644 /etc/${BINARY_NAME}/${BINARY_NAME}.toml
+
+uninstall:
+	userdel ${BINARY_NAME}
+	rm /usr/bin/${BINARY_NAME}
+	rm /etc/systemd/system/${BINARY_NAME}.service
+	rm -rf /etc/${BINARY_NAME}
+ 
+clean:
+	go clean
+	rm ${BINARY_NAME}

README.md

@@ -1,3 +1,24 @@
 # adept
 
-Bit Goblin video transcoder service
+Bit Goblin video transcoder service
+
+## Installation
+
+**Note: currently Adept will only run on Linux as `lsof` is a runtime dependency, which doesn't have a Windows equivalent. This will be rectified in the future.**
+
+Build dependencies:
+* go
+* make
+
+Runtime dependencies:
+* ffmpeg
+* lsof
+
+To install dependencies on Ubuntu: `apt install golang make ffmpeg lsof`
+To install dependencies on Red Hat/AlmaLinux: `dnf install go make ffmpeg lsof`
+
+To install Adept as a system service: `make build && sudo make install`
+
+## Uninstallation
+
+To uninstall Adept (if it was installed through make): `sudo make uninstall`

adept.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-  "time"
-
   "github.com/spf13/viper"
 
   "git.metaunix.net/BitGoblin/adept/config"
@@ -10,25 +8,19 @@ import (
 )
 
 func main() {
+  // load configuration via Viper
   config.LoadConfig()
 
+  // configure our app logging
+  logHandle := config.InitLogging()
+  if logHandle != nil {
+    defer logHandle.Close()
+  }
+
+  // initialize the video repository
   r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
 
-  // main program loop - runs infinitely until externally terminated
+  // create transcoder object and start the main loop
-  for {
+  t := transcoder.NewTranscoder(*r)
-    ingestFiles := r.SearchIngest()
+  t.Start()
-
-    for _, i := range ingestFiles {
-      // archive file
-      r.ArchiveFile(i.Name())
-      // transcode file
-      transcoder.Transcode(i.Name())
-      // clean up source file
-      r.CleanupFile(i.Name())
-    }
-
-    // sleep for X minutes - specified in the adept.toml config file
-    interval := viper.GetInt("transcoder.interval")
-    time.Sleep(time.Duration(interval) * time.Minute)
-  }
 }

build/etc/adept/adept.toml

@@ -0,0 +1,15 @@
+log_to_file = true
+log_file = '~/adept/adept.log'
+log_level = 'info'
+
+[transcoder]
+repository = '~/adept'
+interval = 15
+video_format = 'mov'
+video_codec = 'dnxhd'
+video_profile = 'dnxhr_hq'
+video_resolution = '1920x1080'
+video_framerate = 60
+video_color = 'yuv422p'
+audio_codec = 'pcm_s16le'
+

build/etc/systemd/system/adept.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Adept video transcoder service
+
+[Service]
+User=adept
+Group=adept
+ExecStart=/usr/bin/adept
+SuccessExitStatus=143
+
+[Install]
+WantedBy=multi-user.target

config/log.go

@@ -0,0 +1,32 @@
+package config
+
+import (
+	"io"
+	"log"
+	"os"
+
+	"github.com/spf13/viper"
+
+	"git.metaunix.net/BitGoblin/adept/util"
+)
+
+func InitLogging() *os.File {
+	var fileHandle *os.File = nil
+
+	if viper.GetBool("log_to_file") {
+		// open a file
+		var err error
+		fileHandle, err = os.OpenFile(util.ResolveTilde(viper.GetString("log_file")), os.O_APPEND | os.O_CREATE | os.O_RDWR, 0644)
+		if err != nil {
+		  log.Fatalf("Error opening log file: %v", err)
+		  os.Exit(1)
+		}
+	}
+
+	// create a MultiWriter instance so we can write to both console AND file
+	mw := io.MultiWriter(os.Stdout, fileHandle)
+	// set our multiwriter object as the output for logging
+	log.SetOutput(mw)
+
+	return fileHandle
+}

go.mod

@@ -14,10 +14,10 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.12.0
+	github.com/stretchr/testify v1.8.0 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
 )

go.sum

@@ -95,6 +95,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -301,6 +302,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
@@ -344,6 +346,8 @@ github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=
 github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -351,6 +355,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI=
 github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -842,6 +848,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

transcoder/repository.go

@@ -8,31 +8,41 @@ import (
 )
 
 type Repository struct {
-  basePath string
+  basePath    string
+  ingestPath  string
+  archivePath string
+  outputPath  string
 }
 
 func NewRepository(path string) *Repository {
-  // make sure repository base directory exists
-  create_repo_dir(path)
-
-  // make sure the ingest, archive, and output directories exist
-  subDirs := []string{"ingest", "archive", "output"}
-  for _, d := range subDirs {
-    subPath := filepath.Join(path, d)
-    create_repo_dir(subPath)
-  }
-
   r := new(Repository)
   r.basePath = path
+  r.ingestPath = filepath.Join(path, "ingest")
+  r.archivePath = filepath.Join(path, "archive")
+  r.outputPath = filepath.Join(path, "output")
+
+  // make sure repository base directory exists
+  create_repo_dir(r.basePath)
+  // make sure the folder structure is setup
+  create_repo_dir(r.ingestPath)
+  create_repo_dir(r.archivePath)
+  create_repo_dir(r.archivePath)
+
   return r
 }
 
-func (r *Repository) SearchIngest() []os.FileInfo {
+// Repository getters
-  ingestPath := filepath.Join(r.basePath, "ingest")
+func (r *Repository) GetIngestPath() string {
+  return r.ingestPath
+}
+func (r *Repository) GetOutputPath() string {
+  return r.outputPath
+}
 
+func (r *Repository) SearchIngest() []os.FileInfo {
   log.Printf("Searching ingest directory for files to transcode...")
 
-  ingestDir, err := os.Open(ingestPath)
+  ingestDir, err := os.Open(r.ingestPath)
   if err != nil {
     log.Fatalf("Error opening ingest directory: %s", err)
     os.Exit(1)
@@ -48,13 +58,13 @@ func (r *Repository) SearchIngest() []os.FileInfo {
 
 func (r *Repository) ArchiveFile(inFile string) {
   // create ingest and archive paths
-  ingestPath := filepath.Join(r.basePath, "ingest", inFile)
+  ingestFilePath := filepath.Join(r.ingestPath, inFile)
-  archivePath := filepath.Join(r.basePath, "archive", inFile)
+  archiveFilePath := filepath.Join(r.archivePath, inFile)
 
-  log.Printf("Copying %s to the archive.", ingestPath)
+  log.Printf("Copying %s to the archive.", ingestFilePath)
 
   // open ingest file
-  source, err := os.Open(ingestPath)
+  source, err := os.Open(ingestFilePath)
   if err != nil {
     log.Fatalf("Error opening file in ingest: %s.", err)
     os.Exit(1)
@@ -62,7 +72,7 @@ func (r *Repository) ArchiveFile(inFile string) {
   defer source.Close()
 
   // attempt to create destination file
-  destination, err := os.Create(archivePath)
+  destination, err := os.Create(archiveFilePath)
   if err != nil {
     log.Fatalf("Error opening archive file: %s.", err)
     os.Exit(1)
@@ -80,10 +90,10 @@ func (r *Repository) ArchiveFile(inFile string) {
 
 func (r *Repository) CleanupFile(inFile string) {
   // create ingest path
-  ingestPath := filepath.Join(r.basePath, "ingest", inFile)
+  ingestFilePath := filepath.Join(r.ingestPath, inFile)
 
   // remove the file
-  os.Remove(ingestPath)
+  os.Remove(ingestFilePath)
 }
 
 func create_repo_dir(path string) {

transcoder/transcoder.go

@@ -6,14 +6,61 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/spf13/viper"
+
+	"git.metaunix.net/BitGoblin/adept/util"
 )
 
-func Transcode(inFile string) {
+type Transcoder struct {
+	repo Repository
+}
+
+// Transcoder object constructor
+func NewTranscoder(r Repository) *Transcoder {
+	t := new(Transcoder)
+	t.repo = r
+	return t
+}
+
+// Start the main transcoder loop
+func (t *Transcoder) Start() {
+	// main program loop - runs infinitely until externally terminated
+  for {
+    ingestFiles := t.repo.SearchIngest()
+
+    if len(ingestFiles) < 1 {
+      log.Println("There were no files found in ingest to transcode; skipping run.")
+    } else {
+      for _, i := range ingestFiles {
+        // check if the file is open in another program (e.g. still being written to)
+        if util.IsFileLocked(filepath.Join(t.repo.GetIngestPath(), i.Name())) {
+          log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
+          continue
+        }
+
+        // archive file
+        t.repo.ArchiveFile(i.Name())
+        // transcode file
+        t.Transcode(i.Name())
+        // clean up source file
+        t.repo.CleanupFile(i.Name())
+      }
+    }
+
+    // sleep for X minutes - specified in the adept.toml config file
+    interval := viper.GetInt("transcoder.interval")
+    time.Sleep(time.Duration(interval) * time.Minute)
+  }
+}
+
+func (t *Transcoder) Transcode(inFile string) {
 	// create ingest and archive paths
-  ingestPath := filepath.Join(viper.GetString("transcoder.repository"), "ingest", inFile)
+  ingestPath := filepath.Join(t.repo.GetIngestPath(), inFile)
-  outputPath := filepath.Join(viper.GetString("transcoder.repository"), "output", inFile)
+	outputName := strings.Join([]string{util.FilenameWithoutExtension(inFile), viper.GetString("transcoder.video_format")}, ".")
+  outputPath := filepath.Join(t.repo.GetOutputPath(), outputName)
 
   log.Printf("Transcoding video file %s.", ingestPath)
 

util/env_test.go

@@ -0,0 +1,29 @@
+package util
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+  "github.com/stretchr/testify/suite"
+)
+
+// define our test suite struct
+type EnvTestSuite struct {
+	suite.Suite
+}
+
+// the tilde should expand to user's home directory
+func (s *EnvTestSuite) TestResolveTilde() {
+	resolvedPath := ResolveTilde("~")
+	assert.NotEqual(s.T(), resolvedPath, "~")
+}
+
+// ensure the tilde + relative path gets expanded fully
+func (s *EnvTestSuite) TestResolveTildePath() {
+	resolvedPath := ResolveTilde("~/test")
+	assert.NotEqual(s.T(), resolvedPath, "~/test")
+}
+
+// this is needed to run the test suite
+func TestEnvTestSuite(t *testing.T) {
+	suite.Run(t, new(EnvTestSuite))
+}

util/file.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+  "os/exec"
   "path"
   "strings"
 )
@@ -8,3 +9,14 @@ import (
 func FilenameWithoutExtension(filename string) string {
   return strings.TrimSuffix(filename, path.Ext(filename))
 }
+
+func IsFileLocked(filepath string) bool {
+  cmd := exec.Command("/usr/bin/lsof", filepath)
+  stdout, _ := cmd.Output()
+
+  if strings.Contains(string(stdout), filepath) {
+    return true
+  }
+
+  return false
+}

util/file_test.go

@@ -0,0 +1,64 @@
+package util
+
+import (
+	"os"
+	"testing"
+	"github.com/stretchr/testify/assert"
+  "github.com/stretchr/testify/suite"
+)
+
+// define our test suite struct
+type FileTestSuite struct {
+	suite.Suite
+	TestFile string
+}
+
+// run before tests to set up
+func (s *FileTestSuite) SetupSuite() {
+	s.TestFile = "testfile.txt"
+	// create the test file for file lock testing
+	file, _ := os.Create(s.TestFile)
+	file.Close() // do this just to make extra sure the file handle is closed
+}
+// run after tests to clean up
+func (s *FileTestSuite) TearDownSuite() {
+	// remove the test file since it's no longer needed
+	os.Remove(s.TestFile)
+}
+
+// test the filename extension removal works
+func (s *FileTestSuite) TestFilenameWithoutExtension() {
+	filename := FilenameWithoutExtension(s.TestFile)
+
+	if filename != "testfile" {
+		s.T().Logf("FilenameWithoutExtension returned '%s'; it should be 'testfile'.", filename)
+	}
+}
+
+// test that IsFileLocked returns true when the file is active
+func (s *FileTestSuite) TestFileShouldBeLocked() {
+	file, err := os.Open(s.TestFile)
+	if err != nil {
+		s.T().Logf("Unable to open file %s: %s", s.TestFile, err)
+	}
+
+	assert.True(s.T(), IsFileLocked(s.TestFile))
+
+	file.Close()
+}
+
+// test that IsFileLocked returns false when the file is not active
+func (s *FileTestSuite) TestFileShouldNotBeLocked() {
+	file, err := os.Open(s.TestFile)
+	if err != nil {
+		s.T().Logf("Unable to open file %s: %s", s.TestFile, err)
+	}
+	file.Close() // we want this closed now so it's NOT open!
+
+	assert.False(s.T(), IsFileLocked(s.TestFile))
+}
+
+// this is needed to run the test suite
+func TestFileTestSuite(t *testing.T) {
+	suite.Run(t, new(FileTestSuite))
+}