Added functionality to clone file permissions when archiving them

.gitignore

@@ -16,7 +16,7 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 
 # Go workspace file
 go.work

.woodpecker.yml

@@ -15,8 +15,8 @@ pipeline:
     image: golang:1.16
     commands:
       - go mod vendor
-      - GOOS=linux GOARCH=amd64 go build -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
-      - GOOS=windows GOARCH=amd64 go build -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
+      - GOOS=linux GOARCH=amd64 go build -ldflags "-X git.metaunix.net/BitGoblin/adept/cmd.version=${CI_COMMIT_TAG}" -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
+      - GOOS=windows GOARCH=amd64 go build -ldflags "-X git.metaunix.net/BitGoblin/adept/cmd.version=${CI_COMMIT_TAG}" -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
     when:
       event: tag
 

Makefile

@@ -2,7 +2,7 @@ BINARY_NAME=adept
 
 all: build test
 
-build:
+compile:
 	go build -o ${BINARY_NAME} adept.go
 
 test:
@@ -34,4 +34,5 @@ uninstall:
 
 clean:
 	go clean
-	rm ${BINARY_NAME}
+
+.PHONY: all test clean

adept.go

@@ -1,17 +1,11 @@
 package main
 
 import (
-  "log"
-  "path/filepath"
-  "time"
-
-  "github.com/spf13/viper"
-
+  "git.metaunix.net/BitGoblin/adept/cmd"
   "git.metaunix.net/BitGoblin/adept/config"
-  "git.metaunix.net/BitGoblin/adept/transcoder"
-  "git.metaunix.net/BitGoblin/adept/util"
 )
 
+// start the app - that's where all the good stuff happens
 func main() {
   // load configuration via Viper
   config.LoadConfig()
@@ -22,34 +16,6 @@ func main() {
     defer logHandle.Close()
   }
 
-  // initialize the video repository
-  r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
-
-  // main program loop - runs infinitely until externally terminated
-  for {
-    ingestFiles := r.SearchIngest()
-
-    if len(ingestFiles) < 1 {
-      log.Println("There were no files found in ingest to transcode; skipping run.")
-    } else {
-      for _, i := range ingestFiles {
-        // check if the file is open in another program (e.g. still being written to)
-        if util.IsFileLocked(filepath.Join(viper.GetString("transcoder.repository"), "ingest", i.Name())) {
-          log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
-          continue
-        }
-
-        // archive file
-        r.ArchiveFile(i.Name())
-        // transcode file
-        transcoder.Transcode(i.Name())
-        // clean up source file
-        r.CleanupFile(i.Name())
-      }
-    }
-
-    // sleep for X minutes - specified in the adept.toml config file
-    interval := viper.GetInt("transcoder.interval")
-    time.Sleep(time.Duration(interval) * time.Minute)
-  }
+  // launch the Cobra CLI framework
+  cmd.Execute()
 }

bin/build_deb.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Check that we've been supplied a valid version
+VERSION="$1"
+if [ "$VERSION" == "" ]; then
+  echo -e "You must supply a version tag like './build_deb.sh 1.2.3'."
+  exit 1
+fi
+# Trim the leading 'v' from version number
+if [[ "${VERSION:0:1}" == "v" ]]; then
+  VERSION="${VERSION:1}"
+fi
+
+# Create workspace for our deb package
+WORKDIR="adept_$VERSION"
+mkdir "$WORKDIR"
+
+# Copy build sources into workspace
+cp -r build/etc "$WORKDIR/"
+mkdir -p "$WORKDIR/usr/bin/"
+cp "dist/adept-linux-amd64-v$VERSION" "$WORKDIR/usr/bin/adept"
+
+# Create debian control file
+mkdir "$WORKDIR/DEBIAN"
+cat > "$WORKDIR/DEBIAN/control"<< EOF
+Package: adept
+Version: $VERSION
+Section: video
+Priority: optional
+Architecture: amd64
+Depends: ffmpeg (>= 4), lsof
+Maintainer: Gregory Ballantine <gballantine@bitgoblin.tech>
+Description: Bit Goblin video transcoding service.
+EOF
+# Copy maintainer scripts
+cp build/scripts/* "$WORKDIR/DEBIAN/"
+#chmod +rx "$WORKDIR"/DEBIAN/pre*
+chmod +rx "$WORKDIR"/DEBIAN/post*
+
+# Build deb package!
+dpkg-deb --build "$WORKDIR"
+
+# Cleanup
+rm -rf "$WORKDIR"

bin/build_rpm.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Check that we've been supplied a valid version
+VERSION="$1"
+if [ "$VERSION" == "" ]; then
+  echo -e "You must supply a version tag like './build_rpm.sh 1.2.3'."
+  exit 1
+fi
+# Trim the leading 'v' from version number
+if [[ "${VERSION:0:1}" == "v" ]]; then
+  VERSION="${VERSION:1}"
+fi
+
+# Tar up source files
+mkdir rpm-temp
+cp -r cmd config transcoder util adept.go go.mod go.sum LICENSE Makefile README.md rpm-temp/
+tar cvzf "adept-$VERSION.tar.gz" rpm-temp/*
+mv "adept-$VERSION.tar.gz" ~/rpmbuild/SOURCES/
+
+# Create debian control file
+cat > "./adept.spec"<< EOF
+Name:           adept
+Version:        $VERSION
+Release:        1%{?dist}
+Summary:        Bit Goblin video transcoding service
+
+License:        BSD 2-Clause
+Source0:        %{name}-%{version}.tar.gz
+
+BuildRequires:  golang
+BuildRequires:  systemd-rpm-macros
+
+Provides:       %{name} = %{version}
+
+%description
+Bit Goblin video transcoding service
+
+%global debug_package %{nil}
+
+%prep
+%autosetup
+
+
+%build
+go build -v -o %{name}
+
+
+%install
+install -Dpm 0755 %{name} %{buildroot}%{_bindir}/%{name}
+install -Dpm 0755 build/etc/%{name}/%{name}.toml %{buildroot}%{_sysconfdir}/%{name}/%{name}.toml
+install -Dpm 644 build/etc/systemd/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
+
+%check
+# go test should be here... :)
+
+%post
+%systemd_post %{name}.service
+
+%preun
+%systemd_preun %{name}.service
+
+%files
+%dir %{_sysconfdir}/%{name}
+%{_bindir}/%{name}
+%{_unitdir}/%{name}.service
+%config(noreplace) %{_sysconfdir}/%{name}/%{name}.toml
+
+
+%changelog
+* Thu Nov 17 2022 Gregory Ballantine $VERSION
+- check git.metaunix.net/BitGoblin/adept for a changelog
+EOF
+
+rpmbuild -ba adept.spec

build/etc/adept/adept.toml

@@ -0,0 +1,15 @@
+log_to_file = true
+log_file = '/var/log/adept/adept.log'
+log_level = 'info'
+
+[transcoder]
+repository = '/srv/adept'
+interval = 15
+video_format = 'mov'
+video_codec = 'dnxhd'
+video_profile = 'dnxhr_hq'
+video_resolution = '1920x1080'
+video_framerate = 60
+video_color = 'yuv422p'
+audio_codec = 'pcm_s16le'
+

build/scripts/postinst

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+GETENT_USER=$(getent passwd adept)
+# Create the adept user if it doesn't already exist
+if [ "$GETENT_USER" = "" ]; then
+	echo "Creating the 'adept' user."
+	useradd -r adept
+else
+	echo "The 'adept' user already exists, skipping creation."
+fi
+
+GETENT_GROUP=$(getent group adept)
+# Create the adept group if it doesn't already exist
+if [ "$GETENT_GROUP" = "" ]; then
+	echo "Creating the 'adept' group."
+	groupadd adept
+	usermod -aG adept adept
+else
+	echo "The 'adept' group already exists, skipping creation."
+fi
+
+# Change the directory ownership of /etc
+chown -R adept:adept /etc/adept
+
+# Create the log directory under /var/log
+if [ ! -d /var/log/adept ]; then
+	echo "Creating /var/log/adept to store log files."
+	mkdir /var/log/adept
+	chown adept:adept /var/log/adept
+else
+	echo "/var/log/adept already exists, skipping creation."
+fi
+
+#DEBHELPER#
+
+exit 0

cmd/root.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+  "fmt"
+  "os"
+
+  "github.com/spf13/cobra"
+  "github.com/spf13/viper"
+
+  "git.metaunix.net/BitGoblin/adept/transcoder"
+)
+
+// ./adept - this is the primary command and is how the service is intended to be launched
+var rootCmd = &cobra.Command{
+  Use:   "adept",
+  Short: "Adept is a video transcoder service",
+  Long: `An automated video transcoder service that archives ingested footage.
+          https://git.metaunix.net/BitGoblin/adept`,
+  Run: func(cmd *cobra.Command, args []string) {
+    // initialize the video repository
+    r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
+
+    // create transcoder object and start the main loop
+    t := transcoder.NewTranscoder(*r)
+    t.Start()
+  },
+}
+
+// this is the main function to launch Cobra
+func Execute() {
+  if err := rootCmd.Execute(); err != nil {
+    fmt.Println(err)
+    os.Exit(1)
+  }
+}

cmd/setup.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+  "github.com/spf13/cobra"
+  "github.com/spf13/viper"
+
+	"git.metaunix.net/BitGoblin/adept/transcoder"
+)
+
+// initializes the sub-command
+func init() {
+  rootCmd.AddCommand(setupCmd)
+}
+
+var setupCmd = &cobra.Command{
+  Use:   "setup",
+  Short: "Initialize the video directories.",
+  Long:  `Adept can't run gif it doesn't have a place to ingest/transcode files from.`,
+  Run: func(cmd *cobra.Command, args []string) {
+    // initialize the video repository
+    r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
+		r.Setup()
+  },
+}

cmd/version.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"log"
+	"runtime"
+
+  "github.com/spf13/cobra"
+)
+
+var (
+	version string
+)
+
+// initializes the sub-command
+func init() {
+  rootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+  Use:   "version",
+  Short: "Display version info.",
+  Long:  `In case you're curious, run this command to find out the version info.`,
+  Run: func(cmd *cobra.Command, args []string) {
+    log.Printf("Adept version %s", version)
+		log.Printf("Built with Go %s", runtime.Version())
+		log.Printf("View source at https://git.metaunix.net/BitGoblin/adept")
+  },
+}

config/log.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"io"
 	"log"
 	"os"
 
@@ -20,10 +21,12 @@ func InitLogging() *os.File {
 		  log.Fatalf("Error opening log file: %v", err)
 		  os.Exit(1)
 		}
-
-		// set logging to file handle
-		log.SetOutput(fileHandle)
 	}
 
+	// create a MultiWriter instance so we can write to both console AND file
+	mw := io.MultiWriter(os.Stdout, fileHandle)
+	// set our multiwriter object as the output for logging
+	log.SetOutput(mw)
+
 	return fileHandle
 }

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cobra v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.12.0

go.sum

@@ -93,6 +93,7 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -239,6 +240,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
 github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -327,6 +330,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -338,6 +342,8 @@ github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
 github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
 github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

transcoder/repository.go

@@ -9,30 +9,44 @@ import (
 
 type Repository struct {
   basePath    string
+  ingestPath  string
+  archivePath string
+  outputPath  string
 }
 
+// Repository struct constructor
 func NewRepository(path string) *Repository {
-  // make sure repository base directory exists
-  create_repo_dir(path)
-
-  // make sure the ingest, archive, and output directories exist
-  subDirs := []string{"ingest", "archive", "output"}
-  for _, d := range subDirs {
-    subPath := filepath.Join(path, d)
-    create_repo_dir(subPath)
-  }
-
   r := new(Repository)
   r.basePath = path
+  r.ingestPath = filepath.Join(path, "ingest")
+  r.archivePath = filepath.Join(path, "archive")
+  r.outputPath = filepath.Join(path, "output")
+
   return r
 }
 
-func (r *Repository) SearchIngest() []os.FileInfo {
-  ingestPath := filepath.Join(r.basePath, "ingest")
+// Repository getters
+func (r *Repository) GetIngestPath() string {
+  return r.ingestPath
+}
+func (r *Repository) GetOutputPath() string {
+  return r.outputPath
+}
 
+// Initializes the video repository by ensuring the directories are available
+func (r *Repository) Setup() {
+  // make sure repository base directory exists
+  create_repo_dir(r.basePath)
+  // make sure the folder structure is setup
+  create_repo_dir(r.ingestPath)
+  create_repo_dir(r.archivePath)
+  create_repo_dir(r.archivePath)
+}
+
+func (r *Repository) SearchIngest() []os.FileInfo {
   log.Printf("Searching ingest directory for files to transcode...")
 
-  ingestDir, err := os.Open(ingestPath)
+  ingestDir, err := os.Open(r.ingestPath)
   if err != nil {
     log.Fatalf("Error opening ingest directory: %s", err)
     os.Exit(1)
@@ -48,27 +62,29 @@ func (r *Repository) SearchIngest() []os.FileInfo {
 
 func (r *Repository) ArchiveFile(inFile string) {
   // create ingest and archive paths
-  ingestPath := filepath.Join(r.basePath, "ingest", inFile)
-  archivePath := filepath.Join(r.basePath, "archive", inFile)
+  ingestFilePath := filepath.Join(r.ingestPath, inFile)
+  archiveFilePath := filepath.Join(r.archivePath, inFile)
 
-  log.Printf("Copying %s to the archive.", ingestPath)
+  log.Printf("Copying %s to the archive.", ingestFilePath)
 
   // open ingest file
-  source, err := os.Open(ingestPath)
+  source, err := os.Open(ingestFilePath)
   if err != nil {
     log.Fatalf("Error opening file in ingest: %s.", err)
     os.Exit(1)
   }
+  sourceStat, _ := source.Stat()
+  sourceMode := sourceStat.Mode()
   defer source.Close()
 
   // attempt to create destination file
-  destination, err := os.Create(archivePath)
+  destination, err := os.Create(archiveFilePath)
   if err != nil {
     log.Fatalf("Error opening archive file: %s.", err)
     os.Exit(1)
   }
   defer destination.Close()
-  destination.Chmod(0755)
+  destination.Chmod(sourceMode)
 
   // perform the file copy
   _, err = io.Copy(destination, source)
@@ -80,10 +96,10 @@ func (r *Repository) ArchiveFile(inFile string) {
 
 func (r *Repository) CleanupFile(inFile string) {
   // create ingest path
-  ingestPath := filepath.Join(r.basePath, "ingest", inFile)
+  ingestFilePath := filepath.Join(r.ingestPath, inFile)
 
   // remove the file
-  os.Remove(ingestPath)
+  os.Remove(ingestFilePath)
 }
 
 func create_repo_dir(path string) {

transcoder/transcoder.go

@@ -6,14 +6,61 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/spf13/viper"
+
+	"git.metaunix.net/BitGoblin/adept/util"
 )
 
-func Transcode(inFile string) {
+type Transcoder struct {
+	repo Repository
+}
+
+// Transcoder object constructor
+func NewTranscoder(r Repository) *Transcoder {
+	t := new(Transcoder)
+	t.repo = r
+	return t
+}
+
+// Start the main transcoder loop
+func (t *Transcoder) Start() {
+	// main program loop - runs infinitely until externally terminated
+  for {
+    ingestFiles := t.repo.SearchIngest()
+
+    if len(ingestFiles) < 1 {
+      log.Println("There were no files found in ingest to transcode; skipping run.")
+    } else {
+      for _, i := range ingestFiles {
+        // check if the file is open in another program (e.g. still being written to)
+        if util.IsFileLocked(filepath.Join(t.repo.GetIngestPath(), i.Name())) {
+          log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
+          continue
+        }
+
+        // archive file
+        t.repo.ArchiveFile(i.Name())
+        // transcode file
+        t.Transcode(i.Name())
+        // clean up source file
+        t.repo.CleanupFile(i.Name())
+      }
+    }
+
+    // sleep for X minutes - specified in the adept.toml config file
+    interval := viper.GetInt("transcoder.interval")
+    time.Sleep(time.Duration(interval) * time.Minute)
+  }
+}
+
+func (t *Transcoder) Transcode(inFile string) {
 	// create ingest and archive paths
-  ingestPath := filepath.Join(viper.GetString("transcoder.repository"), "ingest", inFile)
-  outputPath := filepath.Join(viper.GetString("transcoder.repository"), "output", inFile)
+  ingestPath := filepath.Join(t.repo.GetIngestPath(), inFile)
+	outputName := strings.Join([]string{util.FilenameWithoutExtension(inFile), viper.GetString("transcoder.video_format")}, ".")
+  outputPath := filepath.Join(t.repo.GetOutputPath(), outputName)
 
   log.Printf("Transcoding video file %s.", ingestPath)