Compare commits

..

10 Commits

Author SHA1 Message Date
797d6cf1d3 Added functionality to clone file permissions when archiving them
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/tag/woodpecker Pipeline was successful
2023-04-16 16:11:07 -04:00
fa4e51a0e8 Updated Makefile to change 'build' command to 'compile'
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2023-02-11 15:33:08 -05:00
b52a1ef08e Added script to build deb package
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-10-26 19:35:23 -04:00
9b59d7485e Added a version subcommand
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-09-16 23:21:47 -04:00
1ccd481f9e Refactored a bit to put the configuration and logging setup before Cobra launches, so it doesn't need to be handled in every sub-command
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/tag/woodpecker Pipeline was successful
2022-09-15 10:09:10 -04:00
4d77a0e2bc Separated the repository initialization into a separate sub-command
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-09-15 10:04:39 -04:00
076947ebf9 Fixed the transcode job to rename the output file to the proper extension
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/tag/woodpecker Pipeline was successful
2022-09-03 19:55:50 -04:00
611d0dba9a Refactored the code a bit so that the Transcoder object handles more of its own setup 2022-09-03 19:51:37 -04:00
674327e0cf Added a multi-writer so that log messages go to both the console and log file
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-09-03 10:51:25 -04:00
602ddb1a00 Added example config file to the build files
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-09-03 10:46:01 -04:00
16 changed files with 368 additions and 72 deletions

2
.gitignore vendored
View File

@ -16,7 +16,7 @@
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
vendor/
# Go workspace file
go.work

View File

@ -15,8 +15,8 @@ pipeline:
image: golang:1.16
commands:
- go mod vendor
- GOOS=linux GOARCH=amd64 go build -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
- GOOS=windows GOARCH=amd64 go build -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
- GOOS=linux GOARCH=amd64 go build -ldflags "-X git.metaunix.net/BitGoblin/adept/cmd.version=${CI_COMMIT_TAG}" -o "dist/adept-linux-amd64-${CI_COMMIT_TAG}"
- GOOS=windows GOARCH=amd64 go build -ldflags "-X git.metaunix.net/BitGoblin/adept/cmd.version=${CI_COMMIT_TAG}" -o "dist/adept-windows-amd64-${CI_COMMIT_TAG}.exe"
when:
event: tag

View File

@ -2,7 +2,7 @@ BINARY_NAME=adept
all: build test
build:
compile:
go build -o ${BINARY_NAME} adept.go
test:
@ -31,7 +31,8 @@ uninstall:
rm /usr/bin/${BINARY_NAME}
rm /etc/systemd/system/${BINARY_NAME}.service
rm -rf /etc/${BINARY_NAME}
clean:
go clean
rm ${BINARY_NAME}
.PHONY: all test clean

View File

@ -1,17 +1,11 @@
package main
import (
"log"
"path/filepath"
"time"
"github.com/spf13/viper"
"git.metaunix.net/BitGoblin/adept/cmd"
"git.metaunix.net/BitGoblin/adept/config"
"git.metaunix.net/BitGoblin/adept/transcoder"
"git.metaunix.net/BitGoblin/adept/util"
)
// start the app - that's where all the good stuff happens
func main() {
// load configuration via Viper
config.LoadConfig()
@ -22,34 +16,6 @@ func main() {
defer logHandle.Close()
}
// initialize the video repository
r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
// main program loop - runs infinitely until externally terminated
for {
ingestFiles := r.SearchIngest()
if len(ingestFiles) < 1 {
log.Println("There were no files found in ingest to transcode; skipping run.")
} else {
for _, i := range ingestFiles {
// check if the file is open in another program (e.g. still being written to)
if util.IsFileLocked(filepath.Join(viper.GetString("transcoder.repository"), "ingest", i.Name())) {
log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
continue
}
// archive file
r.ArchiveFile(i.Name())
// transcode file
transcoder.Transcode(i.Name())
// clean up source file
r.CleanupFile(i.Name())
}
}
// sleep for X minutes - specified in the adept.toml config file
interval := viper.GetInt("transcoder.interval")
time.Sleep(time.Duration(interval) * time.Minute)
}
// launch the Cobra CLI framework
cmd.Execute()
}

44
bin/build_deb.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
# Check that we've been supplied a valid version
VERSION="$1"
if [ "$VERSION" == "" ]; then
echo -e "You must supply a version tag like './build_deb.sh 1.2.3'."
exit 1
fi
# Trim the leading 'v' from version number
if [[ "${VERSION:0:1}" == "v" ]]; then
VERSION="${VERSION:1}"
fi
# Create workspace for our deb package
WORKDIR="adept_$VERSION"
mkdir "$WORKDIR"
# Copy build sources into workspace
cp -r build/etc "$WORKDIR/"
mkdir -p "$WORKDIR/usr/bin/"
cp "dist/adept-linux-amd64-v$VERSION" "$WORKDIR/usr/bin/adept"
# Create debian control file
mkdir "$WORKDIR/DEBIAN"
cat > "$WORKDIR/DEBIAN/control"<< EOF
Package: adept
Version: $VERSION
Section: video
Priority: optional
Architecture: amd64
Depends: ffmpeg (>= 4), lsof
Maintainer: Gregory Ballantine <gballantine@bitgoblin.tech>
Description: Bit Goblin video transcoding service.
EOF
# Copy maintainer scripts
cp build/scripts/* "$WORKDIR/DEBIAN/"
#chmod +rx "$WORKDIR"/DEBIAN/pre*
chmod +rx "$WORKDIR"/DEBIAN/post*
# Build deb package!
dpkg-deb --build "$WORKDIR"
# Cleanup
rm -rf "$WORKDIR"

74
bin/build_rpm.sh Executable file
View File

@ -0,0 +1,74 @@
#!/bin/bash
# Check that we've been supplied a valid version
VERSION="$1"
if [ "$VERSION" == "" ]; then
echo -e "You must supply a version tag like './build_rpm.sh 1.2.3'."
exit 1
fi
# Trim the leading 'v' from version number
if [[ "${VERSION:0:1}" == "v" ]]; then
VERSION="${VERSION:1}"
fi
# Tar up source files
mkdir rpm-temp
cp -r cmd config transcoder util adept.go go.mod go.sum LICENSE Makefile README.md rpm-temp/
tar cvzf "adept-$VERSION.tar.gz" rpm-temp/*
mv "adept-$VERSION.tar.gz" ~/rpmbuild/SOURCES/
# Create debian control file
cat > "./adept.spec"<< EOF
Name: adept
Version: $VERSION
Release: 1%{?dist}
Summary: Bit Goblin video transcoding service
License: BSD 2-Clause
Source0: %{name}-%{version}.tar.gz
BuildRequires: golang
BuildRequires: systemd-rpm-macros
Provides: %{name} = %{version}
%description
Bit Goblin video transcoding service
%global debug_package %{nil}
%prep
%autosetup
%build
go build -v -o %{name}
%install
install -Dpm 0755 %{name} %{buildroot}%{_bindir}/%{name}
install -Dpm 0755 build/etc/%{name}/%{name}.toml %{buildroot}%{_sysconfdir}/%{name}/%{name}.toml
install -Dpm 644 build/etc/systemd/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
%check
# go test should be here... :)
%post
%systemd_post %{name}.service
%preun
%systemd_preun %{name}.service
%files
%dir %{_sysconfdir}/%{name}
%{_bindir}/%{name}
%{_unitdir}/%{name}.service
%config(noreplace) %{_sysconfdir}/%{name}/%{name}.toml
%changelog
* Thu Nov 17 2022 Gregory Ballantine $VERSION
- check git.metaunix.net/BitGoblin/adept for a changelog
EOF
rpmbuild -ba adept.spec

View File

@ -0,0 +1,15 @@
log_to_file = true
log_file = '/var/log/adept/adept.log'
log_level = 'info'
[transcoder]
repository = '/srv/adept'
interval = 15
video_format = 'mov'
video_codec = 'dnxhd'
video_profile = 'dnxhr_hq'
video_resolution = '1920x1080'
video_framerate = 60
video_color = 'yuv422p'
audio_codec = 'pcm_s16le'

36
build/scripts/postinst Executable file
View File

@ -0,0 +1,36 @@
#!/bin/sh
GETENT_USER=$(getent passwd adept)
# Create the adept user if it doesn't already exist
if [ "$GETENT_USER" = "" ]; then
echo "Creating the 'adept' user."
useradd -r adept
else
echo "The 'adept' user already exists, skipping creation."
fi
GETENT_GROUP=$(getent group adept)
# Create the adept group if it doesn't already exist
if [ "$GETENT_GROUP" = "" ]; then
echo "Creating the 'adept' group."
groupadd adept
usermod -aG adept adept
else
echo "The 'adept' group already exists, skipping creation."
fi
# Change the directory ownership of /etc
chown -R adept:adept /etc/adept
# Create the log directory under /var/log
if [ ! -d /var/log/adept ]; then
echo "Creating /var/log/adept to store log files."
mkdir /var/log/adept
chown adept:adept /var/log/adept
else
echo "/var/log/adept already exists, skipping creation."
fi
#DEBHELPER#
exit 0

35
cmd/root.go Normal file
View File

@ -0,0 +1,35 @@
package cmd
import (
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"git.metaunix.net/BitGoblin/adept/transcoder"
)
// ./adept - this is the primary command and is how the service is intended to be launched
var rootCmd = &cobra.Command{
Use: "adept",
Short: "Adept is a video transcoder service",
Long: `An automated video transcoder service that archives ingested footage.
https://git.metaunix.net/BitGoblin/adept`,
Run: func(cmd *cobra.Command, args []string) {
// initialize the video repository
r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
// create transcoder object and start the main loop
t := transcoder.NewTranscoder(*r)
t.Start()
},
}
// this is the main function to launch Cobra
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}

24
cmd/setup.go Normal file
View File

@ -0,0 +1,24 @@
package cmd
import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"git.metaunix.net/BitGoblin/adept/transcoder"
)
// initializes the sub-command
func init() {
rootCmd.AddCommand(setupCmd)
}
var setupCmd = &cobra.Command{
Use: "setup",
Short: "Initialize the video directories.",
Long: `Adept can't run gif it doesn't have a place to ingest/transcode files from.`,
Run: func(cmd *cobra.Command, args []string) {
// initialize the video repository
r := transcoder.NewRepository(viper.GetString("transcoder.repository"))
r.Setup()
},
}

28
cmd/version.go Normal file
View File

@ -0,0 +1,28 @@
package cmd
import (
"log"
"runtime"
"github.com/spf13/cobra"
)
var (
version string
)
// initializes the sub-command
func init() {
rootCmd.AddCommand(versionCmd)
}
var versionCmd = &cobra.Command{
Use: "version",
Short: "Display version info.",
Long: `In case you're curious, run this command to find out the version info.`,
Run: func(cmd *cobra.Command, args []string) {
log.Printf("Adept version %s", version)
log.Printf("Built with Go %s", runtime.Version())
log.Printf("View source at https://git.metaunix.net/BitGoblin/adept")
},
}

View File

@ -1,6 +1,7 @@
package config
import (
"io"
"log"
"os"
@ -20,10 +21,12 @@ func InitLogging() *os.File {
log.Fatalf("Error opening log file: %v", err)
os.Exit(1)
}
// set logging to file handle
log.SetOutput(fileHandle)
}
// create a MultiWriter instance so we can write to both console AND file
mw := io.MultiWriter(os.Stdout, fileHandle)
// set our multiwriter object as the output for logging
log.SetOutput(mw)
return fileHandle
}

1
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.12.0

6
go.sum
View File

@ -93,6 +93,7 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -239,6 +240,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -327,6 +330,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@ -338,6 +342,8 @@ github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

View File

@ -8,31 +8,45 @@ import (
)
type Repository struct {
basePath string
basePath string
ingestPath string
archivePath string
outputPath string
}
// Repository struct constructor
func NewRepository(path string) *Repository {
// make sure repository base directory exists
create_repo_dir(path)
// make sure the ingest, archive, and output directories exist
subDirs := []string{"ingest", "archive", "output"}
for _, d := range subDirs {
subPath := filepath.Join(path, d)
create_repo_dir(subPath)
}
r := new(Repository)
r.basePath = path
r.ingestPath = filepath.Join(path, "ingest")
r.archivePath = filepath.Join(path, "archive")
r.outputPath = filepath.Join(path, "output")
return r
}
func (r *Repository) SearchIngest() []os.FileInfo {
ingestPath := filepath.Join(r.basePath, "ingest")
// Repository getters
func (r *Repository) GetIngestPath() string {
return r.ingestPath
}
func (r *Repository) GetOutputPath() string {
return r.outputPath
}
// Initializes the video repository by ensuring the directories are available
func (r *Repository) Setup() {
// make sure repository base directory exists
create_repo_dir(r.basePath)
// make sure the folder structure is setup
create_repo_dir(r.ingestPath)
create_repo_dir(r.archivePath)
create_repo_dir(r.archivePath)
}
func (r *Repository) SearchIngest() []os.FileInfo {
log.Printf("Searching ingest directory for files to transcode...")
ingestDir, err := os.Open(ingestPath)
ingestDir, err := os.Open(r.ingestPath)
if err != nil {
log.Fatalf("Error opening ingest directory: %s", err)
os.Exit(1)
@ -48,27 +62,29 @@ func (r *Repository) SearchIngest() []os.FileInfo {
func (r *Repository) ArchiveFile(inFile string) {
// create ingest and archive paths
ingestPath := filepath.Join(r.basePath, "ingest", inFile)
archivePath := filepath.Join(r.basePath, "archive", inFile)
ingestFilePath := filepath.Join(r.ingestPath, inFile)
archiveFilePath := filepath.Join(r.archivePath, inFile)
log.Printf("Copying %s to the archive.", ingestPath)
log.Printf("Copying %s to the archive.", ingestFilePath)
// open ingest file
source, err := os.Open(ingestPath)
source, err := os.Open(ingestFilePath)
if err != nil {
log.Fatalf("Error opening file in ingest: %s.", err)
os.Exit(1)
}
sourceStat, _ := source.Stat()
sourceMode := sourceStat.Mode()
defer source.Close()
// attempt to create destination file
destination, err := os.Create(archivePath)
destination, err := os.Create(archiveFilePath)
if err != nil {
log.Fatalf("Error opening archive file: %s.", err)
os.Exit(1)
}
defer destination.Close()
destination.Chmod(0755)
destination.Chmod(sourceMode)
// perform the file copy
_, err = io.Copy(destination, source)
@ -80,10 +96,10 @@ func (r *Repository) ArchiveFile(inFile string) {
func (r *Repository) CleanupFile(inFile string) {
// create ingest path
ingestPath := filepath.Join(r.basePath, "ingest", inFile)
ingestFilePath := filepath.Join(r.ingestPath, inFile)
// remove the file
os.Remove(ingestPath)
os.Remove(ingestFilePath)
}
func create_repo_dir(path string) {

View File

@ -6,14 +6,61 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/spf13/viper"
"git.metaunix.net/BitGoblin/adept/util"
)
func Transcode(inFile string) {
type Transcoder struct {
repo Repository
}
// Transcoder object constructor
func NewTranscoder(r Repository) *Transcoder {
t := new(Transcoder)
t.repo = r
return t
}
// Start the main transcoder loop
func (t *Transcoder) Start() {
// main program loop - runs infinitely until externally terminated
for {
ingestFiles := t.repo.SearchIngest()
if len(ingestFiles) < 1 {
log.Println("There were no files found in ingest to transcode; skipping run.")
} else {
for _, i := range ingestFiles {
// check if the file is open in another program (e.g. still being written to)
if util.IsFileLocked(filepath.Join(t.repo.GetIngestPath(), i.Name())) {
log.Printf("%s appears to be open in another program; skipping it for this run.", i.Name())
continue
}
// archive file
t.repo.ArchiveFile(i.Name())
// transcode file
t.Transcode(i.Name())
// clean up source file
t.repo.CleanupFile(i.Name())
}
}
// sleep for X minutes - specified in the adept.toml config file
interval := viper.GetInt("transcoder.interval")
time.Sleep(time.Duration(interval) * time.Minute)
}
}
func (t *Transcoder) Transcode(inFile string) {
// create ingest and archive paths
ingestPath := filepath.Join(viper.GetString("transcoder.repository"), "ingest", inFile)
outputPath := filepath.Join(viper.GetString("transcoder.repository"), "output", inFile)
ingestPath := filepath.Join(t.repo.GetIngestPath(), inFile)
outputName := strings.Join([]string{util.FilenameWithoutExtension(inFile), viper.GetString("transcoder.video_format")}, ".")
outputPath := filepath.Join(t.repo.GetOutputPath(), outputName)
log.Printf("Transcoding video file %s.", ingestPath)