Compare commits

...

No commits in common. "7b169dc5065ff30f503f986ffda8d04478201d94" and "cab06fc35000aa75a149f258c1b95f3ee6732f23" have entirely different histories.

23 changed files with 435 additions and 273 deletions

65
.gitignore vendored
View File

@ -1,58 +1,21 @@
# ---> Ruby
*.gem
*.rbc
/.config
/coverage/
/InstalledFiles
/pkg/
/spec/reports/
/spec/examples.txt
/test/tmp/
/test/version_tmp/
/tmp/
# ---> Rust
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Used by dotenv library to load environment variables.
# .env
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# Ignore Byebug command history file.
.byebug_history
# These are backup files generated by rustfmt
**/*.rs.bk
## Specific to RubyMotion:
.dat*
.repl_history
build/
*.bridgesupport
build-iPhoneOS/
build-iPhoneSimulator/
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
## Specific to RubyMotion (use of CocoaPods):
#
# We recommend against adding the Pods directory to your .gitignore. However
# you should judge for yourself, the pros and cons are mentioned at:
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
#
# vendor/Pods/
## Documentation cache and generated files:
/.yardoc/
/_yardoc/
/doc/
/rdoc/
## Environment normalization:
/.bundle/
/vendor/bundle
/lib/bundler/man/
# for a library or gem, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# Gemfile.lock
# .ruby-version
# .ruby-gemset
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
.rvmrc
# Used by RuboCop. Remote config files pulled in from inherit_from directive.
# .rubocop-https?--*
# Added by cargo
/target

View File

@ -1,11 +0,0 @@
Layout/EmptyLinesAroundClassBody:
EnforcedStyle: empty_lines
Layout/MultilineOperationIndentation:
EnforcedStyle: indented
Style/BlockDelimiters:
EnforcedStyle: always_braces
Style/RedundantReturn:
Enabled: false

30
.woodpecker.yml Normal file
View File

@ -0,0 +1,30 @@
pipeline:
tests:
image: rust:1.63
commands:
- "cargo test"
build_release:
image: rust:1.63
commands:
- "cargo install cargo-deb cargo-generate-rpm"
- "cargo build --release"
- "cargo deb"
- "cargo generate-rpm"
- "mv target/release/zealot target/release/zealot-${CI_COMMIT_TAG}-linux-x86_64"
when:
event: tag
gitea_release:
image: plugins/gitea-release
settings:
api_key:
from_secret: gitea_api_key
base_url: https://git.metaunix.net
files:
- "target/release/*${CI_COMMIT_TAG}-linux-x86_64"
- "target/debian/zealot*.deb"
- "target/generate-rpm/zealot*.rpm"
title: "${CI_COMMIT_TAG}"
when:
event: tag

42
Cargo.toml Normal file
View File

@ -0,0 +1,42 @@
[package]
name = "zealot"
description = "Bit Goblin automated video transcoding service."
version = "0.1.3"
edition = "2021"
readme = "README.md"
license = "BSD 2-Clause"
authors = ["Gregory Ballantine <gballantine@bitgoblin.tech>"]
[dependencies]
log = "0.4"
log4rs = "1.1"
toml = "0.5"
serde = "1.0"
serde_derive = "1.0"
shellexpand = "2.1"
[package.metadata.deb]
license-file = "LICENSE"
maintainer-scripts = "build/scripts"
depends = "ffmpeg (>= 4)"
section = "video"
assets = [
["target/release/zealot", "usr/bin/zealot", "755"],
["build/etc/example.toml", "etc/zealot/example.toml", "644"],
["README.md", "usr/share/doc/zealot/README", "644"]
]
[package.metadata.deb.systemd-units]
unit-scripts = "build/debian/service"
unit-name = "zealot"
enable = true
start = true
[package.metadata.generate-rpm]
assets = [
{ source = "target/release/zealot", dest = "/usr/bin/zealot", mode = "755" },
{ source = "build/etc/example.toml", dest = "/etc/zealot/example.toml", mode = "755"},
{ source = "README.md", dest = "/usr/share/doc/zealot/README", mode = "644"}
]
post_install_script = "build/scripts/postinst"
[package.metadata.generate-rpm.requires]
ffmpeg = ">= 4"

View File

@ -1,8 +0,0 @@
# frozen_string_literal: true
source "https://rubygems.org"
gem "toml", "~> 0.3.0"
gem "rubocop", "~> 1.35.1", require: false

View File

@ -1,38 +0,0 @@
GEM
remote: https://rubygems.org/
specs:
ast (2.4.2)
json (2.6.2)
parallel (1.22.1)
parser (3.1.2.1)
ast (~> 2.4.1)
parslet (2.0.0)
rainbow (3.1.1)
regexp_parser (2.5.0)
rexml (3.2.5)
rubocop (1.35.1)
json (~> 2.3)
parallel (~> 1.10)
parser (>= 3.1.2.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.20.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.21.0)
parser (>= 3.1.1.0)
ruby-progressbar (1.11.0)
toml (0.3.0)
parslet (>= 1.8.0, < 3.0.0)
unicode-display_width (2.2.0)
PLATFORMS
x86_64-linux
DEPENDENCIES
rubocop (~> 1.35.1)
toml (~> 0.3.0)
BUNDLED WITH
2.3.19

View File

@ -1,3 +1,3 @@
# Zealot
# zealot
Bit Goblin video transcoder service

View File

@ -0,0 +1,11 @@
[Unit]
Description=Zealot video transcoder service
[Service]
User=zealot
Group=zealot
ExecStart=/usr/bin/zealot
SuccessExitStatus=143
[Install]
WantedBy=multi-user.target

11
build/etc/example.toml Normal file
View File

@ -0,0 +1,11 @@
# This example transcodes footage to DNxHD 1080p60 for use in video editors like DaVinci Resolve.
[transcoder]
repository = "~/zealot" # location of the videos to transcode
interval = 15
video_format = "mov" # video container format
video_codec = "dnxhd" # video codec to use
video_profile = "dnxhr_hq" # DNxHD has multiple presets for various video qualities
video_resolution = "1920x1080"
video_framerate = 60
video_color = "yuv422p"
audio_codec = "pcm_s16le" # audio codec to use

26
build/scripts/postinst Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
GETENT_USER=$(getent passwd zealot)
GETENT_GROUP=$(getent group zealot)
# Create the zealot user if it doesn't already exist
if [ "$GETENT_USER" = "" ]; then
useradd -r zealot
else
echo "The 'zealot' user already exists, skipping creation."
fi
# Create the zealot group if it doesn't already exist
if [ "$GETENT_GROUP" = "" ]; then
groupadd zealot
usermod -aG zealot zealot
else
echo "The 'zealot' group already exists, skipping creation."
fi
# Change the directory ownership of /etc
chown -R zealot:zealot /etc/zealot
#DEBHELPER#
exit 0

12
log4rs.yaml Normal file
View File

@ -0,0 +1,12 @@
appenders:
stdout:
kind: console
encoder:
pattern: "{d(%+)(utc)} {h({l})}: {m}{n}"
filters:
- kind: threshold
level: info
root:
level: info
appenders:
- stdout

View File

@ -1,32 +0,0 @@
# frozen_string_literal: true
require 'toml'
# Loads and handles the application configuration.
class Config
# class constructor
def initialize(config_path)
expanded_path = File.expand_path(config_path)
@config = TOML.load_file(expanded_path)
# just in case the user wants to use a tilde (~) in the repository path...
@config['transcoder']['repository'] = File.expand_path(get('transcoder.repository'))
end
# returns a configuration value from a dot-seperated string, like 'transcoder.interval'
def get(path)
value = @config
bits = path.split('.')
bits.each { |bit|
if value.key?(bit)
value = value[bit]
else
abort("Configuration value #{path} does exist.")
end
}
return value
end
end

34
src/config.rs Normal file
View File

@ -0,0 +1,34 @@
use serde_derive::Deserialize;
use std::fs;
#[derive(Deserialize)]
pub struct Config {
pub transcoder: Transcoder,
}
impl Config {
pub fn new(config_path: &str) -> Config {
let resolved_path = shellexpand::tilde(config_path);
let config_text = fs::read_to_string(&*resolved_path).unwrap();
let c: Config = toml::from_str(&config_text).unwrap();
return c;
}
pub fn get_repository(&self) -> String {
let resolved_path = shellexpand::tilde(&self.transcoder.repository);
return String::from(&*resolved_path);
}
}
#[derive(Deserialize)]
pub struct Transcoder {
pub repository: String,
pub interval: u16,
pub video_format: String,
pub video_codec: String,
pub video_profile: String,
pub video_resolution: String,
pub video_framerate: u8,
pub video_color: String,
pub audio_codec: String,
}

21
src/main.rs Normal file
View File

@ -0,0 +1,21 @@
use log4rs;
use config::Config;
use transcoder::repository::Repository;
use transcoder::transcoder::Transcoder;
mod config;
mod transcoder;
mod util;
fn main() {
// initialize the log4rs logger
log4rs::init_file("./log4rs.yaml", Default::default()).unwrap();
// create and initialize our config and repository objects
let c: Config = Config::new("~/.config/zealot.toml");
let r: Repository = Repository::new(&c.get_repository());
// create and start the video transcoder object
let t: Transcoder = Transcoder::new(c, r);
t.start();
}

View File

@ -1,48 +0,0 @@
# frozen_string_literal: true
require_relative 'util'
require 'fileutils'
# Handles anything pertaining to the video file repository.
class Repository
def initialize(path)
@base_path = path
# create repository base directory
create_directory(@base_path)
# create sub-directories
sub_paths = %w[ingest archive output]
sub_paths.each { |p|
create_directory(File.join(@base_path, p))
}
end
def search_ingest
# create our ingest directory path
ingest_path = File.join(@base_path, 'ingest')
# search for files in ingest; ignore non-files (e.g. directories)
ingest_files = Dir.entries(ingest_path).select { |f| File.file? File.join(ingest_path, f) }
return ingest_files
end
def archive_file(filename)
# create source and destination paths for the copy
ingest_path = File.join(@base_path, 'ingest', filename)
archive_path = File.join(@base_path, 'archive', filename)
# perform the copy, preserving file attributes
FileUtils.cp(ingest_path, archive_path, preserve: true)
end
def cleanup_file(filename)
# create ingest path
ingest_path = File.join(@base_path, 'ingest', filename)
# remove the file
FileUtils.remove_file(ingest_path)
end
end

View File

@ -1,56 +0,0 @@
# frozen_string_literal: true
require 'pathname'
# Handles the actual transcoding.
class Transcoder
def initialize(config, repository)
@config = config
@repository = repository
end
def start
puts 'Starting transcoder...'
# search for files in ingest
ingest_files = @repository.search_ingest
ingest_files.each { |ifile|
# archive the file
@repository.archive_file(ifile)
# perform the transcode
transcode(ifile)
# clean up the file from ingest
@repository.cleanup_file(ifile)
}
end
def transcode(ingest_file)
# create the ingest and output file paths
ingest_path = File.join(@config.get('transcoder.repository'), 'ingest', ingest_file)
out_name = "#{File.basename(ingest_file, File.extname(ingest_file))}.#{@config.get('transcoder.video_format')}"
output_path = File.join(@config.get('transcoder.repository'), 'output', out_name)
ffmpeg_cmd = build_ffmpeg_cmd(ingest_path, output_path)
# execute the FFMPEG command
puts "Transcoding #{ingest_path} to #{output_path}."
result = `#{ffmpeg_cmd} 2>&1 >/dev/null`
abort "Transcoding failed with error:\n#{result}" if result.end_with?("Conversion failed!\n")
end
def build_ffmpeg_cmd(ingest_path, output_path)
# put together the ffmpeg command
"/usr/bin/ffmpeg -i #{ingest_path} -y" \
" -f #{@config.get('transcoder.video_format')}" \
" -c:v #{@config.get('transcoder.video_codec')}" \
" -s #{@config.get('transcoder.video_resolution')}" \
" -r #{@config.get('transcoder.video_framerate')}" \
" -vf 'format=#{@config.get('transcoder.video_color')}'" \
" -profile:v #{@config.get('transcoder.video_profile')}" \
" -c:a #{@config.get('transcoder.audio_codec')}" \
" #{output_path}"
end
end

2
src/transcoder/mod.rs Normal file
View File

@ -0,0 +1,2 @@
pub mod repository;
pub mod transcoder;

View File

@ -0,0 +1,88 @@
use std::fs;
use std::path::Path;
use log::{error, info};
pub struct Repository {
pub base_dir: String,
pub ingest_dir: String,
pub archive_dir: String,
pub output_dir: String,
}
impl Repository {
pub fn new(base_path: &str) -> Repository {
// create the base directory path
create_directory(base_path);
// create the needed sub-directories
let ingest_path = Path::new(base_path).join("ingest");
create_directory(ingest_path.to_str().unwrap());
let archive_path = Path::new(base_path).join("archive");
create_directory(archive_path.to_str().unwrap());
let output_path = Path::new(base_path).join("output");
create_directory(output_path.to_str().unwrap());
return Repository {
base_dir: String::from(base_path),
ingest_dir: String::from(ingest_path.to_str().unwrap()),
archive_dir: String::from(archive_path.to_str().unwrap()),
output_dir: String::from(output_path.to_str().unwrap()),
};
}
pub fn search_ingest(&self) -> Vec<String> {
// read file entries from ingest
let files = fs::read_dir(&self.ingest_dir).unwrap();
// create vec object and loop through entries to find what we want
let mut ingest_files: Vec<String> = vec![];
for f in files {
let f = f.unwrap();
let path = f.path();
if path.is_file() {
let file_path = path.file_name().unwrap().to_str();
ingest_files.push(String::from(file_path.unwrap()));
}
}
return ingest_files;
}
pub fn archive_file(&self, file: &str) {
let ingest_file = Path::new(&self.ingest_dir).join(file);
let archive_file = Path::new(&self.archive_dir).join(file);
match fs::copy(&ingest_file, &archive_file) {
Ok(_) => {
info!("Archiving video file {}.", ingest_file.to_str().unwrap());
},
Err(e) => {
error!("Error archiving file {}: {}", ingest_file.to_str().unwrap(), e);
std::process::exit(1);
}
}
}
pub fn cleanup_file(&self, file: &str) {
let ingest_file = Path::new(&self.ingest_dir).join(file);
fs::remove_file(&ingest_file)
.expect("File deletion failed.");
}
}
fn create_directory(path: &str) {
let d = Path::new(path);
if d.is_dir() {
info!("Directory {} already exists.", path);
} else {
match fs::create_dir(path) {
Ok(_) => {
info!("Creating directory {}.", path);
},
Err(e) => {
error!("Error creating {}: {}", path, e);
std::process::exit(1);
}
}
}
}

View File

@ -0,0 +1,81 @@
use std::path::Path;
use std::process;
use std::{thread, time};
use log::{info};
use crate::config::Config;
use crate::util::io;
use super::repository::Repository;
pub struct Transcoder {
config: Config,
repository: Repository,
}
impl Transcoder {
pub fn new(config: Config, repository: Repository) -> Transcoder {
return Transcoder{
config: config,
repository: repository,
}
}
pub fn start(self) {
info!("Starting transcoder...");
loop {
// search for files in ingest
let ingest_files = self.repository.search_ingest();
// check if we found any files to transcode
if ingest_files.len() < 1 {
info!("There were no files found in ingest to transcode; skipping run.");
} else {
for i in ingest_files {
let ingest_path = Path::new(&self.repository.ingest_dir).join(&i);
if io::is_file_locked(&ingest_path.to_str().unwrap()) {
info!("{} is currently open in another program; skipping it for this run.", &i);
continue;
}
// copy the file to the archive
self.repository.archive_file(&i);
// perform the video transcode
self.transcode(&i);
// remove the source file
self.repository.cleanup_file(&i);
}
}
// put the loop to sleep for X minutes
let sleep_minutes = time::Duration::from_secs((self.config.transcoder.interval * 60).into());
thread::sleep(sleep_minutes);
}
}
fn transcode(&self, file: &str) {
let ingest_file = Path::new(&self.repository.ingest_dir).join(file);
let output_file = Path::new(&self.repository.output_dir).join(file);
let cmd_output = process::Command::new("/usr/bin/ffmpeg")
.arg("-i") .arg(&*ingest_file.to_string_lossy())
.arg("-y")
.arg("-f") .arg(&self.config.transcoder.video_format)
.arg("-c:v") .arg(&self.config.transcoder.video_codec)
.arg("-s") .arg(&self.config.transcoder.video_resolution)
.arg("-r") .arg(format!("{}", self.config.transcoder.video_framerate))
.arg("-vf") .arg(format!("format={}", &self.config.transcoder.video_color))
.arg("-profile:v").arg(&self.config.transcoder.video_profile)
.arg("-c:a") .arg(&self.config.transcoder.audio_codec)
.arg(&*output_file.to_string_lossy())
.output()
.expect("Failed to execute command");
assert!(cmd_output.status.success());
let results_raw = &String::from_utf8_lossy(&cmd_output.stderr);
info!("{}", results_raw);
}
}

View File

@ -1,9 +0,0 @@
# frozen_string_literal: true
def create_directory(path)
if Dir.exist?(path)
puts "Directory #{path} already exists."
else
puts "Creating directory #{path}."
end
end

61
src/util/io.rs Normal file
View File

@ -0,0 +1,61 @@
use std::process;
// checks whether a file is currently open in another program (e.g. still being written)
pub fn is_file_locked(filepath: &str) -> bool {
let cmd = process::Command::new("/usr/bin/lsof")
.arg(filepath)
.output()
.expect("Failed to execute command");
let results = &String::from_utf8_lossy(&cmd.stdout);
println!("{}", results);
if results.contains(filepath) {
return true;
}
return false;
}
#[cfg(test)]
mod tests {
use std::fs;
use std::mem::drop;
use std::path::Path;
use super::*;
#[test]
fn file_should_not_be_locked() {
setup("test-not-locked.txt");
assert!(!is_file_locked("test-not-locked.txt"));
teardown("test-not-locked.txt");
}
#[test]
fn file_should_be_locked() {
setup("test-locked.txt");
let path = Path::new("test-locked.txt");
match fs::File::open(&path) {
Err(e) => panic!("couldn't open {}: {}", "test-locked.txt", e),
Ok(_) => assert!(is_file_locked("test-locked.txt")),
};
teardown("test-locked.txt");
}
// get things ready for our tests
fn setup(test_file: &str) {
let f = match fs::File::create(test_file) {
Ok(file) => file,
Err(e) => panic!("{:?}", e),
};
drop(f);
}
// clean up after running tests
fn teardown(test_file: &str) {
match fs::remove_file(test_file) {
Ok(_) => {},
Err(e) => panic!("{:?}", e),
};
}
}

1
src/util/mod.rs Normal file
View File

@ -0,0 +1 @@
pub mod io;

View File

@ -1,19 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
require_relative 'config'
require_relative 'repository'
require_relative 'transcoder'
# create new configuration instance
c = Config.new('~/.config/zealot.toml')
# create new repository instance
r = Repository.new(c.get('transcoder.repository'))
# create new transcoder instance
t = Transcoder.new(c, r)
# start the transcoder
t.start