Upping to v0.2.7

.woodpecker.yml

@@ -1,17 +1,19 @@
 pipeline:
-  test_build:
+  tests:
-    image: rust:1.63
+    image: rust:1.75
     commands:
-      - "cargo build"
+      - "apt update"
+      - "apt install -y lsof"
+      - "cargo test"
 
   build_release:
-    image: rust:1.63
+    image: rust:1.75
     commands:
       - "cargo install cargo-deb cargo-generate-rpm"
       - "cargo build --release"
       - "cargo deb"
       - "cargo generate-rpm"
-      - "mv target/release/adept target/release/adept-${CI_COMMIT_TAG}-linux-x86_64"
+      - "mv target/release/zealot target/release/zealot-${CI_COMMIT_TAG}-linux-x86_64"
     when:
       event: tag
 
@@ -23,8 +25,54 @@ pipeline:
       base_url: https://git.metaunix.net
       files:
         - "target/release/*${CI_COMMIT_TAG}-linux-x86_64"
-        - "target/debian/adept*.deb"
+        - "target/debian/zealot*.deb"
-        - "target/generate-rpm/adept*.rpm"
+        - "target/generate-rpm/zealot*.rpm"
       title: "${CI_COMMIT_TAG}"
     when:
       event: tag
+
+  copy_deb_package:
+    image: appleboy/drone-scp
+    settings:
+      host: "repo.int.metaunix.net"
+      username:
+        from_secret: repo_admin
+      password:
+        from_secret: repo_password
+      port: 22
+      target: /srv/repo/apt/zealot/
+      source: target/debian/zealot*.deb
+      strip_components: 2
+    when:
+      event: tag
+
+  copy_rpm_package:
+    image: appleboy/drone-scp
+    settings:
+      host: "repo.int.metaunix.net"
+      username:
+        from_secret: repo_admin
+      password:
+        from_secret: repo_password
+      port: 22
+      target: /srv/repo/dnf/zealot/
+      source: target/generate-rpm/zealot*.rpm
+      strip_components: 2
+    when:
+      event: tag
+
+  update_repos:
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - repo.int.metaunix.net
+      username:
+        from_secret: repo_admin
+      password:
+        from_secret: repo_password
+      port: 22
+      command_timeout: 2m
+      script:
+        - "sudo /home/xadmin/scripts/update_repo.sh"
+    when:
+      event: tag

Cargo.toml

@@ -1,13 +1,17 @@
 [package]
-name = "adept"
+name = "zealot"
 description = "Bit Goblin automated video transcoding service."
-version = "0.1.1"
+version = "0.2.7"
 edition = "2021"
 readme = "README.md"
 license = "BSD 2-Clause"
 authors = ["Gregory Ballantine <gballantine@bitgoblin.tech>"]
+repository = "https://git.metaunix.net/BitGoblin/zealot"
 
 [dependencies]
+clap = { version = "3.2", features = ['derive'] }
+config = { version = "0.13", features = ['toml'] }
+filetime = "0.2"
 log = "0.4"
 log4rs = "1.1"
 toml = "0.5"
@@ -21,21 +25,22 @@ maintainer-scripts = "build/scripts"
 depends = "ffmpeg (>= 4)"
 section = "video"
 assets = [
-	["target/release/adept", "usr/bin/adept", "755"],
+	["target/release/zealot", "usr/bin/zealot", "755"],
-	["build/etc/example.toml", "etc/adept/example.toml", "644"],
+	["build/etc/example.toml", "etc/zealot/example.toml", "644"],
-	["README.md", "usr/share/doc/adept/README", "644"]
+	["build/etc/log4rs.yaml", "etc/zealot/log4rs.yaml", "755"],
+	["README.md", "usr/share/doc/zealot/README", "644"]
 ]
 [package.metadata.deb.systemd-units]
 unit-scripts = "build/debian/service"
-unit-name = "adept"
+unit-name = "zealot"
 enable = true
 start = true
 
 [package.metadata.generate-rpm]
 assets = [
-  { source = "target/release/adept", dest = "/usr/bin/adept", mode = "755" },
+  { source = "target/release/zealot", dest = "/usr/bin/zealot", mode = "755" },
-  { source = "build/etc/example.toml", dest = "/etc/adept/example.toml", mode = "755"},
+  { source = "build/etc/example.toml", dest = "/etc/zealot/example.toml", mode = "755"},
-  { source = "README.md", dest = "/usr/share/doc/adept/README", mode = "644"}
+  { source = "README.md", dest = "/usr/share/doc/zealot/README", mode = "644"}
 ]
 post_install_script = "build/scripts/postinst"
 [package.metadata.generate-rpm.requires]

README.md

@@ -1,3 +1,3 @@
-# adept
+# zealot
 
 Bit Goblin video transcoder service

build/debian/service/adept.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Adept video transcoder service
-
-[Service]
-User=adept
-Group=adept
-ExecStart=/usr/bin/adept
-SuccessExitStatus=143
-
-[Install]
-WantedBy=multi-user.target

build/debian/service/zealot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Zealot video transcoder service
+
+[Service]
+User=zealot
+Group=zealot
+ExecStart=/usr/bin/zealot -l /etc/zealot/log4rs.yaml
+SuccessExitStatus=143
+
+[Install]
+WantedBy=multi-user.target

build/etc/example.toml

@@ -1,6 +1,6 @@
 # This example transcodes footage to DNxHD 1080p60 for use in video editors like DaVinci Resolve.
 [transcoder]
-repository = "~/adept" # location of the videos to transcode
+repository = "~/zealot" # location of the videos to transcode
 interval = 15
 video_format = "mov" # video container format
 video_codec = "dnxhd" # video codec to use

build/etc/log4rs.yaml

@@ -0,0 +1,12 @@
+appenders:
+  stdout:
+    kind: console
+    encoder:
+      pattern: "{d(%+)(utc)} {h({l})}: {m}{n}"
+    filters:
+      - kind: threshold
+        level: info
+root:
+  level: info
+  appenders:
+    - stdout

build/scripts/postinst

@@ -1,25 +1,36 @@
 #!/bin/sh
 
-GETENT_USER=$(getent passwd adept)
+GETENT_USER=$(getent passwd zealot)
-GETENT_GROUP=$(getent group adept)
+GETENT_GROUP=$(getent group zealot)
 
-# Create the adept user if it doesn't already exist
+# Create the zealot user if it doesn't already exist
 if [ "$GETENT_USER" = "" ]; then
-	useradd -r adept
+	echo "Creating the 'zealot' user."
+	useradd -r zealot
 else
-	echo "The 'adept' user already exists, skipping creation."
+	echo "The 'zealot' user already exists, skipping creation."
 fi
 
-# Create the adept group if it doesn't already exist
+# Create the zealot group if it doesn't already exist
 if [ "$GETENT_GROUP" = "" ]; then
-	groupadd adept
+	echo "Creating the 'zealot' group."
-	usermod -aG adept adept
+	groupadd zealot
+	usermod -aG zealot zealot
 else
-	echo "The 'adept' group already exists, skipping creation."
+	echo "The 'zealot' group already exists, skipping creation."
 fi
 
 # Change the directory ownership of /etc
-chown -R adept:adept /etc/adept
+chown -R zealot:zealot /etc/zealot
+
+# Create the log directory under /var/log
+if [ ! -d /var/log/zealot ]; then
+	echo "Creating /var/log/zealot to store log files."
+	mkdir /var/log/zealot
+	chown zealot:zealot /var/log/zealot
+else
+	echo "/var/log/zealot already exists, skipping creation."
+fi
 
 #DEBHELPER#
 

example.config

@@ -0,0 +1,10 @@
+[transcoder]
+repo_path = '~/zealot'
+interval = 15
+video_format = 'mov'
+video_codec = 'dnxhd'
+video_profile = 'dnxhr_hq'
+video_resolution = '1920x1080'
+video_framerate = '60'
+video_color = 'yuv422p'
+audio_codec = 'pcm_s16le'

src/cmd/core.rs

@@ -0,0 +1,38 @@
+use config::Config;
+use crate::settings;
+use crate::transcoder::repository::Repository;
+
+pub fn setup_command() {
+  // load configuration
+  let c: Config = settings::load_config();
+
+  // resolve repository path
+  let repository_dir_raw: &str = &c.get_string("transcoder.repository").unwrap();
+  let repository_dir: &str = &shellexpand::tilde(repository_dir_raw);
+
+  // alert the user to what's happening
+  println!("Initializing video repository at {}...", repository_dir);
+
+  // create and initialize our repository object
+  let r: Repository = Repository::new(repository_dir);
+  // initialize the video repository
+  r.initialize();
+}
+
+pub fn version_command() {
+  println!("===================================================================================");
+
+  // output the program and version
+  println!("Zealot version v{}", env!("CARGO_PKG_VERSION"));
+
+  // output project description
+  println!("{}", env!("CARGO_PKG_DESCRIPTION"));
+
+  // display license info
+  println!("\nThis code is licensed under the {} license.", env!("CARGO_PKG_LICENSE"));
+
+  // output the project source URL
+  println!("View the source code or file issues at {}.", env!("CARGO_PKG_REPOSITORY"));
+
+  println!("===================================================================================");
+}

src/cmd/mod.rs

@@ -0,0 +1 @@
+pub mod core;

src/config.rs

@@ -1,34 +0,0 @@
-use serde_derive::Deserialize;
-use std::fs;
-
-#[derive(Deserialize)]
-pub struct Config {
-  pub transcoder: Transcoder,
-}
-
-impl Config {
-  pub fn new(config_path: &str) -> Config {
-    let resolved_path = shellexpand::tilde(config_path);
-    let config_text = fs::read_to_string(&*resolved_path).unwrap();
-    let c: Config = toml::from_str(&config_text).unwrap();
-    return c;
-  }
-
-  pub fn get_repository(&self) -> String {
-    let resolved_path = shellexpand::tilde(&self.transcoder.repository);
-    return String::from(&*resolved_path);
-  }
-}
-
-#[derive(Deserialize)]
-pub struct Transcoder {
-  pub repository:       String,
-  pub interval:         u16,
-  pub video_format:     String,
-  pub video_codec:      String,
-  pub video_profile:    String,
-  pub video_resolution: String,
-  pub video_framerate:  u8,
-  pub video_color:      String,
-  pub audio_codec:      String,
-}

src/main.rs

@@ -1,21 +1,56 @@
+use clap::{Parser, Subcommand};
 use log4rs;
 use config::Config;
-use repository::Repository;
+use transcoder::repository::Repository;
-use transcoder::Transcoder;
+use transcoder::transcoder::Transcoder;
 
-mod config;
+mod cmd;
-mod repository;
+mod settings;
 mod transcoder;
+mod util;
+
+#[derive(Parser)]
+#[clap(author, version, about, long_about = None)]
+struct Cli {
+  /// Number of times to greet
+  #[clap(short = 'l', long, default_value_t = String::from("./log4rs.yaml"))]
+  log_config: String,
+
+  #[clap(subcommand)]
+  command: Option<Commands>,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+  // Runs the setup command
+  Setup {},
+
+  // Displays version info about the program
+  Version {},
+}
 
 fn main() {
-  // initialize the log4rs logger
+  // initialize the clap CLI
-  log4rs::init_file("./log4rs.yaml", Default::default()).unwrap();
+  let cli = Cli::parse();
 
+  // grab the log4rs config file path, then initialize log4rs
+  let log4rs_config: String = cli.log_config;
+  log4rs::init_file(&log4rs_config, Default::default()).unwrap();
+
+  match &cli.command {
+    // sub-commands will be handled here
+    Some(Commands::Setup {}) => cmd::core::setup_command(),
+    Some(Commands::Version {}) => cmd::core::version_command(),
+
+    // run the main program without any commands
+    None => {
       // create and initialize our config and repository objects
-  let c: Config = Config::new("~/.config/adept.toml");
+      let c: Config = settings::load_config();
-  let r: Repository = Repository::new(&c.get_repository());
+      let r: Repository = Repository::new(&shellexpand::tilde(&c.get_string("transcoder.repository").unwrap()));
 
       // create and start the video transcoder object
       let t: Transcoder = Transcoder::new(c, r);
       t.start();
+    },
+  }
 }

src/settings.rs

@@ -0,0 +1,33 @@
+use config::Config;
+
+pub fn load_config() -> Config {
+  let global_config_path: String = find_global_config_path();
+  let home_config_path: String = find_home_config_path();
+
+  let settings = Config::builder()
+    // Add in `./Settings.toml`
+    .add_source(config::File::with_name(&global_config_path).required(false))
+    .add_source(config::File::with_name(&home_config_path).required(false))
+    .build()
+    .unwrap();
+
+  return settings;
+}
+
+fn find_global_config_path() -> String {
+  if cfg!(windows) {
+    return String::from("C:\\Program Files\\Zealot\\config.toml");
+  }
+
+  return String::from("/etc/zealot/config.toml");
+}
+
+fn find_home_config_path() -> String {
+  let home_path: &str = &shellexpand::tilde("~/.config/zealot.toml");
+
+  if cfg!(windows) {
+    return String::from(home_path.replace("/", "\\"));
+  }
+
+  return String::from(home_path);
+}

src/transcoder.rs

@@ -1,74 +0,0 @@
-use std::path::Path;
-use std::process;
-use std::{thread, time};
-use log::{info};
-
-use crate::config::Config;
-use crate::repository::Repository;
-
-pub struct Transcoder {
-  config: Config,
-  repository: Repository,
-}
-
-impl Transcoder {
-  pub fn new(config: Config, repository: Repository) -> Transcoder {
-    return Transcoder{
-      config: config,
-      repository: repository,
-    }
-  }
-
-  pub fn start(self) {
-    info!("Starting transcoder...");
-
-    loop {
-      // search for files in ingest
-      let ingest_files = self.repository.search_ingest();
-
-      // check if we found any files to transcode
-      if ingest_files.len() < 1 {
-        info!("There were no files found in ingest to transcode; skipping run.");
-      } else {
-        for i in ingest_files {
-          // copy the file to the archive
-          self.repository.archive_file(&i);
-
-          // perform the video transcode
-          self.transcode(&i);
-
-          // remove the source file
-          self.repository.cleanup_file(&i);
-        }
-      }
-
-      // put the loop to sleep for X minutes
-      let sleep_minutes = time::Duration::from_secs((self.config.transcoder.interval * 60).into());
-      thread::sleep(sleep_minutes);
-    }
-  }
-
-  fn transcode(&self, file: &str) {
-    let ingest_file = Path::new(&self.repository.ingest_dir).join(file);
-    let output_file = Path::new(&self.repository.output_dir).join(file);
-
-    let cmd_output = process::Command::new("/usr/bin/ffmpeg")
-      .arg("-i")        .arg(&*ingest_file.to_string_lossy())
-      .arg("-y")
-      .arg("-f")        .arg(&self.config.transcoder.video_format)
-      .arg("-c:v")      .arg(&self.config.transcoder.video_codec)
-      .arg("-s")        .arg(&self.config.transcoder.video_resolution)
-      .arg("-r")        .arg(format!("{}", self.config.transcoder.video_framerate))
-      .arg("-vf")       .arg(format!("format={}", &self.config.transcoder.video_color))
-      .arg("-profile:v").arg(&self.config.transcoder.video_profile)
-      .arg("-c:a")      .arg(&self.config.transcoder.audio_codec)
-      .arg(&*output_file.to_string_lossy())
-      .output()
-      .expect("Failed to execute command");
-
-    assert!(cmd_output.status.success());
-
-    let results_raw = &String::from_utf8_lossy(&cmd_output.stderr);
-    info!("{}", results_raw);
-  }
-}

src/transcoder/mod.rs

@@ -0,0 +1,2 @@
+pub mod repository;
+pub mod transcoder;

src/transcoder/repository.rs

@@ -1,6 +1,8 @@
 use std::fs;
 use std::path::Path;
 use log::{error, info};
+use filetime;
+use filetime::FileTime;
 
 pub struct Repository {
   pub base_dir: String,
@@ -11,16 +13,9 @@ pub struct Repository {
 
 impl Repository {
   pub fn new(base_path: &str) -> Repository {
-    // create the base directory path
-    create_directory(base_path);
-
-    // create the needed sub-directories
     let ingest_path = Path::new(base_path).join("ingest");
-    create_directory(ingest_path.to_str().unwrap());
     let archive_path = Path::new(base_path).join("archive");
-    create_directory(archive_path.to_str().unwrap());
     let output_path = Path::new(base_path).join("output");
-    create_directory(output_path.to_str().unwrap());
 
     return Repository {
       base_dir: String::from(base_path),
@@ -30,6 +25,16 @@ impl Repository {
     };
   }
 
+  pub fn initialize(&self) {
+    // create the base directory path
+    create_directory(&self.base_dir);
+
+    // create the needed sub-directories
+    create_directory(&self.ingest_dir.as_str());
+    create_directory(&self.archive_dir.as_str());
+    create_directory(&self.output_dir.as_str());
+  }
+
   pub fn search_ingest(&self) -> Vec<String> {
     // read file entries from ingest
     let files = fs::read_dir(&self.ingest_dir).unwrap();
@@ -55,6 +60,7 @@ impl Repository {
     match fs::copy(&ingest_file, &archive_file) {
       Ok(_) => {
         info!("Archiving video file {}.", ingest_file.to_str().unwrap());
+        copy_file_metadata(ingest_file.to_str().unwrap(), archive_file.to_str().unwrap());
       },
       Err(e) => {
         error!("Error archiving file {}: {}", ingest_file.to_str().unwrap(), e);
@@ -86,3 +92,16 @@ fn create_directory(path: &str) {
     }
   }
 }
+
+fn copy_file_metadata(old_file: &str, new_file: &str) {
+  // grab old file metadata
+  let old_metadata = fs::metadata(old_file).unwrap();
+  let old_permissions = old_metadata.permissions();
+  let mtime = FileTime::from_last_modification_time(&old_metadata);
+  let atime = FileTime::from_last_access_time(&old_metadata);
+
+  // set metadata on the new file
+  let new_path = Path::new(new_file);
+  fs::set_permissions(new_file, old_permissions).expect("Failed to set new file permissions.");
+  filetime::set_file_times(new_path, atime, mtime).expect("Failed to set atime and mtime.");
+}

src/transcoder/transcoder.rs

@@ -0,0 +1,106 @@
+use config::Config;
+use std::path::Path;
+use std::process;
+use std::{thread, time};
+use log::{debug, info};
+
+use crate::util::io;
+use super::repository::Repository;
+
+pub struct Transcoder {
+  config: Config,
+  repository: Repository,
+}
+
+impl Transcoder {
+  pub fn new(config: Config, repository: Repository) -> Transcoder {
+    return Transcoder{
+      config: config,
+      repository: repository,
+    }
+  }
+
+  pub fn start(self) {
+    info!("Starting transcoder...");
+
+    loop {
+      // search for files in ingest
+      let ingest_files = self.repository.search_ingest();
+
+      // check if we found any files to transcode
+      if ingest_files.len() < 1 {
+        info!("There were no files found in ingest to transcode; skipping run.");
+      } else {
+        // log that the transcoder is starting up
+        info!("Found {} files in ingest to transcode. Standby...", ingest_files.len());
+
+        // loop through each file found in ingest
+        for i in ingest_files {
+          let ingest_path = Path::new(&self.repository.ingest_dir).join(&i);
+          if io::is_file_locked(&ingest_path.to_str().unwrap()) {
+            info!("{} is currently open in another program; skipping it for this run.", &i);
+            continue;
+          }
+
+          // copy the file to the archive
+          self.repository.archive_file(&i);
+
+          // perform the video transcode
+          self.transcode(&i);
+
+          // remove the source file
+          self.repository.cleanup_file(&i);
+        }
+
+        // let the user know the transcode has finished
+        info!("Finished transcoding. Sleeping...");
+      }
+
+      // put the loop to sleep for X minutes
+      let sleep_minutes = time::Duration::from_secs((self.config.get_int("transcoder.interval").unwrap() as u64 * 60).into());
+      thread::sleep(sleep_minutes);
+    }
+  }
+
+  fn transcode(&self, file: &str) {
+    let ingest_file = Path::new(&self.repository.ingest_dir).join(file);
+    let video_format = &self.config.get_string("transcoder.video_format").unwrap();
+    let output_stem = Path::new(file).file_stem().unwrap();
+    let output_bits = vec![output_stem.to_str().unwrap(), &video_format];
+    let output_name = output_bits.join(".");
+    let output_file = Path::new(&self.repository.output_dir).join(output_name);
+
+    let video_codec = &self.config.get_string("transcoder.video_codec").unwrap();
+    info!("Transcoding {} to {} with the {} encoder.", ingest_file.display(), output_file.display(), video_codec);
+
+    let binding = process::Command::new("/usr/bin/ffmpeg");
+    let mut cmd = binding;
+
+    // start building the command
+    cmd.arg("-i")        .arg(&*ingest_file.to_string_lossy())
+      .arg("-y")
+      .arg("-f")        .arg(&video_format)
+      .arg("-c:v")      .arg(&video_codec);
+
+    // add video resolution if it's available
+    if self.config.get_string("transcoder.video_resolution").is_ok() {
+        cmd.arg("-s").arg(&self.config.get_string("transcoder.video_resolution").unwrap());
+    };
+
+    // finish out command
+    cmd.arg("-r").arg(format!("{}", self.config.get_string("transcoder.video_framerate").unwrap()))
+      .arg("-vf")       .arg(format!("format={}", &self.config.get_string("transcoder.video_color").unwrap()))
+      .arg("-profile:v").arg(&self.config.get_string("transcoder.video_profile").unwrap())
+      .arg("-c:a")      .arg(&self.config.get_string("transcoder.audio_codec").unwrap());
+
+    // finish the command and run it
+    let cmd_output = cmd.arg(&*output_file.to_string_lossy())
+      .output()
+      .expect("Failed to execute command");
+
+    assert!(cmd_output.status.success());
+
+    let results_raw = &String::from_utf8_lossy(&cmd_output.stderr);
+    debug!("{}", results_raw);
+  }
+}

src/util/io.rs

@@ -0,0 +1,61 @@
+use std::process;
+
+// checks whether a file is currently open in another program (e.g. still being written)
+pub fn is_file_locked(filepath: &str) -> bool {
+  let cmd = process::Command::new("/usr/bin/lsof")
+    .arg(filepath)
+    .output()
+    .expect("Failed to execute command");
+
+  let results = &String::from_utf8_lossy(&cmd.stdout);
+  println!("{}", results);
+  if results.contains(filepath) {
+    return true;
+  }
+
+  return false;
+}
+
+#[cfg(test)]
+mod tests {
+  use std::fs;
+  use std::mem::drop;
+  use std::path::Path;
+
+  use super::*;
+
+  #[test]
+  fn file_should_not_be_locked() {
+    setup("test-not-locked.txt");
+    assert!(!is_file_locked("test-not-locked.txt"));
+    teardown("test-not-locked.txt");
+  }
+
+  #[test]
+  fn file_should_be_locked() {
+    setup("test-locked.txt");
+    let path = Path::new("test-locked.txt");
+    match fs::File::open(&path) {
+      Err(e) => panic!("couldn't open {}: {}", "test-locked.txt", e),
+      Ok(_) => assert!(is_file_locked("test-locked.txt")),
+    };
+    teardown("test-locked.txt");
+  }
+
+  // get things ready for our tests
+  fn setup(test_file: &str) {
+    let f = match fs::File::create(test_file) {
+      Ok(file) => file,
+      Err(e) => panic!("{:?}", e),
+    };
+    drop(f);
+  }
+
+  // clean up after running tests
+  fn teardown(test_file: &str) {
+    match fs::remove_file(test_file) {
+      Ok(_) => {},
+      Err(e) => panic!("{:?}", e),
+    };
+  }
+}

src/util/mod.rs

@@ -0,0 +1 @@
+pub mod io;