Started rewriting project in TypeScript
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed

This commit is contained in:
Gregory Ballantine 2022-11-17 00:07:39 -05:00
parent 6da120f632
commit 51fe19f12c
22 changed files with 3089 additions and 503 deletions

6
.eslintrc.cjs Normal file
View File

@ -0,0 +1,6 @@
module.exports = {
extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'],
parser: '@typescript-eslint/parser',
plugins: ['@typescript-eslint'],
root: true,
};

22
.gitignore vendored
View File

@ -1,21 +1,3 @@
# ---> Rust
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# NPM modules
node_modules/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# Added by cargo
/target

View File

@ -1,21 +1,9 @@
pipeline:
tests:
image: rust:1.63
image: node:1.18
commands:
- "apt update"
- "apt install -y lsof"
- "cargo test"
build_release:
image: rust:1.63
commands:
- "cargo install cargo-deb cargo-generate-rpm"
- "cargo build --release"
- "cargo deb"
- "cargo generate-rpm"
- "mv target/release/zealot target/release/zealot-${CI_COMMIT_TAG}-linux-x86_64"
when:
event: tag
- "npm install"
- "npm run lint"
gitea_release:
image: plugins/gitea-release
@ -24,55 +12,6 @@ pipeline:
from_secret: gitea_api_key
base_url: https://git.metaunix.net
files:
- "target/release/*${CI_COMMIT_TAG}-linux-x86_64"
- "target/debian/zealot*.deb"
- "target/generate-rpm/zealot*.rpm"
title: "${CI_COMMIT_TAG}"
when:
event: tag
copy_deb_package:
image: appleboy/drone-scp
settings:
host: "repo.int.metaunix.net"
username:
from_secret: repo_admin
password:
from_secret: repo_password
port: 22
target: /srv/repo/apt/zealot/
source: target/debian/zealot*.deb
strip_components: 2
when:
event: tag
copy_rpm_package:
image: appleboy/drone-scp
settings:
host: "repo.int.metaunix.net"
username:
from_secret: repo_admin
password:
from_secret: repo_password
port: 22
target: /srv/repo/dnf/zealot/
source: target/generate-rpm/zealot*.rpm
strip_components: 2
when:
event: tag
update_repos:
image: appleboy/drone-ssh
settings:
host:
- repo.int.metaunix.net
username:
from_secret: repo_admin
password:
from_secret: repo_password
port: 22
command_timeout: 2m
script:
- "sudo /home/xadmin/scripts/update_repo.sh"
when:
event: tag

View File

@ -1,45 +0,0 @@
[package]
name = "zealot"
description = "Bit Goblin automated video transcoding service."
version = "0.2.3"
edition = "2021"
readme = "README.md"
license = "BSD 2-Clause"
authors = ["Gregory Ballantine <gballantine@bitgoblin.tech>"]
[dependencies]
clap = { version = "3.2", features = ['derive'] }
config = { version = "0.13", features = ['toml'] }
log = "0.4"
log4rs = "1.1"
toml = "0.5"
serde = "1.0"
serde_derive = "1.0"
shellexpand = "2.1"
[package.metadata.deb]
license-file = "LICENSE"
maintainer-scripts = "build/scripts"
depends = "ffmpeg (>= 4)"
section = "video"
assets = [
["target/release/zealot", "usr/bin/zealot", "755"],
["build/etc/example.toml", "etc/zealot/example.toml", "644"],
["build/etc/log4rs.yaml", "etc/zealot/log4rs.yaml", "755"],
["README.md", "usr/share/doc/zealot/README", "644"]
]
[package.metadata.deb.systemd-units]
unit-scripts = "build/debian/service"
unit-name = "zealot"
enable = true
start = true
[package.metadata.generate-rpm]
assets = [
{ source = "target/release/zealot", dest = "/usr/bin/zealot", mode = "755" },
{ source = "build/etc/example.toml", dest = "/etc/zealot/example.toml", mode = "755"},
{ source = "README.md", dest = "/usr/share/doc/zealot/README", mode = "644"}
]
post_install_script = "build/scripts/postinst"
[package.metadata.generate-rpm.requires]
ffmpeg = ">= 4"

4
bin/run Executable file
View File

@ -0,0 +1,4 @@
#!/bin/sh
npm run ts index.ts

View File

@ -1,12 +0,0 @@
appenders:
stdout:
kind: console
encoder:
pattern: "{d(%+)(utc)} {h({l})}: {m}{n}"
filters:
- kind: threshold
level: info
root:
level: info
appenders:
- stdout

View File

@ -1,5 +1,8 @@
[ffmpeg]
bin_path = '/usr/bin/ffmpeg'
[transcoder]
repo_path = '~/zealot'
repository = '~/zealot'
interval = 15
video_format = 'mov'
video_codec = 'dnxhd'

11
index.ts Normal file
View File

@ -0,0 +1,11 @@
import fs from 'fs';
import toml from 'toml';
import Repository from './src/repository';
// load config file
const config = toml.parse(fs.readFileSync('./config/default.toml', 'utf-8'));
// create and initialize repository object
const repo = new Repository(config.transcoder.repository);
repo.initialize();

View File

@ -1,12 +0,0 @@
appenders:
stdout:
kind: console
encoder:
pattern: "{d(%+)(utc)} {h({l})}: {m}{n}"
filters:
- kind: threshold
level: info
root:
level: info
appenders:
- stdout

2976
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

32
package.json Normal file
View File

@ -0,0 +1,32 @@
{
"name": "zealot",
"version": "0.3.0",
"description": "Bit Goblin video transcoder service",
"main": "index.js",
"scripts": {
"ts": "ts-node",
"lint": "eslint index.ts src/**/*",
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "https://git.metaunix.net/BitGoblin/zealot"
},
"keywords": [
"video",
"transcoder",
"ffmpeg"
],
"author": "Gregory Ballantine <gballantine@bitgoblin.tech>",
"license": "BSD-2-Clause",
"dependencies": {
"toml": "^3.0.0",
"ts-node": "^10.9.1",
"typescript": "^4.9.3"
},
"devDependencies": {
"@typescript-eslint/eslint-plugin": "^5.43.0",
"@typescript-eslint/parser": "^5.43.0",
"eslint": "^8.27.0"
}
}

View File

@ -1,20 +0,0 @@
use config::Config;
use crate::settings;
use crate::transcoder::repository::Repository;
pub fn setup_command() {
// load configuration
let c: Config = settings::load_config();
// resolve repository path
let repository_dir_raw: &str = &c.get_string("transcoder.repository").unwrap();
let repository_dir: &str = &shellexpand::tilde(repository_dir_raw);
// alert the user to what's happening
println!("Initializing video repository at {}...", repository_dir);
// create and initialize our repository object
let r: Repository = Repository::new(repository_dir);
// initialize the video repository
r.initialize();
}

View File

@ -1 +0,0 @@
pub mod core;

View File

@ -1,52 +0,0 @@
use clap::{Parser, Subcommand};
use log4rs;
use config::Config;
use transcoder::repository::Repository;
use transcoder::transcoder::Transcoder;
mod cmd;
mod settings;
mod transcoder;
mod util;
#[derive(Parser)]
#[clap(author, version, about, long_about = None)]
struct Cli {
/// Number of times to greet
#[clap(short = 'l', long, default_value_t = String::from("./log4rs.yaml"))]
log_config: String,
#[clap(subcommand)]
command: Option<Commands>,
}
#[derive(Subcommand)]
enum Commands {
/// displays version info about this program
Setup {},
}
fn main() {
// initialize the clap CLI
let cli = Cli::parse();
// grab the log4rs config file path, then initialize log4rs
let log4rs_config: String = cli.log_config;
log4rs::init_file(&log4rs_config, Default::default()).unwrap();
match &cli.command {
// sub-commands will be handled here
Some(Commands::Setup {}) => cmd::core::setup_command(),
// run the main program without any commands
None => {
// create and initialize our config and repository objects
let c: Config = settings::load_config();
let r: Repository = Repository::new(&shellexpand::tilde(&c.get_string("transcoder.repository").unwrap()));
// create and start the video transcoder object
let t: Transcoder = Transcoder::new(c, r);
t.start();
},
}
}

40
src/repository.ts Normal file
View File

@ -0,0 +1,40 @@
import fs from 'fs';
import path from 'path';
import { resolveTilde } from './util';
export default class Repository {
base_path: string;
ingest_path: string;
archive_path: string;
output_path: string;
constructor(base_path: string) {
// set base path
this.base_path = resolveTilde(base_path);
// set sub-directory paths
this.ingest_path = path.join(this.base_path, 'ingest');
this.archive_path = path.join(this.base_path, 'archive');
this.output_path = path.join(this.base_path, 'output');
}
initialize(): void {
console.log(`Initializing video repository at ${this.base_path}.`);
// ensure each directory exists
if (!fs.existsSync(this.base_path)) {
fs.mkdirSync(this.base_path);
}
if (!fs.existsSync(this.ingest_path)) {
fs.mkdirSync(this.ingest_path);
}
if (!fs.existsSync(this.archive_path)) {
fs.mkdirSync(this.archive_path);
}
if (!fs.existsSync(this.output_path)) {
fs.mkdirSync(this.output_path);
}
}
}

View File

@ -1,33 +0,0 @@
use config::Config;
pub fn load_config() -> Config {
let global_config_path: String = find_global_config_path();
let home_config_path: String = find_home_config_path();
let settings = Config::builder()
// Add in `./Settings.toml`
.add_source(config::File::with_name(&global_config_path).required(false))
.add_source(config::File::with_name(&home_config_path).required(false))
.build()
.unwrap();
return settings;
}
fn find_global_config_path() -> String {
if cfg!(windows) {
return String::from("C:\\Program Files\\Zealot\\config.toml");
}
return String::from("/etc/zealot/config.toml");
}
fn find_home_config_path() -> String {
let home_path: &str = &shellexpand::tilde("~/.config/zealot.toml");
if cfg!(windows) {
return String::from(home_path.replace("/", "\\"));
}
return String::from(home_path);
}

View File

@ -1,2 +0,0 @@
pub mod repository;
pub mod transcoder;

View File

@ -1,91 +0,0 @@
use std::fs;
use std::path::Path;
use log::{error, info};
pub struct Repository {
pub base_dir: String,
pub ingest_dir: String,
pub archive_dir: String,
pub output_dir: String,
}
impl Repository {
pub fn new(base_path: &str) -> Repository {
let ingest_path = Path::new(base_path).join("ingest");
let archive_path = Path::new(base_path).join("archive");
let output_path = Path::new(base_path).join("output");
return Repository {
base_dir: String::from(base_path),
ingest_dir: String::from(ingest_path.to_str().unwrap()),
archive_dir: String::from(archive_path.to_str().unwrap()),
output_dir: String::from(output_path.to_str().unwrap()),
};
}
pub fn initialize(&self) {
// create the base directory path
create_directory(&self.base_dir);
// create the needed sub-directories
create_directory(&self.ingest_dir.as_str());
create_directory(&self.archive_dir.as_str());
create_directory(&self.output_dir.as_str());
}
pub fn search_ingest(&self) -> Vec<String> {
// read file entries from ingest
let files = fs::read_dir(&self.ingest_dir).unwrap();
// create vec object and loop through entries to find what we want
let mut ingest_files: Vec<String> = vec![];
for f in files {
let f = f.unwrap();
let path = f.path();
if path.is_file() {
let file_path = path.file_name().unwrap().to_str();
ingest_files.push(String::from(file_path.unwrap()));
}
}
return ingest_files;
}
pub fn archive_file(&self, file: &str) {
let ingest_file = Path::new(&self.ingest_dir).join(file);
let archive_file = Path::new(&self.archive_dir).join(file);
match fs::copy(&ingest_file, &archive_file) {
Ok(_) => {
info!("Archiving video file {}.", ingest_file.to_str().unwrap());
},
Err(e) => {
error!("Error archiving file {}: {}", ingest_file.to_str().unwrap(), e);
std::process::exit(1);
}
}
}
pub fn cleanup_file(&self, file: &str) {
let ingest_file = Path::new(&self.ingest_dir).join(file);
fs::remove_file(&ingest_file)
.expect("File deletion failed.");
}
}
fn create_directory(path: &str) {
let d = Path::new(path);
if d.is_dir() {
info!("Directory {} already exists.", path);
} else {
match fs::create_dir(path) {
Ok(_) => {
info!("Creating directory {}.", path);
},
Err(e) => {
error!("Error creating {}: {}", path, e);
std::process::exit(1);
}
}
}
}

View File

@ -1,88 +0,0 @@
use config::Config;
use std::path::Path;
use std::process;
use std::{thread, time};
use log::{debug, info};
use crate::util::io;
use super::repository::Repository;
pub struct Transcoder {
config: Config,
repository: Repository,
}
impl Transcoder {
pub fn new(config: Config, repository: Repository) -> Transcoder {
return Transcoder{
config: config,
repository: repository,
}
}
pub fn start(self) {
info!("Starting transcoder...");
loop {
// search for files in ingest
let ingest_files = self.repository.search_ingest();
// check if we found any files to transcode
if ingest_files.len() < 1 {
info!("There were no files found in ingest to transcode; skipping run.");
} else {
for i in ingest_files {
let ingest_path = Path::new(&self.repository.ingest_dir).join(&i);
if io::is_file_locked(&ingest_path.to_str().unwrap()) {
info!("{} is currently open in another program; skipping it for this run.", &i);
continue;
}
// copy the file to the archive
self.repository.archive_file(&i);
// perform the video transcode
self.transcode(&i);
// remove the source file
self.repository.cleanup_file(&i);
}
}
// put the loop to sleep for X minutes
let sleep_minutes = time::Duration::from_secs((self.config.get_int("transcoder.interval").unwrap() as u64 * 60).into());
thread::sleep(sleep_minutes);
}
}
fn transcode(&self, file: &str) {
let ingest_file = Path::new(&self.repository.ingest_dir).join(file);
let video_format = &self.config.get_string("transcoder.video_format").unwrap();
let output_stem = Path::new(file).file_stem().unwrap();
let output_bits = vec![output_stem.to_str().unwrap(), &video_format];
let output_name = output_bits.join(".");
let output_file = Path::new(&self.repository.output_dir).join(output_name);
let video_codec = &self.config.get_string("transcoder.video_codec").unwrap();
info!("Transcoding {} to {} with the {} encoder.", ingest_file.display(), output_file.display(), video_codec);
let cmd_output = process::Command::new("/usr/bin/ffmpeg")
.arg("-i") .arg(&*ingest_file.to_string_lossy())
.arg("-y")
.arg("-f") .arg(&video_format)
.arg("-c:v") .arg(&video_codec)
.arg("-s") .arg(&self.config.get_string("transcoder.video_resolution").unwrap())
.arg("-r") .arg(format!("{}", self.config.get_string("transcoder.video_framerate").unwrap()))
.arg("-vf") .arg(format!("format={}", &self.config.get_string("transcoder.video_color").unwrap()))
.arg("-profile:v").arg(&self.config.get_string("transcoder.video_profile").unwrap())
.arg("-c:a") .arg(&self.config.get_string("transcoder.audio_codec").unwrap())
.arg(&*output_file.to_string_lossy())
.output()
.expect("Failed to execute command");
assert!(cmd_output.status.success());
let results_raw = &String::from_utf8_lossy(&cmd_output.stderr);
debug!("{}", results_raw);
}
}

11
src/util.ts Normal file
View File

@ -0,0 +1,11 @@
import path from 'path';
export function resolveTilde(filepath: string): string {
// if the first character is a tilde, replace it with the user's home
if (filepath[0] === '~') {
return path.join(process.env.HOME, filepath.slice(1));
}
// return the path untouched
return filepath;
}

View File

@ -1,61 +0,0 @@
use std::process;
// checks whether a file is currently open in another program (e.g. still being written)
pub fn is_file_locked(filepath: &str) -> bool {
let cmd = process::Command::new("/usr/bin/lsof")
.arg(filepath)
.output()
.expect("Failed to execute command");
let results = &String::from_utf8_lossy(&cmd.stdout);
println!("{}", results);
if results.contains(filepath) {
return true;
}
return false;
}
#[cfg(test)]
mod tests {
use std::fs;
use std::mem::drop;
use std::path::Path;
use super::*;
#[test]
fn file_should_not_be_locked() {
setup("test-not-locked.txt");
assert!(!is_file_locked("test-not-locked.txt"));
teardown("test-not-locked.txt");
}
#[test]
fn file_should_be_locked() {
setup("test-locked.txt");
let path = Path::new("test-locked.txt");
match fs::File::open(&path) {
Err(e) => panic!("couldn't open {}: {}", "test-locked.txt", e),
Ok(_) => assert!(is_file_locked("test-locked.txt")),
};
teardown("test-locked.txt");
}
// get things ready for our tests
fn setup(test_file: &str) {
let f = match fs::File::create(test_file) {
Ok(file) => file,
Err(e) => panic!("{:?}", e),
};
drop(f);
}
// clean up after running tests
fn teardown(test_file: &str) {
match fs::remove_file(test_file) {
Ok(_) => {},
Err(e) => panic!("{:?}", e),
};
}
}

View File

@ -1 +0,0 @@
pub mod io;