Added ability for the transcoder to determine if a video file is open in another program to avoid trying to transcode/remove a partially written file; reworked the main transcode loop to handle one file at a time instead of archiving everything, then transcoding, then cleanup
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
ci/woodpecker/tag/woodpecker Pipeline failed

This commit is contained in:
Gregory Ballantine 2022-05-20 07:58:52 -04:00
parent fe29664a83
commit 5bc2acac1d
5 changed files with 110 additions and 57 deletions

View File

@ -1,6 +1,8 @@
package tech.bitgoblin.io; package tech.bitgoblin.io;
import java.io.File; import tech.bitgoblin.Logger;
import java.io.*;
public class IOUtils { public class IOUtils {
@ -24,4 +26,23 @@ public class IOUtils {
return path; return path;
} }
// checks to see if a file is currently locked/being written to.
public static boolean isFileLocked(File file) throws IOException {
String[] cmd = {"lsof", file.toString()};
Process process = Runtime.getRuntime().exec(cmd);
BufferedReader reader = new BufferedReader(new InputStreamReader(
process.getInputStream()));
boolean isOpen = false; // we'll change this if lsof returns that the file is open
String s;
while ((s = reader.readLine()) != null) {
if (s.endsWith(file.toString())) {
isOpen = true;
}
}
return isOpen;
}
} }

View File

@ -40,9 +40,8 @@ public class Repository {
} }
// archives files in the ingest directory // archives files in the ingest directory
public void archiveIngest(File[] sourceFiles) { public void archiveFile(File sourceFile) {
for (File f : sourceFiles) { Path filePath = Path.of(sourceFile.toString());
Path filePath = Path.of(f.toString());
String filename = filePath.getFileName().toString(); String filename = filePath.getFileName().toString();
String archivePath = Paths.get(this.archivePath, filename).toString(); String archivePath = Paths.get(this.archivePath, filename).toString();
@ -52,13 +51,10 @@ public class Repository {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
}
// clean up the ingest directory once we're done // clean up the ingest directory once we're done
public void cleanupIngest(File[] sourceFiles) { public void cleanupFile(File sourceFile) {
for (File f : sourceFiles) { sourceFile.delete();
f.delete();
}
} }
// returns the repository's path // returns the repository's path

View File

@ -1,5 +1,6 @@
package tech.bitgoblin.transcoder; package tech.bitgoblin.transcoder;
import java.io.IOException;
import java.util.TimerTask; import java.util.TimerTask;
public class RunTranscoderTask extends TimerTask { public class RunTranscoderTask extends TimerTask {
@ -13,7 +14,11 @@ public class RunTranscoderTask extends TimerTask {
@Override @Override
public void run() { public void run() {
// archive the files // archive the files
try {
transcoder.run(); transcoder.run();
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
} }

View File

@ -31,36 +31,45 @@ public class Transcoder {
} }
// create a periodic timer task and start it // create a periodic timer task and start it
public void run() { public void run() throws IOException {
// pull list of files to transcode // pull list of files to transcode
File[] sourceFiles = this.repo.searchIngest(); File[] sourceFiles = this.repo.searchIngest();
// check if we have anything to process
if (sourceFiles.length == 0) {
Logger.logger.info("There is nothing in ingest, skipping transcode run.");
}
// loop through each file
for (File sf : sourceFiles) {
// check if the file is open before attempting to transcode it
if (IOUtils.isFileLocked(sf)) {
Logger.logger.info(String.format("Skipping %s because it is open in another program.", sf.toString()));
continue;
}
// archive files // archive files
this.repo.archiveIngest(sourceFiles); this.repo.archiveFile(sf);
// transcode files // transcode files
this.transcode(sourceFiles); this.transcodeFile(sf);
// cleanup old files // cleanup old files
this.repo.cleanupIngest(sourceFiles); this.repo.cleanupFile(sf);
}
// end output
Logger.logger.info("------------ End of transcoding ------------");
Logger.logger.info("");
} }
// transcode files in the working directory // transcode files in the working directory
public void transcode(File[] sourceFiles) { public void transcodeFile(File sourceFile) throws IOException {
// check if the ingest directory is empty String filePath = sourceFile.toString().substring(0, sourceFile.toString().lastIndexOf("."));
if (sourceFiles.length == 0) {
Logger.logger.info("There is nothing to transcode in ingest.");
return;
}
// transcode
Logger.logger.info("Transcoding video files ingest...");
for (File f : sourceFiles) {
String filePath = f.toString().substring(0, f.toString().lastIndexOf("."));
String filename = Paths.get(filePath).getFileName().toString(); String filename = Paths.get(filePath).getFileName().toString();
String outputPath = Paths.get(this.repo.getOutputPath(), String.format("%s.mov", filename)).toString(); String outputPath = Paths.get(this.repo.getOutputPath(), String.format("%s.mov", filename)).toString();
String cmd = String.format("%s -i %s -y -f %s -c:v %s -vf %s -profile:v %s -c:a %s %s", String cmd = String.format("%s -i %s -y -f %s -c:v %s -vf %s -profile:v %s -c:a %s %s",
this.ffmpeg_path, // FFMPEG binary path this.ffmpeg_path, // FFMPEG binary path
f.toString(), // input file sourceFile.toString(), // input file
this.config.getString("transcoder.video_format"), // video container format this.config.getString("transcoder.video_format"), // video container format
this.config.getString("transcoder.video_codec"), // video codec this.config.getString("transcoder.video_codec"), // video codec
this.config.getString("transcoder.video_parameters"), // video format this.config.getString("transcoder.video_parameters"), // video format
@ -74,16 +83,11 @@ public class Transcoder {
try { try {
Process process = pb.start(); Process process = pb.start();
int ret = process.waitFor(); int ret = process.waitFor();
Logger.logger.info("Program exited with code: %d", ret); Logger.logger.info(String.format("Program exited with code: %d", ret));
Logger.logger.info(""); Logger.logger.info("");
} catch (IOException | InterruptedException e) { } catch (IOException | InterruptedException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
// end output
Logger.logger.info("------------ End of transcoding ------------");
Logger.logger.info("");
}
} }

View File

@ -1,15 +1,21 @@
package tech.bitgoblin.io; package tech.bitgoblin.io;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
public class IOUtilsTest { public class IOUtilsTest {
private static String testDir = "test-temp"; private static String testDir = "test-temp";
private static String testFile = "test.txt";
@Test @Test
public void shouldCreateDirectory() { public void shouldCreateDirectory() {
@ -28,9 +34,30 @@ public class IOUtilsTest {
IOUtils.resolveTilda("~test"); IOUtils.resolveTilda("~test");
} }
@Test
public void fileShouldBeLocked() throws IOException {
RandomAccessFile raf = new RandomAccessFile(testFile, "rw");
assertTrue(IOUtils.isFileLocked(new File(testFile)));
}
@Test
public void fileShouldNotBeLocked() throws IOException {
RandomAccessFile raf = new RandomAccessFile("test.txt", "rw");
raf.close();
assertFalse(IOUtils.isFileLocked(new File(testFile)));
}
@BeforeClass
// ensure that test files are created before running tests
public static void setup() throws IOException {
new File(testFile).createNewFile();
}
@AfterClass @AfterClass
// ensure test files are cleaned up from the environment
public static void cleanup() { public static void cleanup() {
new File(testDir).delete(); new File(testDir).delete();
new File(testFile).delete();
} }
} }