From 5bc2acac1de3c6328c9c9f2036a6746252b6a1f7 Mon Sep 17 00:00:00 2001 From: Gregory Ballantine Date: Fri, 20 May 2022 07:58:52 -0400 Subject: [PATCH] Added ability for the transcoder to determine if a video file is open in another program to avoid trying to transcode/remove a partially written file; reworked the main transcode loop to handle one file at a time instead of archiving everything, then transcoding, then cleanup --- src/main/java/tech/bitgoblin/io/IOUtils.java | 23 ++++- .../tech/bitgoblin/transcoder/Repository.java | 24 +++--- .../transcoder/RunTranscoderTask.java | 7 +- .../tech/bitgoblin/transcoder/Transcoder.java | 86 ++++++++++--------- .../java/tech/bitgoblin/io/IOUtilsTest.java | 27 ++++++ 5 files changed, 110 insertions(+), 57 deletions(-) diff --git a/src/main/java/tech/bitgoblin/io/IOUtils.java b/src/main/java/tech/bitgoblin/io/IOUtils.java index a5363de..97d579b 100644 --- a/src/main/java/tech/bitgoblin/io/IOUtils.java +++ b/src/main/java/tech/bitgoblin/io/IOUtils.java @@ -1,6 +1,8 @@ package tech.bitgoblin.io; -import java.io.File; +import tech.bitgoblin.Logger; + +import java.io.*; public class IOUtils { @@ -24,4 +26,23 @@ public class IOUtils { return path; } + // checks to see if a file is currently locked/being written to. + public static boolean isFileLocked(File file) throws IOException { + String[] cmd = {"lsof", file.toString()}; + Process process = Runtime.getRuntime().exec(cmd); + BufferedReader reader = new BufferedReader(new InputStreamReader( + process.getInputStream())); + + boolean isOpen = false; // we'll change this if lsof returns that the file is open + + String s; + while ((s = reader.readLine()) != null) { + if (s.endsWith(file.toString())) { + isOpen = true; + } + } + + return isOpen; + } + } diff --git a/src/main/java/tech/bitgoblin/transcoder/Repository.java b/src/main/java/tech/bitgoblin/transcoder/Repository.java index 128c1ae..54704de 100644 --- a/src/main/java/tech/bitgoblin/transcoder/Repository.java +++ b/src/main/java/tech/bitgoblin/transcoder/Repository.java @@ -40,25 +40,21 @@ public class Repository { } // archives files in the ingest directory - public void archiveIngest(File[] sourceFiles) { - for (File f : sourceFiles) { - Path filePath = Path.of(f.toString()); - String filename = filePath.getFileName().toString(); - String archivePath = Paths.get(this.archivePath, filename).toString(); + public void archiveFile(File sourceFile) { + Path filePath = Path.of(sourceFile.toString()); + String filename = filePath.getFileName().toString(); + String archivePath = Paths.get(this.archivePath, filename).toString(); - try { - Files.copy(filePath, Paths.get(archivePath), StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - throw new RuntimeException(e); - } + try { + Files.copy(filePath, Paths.get(archivePath), StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new RuntimeException(e); } } // clean up the ingest directory once we're done - public void cleanupIngest(File[] sourceFiles) { - for (File f : sourceFiles) { - f.delete(); - } + public void cleanupFile(File sourceFile) { + sourceFile.delete(); } // returns the repository's path diff --git a/src/main/java/tech/bitgoblin/transcoder/RunTranscoderTask.java b/src/main/java/tech/bitgoblin/transcoder/RunTranscoderTask.java index 245c113..0cb4b1f 100644 --- a/src/main/java/tech/bitgoblin/transcoder/RunTranscoderTask.java +++ b/src/main/java/tech/bitgoblin/transcoder/RunTranscoderTask.java @@ -1,5 +1,6 @@ package tech.bitgoblin.transcoder; +import java.io.IOException; import java.util.TimerTask; public class RunTranscoderTask extends TimerTask { @@ -13,7 +14,11 @@ public class RunTranscoderTask extends TimerTask { @Override public void run() { // archive the files - transcoder.run(); + try { + transcoder.run(); + } catch (IOException e) { + throw new RuntimeException(e); + } } } diff --git a/src/main/java/tech/bitgoblin/transcoder/Transcoder.java b/src/main/java/tech/bitgoblin/transcoder/Transcoder.java index 4474e0b..82282c9 100644 --- a/src/main/java/tech/bitgoblin/transcoder/Transcoder.java +++ b/src/main/java/tech/bitgoblin/transcoder/Transcoder.java @@ -31,54 +31,29 @@ public class Transcoder { } // create a periodic timer task and start it - public void run() { + public void run() throws IOException { // pull list of files to transcode File[] sourceFiles = this.repo.searchIngest(); - // archive files - this.repo.archiveIngest(sourceFiles); - // transcode files - this.transcode(sourceFiles); - // cleanup old files - this.repo.cleanupIngest(sourceFiles); - } - - // transcode files in the working directory - public void transcode(File[] sourceFiles) { - // check if the ingest directory is empty + // check if we have anything to process if (sourceFiles.length == 0) { - Logger.logger.info("There is nothing to transcode in ingest."); - return; + Logger.logger.info("There is nothing in ingest, skipping transcode run."); } - // transcode - Logger.logger.info("Transcoding video files ingest..."); - for (File f : sourceFiles) { - String filePath = f.toString().substring(0, f.toString().lastIndexOf(".")); - String filename = Paths.get(filePath).getFileName().toString(); - String outputPath = Paths.get(this.repo.getOutputPath(), String.format("%s.mov", filename)).toString(); - - String cmd = String.format("%s -i %s -y -f %s -c:v %s -vf %s -profile:v %s -c:a %s %s", - this.ffmpeg_path, // FFMPEG binary path - f.toString(), // input file - this.config.getString("transcoder.video_format"), // video container format - this.config.getString("transcoder.video_codec"), // video codec - this.config.getString("transcoder.video_parameters"), // video format - this.config.getString("transcoder.video_profile"), // video profile - this.config.getString("transcoder.audio_codec"), // audio codec - outputPath // output file path - ); - - ProcessBuilder pb = new ProcessBuilder(cmd.split("\\s+")); - pb.inheritIO(); // use the java processes' input and output streams - try { - Process process = pb.start(); - int ret = process.waitFor(); - Logger.logger.info("Program exited with code: %d", ret); - Logger.logger.info(""); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); + // loop through each file + for (File sf : sourceFiles) { + // check if the file is open before attempting to transcode it + if (IOUtils.isFileLocked(sf)) { + Logger.logger.info(String.format("Skipping %s because it is open in another program.", sf.toString())); + continue; } + + // archive files + this.repo.archiveFile(sf); + // transcode files + this.transcodeFile(sf); + // cleanup old files + this.repo.cleanupFile(sf); } // end output @@ -86,4 +61,33 @@ public class Transcoder { Logger.logger.info(""); } + // transcode files in the working directory + public void transcodeFile(File sourceFile) throws IOException { + String filePath = sourceFile.toString().substring(0, sourceFile.toString().lastIndexOf(".")); + String filename = Paths.get(filePath).getFileName().toString(); + String outputPath = Paths.get(this.repo.getOutputPath(), String.format("%s.mov", filename)).toString(); + + String cmd = String.format("%s -i %s -y -f %s -c:v %s -vf %s -profile:v %s -c:a %s %s", + this.ffmpeg_path, // FFMPEG binary path + sourceFile.toString(), // input file + this.config.getString("transcoder.video_format"), // video container format + this.config.getString("transcoder.video_codec"), // video codec + this.config.getString("transcoder.video_parameters"), // video format + this.config.getString("transcoder.video_profile"), // video profile + this.config.getString("transcoder.audio_codec"), // audio codec + outputPath // output file path + ); + + ProcessBuilder pb = new ProcessBuilder(cmd.split("\\s+")); + pb.inheritIO(); // use the java processes' input and output streams + try { + Process process = pb.start(); + int ret = process.waitFor(); + Logger.logger.info(String.format("Program exited with code: %d", ret)); + Logger.logger.info(""); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + } diff --git a/src/test/java/tech/bitgoblin/io/IOUtilsTest.java b/src/test/java/tech/bitgoblin/io/IOUtilsTest.java index 8bfe43d..22b0ca4 100644 --- a/src/test/java/tech/bitgoblin/io/IOUtilsTest.java +++ b/src/test/java/tech/bitgoblin/io/IOUtilsTest.java @@ -1,15 +1,21 @@ package tech.bitgoblin.io; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; public class IOUtilsTest { private static String testDir = "test-temp"; + private static String testFile = "test.txt"; @Test public void shouldCreateDirectory() { @@ -28,9 +34,30 @@ public class IOUtilsTest { IOUtils.resolveTilda("~test"); } + @Test + public void fileShouldBeLocked() throws IOException { + RandomAccessFile raf = new RandomAccessFile(testFile, "rw"); + assertTrue(IOUtils.isFileLocked(new File(testFile))); + } + + @Test + public void fileShouldNotBeLocked() throws IOException { + RandomAccessFile raf = new RandomAccessFile("test.txt", "rw"); + raf.close(); + assertFalse(IOUtils.isFileLocked(new File(testFile))); + } + + @BeforeClass + // ensure that test files are created before running tests + public static void setup() throws IOException { + new File(testFile).createNewFile(); + } + @AfterClass + // ensure test files are cleaned up from the environment public static void cleanup() { new File(testDir).delete(); + new File(testFile).delete(); } }