Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

After requested refactorings #246

Merged
merged 27 commits into from
Apr 25, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
3bb699a
Server mode added
RezviyBelorus Jan 23, 2018
9d2e5e1
minor fixes
RezviyBelorus Feb 2, 2018
90e7df7
veraPDF multi threads version
RezviyBelorus Feb 2, 2018
905d170
Refactor constructor of MultiThreadProcessor
RezviyBelorus Feb 2, 2018
02368e6
Clear unused imports
RezviyBelorus Feb 2, 2018
5a21e9d
Redirect error stream in BaseCliRunner
RezviyBelorus Feb 5, 2018
87303fa
Changes in StaticContainers
RezviyBelorus Feb 27, 2018
8bc22a4
Merge branch 'integration' into multithread-cli
RezviyBelorus Mar 7, 2018
06af178
Multithreading test added
RezviyBelorus Mar 14, 2018
9687abd
Merge remote-tracking branch 'origin/multithread-cli' into multithrea…
RezviyBelorus Mar 14, 2018
78c6b35
Merge pull request #1 from RezviyBelorus/multithread-cli
RezviyBelorus Mar 14, 2018
b1f3890
Multithreaded validator test added
RezviyBelorus Mar 16, 2018
c3db1bb
Merge pull request #2 from RezviyBelorus/multithread-cli
RezviyBelorus Mar 16, 2018
6959593
Fixes
RezviyBelorus Mar 19, 2018
8258470
NPE in ReportParserEventHandler fixed
RezviyBelorus Mar 19, 2018
7418570
Refatorings
RezviyBelorus Mar 19, 2018
2c2b3a8
Merge pull request #3 from RezviyBelorus/multithread-cli
RezviyBelorus Mar 20, 2018
23e849d
Fixes
RezviyBelorus Apr 5, 2018
551b725
Merge branch 'integration' into multithread-cli
RezviyBelorus Apr 5, 2018
c34b12a
White space fixes
RezviyBelorus Apr 6, 2018
7b1bdb4
White space fixes
RezviyBelorus Apr 6, 2018
dabc309
Refactorings
RezviyBelorus Apr 6, 2018
6fad77c
Merge pull request #4 from RezviyBelorus/multithread-cli
RezviyBelorus Apr 6, 2018
2faece6
Refactorings
RezviyBelorus Apr 9, 2018
602c38d
Refactorings
RezviyBelorus Apr 9, 2018
91baef1
Merge pull request #5 from RezviyBelorus/multithread-cli
RezviyBelorus Apr 9, 2018
f8186a8
Merge branch 'integration' into integration
carlwilson Apr 24, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package org.verapdf.apps;

import org.junit.Before;
import org.junit.Test;
import org.verapdf.pdfa.VeraGreenfieldFoundryProvider;
import org.verapdf.pdfa.results.ValidationResult;
import org.verapdf.pdfa.validation.validators.CallableValidatorForTest;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import static junit.framework.TestCase.assertTrue;

public class MultithreadingTest {
private static final String TEST_FILE = "src/test/resources/veraPDFtest-pass-a.pdf";

@Before
public void init() {
VeraGreenfieldFoundryProvider.initialise();
}

@Test
public void shouldCreateXmlReportsInGivenDirectory() throws Exception {
File fileToValidate = new File(TEST_FILE);

int numberOfThreads = 4;

List<Future<ValidationResult>> futureResult = startValidation(fileToValidate, numberOfThreads);
List<ValidationResult> validationResults = getValidationResult(futureResult);

Boolean isExpectedResults = compareResultsFromDifferentThreads(validationResults);

assertTrue(isExpectedResults);
}

private boolean compareResultsFromDifferentThreads(List<ValidationResult> validationResults) {
boolean isExpectedResults = true;
for (ValidationResult result : validationResults) {
for (ValidationResult comparingResult : validationResults) {
boolean isCompliantResultsEquals = result.isCompliant() == comparingResult.isCompliant();
boolean isTestAssertionsEquals = result.getTestAssertions().equals(comparingResult.getTestAssertions());
boolean isTotalAssertionsEquals = result.getTotalAssertions() == comparingResult.getTotalAssertions();
if (!isCompliantResultsEquals || !isTestAssertionsEquals || !isTotalAssertionsEquals) {
isExpectedResults = false;
}
}
}
return isExpectedResults;
}

private List<ValidationResult> getValidationResult(List<Future<ValidationResult>> results) throws InterruptedException, java.util.concurrent.ExecutionException {
List<ValidationResult> validationResults = new ArrayList<>();
for (Future<ValidationResult> result : results) {
ValidationResult validationResult = result.get();
validationResults.add(validationResult);
}
return validationResults;
}

private List<Future<ValidationResult>> startValidation(File fileToValidate, int numberOfThreads) {
List<Future<ValidationResult>> futureResult = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
for (int i = 0; i < numberOfThreads; i++) {
CallableValidatorForTest validator = new CallableValidatorForTest(fileToValidate);
Future<ValidationResult> submit = executor.submit(validator);
futureResult.add(submit);
}
return futureResult;
}
}
Binary file not shown.
30 changes: 28 additions & 2 deletions gui/src/main/java/org/verapdf/apps/Applications.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* This file is part of VeraPDF Library GUI, a module of the veraPDF project.
* Copyright (c) 2015, veraPDF Consortium <info@verapdf.org> All rights
* Copyright (c) 2015, veraPDF Consortium <info@VERAPDF.org> All rights
* reserved. VeraPDF Library GUI is free software: you can redistribute it
* and/or modify it under the terms of either: The GNU General public license
* GPLv3+. You should have received a copy of the GNU General Public License
Expand Down Expand Up @@ -46,11 +46,37 @@ public final class Applications {
public static final String UPDATE_SERVICE_NOT_AVAILABLE = "Update Service not available"; //$NON-NLS-1$
public static final String UPDATE_LATEST_VERSION = "You are currently running the latest version of veraPDF%s v%s"; //$NON-NLS-1$
public static final String UPDATE_OLD_VERSION = "You are NOT running the latest version of veraPDF.\nYou are running version %s, the latest version is %s.\n"; //$NON-NLS-1$
public static final String UPDATE_URI = "http://downloads.verapdf.org/rel/verapdf-installer.zip"; //$NON-NLS-1$
public static final String UPDATE_URI = "http://downloads.VERAPDF.org/rel/VERAPDF-installer.zip"; //$NON-NLS-1$

private static final String VERAPDF = "verapdf";
private static final String BAT_EXTENSION = ".bat";

private Applications() {
assert (false);
}

public static File getVeraScriptFile() {
File startFile = null;
String appHome = System.getProperty(APP_HOME_PROPERTY);

if (appHome != null) {
File veraPdfDirectory = new File(appHome);
startFile = getStartFile(veraPdfDirectory);
}
return startFile;
}

private static File getStartFile(File veraPdfDirectory) {
File unixStartFile = new File(veraPdfDirectory, VERAPDF);
if (unixStartFile.isFile()) {
return unixStartFile;
}
File windowsStartFile = new File(veraPdfDirectory, VERAPDF+BAT_EXTENSION);
if (windowsStartFile.isFile()) {
return windowsStartFile;
}
return null;
}

/**
* @return the Application Release details for the GUI
Expand Down
70 changes: 54 additions & 16 deletions gui/src/main/java/org/verapdf/cli/VeraPdfCli.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@
*/
package org.verapdf.cli;

import java.io.*;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -29,6 +33,7 @@
import org.verapdf.apps.ConfigManager;
import org.verapdf.apps.SoftwareUpdater;
import org.verapdf.cli.commands.VeraCliArgParser;
import org.verapdf.cli.multithread.MultiThreadProcessor;
import org.verapdf.core.VeraPDFException;
import org.verapdf.pdfa.flavours.PDFAFlavour;
import org.verapdf.pdfa.validation.profiles.ProfileDirectory;
Expand All @@ -48,16 +53,17 @@ public final class VeraPdfCli {
private static final String FLAVOURS_HEADING = CliConstants.APP_NAME + " supported PDF/A profiles:"; //$NON-NLS-1$
private static final ProfileDirectory PROFILES = Profiles.getVeraProfileDirectory();

public static final String EXIT = "q";

private VeraPdfCli() {
// disable default constructor
}

/**
* Main CLI entry point, process the command line arguments
*
* @param args
* Java.lang.String array of command line args, to be processed
* using Apache commons CLI.
* @param args Java.lang.String array of command line args, to be processed
* using Apache commons CLI.
*/
public static void main(final String[] args) throws VeraPDFException {
MemoryMXBean memoryMan = ManagementFactory.getMemoryMXBean();
Expand All @@ -71,24 +77,22 @@ public static void main(final String[] args) throws VeraPDFException {
jCommander.parse(args);
} catch (ParameterException e) {
System.err.println(e.getMessage());
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(1);
displayHelpAndExit(cliArgParser, jCommander, 1);
}
if (cliArgParser.isHelp()) {
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(0);
displayHelpAndExit(cliArgParser, jCommander, 0);
}
messagesFromParser(cliArgParser);
if (isProcess(cliArgParser)) {
try (VeraPdfCliProcessor processor = VeraPdfCliProcessor.createProcessorFromArgs(cliArgParser,
configManager)) {
if (args.length == 0)
jCommander.usage();
// FIXME: trap policy IO Exception (deliberately left un-caught
// for development)
processor.processPaths(cliArgParser.getPdfPaths());
if (args.length == 0) {
jCommander.usage();
}
try {
if (cliArgParser.isServerMode() || cliArgParser.getNumberOfProcesses() < 2) {
singleThreadProcess(cliArgParser);
} else {
MultiThreadProcessor.process(cliArgParser);
}
} catch (OutOfMemoryError oome) {
final String message = "The JVM appears to have run out of memory"; //$NON-NLS-1$
logger.log(Level.WARNING, message, oome);
Expand All @@ -109,6 +113,40 @@ public static void main(final String[] args) throws VeraPDFException {
}
}

private static void singleThreadProcess(VeraCliArgParser cliArgParser) throws VeraPDFException {
try (VeraPdfCliProcessor processor = VeraPdfCliProcessor.createProcessorFromArgs(cliArgParser,
configManager)) {
// FIXME: trap policy IO Exception (deliberately left un-caught for development)
processor.processPaths(cliArgParser.getPdfPaths());
if (cliArgParser.isServerMode()) {
File tempFile = processor.getTempFile();
if (tempFile != null) {
System.out.println(tempFile.getAbsoluteFile());
}
Scanner scanner = new Scanner(System.in);
while (scanner.hasNextLine()) {
String path = scanner.nextLine();
if (path != null) {
if (path.equals(EXIT)) {
break;
} else {
List<String> pathes = new ArrayList<>();
pathes.add(path);
processor.processPaths(pathes);
System.out.println(processor.getTempFile().getAbsolutePath());
}
}
}
}
}
}

public static void displayHelpAndExit(VeraCliArgParser cliArgParser, JCommander jCommander, int i) {
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(i);
}

private static void messagesFromParser(final VeraCliArgParser parser) {

if (parser.listProfiles()) {
Expand Down
31 changes: 26 additions & 5 deletions gui/src/main/java/org/verapdf/cli/VeraPdfCliProcessor.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.verapdf.cli;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand Down Expand Up @@ -51,16 +52,21 @@ final class VeraPdfCliProcessor implements Closeable {
private final VeraAppConfig appConfig;
private final boolean isPolicy;
private final boolean isRecursive;
private final boolean isServerMode;
private final File tempMrrFile;
private final File policyFile;
private boolean isStdOut = true;
private boolean appendData = true;
private String baseDirectory = ""; //$NON-NLS-1$
private OutputStream os;
private File tempFile;

private VeraPdfCliProcessor(final VeraCliArgParser args, ConfigManager configManager) throws VeraPDFException {
this.configManager = configManager;
this.isPolicy = args.isPolicy();
this.isRecursive = args.isRecurse();
this.isServerMode = args.isServerMode();

try {
this.tempMrrFile = (this.isPolicy) ? File.createTempFile("mrr", "veraPDF") : null; //$NON-NLS-1$//$NON-NLS-2$
} catch (IOException excep) {
Expand Down Expand Up @@ -93,6 +99,16 @@ ProcessorConfig getProcessorConfig() {
}

void processPaths(final List<String> pdfPaths) throws VeraPDFException {
if (isServerMode) {
try {
this.tempFile = Files.createTempFile("tempReport", ".xml").toFile();
this.os = new FileOutputStream(tempFile);
} catch (IOException e) {
logger.log(Level.SEVERE, "Can't create temp file", e);
}
} else {
this.os = System.out;
}
// If the path list is empty then process the STDIN stream
if (pdfPaths.isEmpty()) {
processStdIn();
Expand Down Expand Up @@ -135,6 +151,7 @@ private void processFilePaths(final List<String> paths) {
ProcessorFactory.getHandler(this.appConfig.getFormat(), this.appConfig.isVerbose(), reportStream,
this.appConfig.getMaxFailsDisplayed(),
this.processorConfig.getValidatorConfig().isRecordPasses()));
reportStream.flush();
} catch (VeraPDFException excep) {
String message = CliConstants.EXCEP_VERA_BATCH;
System.err.println(message);
Expand All @@ -151,9 +168,9 @@ private void processStream(final ItemDetails item, final InputStream toProcess)

OutputStream outputReportStream = this.getReportStream();
try {
if (result.isPdf() && !result.isEncryptedPdf())
if (result.isPdf() && !result.isEncryptedPdf()) {
ProcessorFactory.resultToXml(result, outputReportStream, true);
else {
} else {
String message = String.format(
(result.isPdf()) ? CliConstants.MESS_PDF_ENCRYPTED : CliConstants.MESS_PDF_NOT_VALID,
item.getName());
Expand Down Expand Up @@ -188,11 +205,11 @@ private OutputStream getReportStream() {
throw new IllegalStateException("Policy enabled BUT no temp destination", excep);
}
}
return System.out;
return this.os;
}

private void applyPolicy() throws VeraPDFException {
File tempPolicyResult = null;
File tempPolicyResult;
try {
tempPolicyResult = File.createTempFile("policyResult", "veraPDF");
} catch (IOException excep) {
Expand All @@ -201,7 +218,7 @@ private void applyPolicy() throws VeraPDFException {
try (InputStream mrrIs = new FileInputStream(this.tempMrrFile);
OutputStream policyResultOs = new FileOutputStream(tempPolicyResult)) {
PolicyChecker.applyPolicy(this.policyFile, mrrIs, policyResultOs);
PolicyChecker.insertPolicyReport(tempPolicyResult, this.tempMrrFile, System.out);
PolicyChecker.insertPolicyReport(tempPolicyResult, this.tempMrrFile, os);
} catch (FileNotFoundException excep) {
throw new VeraPDFException("Could not find temporary policy result file.", excep);
} catch (IOException excep) {
Expand Down Expand Up @@ -260,4 +277,8 @@ public void close() {
this.tempMrrFile.deleteOnExit();
}
}

public File getTempFile() {
return tempFile;
}
}
Loading