From a1ab1a685d85fe3aea9fd076cbeb29fdc2be4f9c Mon Sep 17 00:00:00 2001 From: Veniamin Fernandes Date: Sun, 22 Jan 2023 18:37:39 +0200 Subject: [PATCH 1/5] remove unused code from ClientRequestSelectorTest Signed-off-by: Veniamin Fernandes --- .../async/ClientRequestSelectorTest.java | 193 +++--------------- 1 file changed, 23 insertions(+), 170 deletions(-) diff --git a/test/freenet/client/async/ClientRequestSelectorTest.java b/test/freenet/client/async/ClientRequestSelectorTest.java index 4630c6fcf65..4c93888fa1b 100644 --- a/test/freenet/client/async/ClientRequestSelectorTest.java +++ b/test/freenet/client/async/ClientRequestSelectorTest.java @@ -16,7 +16,6 @@ import freenet.client.Metadata; import freenet.client.InsertException.InsertExceptionMode; import freenet.client.async.SplitFileInserterSegmentStorage.BlockInsert; -import freenet.client.async.SplitFileInserterSegmentStorage.MissingKeyException; import freenet.client.async.SplitFileInserterStorage.Status; import freenet.client.events.SimpleEventProducer; import freenet.crypt.CRCChecksumChecker; @@ -26,15 +25,7 @@ import freenet.crypt.HashType; import freenet.crypt.MultiHashInputStream; import freenet.crypt.RandomSource; -import freenet.keys.ClientKey; -import freenet.keys.FreenetURI; import freenet.keys.Key; -import freenet.node.KeysFetchingLocally; -import freenet.node.LowLevelPutException; -import freenet.node.RequestClient; -import freenet.node.SendableInsert; -import freenet.node.SendableRequestItem; -import freenet.node.SendableRequestSender; import freenet.support.CheatingTicker; import freenet.support.DummyJobRunner; import freenet.support.MemoryLimitedJobRunner; @@ -55,12 +46,11 @@ import freenet.support.io.PooledFileRandomAccessBufferFactory; import freenet.support.io.RAFInputStream; import freenet.support.io.ReadOnlyRandomAccessBuffer; -import freenet.support.io.ResumeFailedException; import freenet.support.io.TempBucketFactory; import freenet.support.io.TrivialPersistentFileTracker; public class ClientRequestSelectorTest { - + final LockableRandomAccessBufferFactory smallRAFFactory = new ByteArrayRandomAccessBufferFactory(); final FilenameGenerator fg; final PersistentFileTracker persistentFileTracker; @@ -76,16 +66,7 @@ public class ClientRequestSelectorTest { final ChecksumChecker checker; final MemoryLimitedJobRunner memoryLimitedJobRunner; final PersistentJobRunner jobRunner; - final KeySalter salt = new KeySalter() { - @Override - public byte[] saltKey(Key key) { - return key.getRoutingKey(); - } - - }; - private final FreenetURI URI; - public ClientRequestSelectorTest() throws IOException { dir = new File("split-file-inserter-storage-test"); dir.mkdir(); @@ -101,23 +82,15 @@ public ClientRequestSelectorTest() throws IOException { cryptoKey = new byte[32]; r.nextBytes(cryptoKey); checker = new CRCChecksumChecker(); - memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 20, executor, NativeThread.JAVA_PRIORITY_RANGE); + memoryLimitedJobRunner = new MemoryLimitedJobRunner(9 * 1024 * 1024L, 20, executor, NativeThread.JAVA_PRIORITY_RANGE); jobRunner = new DummyJobRunner(executor, null); - URI = FreenetURI.generateRandomCHK(r); } - - class MyCallback implements SplitFileInserterStorageCallback { - - MyCallback(SendableInsert sender) { - this.sender = sender; - } - + + private static class MyCallback implements SplitFileInserterStorageCallback { + private boolean finishedEncode; - private boolean hasKeys; private boolean succeededInsert; private InsertException failed; - private Metadata metadata; - private final SendableInsert sender; @Override public synchronized void onFinishedEncode() { @@ -127,33 +100,21 @@ public synchronized void onFinishedEncode() { @Override public synchronized void onHasKeys() { - hasKeys = true; notifyAll(); } - + @Override public void encodingProgress() { // Ignore. } - + private void checkFailed() throws InsertException { - if(failed != null) + if (failed != null) throw failed; } public synchronized void waitForFinishedEncode() throws InsertException { - while(!finishedEncode) { - checkFailed(); - try { - wait(); - } catch (InterruptedException e) { - // Ignore. - } - } - } - - public synchronized void waitForHasKeys() throws InsertException { - while(!hasKeys) { + while (!finishedEncode) { checkFailed(); try { wait(); @@ -166,12 +127,11 @@ public synchronized void waitForHasKeys() throws InsertException { @Override public synchronized void onSucceeded(Metadata metadata) { succeededInsert = true; - this.metadata = metadata; notifyAll(); } - - public synchronized Metadata waitForSucceededInsert() throws InsertException { - while(!succeededInsert) { + + public synchronized void waitForSucceededInsert() throws InsertException { + while (!succeededInsert) { checkFailed(); try { wait(); @@ -179,7 +139,6 @@ public synchronized Metadata waitForSucceededInsert() throws InsertException { // Ignore. } } - return metadata; } @Override @@ -198,21 +157,13 @@ public void clearCooldown() { // Ignore. } - public synchronized boolean hasFailed() { - return failed != null; - } - - public synchronized boolean hasFinishedEncode() { - return finishedEncode; - } - @Override public short getPriorityClass() { return 0; } } - + private HashResult[] getHashes(LockableRandomAccessBuffer data) throws IOException { InputStream is = new RAFInputStream(data, 0, data.size()); MultiHashInputStream hashStream = new MultiHashInputStream(is, HashType.SHA256.bitmask); @@ -222,115 +173,19 @@ private HashResult[] getHashes(LockableRandomAccessBuffer data) throws IOExcepti } private LockableRandomAccessBuffer generateData(Random random, long size, - LockableRandomAccessBufferFactory smallRAFFactory) throws IOException { + LockableRandomAccessBufferFactory smallRAFFactory) throws IOException { LockableRandomAccessBuffer thing = smallRAFFactory.makeRAF(size); BucketTools.fill(thing, random, 0, size); return new ReadOnlyRandomAccessBuffer(thing); } - - class NullSendableInsert extends SendableInsert { - - public NullSendableInsert(boolean persistent, boolean realTimeFlag) { - super(persistent, realTimeFlag); - } - @Override - public void onSuccess(SendableRequestItem keyNum, ClientKey key, ClientContext context) { - // Ignore - } - - @Override - public void onFailure(LowLevelPutException e, SendableRequestItem keyNum, - ClientContext context) { - // Ignore - } - - @Override - public boolean canWriteClientCache() { - return false; - } - - @Override - public boolean localRequestOnly() { - return false; - } - - @Override - public boolean forkOnCacheable() { - return false; - } - - @Override - public void onEncode(SendableRequestItem token, ClientKey key, ClientContext context) { - // Ignore - } - - @Override - public boolean isEmpty() { - return false; - } - - @Override - protected void innerOnResume(ClientContext context) throws InsertException, - ResumeFailedException { - // Ignore - } - - @Override - public short getPriorityClass() { - return 0; - } - - @Override - public SendableRequestItem chooseKey(KeysFetchingLocally keys, ClientContext context) { - return null; - } - - @Override - public long countAllKeys(ClientContext context) { - return 0; - } - - @Override - public long countSendableKeys(ClientContext context) { - return 0; - } - - @Override - public SendableRequestSender getSender(ClientContext context) { - return null; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public RequestClient getClient() { - return null; - } - - @Override - public ClientRequester getClientRequest() { - return null; - } - - @Override - public boolean isSSK() { - return false; - } - - } - @Test - public void testSmallSplitfileChooseCompletion() throws IOException, InsertException, MissingKeyException { + public void testSmallSplitfileChooseCompletion() throws IOException, InsertException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory); HashResult[] hashes = getHashes(data); - NullSendableInsert insert = new NullSendableInsert(false, false); - MyCallback cb = new MyCallback(insert); + MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.maxInsertRetries = 2; ClientRequestSelector keys = new ClientRequestSelector(true, false, false, null); @@ -348,24 +203,25 @@ public void testSmallSplitfileChooseCompletion() throws IOException, InsertExcep assertEquals(storage.getStatus(), Status.ENCODED); boolean[] chosenBlocks = new boolean[segment.totalBlockCount]; // Choose and fail all blocks. - for(int i=0;i Date: Sun, 22 Jan 2023 18:59:05 +0200 Subject: [PATCH 2/5] make inner classes to be static in PersistentJobRunnerImplTest Signed-off-by: Veniamin Fernandes --- .../async/PersistentJobRunnerImplTest.java | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/test/freenet/client/async/PersistentJobRunnerImplTest.java b/test/freenet/client/async/PersistentJobRunnerImplTest.java index 6d106c4027f..87d5ac3b6e2 100644 --- a/test/freenet/client/async/PersistentJobRunnerImplTest.java +++ b/test/freenet/client/async/PersistentJobRunnerImplTest.java @@ -12,12 +12,12 @@ import freenet.support.io.NativeThread; public class PersistentJobRunnerImplTest { - + final WaitableExecutor exec = new WaitableExecutor(new PooledExecutor()); final Ticker ticker = new CheatingTicker(exec); final JobRunner jobRunner; final ClientContext context; - + public PersistentJobRunnerImplTest() { jobRunner = new JobRunner(exec, ticker, 1000); context = new ClientContext(0, null, exec, null, null, null, null, null, null, null, null, ticker, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); @@ -26,18 +26,18 @@ public PersistentJobRunnerImplTest() { exec.waitForIdle(); jobRunner.grabHasCheckpointed(); } - - private class WakeableJob implements PersistentJob { + + private static class WakeableJob implements PersistentJob { private boolean wake; private boolean started; private boolean finished; @Override public boolean run(ClientContext context) { - synchronized(this) { + synchronized (this) { started = true; notifyAll(); - while(!wake) { + while (!wake) { try { wait(); } catch (InterruptedException e) { @@ -49,22 +49,22 @@ public boolean run(ClientContext context) { } return false; } - + public synchronized void wakeUp() { wake = true; notifyAll(); } - + public synchronized boolean started() { return started; } - + public synchronized boolean finished() { return finished; } public synchronized void waitForStarted() { - while(!started) { + while (!started) { try { wait(); } catch (InterruptedException e) { @@ -72,11 +72,10 @@ public synchronized void waitForStarted() { } } } - } - - private class JobRunner extends PersistentJobRunnerImpl { - + + private static class JobRunner extends PersistentJobRunnerImpl { + private boolean hasCheckpointed; public JobRunner(Executor executor, Ticker ticker, long interval) { @@ -95,7 +94,7 @@ protected synchronized void innerCheckpoint(boolean shutdown) { hasCheckpointed = true; notifyAll(); } - + public synchronized boolean grabHasCheckpointed() { boolean ret = hasCheckpointed; hasCheckpointed = false; @@ -103,42 +102,40 @@ public synchronized boolean grabHasCheckpointed() { } } - - private class WaitAndCheckpoint implements Runnable { + + private static class WaitAndCheckpoint implements Runnable { private final JobRunner jobRunner; private boolean started; private boolean finished; - + public WaitAndCheckpoint(JobRunner jobRunner2) { jobRunner = jobRunner2; } @Override public void run() { - synchronized(this) { + synchronized (this) { started = true; notifyAll(); } try { jobRunner.waitAndCheckpoint(); } catch (PersistenceDisabledException e) { - System.err.println("Impossible: "+e); - return; + throw new IllegalStateException( + JobRunner.class.getSimpleName() + " has failed with unexpected exception", + e + ); } assertTrue(jobRunner.grabHasCheckpointed()); - synchronized(this) { + synchronized (this) { finished = true; notifyAll(); } } - public synchronized boolean hasStarted() { - return started; - } - public synchronized void waitForFinished() { - while(!finished) { + while (!finished) { try { wait(); } catch (InterruptedException e) { @@ -148,7 +145,7 @@ public synchronized void waitForFinished() { } public synchronized void waitForStarted() { - while(!started) { + while (!started) { try { wait(); } catch (InterruptedException e) { @@ -156,9 +153,8 @@ public synchronized void waitForStarted() { } } } - } - + @Test public void testWaitForCheckpoint() throws PersistenceDisabledException { jobRunner.onLoading(); @@ -172,7 +168,7 @@ public void testWaitForCheckpoint() throws PersistenceDisabledException { checkpointer.waitForFinished(); assertTrue(w.finished()); } - + @Test public void testDisabledCheckpointing() throws PersistenceDisabledException { jobRunner.setCheckpointASAP(); @@ -182,16 +178,10 @@ public void testDisabledCheckpointing() throws PersistenceDisabledException { assertFalse(jobRunner.mustCheckpoint()); jobRunner.setCheckpointASAP(); assertFalse(jobRunner.mustCheckpoint()); - + // Run a job which will request a checkpoint. - jobRunner.queue(new PersistentJob() { + jobRunner.queue(context -> true, NativeThread.NORM_PRIORITY); - @Override - public boolean run(ClientContext context) { - return true; - } - - }, NativeThread.NORM_PRIORITY); // Wait for the job to complete. exec.waitForIdle(); assertFalse(jobRunner.mustCheckpoint()); From 2997613d7cdc5cf7e2edf2ed1ebfde726a47cfb5 Mon Sep 17 00:00:00 2001 From: Veniamin Fernandes Date: Mon, 23 Jan 2023 00:45:24 +0200 Subject: [PATCH 3/5] improve SplitFileFetcherStorageTest and SplitFileInserterStorageTest remove duplication reformat fix assertion methods Signed-off-by: Veniamin Fernandes --- .../async/SplitFileFetcherStorageTest.java | 592 ++++--- .../async/SplitFileInserterStorageTest.java | 1510 ++++++++--------- 2 files changed, 1001 insertions(+), 1101 deletions(-) diff --git a/test/freenet/client/async/SplitFileFetcherStorageTest.java b/test/freenet/client/async/SplitFileFetcherStorageTest.java index 1ef8adea368..3389da31e4e 100644 --- a/test/freenet/client/async/SplitFileFetcherStorageTest.java +++ b/test/freenet/client/async/SplitFileFetcherStorageTest.java @@ -11,6 +11,7 @@ import java.util.HashSet; import java.util.List; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -57,24 +58,17 @@ import freenet.support.io.StorageFormatException; public class SplitFileFetcherStorageTest { - + // Setup code is considerable. See below for actual tests ... - - static DummyRandomSource random; - static final KeySalter salt = new KeySalter() { - @Override - public byte[] saltKey(Key key) { - return key.getRoutingKey(); - } - - }; + static DummyRandomSource random; + static final KeySalter salt = Key::getRoutingKey; static BucketFactory bf = new ArrayBucketFactory(); static LockableRandomAccessBufferFactory rafFactory = new ByteArrayRandomAccessBufferFactory(); static final WaitableExecutor exec = new WaitableExecutor(new PooledExecutor()); static final PersistentJobRunner jobRunner = new DummyJobRunner(exec, null); static final Ticker ticker = new CheatingTicker(exec); - static MemoryLimitedJobRunner memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 20, exec, NativeThread.JAVA_PRIORITY_RANGE); + static MemoryLimitedJobRunner memoryLimitedJobRunner = new MemoryLimitedJobRunner(9 * 1024 * 1024L, 20, exec, NativeThread.JAVA_PRIORITY_RANGE); static final int BLOCK_SIZE = CHKBlock.DATA_LENGTH; private static final OnionFECCodec codec = new OnionFECCodec(); private static final int MAX_SEGMENT_SIZE = 256; @@ -82,15 +76,15 @@ public byte[] saltKey(Key key) { static final CompatibilityMode COMPATIBILITY_MODE = InsertContext.CompatibilityMode.COMPAT_1416; static FreenetURI URI; private static final List NO_DECOMPRESSORS = Collections.emptyList(); - + @Before public void setUp() { // Reset RNG for each test (it's static so not reset by junit). random = new DummyRandomSource(1234); URI = FreenetURI.generateRandomCHK(random); } - - static class TestSplitfile { + + private static class TestSplitfile { final Bucket originalData; final Metadata metadata; final byte[][] dataBlocks; @@ -103,11 +97,11 @@ static class TestSplitfile { private final int[] segmentCheckBlockCount; private final boolean persistent; final MyKeysFetchingLocally fetchingKeys; - + private TestSplitfile(Bucket data, Metadata m, byte[][] originalDataBlocks, - byte[][] originalCheckBlocks, ClientCHK[] dataKeys, ClientCHK[] checkKeys, - byte[] cryptoKey, byte cryptoAlgorithm, int[] segmentDataBlockCount, - int[] segmentCheckBlockCount, boolean persistent) { + byte[][] originalCheckBlocks, ClientCHK[] dataKeys, ClientCHK[] checkKeys, + byte[] cryptoKey, byte cryptoAlgorithm, int[] segmentDataBlockCount, + int[] segmentCheckBlockCount, boolean persistent) { this.originalData = data; this.metadata = m; this.dataBlocks = originalDataBlocks; @@ -121,14 +115,14 @@ private TestSplitfile(Bucket data, Metadata m, byte[][] originalDataBlocks, this.persistent = persistent; this.fetchingKeys = new MyKeysFetchingLocally(); } - + void free() { originalData.free(); } - static TestSplitfile constructSingleSegment(long size, int checkBlocks, String mime, boolean persistent) throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { + static TestSplitfile constructSingleSegment(long size, int checkBlocks, boolean persistent) throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { assertTrue(checkBlocks <= MAX_SEGMENT_SIZE); - assertTrue(size < MAX_SEGMENT_SIZE * (long)BLOCK_SIZE); + assertTrue(size < MAX_SEGMENT_SIZE * (long) BLOCK_SIZE); Bucket data = makeRandomBucket(size); byte[][] originalDataBlocks = splitAndPadBlocks(data, size); int dataBlocks = originalDataBlocks.length; @@ -136,17 +130,18 @@ static TestSplitfile constructSingleSegment(long size, int checkBlocks, String m assertTrue(dataBlocks + checkBlocks <= MAX_SEGMENT_SIZE); byte[][] originalCheckBlocks = constructBlocks(checkBlocks); codec.encode(originalDataBlocks, originalCheckBlocks, falseArray(checkBlocks), BLOCK_SIZE); - ClientMetadata cm = new ClientMetadata(mime); + ClientMetadata cm = new ClientMetadata(null); // FIXME no hashes for now. // FIXME no compression for now. byte[] cryptoKey = randomKey(); byte cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256; ClientCHK[] dataKeys = makeKeys(originalDataBlocks, cryptoKey, cryptoAlgorithm); ClientCHK[] checkKeys = makeKeys(originalCheckBlocks, cryptoKey, cryptoAlgorithm); - Metadata m = new Metadata(SplitfileAlgorithm.ONION_STANDARD, dataKeys, checkKeys, dataBlocks, - checkBlocks, 0, cm, size, null, null, size, false, null, null, size, size, dataBlocks, - dataBlocks + checkBlocks, false, COMPATIBILITY_MODE, - cryptoAlgorithm, cryptoKey, true, 0); + Metadata m = new Metadata(SplitfileAlgorithm.ONION_STANDARD, dataKeys, checkKeys, dataBlocks, + checkBlocks, 0, cm, size, null, null, size, false, null, null, size, size, dataBlocks, + dataBlocks + checkBlocks, false, COMPATIBILITY_MODE, + cryptoAlgorithm, cryptoKey, true, 0 + ); // Make sure the metadata is reusable. // FIXME also necessary as the above constructor doesn't set segments. Bucket metaBucket = m.toBucket(bf); @@ -155,36 +150,37 @@ static TestSplitfile constructSingleSegment(long size, int checkBlocks, String m assertTrue(BucketTools.equalBuckets(metaBucket, copyBucket)); metaBucket.free(); copyBucket.free(); - return new TestSplitfile(data, m1, originalDataBlocks, originalCheckBlocks, dataKeys, checkKeys, - cryptoKey, cryptoAlgorithm, null, null, persistent); + return new TestSplitfile(data, m1, originalDataBlocks, originalCheckBlocks, dataKeys, checkKeys, + cryptoKey, cryptoAlgorithm, null, null, persistent + ); } /** - * Create a multi-segment test splitfile. The main complication with multi-segment is that we can't + * Create a multi-segment test splitfile. The main complication with multi-segment is that we can't * choose the number of blocks in each segment arbitrarily; that depends on the metadata format; the * caller must ensure that the number are consistent. + * * @param size - * @param segmentDataBlockCount The actual number of data blocks in each segment. Must be consistent - * with the other parameters; this cannot be chosen freely due to the metadata format. - * @param segmentCheckBlockCount The actual number of check blocks in each segment. Must be consistent - * with the other parameters; this cannot be chosen freely due to the metadata format. - * @param segmentSize The "typical" number of data blocks in a segment. - * @param checkSegmentSize The "typical" number of check blocks in a segment. + * @param segmentDataBlockCount The actual number of data blocks in each segment. Must be consistent + * with the other parameters; this cannot be chosen freely due to the metadata format. + * @param segmentCheckBlockCount The actual number of check blocks in each segment. Must be consistent + * with the other parameters; this cannot be chosen freely due to the metadata format. + * @param segmentSize The "typical" number of data blocks in a segment. + * @param checkSegmentSize The "typical" number of check blocks in a segment. * @param deductBlocksFromSegments The number of segments from which a single block has been deducted. - * This is used when the number of data blocks isn't an exact multiple of the number of segments. - * @param topCompatibilityMode The "short" value of the definitive compatibility mode used to create - * the splitfile. This must again be consistent with the rest, as it is sometimes used in decoding. - * @param mime + * This is used when the number of data blocks isn't an exact multiple of the number of segments. + * @param topCompatibilityMode The "short" value of the definitive compatibility mode used to create + * the splitfile. This must again be consistent with the rest, as it is sometimes used in decoding. * @return * @throws IOException * @throws CHKEncodeException * @throws MetadataUnresolvedException * @throws MetadataParseException */ - static TestSplitfile constructMultipleSegments(long size, int[] segmentDataBlockCount, - int[] segmentCheckBlockCount, int segmentSize, int checkSegmentSize, - int deductBlocksFromSegments, CompatibilityMode topCompatibilityMode, String mime, boolean persistent) - throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { + static TestSplitfile constructMultipleSegments(long size, int[] segmentDataBlockCount, + int[] segmentCheckBlockCount, int segmentSize, int checkSegmentSize, + int deductBlocksFromSegments, CompatibilityMode topCompatibilityMode) + throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { int dataBlocks = sum(segmentDataBlockCount); int checkBlocks = sum(segmentCheckBlockCount); int segments = segmentDataBlockCount.length; @@ -195,25 +191,26 @@ static TestSplitfile constructMultipleSegments(long size, int[] segmentDataBlock byte[][] originalCheckBlocks = constructBlocks(checkBlocks); int startDataBlock = 0; int startCheckBlock = 0; - for(int seg=0;seg= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, false); + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, false); testDataBlocksOnly(test); - if(checkBlocks >= dataBlocks) + if (checkBlocks >= dataBlocks) { testCheckBlocksOnly(test); + } testRandomMixture(test); test.free(); } @@ -646,10 +668,10 @@ private void testDataBlocksOnly(TestSplitfile test) throws IOException, CHKEncod StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; - for(int i=0;i keys = new HashSet(); @Override @@ -886,27 +911,27 @@ public void add(Key k) { public void clear() { keys.clear(); } - + } - + @Test public void testChooseKeyOneTry() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 3, checkBlocks = 3; - TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); + TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks * BLOCK_SIZE, checkBlocks, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 0; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); - boolean[] tried = new boolean[dataBlocks+checkBlocks]; + boolean[] tried = new boolean[dataBlocks + checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); - assertEquals(storage.chooseRandomKey(), null); + assertNull(storage.chooseRandomKey()); cb.waitForFailed(); } - + private void innerChooseKeyTest(int dataBlocks, int checkBlocks, SplitFileFetcherSegmentStorage storage, boolean[] tried, TestSplitfile test, boolean cooldown) { final MyKeysFetchingLocally keys = test.fetchingKeys; keys.clear(); - for(int i=0;i now); - assertFalse(storage.segments[0].getOverallCooldownTime() == Long.MAX_VALUE); + assertNotEquals(Long.MAX_VALUE, storage.segments[0].getOverallCooldownTime()); // Now in cooldown. test.fetchingKeys.clear(); - assertEquals(storage.chooseRandomKey(), null); - Thread.sleep((long)(COOLDOWN_TIME+COOLDOWN_TIME/2+1)); + assertNull(storage.chooseRandomKey()); + Thread.sleep(COOLDOWN_TIME + COOLDOWN_TIME / 2 + 1); cb.checkFailed(); // Should be out of cooldown now. - for(int i=0;i<3;i++) { - boolean[] tried = new boolean[dataBlocks+checkBlocks]; + for (int i = 0; i < 3; i++) { + boolean[] tried = new boolean[dataBlocks + checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, true); } // Now it should fail. cb.waitForFailed(); } - + @Test public void testWriteReadSegmentKeys() throws FetchException, MetadataParseException, IOException, CHKEncodeException, MetadataUnresolvedException, ChecksumFailedException { int dataBlocks = 3, checkBlocks = 3; - TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, true); + TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; SplitFileSegmentKeys keys = segment.getSegmentKeys(); SplitFileSegmentKeys moreKeys = segment.readSegmentKeys(); - assertTrue(keys.equals(moreKeys)); + assertEquals(keys, moreKeys); storage.close(); } - /** Test persistence: Create and then reload. Don't do anything. */ + /** + * Test persistence: Create and then reload. Don't do anything. + */ @Test public void testPersistenceReload() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException, StorageFormatException { int dataBlocks = 2; int checkBlocks = 3; - long size = 32768*2-1; - assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); + long size = 32768 * 2 - 1; + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); - SplitFileFetcherStorage storage = test.createStorage(cb); - // No need to shutdown the old storage. - storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); + SplitFileFetcherStorage storage = createSplitFileFetcherStorageTwice(test, cb); storage.close(); } - + @Test public void testPersistenceReloadThenFetch() throws IOException, StorageFormatException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 2; int checkBlocks = 3; - long size = 32768*2-1; - assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); + long size = 32768 * 2 - 1; + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); - SplitFileFetcherStorage storage = test.createStorage(cb); - // No need to shutdown the old storage. - storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); + SplitFileFetcherStorage storage = createSplitFileFetcherStorageTwice(test, cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; assertFalse(segment.corruptMetadata()); - int total = test.dataBlocks.length+test.checkBlocks.length; - for(int i=0;i= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); + long size = 32768 * 2 - 1; + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; - SplitFileFetcherStorage storage = test.createStorage(cb, ctx); + test.createStorage(cb, ctx); // No need to shutdown the old storage. - storage = test.createStorage(cb, ctx, cb.getRAF()); - for(int i=0;i<3;i++) { - boolean[] tried = new boolean[dataBlocks+checkBlocks]; + SplitFileFetcherStorage storage = test.createStorage(cb, ctx, cb.getRAF()); + for (int i = 0; i < 3; i++) { + boolean[] tried = new boolean[dataBlocks + checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); } test.fetchingKeys.clear(); - assertEquals(storage.chooseRandomKey(), null); + assertNull(storage.chooseRandomKey()); cb.waitForFailed(); } - + @Test public void testPersistenceReloadBetweenChooseKey() throws IOException, StorageFormatException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 2; int checkBlocks = 3; - long size = 32768*2-1; - assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); + long size = 32768 * 2 - 1; + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; - SplitFileFetcherStorage storage = test.createStorage(cb, ctx); + test.createStorage(cb, ctx); // No need to shutdown the old storage. - storage = test.createStorage(cb, ctx, cb.getRAF()); - for(int i=0;i<3;i++) { - boolean[] tried = new boolean[dataBlocks+checkBlocks]; + SplitFileFetcherStorage storage = test.createStorage(cb, ctx, cb.getRAF()); + for (int i = 0; i < 3; i++) { + boolean[] tried = new boolean[dataBlocks + checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); // Reload. exec.waitForIdle(); @@ -1106,33 +1130,32 @@ public void testPersistenceReloadBetweenChooseKey() throws IOException, StorageF assertTrue(i != 2); storage.start(false); } catch (FetchException e) { - if(i != 2) throw e; // Already failed on the final iteration, otherwise is an error. + if (i != 2) throw e; // Already failed on the final iteration, otherwise is an error. return; } } test.fetchingKeys.clear(); - assertEquals(storage.chooseRandomKey(), null); + assertNull(storage.chooseRandomKey()); cb.waitForFailed(); } - + @Test public void testPersistenceReloadBetweenFetches() throws IOException, StorageFormatException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 2; int checkBlocks = 3; - long size = 32768*2-1; - assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); - TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); + long size = 32768 * 2 - 1; + assertTrue(dataBlocks * (long) BLOCK_SIZE >= size); + TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, true); StorageCallback cb = test.createStorageCallback(); - SplitFileFetcherStorage storage = test.createStorage(cb); - // No need to shutdown the old storage. - storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); + SplitFileFetcherStorage storage = createSplitFileFetcherStorageTwice(test, cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; assertFalse(segment.corruptMetadata()); - int total = test.dataBlocks.length+test.checkBlocks.length; - for(int i=0;i cb.waitForSucceededInsert()); + + assertEquals(InsertExceptionMode.TOO_MANY_RETRIES_IN_BLOCKS, e.mode); + assertNotNull(e.errorCodes); + assertEquals(3, e.errorCodes.getErrorCount(InsertExceptionMode.ROUTE_NOT_FOUND)); + assertEquals(3, e.errorCodes.totalCount()); + assertEquals(Status.FAILED, storage.getStatus()); } @Test - public void testSmallSplitfileFailureFatalError() throws IOException, InsertException, MissingKeyException { - Random r = new Random(12121); - long size = 65536; // Exact multiple, so no last block - LockableRandomAccessBuffer data = generateData(r, size); - HashResult[] hashes = getHashes(data); - MyCallback cb = new MyCallback(); - InsertContext context = baseContext.clone(); + public void testSmallSplitfileFailureFatalError() throws Exception { context.maxInsertRetries = 2; - KeysFetchingLocally keys = new MyKeysFetchingLocally(); - SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, - new ClientMetadata(), false, null, smallRAFFactory, false, context, - cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, - r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); - storage.start(); - cb.waitForFinishedEncode(); - assertEquals(storage.segments.length, 1); - SplitFileInserterSegmentStorage segment = storage.segments[0]; - assertEquals(segment.dataBlockCount, 2); - assertEquals(segment.checkBlockCount, 3); - assertEquals(segment.crossCheckBlockCount, 0); - assertEquals(storage.getStatus(), Status.ENCODED); + SplitFileInserterStorage storage = createSplitFileInserterStorage(data, size, cb, false, context, cryptoAlgorithm, cryptoKey, hashes, random, memoryLimitedJobRunner, keys); + SplitFileInserterSegmentStorage segment = assertProperSegmentStateAndGet(storage); assertTrue(InsertException.isFatal(InsertExceptionMode.INTERNAL_ERROR)); segment.onFailure(0, new InsertException(InsertExceptionMode.INTERNAL_ERROR)); - try { - cb.waitForSucceededInsert(); - assertTrue(false); - } catch (InsertException e) { - assertEquals(e.mode, InsertExceptionMode.FATAL_ERRORS_IN_BLOCKS); - assertTrue(e.errorCodes != null); - assertEquals(e.errorCodes.getErrorCount(InsertExceptionMode.INTERNAL_ERROR), 1); - assertEquals(e.errorCodes.totalCount(), 1); - assertEquals(storage.getStatus(), Status.FAILED); - } + + InsertException e = assertThrows(InsertException.class, () -> cb.waitForSucceededInsert()); + assertEquals(InsertExceptionMode.FATAL_ERRORS_IN_BLOCKS, e.mode); + assertNotNull(e.errorCodes); + assertEquals(1, e.errorCodes.getErrorCount(InsertExceptionMode.INTERNAL_ERROR)); + assertEquals(1, e.errorCodes.totalCount()); + assertEquals(Status.FAILED, storage.getStatus()); } private HashResult[] getHashes(LockableRandomAccessBuffer data) throws IOException { - InputStream is = new RAFInputStream(data, 0, data.size()); - MultiHashInputStream hashStream = new MultiHashInputStream(is, HashType.SHA256.bitmask); - FileUtil.copy(is, new NullOutputStream(), data.size()); - is.close(); - return hashStream.getResults(); + try ( + InputStream is = new RAFInputStream(data, 0, data.size()); + MultiHashInputStream hashStream = new MultiHashInputStream(is, HashType.SHA256.bitmask) + ) { + FileUtil.copy(is, new NullOutputStream(), data.size()); + return hashStream.getResults(); + } } private LockableRandomAccessBuffer generateData(Random random, long size) throws IOException { @@ -671,64 +476,64 @@ private LockableRandomAccessBuffer generateData(Random random, long size) throws } private LockableRandomAccessBuffer generateData(Random random, long size, - LockableRandomAccessBufferFactory factory) throws IOException { + LockableRandomAccessBufferFactory factory) throws IOException { LockableRandomAccessBuffer thing = factory.makeRAF(size); BucketTools.fill(thing, random, 0, size); return new ReadOnlyRandomAccessBuffer(thing); } - + @Test - public void testRoundTripSimple() throws FetchException, MetadataParseException, Exception { - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*2, CompatibilityMode.COMPAT_CURRENT); - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*2-1, CompatibilityMode.COMPAT_CURRENT); - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*128, CompatibilityMode.COMPAT_CURRENT); - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*128+1, CompatibilityMode.COMPAT_CURRENT); - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*192, CompatibilityMode.COMPAT_CURRENT); - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*192+1, CompatibilityMode.COMPAT_CURRENT); + public void testRoundTripSimple() throws Exception { + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 2, CompatibilityMode.COMPAT_CURRENT); + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 2 - 1, CompatibilityMode.COMPAT_CURRENT); + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 128, CompatibilityMode.COMPAT_CURRENT); + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 128 + 1, CompatibilityMode.COMPAT_CURRENT); + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 192, CompatibilityMode.COMPAT_CURRENT); + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * 192 + 1, CompatibilityMode.COMPAT_CURRENT); } - + @Test - public void testRoundTripOneBlockSegment() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception { - testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*(128+1)-1, CompatibilityMode.COMPAT_1250_EXACT); + public void testRoundTripOneBlockSegment() throws Exception { + testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH * (128 + 1) - 1, CompatibilityMode.COMPAT_1250_EXACT); } - + @Test - public void testRoundTripCrossSegment() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception { - if(!TestProperty.EXTENSIVE) return; + public void testRoundTripCrossSegment() throws Exception { + if (!TestProperty.EXTENSIVE) return; // Test cross-segment: - testRoundTripCrossSegmentRandom(CHKBlock.DATA_LENGTH*128*21); + testRoundTripCrossSegmentRandom(CHKBlock.DATA_LENGTH * 128 * 21); } - + @Test - public void testRoundTripDataBlocksOnly() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception { - testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH*128*5); - if(!TestProperty.EXTENSIVE) return; + public void testRoundTripDataBlocksOnly() throws Exception { + testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH * 128 * 5); + if (!TestProperty.EXTENSIVE) return; // Test cross-segment: - testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH*128*21); + testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH * 128 * 21); } - + @Test - public void testResumeCrossSegment() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException { - if(!TestProperty.EXTENSIVE) return; - testResumeCrossSegment(CHKBlock.DATA_LENGTH*128*21); + public void testResumeCrossSegment() throws Exception { + if (!TestProperty.EXTENSIVE) return; + testResumeCrossSegment(CHKBlock.DATA_LENGTH * 128 * 21); } - + @Test - public void testEncodeAfterShutdownCrossSegment() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException { - if(!TestProperty.EXTENSIVE) return; - testEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*21); + public void testEncodeAfterShutdownCrossSegment() throws Exception { + if (!TestProperty.EXTENSIVE) return; + testEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH * 128 * 21); } - + @Test - public void testRepeatedEncodeAfterShutdown() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException { - testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*5); // Not cross-segment. - if(!TestProperty.EXTENSIVE) return; - testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*21); // Cross-segment. + public void testRepeatedEncodeAfterShutdown() throws Exception { + testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH * 128 * 5); // Not cross-segment. + if (!TestProperty.EXTENSIVE) return; + testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH * 128 * 21); // Cross-segment. } - - static class MyKeysFetchingLocally implements KeysFetchingLocally { - private final HashSet keys = new HashSet(); - private final HashSet inserts = new HashSet(); + + private static class MyKeysFetchingLocally implements KeysFetchingLocally { + private final HashSet keys = new HashSet<>(); + private final HashSet inserts = new HashSet<>(); @Override public long checkRecentlyFailed(Key key, boolean realTime) { @@ -757,10 +562,9 @@ public void clear() { keys.clear(); inserts.clear(); } - } - - private void testRoundTripSimpleRandom(long size, CompatibilityMode cmode) throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception { + + private void testRoundTripSimpleRandom(long size, CompatibilityMode cmode) throws Exception { RandomSource r = new DummyRandomSource(12123); LockableRandomAccessBuffer data = generateData(r, size); Bucket dataBucket = new RAFBucket(data); @@ -772,18 +576,14 @@ private void testRoundTripSimpleRandom(long size, CompatibilityMode cmode) throw cmode = context.getCompatibilityMode(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); boolean old = cmode.code < CompatibilityMode.COMPAT_1255.code; - byte cryptoAlgorithm = this.cryptoAlgorithm; - if(!(cmode == CompatibilityMode.COMPAT_CURRENT || cmode.ordinal() >= CompatibilityMode.COMPAT_1416.ordinal())) + byte cryptoAlgorithm; + if (!(cmode == CompatibilityMode.COMPAT_CURRENT || cmode.ordinal() >= CompatibilityMode.COMPAT_1416.ordinal())) { cryptoAlgorithm = Key.ALGO_AES_PCFB_256_SHA256; - else + } else { cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256; - SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, - new ClientMetadata(), false, null, smallRAFFactory, false, context, - cryptoAlgorithm, old ? null : cryptoKey, null, hashes, smallBucketFactory, checker, - r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); - storage.start(); - cb.waitForFinishedEncode(); - assertTrue(storage.getStatus() == Status.ENCODED); + } + SplitFileInserterStorage storage = createSplitFileInserterStorage(data, size, cb, false, context, cryptoAlgorithm, old ? null : cryptoKey, hashes, r, memoryLimitedJobRunner, keys); + assertEquals(Status.ENCODED, storage.getStatus()); // Encoded. Now try to decode it ... cb.waitForHasKeys(); Metadata metadata = storage.encodeMetadata(); @@ -794,29 +594,27 @@ private void testRoundTripSimpleRandom(long size, CompatibilityMode cmode) throw Metadata m1 = Metadata.construct(metaBucket); Bucket copyBucket = m1.toBucket(smallBucketFactory); assertTrue(BucketTools.equalBuckets(metaBucket, copyBucket)); - + FetchCallbackForTestingSplitFileInserter fcb = new FetchCallbackForTestingSplitFileInserter(); - - FetchContext fctx = HighLevelSimpleClientImpl.makeDefaultFetchContext(size*2, size*2, smallBucketFactory, new SimpleEventProducer()); - - SplitFileFetcherStorage fetcherStorage = new SplitFileFetcherStorage(m1, fcb, new ArrayList(), - new ClientMetadata(), false, cmode.code, fctx, false, salt, URI, URI, true, new byte[0], - r, smallBucketFactory, smallRAFFactory, jobRunner, ticker, memoryLimitedJobRunner, - checker, false, null, null, keys); - + + FetchContext fctx = HighLevelSimpleClientImpl.makeDefaultFetchContext(size * 2, size * 2, smallBucketFactory, new SimpleEventProducer()); + + SplitFileFetcherStorage fetcherStorage = createFetcherStorage(m1, fcb, new ArrayList<>(), cmode.code, fctx, r); + fetcherStorage.start(false); - + // Fully decode one segment at a time, ignore cross-segment. - - for(int i=0;i(), - new ClientMetadata(), false, cmode, fctx, false, salt, URI, URI, true, new byte[0], - r, smallBucketFactory, smallRAFFactory, jobRunner, ticker, memoryLimitedJobRunner, - checker, false, null, null, keysFetching); - + + SplitFileFetcherStorage fetcherStorage = createFetcherStorage(m1, fcb, new ArrayList<>(), cmode, fctx, r); + fetcherStorage.start(false); - + int segments = storage.segments.length; - for(int i=0;i(), - new ClientMetadata(), false, cmode, fctx, false, salt, URI, URI, true, new byte[0], - r, smallBucketFactory, smallRAFFactory, jobRunner, ticker, memoryLimitedJobRunner, - checker, false, null, null, keysFetching); - + + SplitFileFetcherStorage fetcherStorage = createFetcherStorage(m1, fcb, new ArrayList<>(), cmode, fctx, r); + fetcherStorage.start(false); - - if(storage.crossSegments != null) { + + if (storage.crossSegments != null) { int segments = storage.segments.length; - for(int i=0;i cb.waitForFinishedEncode()); + assertFalse(segment.isEncoding()); + assertEquals(Status.FAILED, storage.getStatus()); } - + @Test - public void testCancelAlt() throws IOException, InsertException, MissingKeyException { + public void testCancelAlt() throws Exception { // We need to check that onFailed() isn't called until after all the cross segment encode threads have finished. - Random r = new Random(12124); - testCancelAlt(r, 32768*6); + testCancelAlt(random, 32768 * 6); } - + @Test - public void testCancelAltCrossSegment() throws IOException, InsertException, MissingKeyException { + public void testCancelAltCrossSegment() throws Exception { // We need to check that onFailed() isn't called until after all the cross segment encode threads have finished. - Random r = new Random(0xb395f44d); - testCancelAlt(r, CHKBlock.DATA_LENGTH*128*21); + testCancelAlt(random, CHKBlock.DATA_LENGTH*128*21); } - private void testCancelAlt(Random r, long size) throws IOException, InsertException { + private void testCancelAlt(Random r, long size) throws Exception { // FIXME tricky to wait for "all threads are in pread()", when # threads != # segments. // So just set max threads to 1 (only affects this test). memoryLimitedJobRunner.setMaxThreads(1); BarrierRandomAccessBuffer data = new BarrierRandomAccessBuffer(generateData(r, size)); HashResult[] hashes = getHashes(data); data.pause(); - MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.earlyEncode = true; - KeysFetchingLocally keys = new MyKeysFetchingLocally(); - SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, - new ClientMetadata(), false, null, smallRAFFactory, false, context, - cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, - r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); + SplitFileInserterStorage storage = new SplitFileInserterStorage( + data, + size, + cb, + null, + new ClientMetadata(), + false, + null, + smallRAFFactory, + false, + context, + cryptoAlgorithm, + cryptoKey, + null, + hashes, + smallBucketFactory, + checker, + r, + memoryLimitedJobRunner, + jobRunner, + ticker, + keys, + false, + 0, + 0, + 0, + 0 + ); storage.start(); - assertEquals(storage.getStatus(), Status.STARTED); - if(storage.crossSegments != null) + assertEquals(Status.STARTED, storage.getStatus()); + if (storage.crossSegments != null) { assertTrue(allCrossSegmentsEncoding(storage)); - else + } else { assertTrue(allSegmentsEncoding(storage)); + } SplitFileInserterSegmentStorage segment = storage.segments[0]; assertTrue(memoryLimitedJobRunner.getRunningThreads() > 0); // Wait for one segment to be in pread(). @@ -1427,83 +1269,85 @@ private void testCancelAlt(Random r, long size) throws IOException, InsertExcept segment.onFailure(0, new InsertException(InsertExceptionMode.INTERNAL_ERROR)); assertFalse(cb.hasFailed()); // Callback must not have been called yet. data.proceed(); // Now it will complete encoding, and then report in, and then fail. - try { - cb.waitForFinishedEncode(); - assertFalse(true); // Should have failed now. - } catch (InsertException e) { - if(storage.segments.length > 2) { - assertFalse(cb.hasFinishedEncode()); - assertTrue(anySegmentNotEncoded(storage)); - } - assertEquals(memoryLimitedJobRunner.getRunningThreads(), 0); - assertFalse(anySegmentEncoding(storage)); - assertEquals(storage.getStatus(), Status.FAILED); + + assertThrows(InsertException.class, ()-> cb.waitForFinishedEncode()); + if (storage.segments.length > 2) { + assertFalse(cb.hasFinishedEncode()); + assertTrue(anySegmentNotEncoded(storage)); } + assertEquals(0, memoryLimitedJobRunner.getRunningThreads()); + assertFalse(anySegmentEncoding(storage)); + assertEquals(Status.FAILED, storage.getStatus()); } - + private boolean allSegmentsEncoding(SplitFileInserterStorage storage) { - for(SplitFileInserterSegmentStorage segment : storage.segments) - if(!segment.isEncoding()) return false; + for (SplitFileInserterSegmentStorage segment : storage.segments) { + if (!segment.isEncoding()) { + return false; + } + } return true; } - + private boolean allCrossSegmentsEncoding(SplitFileInserterStorage storage) { - if(storage.crossSegments != null) { - for(SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) - if(!segment.isEncoding()) return false; + if (storage.crossSegments != null) { + for (SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) { + if (!segment.isEncoding()) { + return false; + } + } } return true; } private boolean anySegmentEncoding(SplitFileInserterStorage storage) { - for(SplitFileInserterSegmentStorage segment : storage.segments) - if(segment.isEncoding()) return true; - if(storage.crossSegments != null) { - for(SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) - if(segment.isEncoding()) return true; + for (SplitFileInserterSegmentStorage segment : storage.segments) { + if (segment.isEncoding()) { + return true; + } + } + if (storage.crossSegments != null) { + for (SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) { + if (segment.isEncoding()) { + return true; + } + } } return false; } private boolean anySegmentNotEncoded(SplitFileInserterStorage storage) { - for(SplitFileInserterSegmentStorage segment : storage.segments) - if(!segment.hasEncoded()) return true; - if(storage.crossSegments != null) { - for(SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) - if(!segment.hasEncodedSuccessfully()) return true; + for (SplitFileInserterSegmentStorage segment : storage.segments) { + if (!segment.hasEncoded()) { + return true; + } + } + if (storage.crossSegments != null) { + for (SplitFileInserterCrossSegmentStorage segment : storage.crossSegments) { + if (!segment.hasEncodedSuccessfully()) { + return true; + } + } } return false; } @Test - public void testPersistentSmallSplitfileNoLastBlockCompletion() throws IOException, InsertException, StorageFormatException, ChecksumFailedException, ResumeFailedException { - Random r = new Random(12121); - long size = 65536; // Exact multiple, so no last block - LockableRandomAccessBuffer data = generateData(r, size, bigRAFFactory); + public void testPersistentSmallSplitfileNoLastBlockCompletion() throws Exception { + LockableRandomAccessBuffer data = generateData(random, size, bigRAFFactory); HashResult[] hashes = getHashes(data); - MyCallback cb = new MyCallback(); - KeysFetchingLocally keys = new MyKeysFetchingLocally(); - SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, - new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), - cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, - r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); - storage.start(); - cb.waitForFinishedEncode(); - assertEquals(storage.segments.length, 1); - assertEquals(storage.segments[0].dataBlockCount, 2); - assertEquals(storage.segments[0].checkBlockCount, 3); - assertEquals(storage.segments[0].crossCheckBlockCount, 0); - assertTrue(storage.getStatus() == Status.ENCODED); + SplitFileInserterStorage storage = createSplitFileInserterStorage(data, size, cb, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, hashes, random, memoryLimitedJobRunner, keys); + assertProperSegmentStateAndGet(storage); executor.waitForIdle(); - SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, - memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); - assertEquals(resumed.segments.length, 1); - SplitFileInserterSegmentStorage segment = resumed.segments[0]; - assertEquals(segment.dataBlockCount, 2); - assertEquals(segment.checkBlockCount, 3); - assertEquals(segment.crossCheckBlockCount, 0); - assertTrue(resumed.getStatus() == Status.ENCODED); - for(int i=0;i cb.waitForSucceededInsert()); + assertEquals(InsertExceptionMode.TOO_MANY_RETRIES_IN_BLOCKS, e.mode); + assertNotNull(e.errorCodes); + assertEquals(3, e.errorCodes.getErrorCount(InsertExceptionMode.ROUTE_NOT_FOUND)); + assertEquals(3, e.errorCodes.totalCount()); assertEquals(Status.FAILED, resumed.getStatus()); } @Test - public void testPersistentSmallSplitfileNoLastBlockChooseAfterResume() throws IOException, InsertException, StorageFormatException, ChecksumFailedException, ResumeFailedException { - Random r = new Random(12121); - long size = 65536; // Exact multiple, so no last block - LockableRandomAccessBuffer data = generateData(r, size, bigRAFFactory); + public void testPersistentSmallSplitfileNoLastBlockChooseAfterResume() throws Exception { + LockableRandomAccessBuffer data = generateData(random, size, bigRAFFactory); HashResult[] hashes = getHashes(data); - MyCallback cb = new MyCallback(); - MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); - InsertContext context = baseContext.clone(); context.consecutiveRNFsCountAsSuccess = 0; context.maxInsertRetries = 1; - SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, - new ClientMetadata(), false, null, smallRAFFactory, true, context, - cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, - r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); - storage.start(); - cb.waitForFinishedEncode(); - assertEquals(storage.segments.length, 1); - assertEquals(storage.segments[0].dataBlockCount, 2); - assertEquals(storage.segments[0].checkBlockCount, 3); - assertEquals(storage.segments[0].crossCheckBlockCount, 0); - assertTrue(storage.getStatus() == Status.ENCODED); + SplitFileInserterStorage storage = createSplitFileInserterStorage(data, size, cb, true, context, cryptoAlgorithm, cryptoKey, hashes, random, memoryLimitedJobRunner, keys); + assertProperSegmentStateAndGet(storage); SplitFileInserterStorage resumed = null; int totalBlockCount = storage.segments[0].totalBlockCount; - + boolean[] chosenBlocks = new boolean[totalBlockCount]; // Choose and fail all blocks. - for(int i=0;i decompressors, + short cmode, + FetchContext fctx, + RandomSource r + ) throws FetchException, MetadataParseException, IOException { + return new SplitFileFetcherStorage( + m1, + fcb, + decompressors, + new ClientMetadata(), + false, + cmode, + fctx, + false, + salt, + URI, + URI, + true, + new byte[0], + r, + smallBucketFactory, + smallRAFFactory, + jobRunner, + ticker, + memoryLimitedJobRunner, + checker, + false, + null, + null, + keys + ); + } + + private SplitFileInserterSegmentStorage assertProperSegmentStateAndGet(SplitFileInserterStorage storage) { + assertEquals(1, storage.segments.length); + SplitFileInserterSegmentStorage segment = storage.segments[0]; + assertEquals(2, segment.dataBlockCount); + assertEquals(3, segment.checkBlockCount); + assertEquals(0, segment.crossCheckBlockCount); + assertEquals(Status.ENCODED, storage.getStatus()); + return segment; + } } From 583e8885966f196a44a1e1db9aab95194fea0b7d Mon Sep 17 00:00:00 2001 From: Veniamin Fernandes Date: Mon, 23 Jan 2023 19:29:27 +0200 Subject: [PATCH 4/5] Improve other tests in the freenet.client package wrap input/output streams with try-with-resources remove duplication use better assertion methods create helper utility ResourceFileUtil Signed-off-by: Veniamin Fernandes --- test/freenet/client/CodeTest.java | 262 +++---- test/freenet/client/DefaultMIMETypesTest.java | 26 +- .../client/FailureCodeTrackerTest.java | 18 +- test/freenet/client/FetchContextTest.java | 36 +- test/freenet/client/OnionFECCodecTest.java | 181 ++--- test/freenet/client/filter/BMPFilterTest.java | 318 ++++---- test/freenet/client/filter/CSSParserTest.java | 336 ++++---- .../client/filter/ContentFilterTest.java | 722 +++++++++--------- .../client/filter/FilterUtilsTest.java | 2 +- test/freenet/client/filter/GIFFilterTest.java | 19 +- .../freenet/client/filter/JPEGFilterTest.java | 11 +- test/freenet/client/filter/M3UFilterTest.java | 28 +- test/freenet/client/filter/MP3FilterTest.java | 33 +- .../client/filter/OggBitStreamFilterTest.java | 49 +- test/freenet/client/filter/OggFilterTest.java | 87 ++- test/freenet/client/filter/OggPageTest.java | 66 +- test/freenet/client/filter/PNGFilterTest.java | 22 +- .../client/filter/ResourceFileUtil.java | 41 + .../client/filter/TagVerifierTest.java | 63 +- .../filter/TheoraBitstreamFilterTest.java | 22 +- 20 files changed, 1186 insertions(+), 1156 deletions(-) create mode 100644 test/freenet/client/filter/ResourceFileUtil.java diff --git a/test/freenet/client/CodeTest.java b/test/freenet/client/CodeTest.java index 181b4feec41..439dd4e3aca 100644 --- a/test/freenet/client/CodeTest.java +++ b/test/freenet/client/CodeTest.java @@ -15,133 +15,137 @@ public class CodeTest { - public static FECMath fecMath = new FECMath(8); - - public static final int KK = 192; - public static final int PACKET_SIZE = 4096; - - /** - * Creates k packets of size sz of random data, encodes them, and tries to decode. Index - * contains the permutation entry. - */ - private static final void encodeDecode(FECCode encode, FECCode decode, int index[]) { - byte[] src = new byte[KK * PACKET_SIZE]; - Util.rand.nextBytes(src); - Buffer[] srcBufs = new Buffer[KK]; - for (int i = 0; i < srcBufs.length; i++) - srcBufs[i] = new Buffer(src, i * PACKET_SIZE, PACKET_SIZE); - - byte[] repair = new byte[KK * PACKET_SIZE]; - Buffer[] repairBufs = new Buffer[KK]; - for (int i = 0; i < repairBufs.length; i++) { - repairBufs[i] = new Buffer(repair, i * PACKET_SIZE, PACKET_SIZE); - } - - encode.encode(srcBufs, repairBufs, index); - decode.decode(repairBufs, index); - - for (int i = 0; i < src.length; i++) - assertEquals(src[i], repair[i]); - } - - @Test - public void testBenchmark() { - if(!TestProperty.BENCHMARK) return; - - int lim = fecMath.gfSize + 1; - FECCode maybeNative = FECCodeFactory.getDefault().createFECCode(KK, lim); - FECCode pureCode = new PureCode(KK, lim); - int[] index = new int[KK]; - - for (int i = 0; i < KK; i++) - index[i] = lim - i - 1; - - byte[] src = new byte[KK * PACKET_SIZE]; - Util.rand.nextBytes(src); - Buffer[] srcBufs = new Buffer[KK]; - for (int i = 0; i < srcBufs.length; i++) - srcBufs[i] = new Buffer(src, i * PACKET_SIZE, PACKET_SIZE); - - byte[] repair = new byte[KK * PACKET_SIZE]; - Buffer[] repairBufs = new Buffer[KK]; - for (int i = 0; i < repairBufs.length; i++) { - repairBufs[i] = new Buffer(repair, i * PACKET_SIZE, PACKET_SIZE); - } - - int[] indexBackup = new int[index.length]; - System.arraycopy(index,0,indexBackup,0,index.length); - - System.out.println("Getting ready for benchmarking encode()"); - long t1 = System.currentTimeMillis(); - maybeNative.encode(srcBufs, repairBufs, index); - long t2 = System.currentTimeMillis(); - pureCode.encode(srcBufs, repairBufs, indexBackup); - long t3 = System.currentTimeMillis(); - - float dNativeEncode = t2 - t1; - float dPureEncode = t3 - t2; - - Buffer[] repairBufs2 = repairBufs.clone(); - System.arraycopy(repairBufs, 0, repairBufs2, 0, repairBufs.length); - System.out.println("Getting ready for benchmarking decode()"); - t1 = System.currentTimeMillis(); - maybeNative.decode(repairBufs, index); - t2 = System.currentTimeMillis(); - pureCode.decode(repairBufs2, indexBackup); - t3 = System.currentTimeMillis(); - - float dNativeDecode = t2 - t1; - float dPureDecode = t3 - t2; - - System.out.println(maybeNative); - System.out.println(pureCode); - System.out.println("Native code took "+dNativeEncode+"ms whereas java's code took "+dPureEncode+"ms to encode()"); - System.out.println("Native code took "+dNativeDecode+"ms whereas java's code took "+dPureDecode+"ms to decode()"); - } - - @Test - public void testSimpleRev() { - int lim = fecMath.gfSize + 1; - FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); - FECCode code2 = new PureCode(KK, lim); - int[] index = new int[KK]; - - for (int i = 0; i < KK; i++) - index[i] = lim - i - 1; - - encodeDecode(code, code2, index); - encodeDecode(code2, code, index); - } - - @Test - public void testSimple() { - int lim = fecMath.gfSize + 1; - FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); - FECCode code2 = new PureCode(KK, lim); - int[] index = new int[KK]; - - for (int i = 0; i < KK; i++) - index[i] = KK - i; - encodeDecode(code, code2, index); - encodeDecode(code2, code, index); - } - - @Test - public void testShifted() { - int lim = fecMath.gfSize + 1; - FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); - FECCode code2 = new PureCode(KK, lim); - int[] index = new int[KK]; - - int max_i0 = KK / 2; - if (max_i0 + KK > lim) - max_i0 = lim - KK; - - for (int s = max_i0 - 2; s <= max_i0; s++) { - for (int i = 0; i < KK; i++) - index[i] = i + s; - encodeDecode(code, code2, index); - encodeDecode(code2, code, index); - } - } + public static FECMath fecMath = new FECMath(8); + + public static final int KK = 192; + public static final int PACKET_SIZE = 4096; + + /** + * Creates k packets of size sz of random data, encodes them, and tries to decode. Index + * contains the permutation entry. + */ + private static void encodeDecode(FECCode encode, FECCode decode, int[] index) { + byte[] src = new byte[KK * PACKET_SIZE]; + Util.rand.nextBytes(src); + Buffer[] srcBufs = createBuffers(src); + + byte[] repair = new byte[KK * PACKET_SIZE]; + Buffer[] repairBufs = createBuffers(repair); + + encode.encode(srcBufs, repairBufs, index); + decode.decode(repairBufs, index); + + assertArrayEquals(src, repair); + } + + @Test + public void testBenchmark() { + if (!TestProperty.BENCHMARK) { + return; + } + + int lim = fecMath.gfSize + 1; + FECCode maybeNative = FECCodeFactory.getDefault().createFECCode(KK, lim); + FECCode pureCode = new PureCode(KK, lim); + int[] index = new int[KK]; + + for (int i = 0; i < KK; i++) { + index[i] = lim - i - 1; + } + + byte[] src = new byte[KK * PACKET_SIZE]; + Util.rand.nextBytes(src); + Buffer[] srcBufs = createBuffers(src); + + byte[] repair = new byte[KK * PACKET_SIZE]; + Buffer[] repairBufs = createBuffers(repair); + + int[] indexBackup = new int[index.length]; + System.arraycopy(index, 0, indexBackup, 0, index.length); + + System.out.println("Getting ready for benchmarking encode()"); + long t1 = System.currentTimeMillis(); + maybeNative.encode(srcBufs, repairBufs, index); + long t2 = System.currentTimeMillis(); + pureCode.encode(srcBufs, repairBufs, indexBackup); + long t3 = System.currentTimeMillis(); + + float dNativeEncode = t2 - t1; + float dPureEncode = t3 - t2; + + Buffer[] repairBufs2 = repairBufs.clone(); + System.arraycopy(repairBufs, 0, repairBufs2, 0, repairBufs.length); + System.out.println("Getting ready for benchmarking decode()"); + t1 = System.currentTimeMillis(); + maybeNative.decode(repairBufs, index); + t2 = System.currentTimeMillis(); + pureCode.decode(repairBufs2, indexBackup); + t3 = System.currentTimeMillis(); + + float dNativeDecode = t2 - t1; + float dPureDecode = t3 - t2; + + System.out.println(maybeNative); + System.out.println(pureCode); + System.out.println("Native code took " + dNativeEncode + "ms whereas java's code took " + dPureEncode + "ms to encode()"); + System.out.println("Native code took " + dNativeDecode + "ms whereas java's code took " + dPureDecode + "ms to decode()"); + } + + @Test + public void testSimpleRev() { + int lim = fecMath.gfSize + 1; + FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); + FECCode code2 = new PureCode(KK, lim); + int[] index = new int[KK]; + + for (int i = 0; i < KK; i++) { + index[i] = lim - i - 1; + } + + encodeDecode(code, code2, index); + encodeDecode(code2, code, index); + } + + @Test + public void testSimple() { + int lim = fecMath.gfSize + 1; + FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); + FECCode code2 = new PureCode(KK, lim); + int[] index = new int[KK]; + + for (int i = 0; i < KK; i++) { + index[i] = KK - i; + } + encodeDecode(code, code2, index); + encodeDecode(code2, code, index); + } + + @Test + public void testShifted() { + int lim = fecMath.gfSize + 1; + FECCode code = FECCodeFactory.getDefault().createFECCode(KK, lim); + FECCode code2 = new PureCode(KK, lim); + int[] index = new int[KK]; + + int max_i0 = KK / 2; + if (max_i0 + KK > lim) { + max_i0 = lim - KK; + } + + for (int s = max_i0 - 2; s <= max_i0; s++) { + for (int i = 0; i < KK; i++) { + index[i] = i + s; + } + encodeDecode(code, code2, index); + encodeDecode(code2, code, index); + } + } + + private static Buffer[] createBuffers(byte[] src) { + Buffer[] srcBufs = new Buffer[KK]; + for (int i = 0; i < srcBufs.length; i++) { + srcBufs[i] = new Buffer(src, i * PACKET_SIZE, PACKET_SIZE); + } + return srcBufs; + } } diff --git a/test/freenet/client/DefaultMIMETypesTest.java b/test/freenet/client/DefaultMIMETypesTest.java index 1b9ca9b0b6a..590759f0b72 100644 --- a/test/freenet/client/DefaultMIMETypesTest.java +++ b/test/freenet/client/DefaultMIMETypesTest.java @@ -5,18 +5,18 @@ import org.junit.Test; public class DefaultMIMETypesTest { - - @Test - public void testFullList() { - for(String mimeType : DefaultMIMETypes.getMIMETypes()) { - assertTrue("Failed: \""+mimeType+"\"", DefaultMIMETypes.isPlausibleMIMEType(mimeType)); - } - } - - @Test - public void testParams() { - assertTrue(DefaultMIMETypes.isPlausibleMIMEType("text/xhtml+xml; charset=ISO-8859-1; blah=blah")); - assertTrue(DefaultMIMETypes.isPlausibleMIMEType("multipart/mixed; boundary=\"---this is a silly boundary---\"")); - } + + @Test + public void testFullList() { + for (String mimeType : DefaultMIMETypes.getMIMETypes()) { + assertTrue("Failed: \"" + mimeType + "\"", DefaultMIMETypes.isPlausibleMIMEType(mimeType)); + } + } + + @Test + public void testParams() { + assertTrue(DefaultMIMETypes.isPlausibleMIMEType("text/xhtml+xml; charset=ISO-8859-1; blah=blah")); + assertTrue(DefaultMIMETypes.isPlausibleMIMEType("multipart/mixed; boundary=\"---this is a silly boundary---\"")); + } } diff --git a/test/freenet/client/FailureCodeTrackerTest.java b/test/freenet/client/FailureCodeTrackerTest.java index 18bc0a21c30..cd8690c3144 100644 --- a/test/freenet/client/FailureCodeTrackerTest.java +++ b/test/freenet/client/FailureCodeTrackerTest.java @@ -12,13 +12,15 @@ public class FailureCodeTrackerTest { - /** Test that the fixed size representation really is fixed size */ + /** + * Test that the fixed size representation really is fixed size + */ @Test public void testSize() throws IOException { testSize(false); testSize(true); } - + public void testSize(boolean insert) throws IOException { FailureCodeTracker f = new FailureCodeTracker(insert); int fixedLength = FailureCodeTracker.getFixedLength(insert); @@ -30,11 +32,13 @@ public void testSize(boolean insert) throws IOException { } private int getStoredLength(FailureCodeTracker f) throws IOException { - CountedOutputStream os = new CountedOutputStream(new NullOutputStream()); - DataOutputStream dos = new DataOutputStream(os); - f.writeFixedLengthTo(dos); - dos.close(); - return (int) os.written(); + try ( + CountedOutputStream os = new CountedOutputStream(new NullOutputStream()); + DataOutputStream dos = new DataOutputStream(os) + ) { + f.writeFixedLengthTo(dos); + return (int) os.written(); + } } } diff --git a/test/freenet/client/FetchContextTest.java b/test/freenet/client/FetchContextTest.java index c5051e0c642..2c4ac174f67 100644 --- a/test/freenet/client/FetchContextTest.java +++ b/test/freenet/client/FetchContextTest.java @@ -14,22 +14,28 @@ import freenet.support.io.StorageFormatException; public class FetchContextTest { - + @Test public void testPersistence() throws IOException, StorageFormatException { - FetchContext context = - HighLevelSimpleClientImpl.makeDefaultFetchContext(Long.MAX_VALUE, Long.MAX_VALUE, - new ArrayBucketFactory(), new SimpleEventProducer()); - ArrayBucket bucket = new ArrayBucket(); - DataOutputStream dos = new DataOutputStream(bucket.getOutputStream()); - context.writeTo(dos); - dos.close(); - assert(bucket.size() != 0); - DataInputStream dis = new DataInputStream(bucket.getInputStream()); - FetchContext ctx = new FetchContext(dis); - dis.close(); - assertTrue(ctx.equals(context)); - bucket.free(); + FetchContext context = HighLevelSimpleClientImpl.makeDefaultFetchContext( + Long.MAX_VALUE, + Long.MAX_VALUE, + new ArrayBucketFactory(), + new SimpleEventProducer() + ); + final ArrayBucket bucket = new ArrayBucket(); + try { + try (DataOutputStream dos = new DataOutputStream(bucket.getOutputStream())) { + context.writeTo(dos); + } + assertNotEquals(0, bucket.size()); + FetchContext ctx; + try (DataInputStream dis = new DataInputStream(bucket.getInputStream())) { + ctx = new FetchContext(dis); + } + assertEquals(ctx, context); + } finally { + bucket.free(); + } } - } diff --git a/test/freenet/client/OnionFECCodecTest.java b/test/freenet/client/OnionFECCodecTest.java index 9fcea82999e..75877642dc1 100644 --- a/test/freenet/client/OnionFECCodecTest.java +++ b/test/freenet/client/OnionFECCodecTest.java @@ -5,16 +5,18 @@ import java.util.Arrays; import java.util.Random; +import org.junit.Before; import org.junit.Test; import freenet.support.TestProperty; -/** Test the new (post db4o) high level FEC API */ +/** + * Test the new (post db4o) high level FEC API + */ public class OnionFECCodecTest { - + private static final int BLOCK_SIZE = 4096; - private static final int MAX_SEGMENT_SIZE = 255; - + private final OnionFECCodec codec = new OnionFECCodec(); private byte[][] originalDataBlocks; private byte[][] dataBlocks; @@ -22,150 +24,151 @@ public class OnionFECCodecTest { private byte[][] checkBlocks; private boolean[] checkBlocksPresent; private boolean[] dataBlocksPresent; - + private Random random; + + @Before + public void setUp() throws Exception { + random = new Random(21482106); + } + @Test public void testDecodeRandomSubset() { - Random r = new Random(19412106); int iterations = TestProperty.EXTENSIVE ? 100 : 10; - for(int i=0;i codec.encode(dataBlocks, checkBlocks, checkBlocksPresent, BLOCK_SIZE) + ); } - + @Test public void testDecodeThrowsOnNotPaddedLastBlock() { - Random r = new Random(21482106); - setup(128, 128, r); + setup(128, 128, random); // Now delete a random selection of blocks - deleteRandomBlocks(r); - dataBlocks[127] = new byte[BLOCK_SIZE/2]; - try { - codec.decode(dataBlocks, checkBlocks, dataBlocksPresent, checkBlocksPresent, BLOCK_SIZE); - assertTrue(false); // Should throw - } catch (IllegalArgumentException e) { - // Ok. - } + deleteRandomBlocks(random); + dataBlocks[127] = new byte[BLOCK_SIZE / 2]; + assertThrows( + IllegalArgumentException.class, + () -> codec.encode(dataBlocks, checkBlocks, checkBlocksPresent, BLOCK_SIZE) + ); } - + @Test public void testDecodeAlreadyDecoded() { - Random r = new Random(21482106); - setup(128, 128, r); + setup(128, 128, random); // Now delete a random selection of blocks deleteAllCheckBlocks(); decode(); // Should be a no-op. } - + @Test public void testDecodeNoneDecoded() { - Random r = new Random(21482106); - setup(128, 128, r); + setup(128, 128, random); // Now delete a random selection of blocks deleteAllDataBlocks(); decode(); } - + @Test public void testManyCheckFewData() { - Random r = new Random(21582106); - inner(2, 253, r); - inner(5, 250, r); - inner(50, 200, r); - inner(2, 3, r); // Common case, include it here. + inner(2, 253, random); + inner(5, 250, random); + inner(50, 200, random); + inner(2, 3, random); // Common case, include it here. } - + @Test public void testManyDataFewCheck() { - Random r = new Random(21592106); - inner(200, 55, r); - inner(253, 2, r); + inner(200, 55, random); + inner(253, 2, random); } - + @Test public void testRandomDataCheckCounts() { - Random r = new Random(21602106); int iterations = TestProperty.EXTENSIVE ? 100 : 10; - for(int i=0;i expected) { - BMPFilter objBMPFilter = new BMPFilter(); - Bucket output = new ArrayBucket(); - - InputStream inStream; - OutputStream outStream; - try { - inStream = input.getInputStream(); - outStream = output.getOutputStream(); - } catch (IOException e) { - e.printStackTrace(); - fail("Caugth unexpected IOException: " + e); - return null; //Convince the compiler that we won't continue - } - - try { - objBMPFilter.readFilter(inStream, outStream, "", null, null, null); - - if(expected != null) { - fail("Filter didn't throw expected exception"); - } - } catch (Exception e) { - if((expected == null) || (!expected.equals(e.getClass()))) { - //Exception is not the one we expected - e.printStackTrace(); - fail("Caugth unexpected exception: " + e.getClass() + ": " + e.getMessage()); - } - } - - try { - inStream.close(); - outStream.close(); - } catch(IOException e) { - e.printStackTrace(); - fail("Caugth unexpected IOException: " + e); - return null; //Convince the compiler that we won't continue - } - - return output; - } - - private Bucket resourceToBucket(String filename) throws IOException { - InputStream is = getClass().getResourceAsStream(filename); - if (is == null) throw new FileNotFoundException(); - Bucket ab = new ArrayBucket(); - BucketTools.copyFrom(ab, is, Long.MAX_VALUE); - return ab; - } + /** + * File of size less than 54 bytes + */ + @Test + public void testTooShortImage() throws IOException { + Bucket input = resourceToBucket("./bmp/small.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Illegal start word (AB instead of BM) + */ + @Test + public void testIllegalStartWord() throws IOException { + Bucket input = resourceToBucket("./bmp/one.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid offset i.e. starting address + */ + @Test + public void testInvalidOffset() throws IOException { + Bucket input = resourceToBucket("./bmp/two.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid size of bitmap info header + */ + @Test + public void testInvalidBitmapInfoHeaderSize() throws IOException { + Bucket input = resourceToBucket("./bmp/three.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Negative image width + */ + @Test + public void testNegativeImageWidth() throws IOException { + Bucket input = resourceToBucket("./bmp/four.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid number of planes + */ + @Test + public void testInvalidNumberOfPlanes() throws IOException { + Bucket input = resourceToBucket("./bmp/five.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid bit depth + */ + @Test + public void testInvalidBitDepth() throws IOException { + Bucket input = resourceToBucket("./bmp/six.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid compression type + */ + @Test + public void testInvalidCompressionType() throws IOException { + Bucket input = resourceToBucket("./bmp/seven.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid image data size (i.e. not satisfying fileSize = headerSize + imagedatasize) + */ + @Test + public void testInvalidImageDataSize() throws IOException { + Bucket input = resourceToBucket("./bmp/eight.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Invalid image resolution + */ + @Test + public void testInvalidImageResolution() throws IOException { + Bucket input = resourceToBucket("./bmp/nine.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * File is shorter than expected + */ + @Test + public void testNotEnoughImageData() throws IOException { + Bucket input = resourceToBucket("./bmp/ten.bmp"); + filterImage(input, DataFilterException.class); + } + + /** + * Tests valid image + */ + @Test + public void testValidImage() throws IOException { + Bucket input = resourceToBucket("./bmp/ok.bmp"); + Bucket output = filterImage(input, null); + + //Filter should return the original + assertEquals("Input and output should be the same length", input.size(), output.size()); + assertArrayEquals("Input and output are not identical", BucketTools.toByteArray(input), BucketTools.toByteArray(output)); + } + + /** + * Checks that the image size calculation works for images with padding + */ + @Test + public void testImageSizeCalculationWithPadding() throws IOException { + Bucket input = resourceToBucket("./bmp/sizeCalculationWithPadding.bmp"); + Bucket output = filterImage(input, null); + + //Filter should return the original + assertEquals("Input and output should be the same length", input.size(), output.size()); + assertArrayEquals("Input and output are not identical", BucketTools.toByteArray(input), BucketTools.toByteArray(output)); + } + + /** + * Checks that the image size calculation works for images without padding + */ + @Test + public void testImageSizeCalculationWithoutPadding() throws IOException { + Bucket input = resourceToBucket("./bmp/sizeCalculationWithoutPadding.bmp"); + Bucket output = filterImage(input, null); + + //Filter should return the original + assertEquals("Input and output should be the same length", input.size(), output.size()); + assertArrayEquals("Input and output are not identical", BucketTools.toByteArray(input), BucketTools.toByteArray(output)); + } + + private Bucket filterImage(Bucket input, Class expected) throws IOException { + BMPFilter objBMPFilter = new BMPFilter(); + Bucket output = new ArrayBucket(); + try ( + InputStream inStream = input.getInputStream(); + OutputStream outStream = output.getOutputStream() + ) { + if (expected != null) { + assertThrows(expected, () -> readFilter(objBMPFilter, inStream, outStream)); + } else { + readFilter(objBMPFilter, inStream, outStream); + } + } + return output; + } + + private static void readFilter(BMPFilter objBMPFilter, InputStream inStream, OutputStream outStream) throws IOException { + objBMPFilter.readFilter(inStream, outStream, "", null, null, null); + } } diff --git a/test/freenet/client/filter/CSSParserTest.java b/test/freenet/client/filter/CSSParserTest.java index 3830f0b2743..b199c8b5c1e 100644 --- a/test/freenet/client/filter/CSSParserTest.java +++ b/test/freenet/client/filter/CSSParserTest.java @@ -1,5 +1,7 @@ package freenet.client.filter; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalToIgnoringCase; import static org.junit.Assert.*; import java.io.IOException; @@ -9,13 +11,10 @@ import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; +import java.util.*; import java.util.Map.Entry; +import org.hamcrest.MatcherAssert; import org.junit.Before; import org.junit.Test; @@ -33,9 +32,8 @@ public class CSSParserTest { // FIXME should specify exact output values /** CSS1 Selectors */ - private final static HashMap CSS1_SELECTOR= new HashMap(); - static - { + private final static HashMap CSS1_SELECTOR= new HashMap<>(); + static { CSS1_SELECTOR.put("h1 {}","h1"); CSS1_SELECTOR.put("h1:link {}","h1:link"); CSS1_SELECTOR.put("h1:visited {}",""); @@ -47,17 +45,12 @@ public class CSSParserTest { CSS1_SELECTOR.put("h1:focus {}" ,"h1:focus"); CSS1_SELECTOR.put("h1:first-line {}" ,"h1:first-line"); CSS1_SELECTOR.put("h1:first-letter {}" ,"h1:first-letter"); - - - - } // FIXME should specify exact output values /** CSS2 Selectors */ - private final static HashMap CSS2_SELECTOR= new HashMap(); - static - { + private final static HashMap CSS2_SELECTOR= new HashMap<>(); + static { CSS2_SELECTOR.put("* {}","*"); CSS2_SELECTOR.put("h1[foo] {}","h1[foo]"); CSS2_SELECTOR.put("h1[foo=\"bar\"] {}", "h1[foo=\"bar\"]"); @@ -120,9 +113,8 @@ public class CSSParserTest { // CONFORMANCE: We combine pseudo-classes and pseudo-elements, so we allow pseudo-elements on earlier selectors. This is against the spec, CSS2 section 5.10. } - private final static HashSet CSS2_BAD_SELECTOR= new HashSet(); - static - { + private final static HashSet CSS2_BAD_SELECTOR= new HashSet<>(); + static { // Doubled = CSS2_BAD_SELECTOR.add("h1[foo=bar=bat] {}"); CSS2_BAD_SELECTOR.add("h1[foo~=bar~=bat] {}"); @@ -147,9 +139,8 @@ public class CSSParserTest { /** CSS3 Selectors */ - private final static HashMap CSS3_SELECTOR= new HashMap(); - static - { + private final static HashMap CSS3_SELECTOR= new HashMap<>(); + static { CSS3_SELECTOR.put("tr:nth-child(odd) { background-color: red; }","tr:nth-child(odd) { background-color: red; }"); CSS3_SELECTOR.put("tr:nth-child(even) { background-color: yellow; }","tr:nth-child(even) { background-color: yellow; }"); CSS3_SELECTOR.put("tr:nth-child(1) {}","tr:nth-child(1)"); @@ -183,9 +174,8 @@ public class CSSParserTest { CSS3_SELECTOR.put("h1:nth-last-of-type(even) {}","h1:nth-last-of-type(even)"); } - private final static HashSet CSS3_BAD_SELECTOR= new HashSet(); - static - { + private final static HashSet CSS3_BAD_SELECTOR= new HashSet<>(); + static { CSS3_BAD_SELECTOR.add("tr:nth-child() {}"); CSS3_BAD_SELECTOR.add("tr:nth-child(-) {}"); CSS3_BAD_SELECTOR.add("tr:nth-child(+) {}"); @@ -254,10 +244,10 @@ public class CSSParserTest { private static final String CSS_IMPORT_SPACE_IN_STRING = "@import url(\"/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page\") screen;"; private static final String CSS_IMPORT_SPACE_IN_STRINGC = "@import url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page?type=text/css&maybecharset=UTF-8\") screen;"; - private static final String CSS_IMPORT_QUOTED_STUFF = "@import url(\"/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page \\) \\\\ \\\' \\\" \") screen;"; + private static final String CSS_IMPORT_QUOTED_STUFF = "@import url(\"/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page \\) \\\\ \\' \\\" \") screen;"; private static final String CSS_IMPORT_QUOTED_STUFFC = "@import url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page%20%29%20%5c%20%27%20%22%20?type=text/css&maybecharset=UTF-8\") screen;"; - private static final String CSS_IMPORT_QUOTED_STUFF2 = "@import url(/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page \\) \\\\ \\\' \\\" ) screen;"; + private static final String CSS_IMPORT_QUOTED_STUFF2 = "@import url(/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page \\) \\\\ \\' \\\" ) screen;"; private static final String CSS_IMPORT_QUOTED_STUFF2C = "@import url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page%20%29%20%5c%20%27%20%22?type=text/css&maybecharset=UTF-8\") screen;"; private static final String CSS_IMPORT_NOURL_TWOMEDIAS = "@import \"/chk@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/1-1.html\" screen tty;"; @@ -324,7 +314,7 @@ public class CSSParserTest { private static final String CSS_INVALID_MEDIA_CASCADE = "@media blah { h1, h2 { color: green;} }"; - private final static LinkedHashMap propertyTests = new LinkedHashMap(); + private final static LinkedHashMap propertyTests = new LinkedHashMap<>(); static { // Check that the last part of a double bar works propertyTests.put("@media speech { h1 { azimuth: behind }; }", "@media speech { h1 { azimuth: behind }}"); @@ -464,7 +454,6 @@ public class CSSParserTest { propertyTests.put("H3 { BACKGROUND: URL(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test page\") }", "H3 { BACKGROUND: url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }"); propertyTests.put("h3 { background: scroll url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }", "h3 { background: scroll url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }"); propertyTests.put("h3 { background: scroll #f00 url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }", "h3 { background: scroll #f00 url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }"); - propertyTests.put("h3 { background: scroll #f00 url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }", "h3 { background: scroll #f00 url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }"); propertyTests.put("h3 { background: scroll rgb(100%, 2%, 1%) url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }", "h3 { background: scroll rgb(100%, 2%, 1%) url(\"/CHK@~~vxVQDfC9m8sR~M9zWJQKzCxLeZRWy6T1pWLM2XX74,2LY7xwOdUGv0AeJ2WKRXZG6NmiUL~oqVLKnh3XdviZU,AAIC--8/test%20page\") }"); propertyTests.put("h3 { background: 3.3cm 20%;}", "h3 { background: 3.3cm 20%;}"); propertyTests.put("h3 { background: scroll 3.3cm 20%;}", "h3 { background: scroll 3.3cm 20%;}"); @@ -825,11 +814,9 @@ public class CSSParserTest { propertyTests.put("nav > ul { display: flex; }", "nav>ul { display: flex; }"); propertyTests.put("nav > ul > li {\n min-width: 100px;\n /* Prevent items from getting too small for their content. */\n }", "nav>ul>li {\n min-width: 100px;\n \n }"); propertyTests.put("nav > ul > #login {\n margin-left: auto;\n}", "nav>ul>#login {\n margin-left: auto;\n}"); - propertyTests.put("nav > ul { display: flex; }", "nav>ul { display: flex; }"); propertyTests.put("div { flex-flow: row nowrap; }", "div { flex-flow: row nowrap; }"); propertyTests.put("div { flex-grow: 5; }", "div { flex-grow: 5; }"); propertyTests.put("div { flex: 64 content; }", "div { flex: 64 content; }"); - propertyTests.put("div { flex-grow: 5; }", "div { flex-grow: 5; }"); propertyTests.put("div { flex-basis: 5px; }", "div { flex-basis: 5px; }"); propertyTests.put("div { flex-basis: content; }", "div { flex-basis: content; }"); propertyTests.put("div { flex: 64 ; }", "div { flex: 64; }"); @@ -880,8 +867,8 @@ public class CSSParserTest { propertyTests.put("body { nav-left: div.bold ''; }", "body { }"); propertyTests.put("button#foo { nav-left: #bar \"sidebar\"; }", "button#foo { nav-left: #bar \"sidebar\"; }"); propertyTests.put("button#foo { nav-left: invalidSelector \"sidebar\"; }", "button#foo { }"); - - // transition-* + + // transition-* // valid, 1 value propertyTests.put("div { transition-duration: 5s; }", "div { transition-duration: 5s; }"); propertyTests.put("div { transition-delay: 1s; }", "div { transition-delay: 1s; }"); @@ -918,104 +905,99 @@ public void setUp() throws InvalidThresholdException { @Test public void testCSS1Selector() throws IOException, URISyntaxException { + testCssSelectorFiltering(CSS1_SELECTOR); + assertEquals( + "key=\"" + CSS_DELETE_INVALID_SELECTOR + "\" value=\"" + filter(CSS_DELETE_INVALID_SELECTOR) + "\" should be \"" + CSS_DELETE_INVALID_SELECTORC + "\"", + CSS_DELETE_INVALID_SELECTORC, + filter(CSS_DELETE_INVALID_SELECTOR) + ); + assertEquals( + "key=\"" + CSS_INVALID_MEDIA_CASCADE + "\" value=\"" + filter(CSS_INVALID_MEDIA_CASCADE) + "\"", + "", + filter(CSS_INVALID_MEDIA_CASCADE) + ); + } - - Collection c = CSS1_SELECTOR.keySet(); - Iterator itr = c.iterator(); - while(itr.hasNext()) - { - - String key=itr.next(); - String value=CSS1_SELECTOR.get(key); - assertTrue("key=\""+key+"\" value=\""+filter(key)+"\" should be \""+value+"\"", filter(key).contains(value)); + private void testCssSelectorFiltering(Map cssSelectorMap) throws IOException, URISyntaxException { + for (Entry entry : cssSelectorMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + assertTrue("key=\"" + key + "\" value=\"" + filter(key) + "\" should be \"" + value + "\"", filter(key).contains(value)); } + } - assertTrue("key=\""+CSS_DELETE_INVALID_SELECTOR+"\" value=\""+filter(CSS_DELETE_INVALID_SELECTOR)+"\" should be \""+CSS_DELETE_INVALID_SELECTORC+"\"", CSS_DELETE_INVALID_SELECTORC.equals(filter(CSS_DELETE_INVALID_SELECTOR))); - assertTrue("key=\""+CSS_INVALID_MEDIA_CASCADE+"\" value=\""+filter(CSS_INVALID_MEDIA_CASCADE)+"\"", "".equals(filter(CSS_INVALID_MEDIA_CASCADE))); + private void testBadSelectorFiltering(Set badSelectorSet) throws IOException, URISyntaxException { + for(String key : badSelectorSet) { + assertEquals( + "Bad selector filtering should produce empty string: '" + key + "'", + "", + filter(key) + ); + } } @Test public void testCSS2Selector() throws IOException, URISyntaxException { - Collection c = CSS2_SELECTOR.keySet(); - Iterator itr = c.iterator(); - int i=0; - while(itr.hasNext()) - { - String key=itr.next(); - String value=CSS2_SELECTOR.get(key); - System.err.println("Test "+(i++)+" : "+key+" -> "+value); - assertTrue("key="+key+" value=\""+filter(key)+"\" should be \""+value+"\"", filter(key).contains(value)); - } - - i=0; - for(String key : CSS2_BAD_SELECTOR) { - System.err.println("Bad selector test "+(i++)); - assertTrue("".equals(filter(key))); - } - + testCssSelectorFiltering(CSS2_SELECTOR); + testBadSelectorFiltering(CSS2_BAD_SELECTOR); } @Test public void testCSS3Selector() throws IOException, URISyntaxException { - Collection c = CSS3_SELECTOR.keySet(); - Iterator itr = c.iterator(); - int i=0; - while(itr.hasNext()) - { - String key=itr.next(); - String value=CSS3_SELECTOR.get(key); - System.err.println("CSS3 test"+(i++)+" : "+key+" -> "+value); - assertTrue("key="+key+" value=\""+filter(key)+"\" should be \""+value+"\"", filter(key).contains(value)); - } - - i=0; - for(String key : CSS3_BAD_SELECTOR) { - System.err.println("CSS3 bad selector test "+(i++)); - assertTrue("".equals(filter(key))); - } - + testCssSelectorFiltering(CSS3_SELECTOR); + testBadSelectorFiltering(CSS3_BAD_SELECTOR); } @Test public void testNewlines() throws IOException, URISyntaxException { - assertTrue("key=\""+CSS_STRING_NEWLINES+"\" value=\""+filter(CSS_STRING_NEWLINES)+"\" should be: \""+CSS_STRING_NEWLINESC+"\"", CSS_STRING_NEWLINESC.equals(filter(CSS_STRING_NEWLINES))); + assertEquals( + "key=\"" + CSS_STRING_NEWLINES + "\" value=\"" + filter(CSS_STRING_NEWLINES) + "\" should be: \"" + CSS_STRING_NEWLINESC + "\"", + CSS_STRING_NEWLINESC, + filter(CSS_STRING_NEWLINES) + ); } @Test public void testBackgroundURL() throws IOException, URISyntaxException { - assertTrue("key="+CSS_BACKGROUND_URL+" value=\""+filter(CSS_BACKGROUND_URL)+"\" should be \""+CSS_BACKGROUND_URLC+"\"", CSS_BACKGROUND_URLC.equals(filter(CSS_BACKGROUND_URL))); - - assertTrue("key="+CSS_LCASE_BACKGROUND_URL+" value=\""+filter(CSS_LCASE_BACKGROUND_URL)+"\"", CSS_LCASE_BACKGROUND_URLC.equals(filter(CSS_LCASE_BACKGROUND_URL))); + assertEquals("key=" + CSS_BACKGROUND_URL + " value=\"" + filter(CSS_BACKGROUND_URL) + "\" should be \"" + CSS_BACKGROUND_URLC + "\"", + CSS_BACKGROUND_URLC, + filter(CSS_BACKGROUND_URL) + ); + assertEquals( + "key=" + CSS_LCASE_BACKGROUND_URL + " value=\"" + filter(CSS_LCASE_BACKGROUND_URL) + "\"", + CSS_LCASE_BACKGROUND_URLC, + filter(CSS_LCASE_BACKGROUND_URL) + ); } @Test public void testImports() throws IOException, URISyntaxException { - assertTrue("key="+CSS_IMPORT+" value=\""+filter(CSS_IMPORT)+"\"", CSS_IMPORTC.equals(filter(CSS_IMPORT))); - assertTrue("key="+CSS_IMPORT2+" value=\""+filter(CSS_IMPORT2)+"\"", CSS_IMPORT2C.equals(filter(CSS_IMPORT2))); - assertTrue("key="+CSS_IMPORT_MULTI_MEDIA+" value=\""+filter(CSS_IMPORT_MULTI_MEDIA)+"\"", CSS_IMPORT_MULTI_MEDIAC.equals(filter(CSS_IMPORT_MULTI_MEDIA))); - assertTrue("key="+CSS_IMPORT_MULTI_MEDIA_BOGUS+" value=\""+filter(CSS_IMPORT_MULTI_MEDIA_BOGUS)+"\"", CSS_IMPORT_MULTI_MEDIA_BOGUSC.equals(filter(CSS_IMPORT_MULTI_MEDIA_BOGUS))); - assertTrue("key="+CSS_IMPORT_MULTI_MEDIA_ALL+" value=\""+filter(CSS_IMPORT_MULTI_MEDIA_ALL)+"\"", CSS_IMPORT_MULTI_MEDIA_ALLC.equals(filter(CSS_IMPORT_MULTI_MEDIA_ALL))); - assertTrue("key="+CSS_IMPORT_TYPE+" value=\""+filter(CSS_IMPORT_TYPE)+"\"", CSS_IMPORT_TYPEC.equals(filter(CSS_IMPORT_TYPE))); - assertTrue("key="+CSS_IMPORT_SPACE_IN_STRING+" value=\""+filter(CSS_IMPORT_SPACE_IN_STRING)+"\"", CSS_IMPORT_SPACE_IN_STRINGC.equals(filter(CSS_IMPORT_SPACE_IN_STRING))); - assertTrue("key="+CSS_IMPORT_QUOTED_STUFF+" value=\""+filter(CSS_IMPORT_QUOTED_STUFF)+"\"", CSS_IMPORT_QUOTED_STUFFC.equals(filter(CSS_IMPORT_QUOTED_STUFF))); - assertTrue("key="+CSS_IMPORT_QUOTED_STUFF2+" value=\""+filter(CSS_IMPORT_QUOTED_STUFF2)+"\"", CSS_IMPORT_QUOTED_STUFF2C.equals(filter(CSS_IMPORT_QUOTED_STUFF2))); - assertTrue("key="+CSS_IMPORT_NOURL_TWOMEDIAS+" value=\""+filter(CSS_IMPORT_NOURL_TWOMEDIAS)+"\"", CSS_IMPORT_NOURL_TWOMEDIASC.equals(filter(CSS_IMPORT_NOURL_TWOMEDIAS))); - assertTrue("key="+CSS_IMPORT_UNQUOTED+" should be empty", "".equals(filter(CSS_IMPORT_UNQUOTED))); - assertTrue("key="+CSS_IMPORT_NOURL+" value=\""+filter(CSS_IMPORT_NOURL)+"\"", CSS_IMPORT_NOURLC.equals(filter(CSS_IMPORT_NOURL))); - assertTrue("key="+CSS_IMPORT_BRACKET+" value=\""+filter(CSS_IMPORT_BRACKET)+"\"", CSS_IMPORT_BRACKETC.equals(filter(CSS_IMPORT_BRACKET))); - assertTrue("key="+CSS_LATE_IMPORT+" value=\""+filter(CSS_LATE_IMPORT)+"\"", CSS_LATE_IMPORTC.equals(filter(CSS_LATE_IMPORT))); - assertTrue("key="+CSS_LATE_IMPORT2+" value=\""+filter(CSS_LATE_IMPORT2)+"\"", CSS_LATE_IMPORT2C.equals(filter(CSS_LATE_IMPORT2))); - assertTrue("key="+CSS_LATE_IMPORT3+" value=\""+filter(CSS_LATE_IMPORT3)+"\"", CSS_LATE_IMPORT3C.equals(filter(CSS_LATE_IMPORT3))); - assertTrue("key="+CSS_BOGUS_AT_RULE+" value=\""+filter(CSS_BOGUS_AT_RULE)+"\"", CSS_BOGUS_AT_RULEC.equals(filter(CSS_BOGUS_AT_RULE))); - assertTrue("key="+PRESERVE_CDO_CDC+" value=\""+filter(PRESERVE_CDO_CDC)+"\"", PRESERVE_CDO_CDCC.equals(filter(PRESERVE_CDO_CDC))); - assertTrue("key="+BROKEN_BEFORE_IMPORT+" value=\""+filter(BROKEN_BEFORE_IMPORT)+"\"", BROKEN_BEFORE_IMPORTC.equals(filter(BROKEN_BEFORE_IMPORT))); - assertTrue("key="+BROKEN_BEFORE_MEDIA+" value=\""+filter(BROKEN_BEFORE_MEDIA)+"\"", BROKEN_BEFORE_MEDIAC.equals(filter(BROKEN_BEFORE_MEDIA))); + assertEquals("key=" + CSS_IMPORT + " value=\"" + filter(CSS_IMPORT) + "\"", CSS_IMPORTC, filter(CSS_IMPORT)); + assertEquals("key=" + CSS_IMPORT2 + " value=\"" + filter(CSS_IMPORT2) + "\"", CSS_IMPORT2C, filter(CSS_IMPORT2)); + assertEquals("key=" + CSS_IMPORT_MULTI_MEDIA + " value=\"" + filter(CSS_IMPORT_MULTI_MEDIA) + "\"", CSS_IMPORT_MULTI_MEDIAC, filter(CSS_IMPORT_MULTI_MEDIA)); + assertEquals("key=" + CSS_IMPORT_MULTI_MEDIA_BOGUS + " value=\"" + filter(CSS_IMPORT_MULTI_MEDIA_BOGUS) + "\"", CSS_IMPORT_MULTI_MEDIA_BOGUSC, filter(CSS_IMPORT_MULTI_MEDIA_BOGUS)); + assertEquals("key=" + CSS_IMPORT_MULTI_MEDIA_ALL + " value=\"" + filter(CSS_IMPORT_MULTI_MEDIA_ALL) + "\"", CSS_IMPORT_MULTI_MEDIA_ALLC, filter(CSS_IMPORT_MULTI_MEDIA_ALL)); + assertEquals("key=" + CSS_IMPORT_TYPE + " value=\"" + filter(CSS_IMPORT_TYPE) + "\"", CSS_IMPORT_TYPEC, filter(CSS_IMPORT_TYPE)); + assertEquals("key=" + CSS_IMPORT_SPACE_IN_STRING + " value=\"" + filter(CSS_IMPORT_SPACE_IN_STRING) + "\"", CSS_IMPORT_SPACE_IN_STRINGC, filter(CSS_IMPORT_SPACE_IN_STRING)); + assertEquals("key=" + CSS_IMPORT_QUOTED_STUFF + " value=\"" + filter(CSS_IMPORT_QUOTED_STUFF) + "\"", CSS_IMPORT_QUOTED_STUFFC, filter(CSS_IMPORT_QUOTED_STUFF)); + assertEquals("key=" + CSS_IMPORT_QUOTED_STUFF2 + " value=\"" + filter(CSS_IMPORT_QUOTED_STUFF2) + "\"", CSS_IMPORT_QUOTED_STUFF2C, filter(CSS_IMPORT_QUOTED_STUFF2)); + assertEquals("key=" + CSS_IMPORT_NOURL_TWOMEDIAS + " value=\"" + filter(CSS_IMPORT_NOURL_TWOMEDIAS) + "\"", CSS_IMPORT_NOURL_TWOMEDIASC, filter(CSS_IMPORT_NOURL_TWOMEDIAS)); + assertEquals("key=" + CSS_IMPORT_UNQUOTED + " should be empty", "", filter(CSS_IMPORT_UNQUOTED)); + assertEquals("key=" + CSS_IMPORT_NOURL + " value=\"" + filter(CSS_IMPORT_NOURL) + "\"", CSS_IMPORT_NOURLC, filter(CSS_IMPORT_NOURL)); + assertEquals("key=" + CSS_IMPORT_BRACKET + " value=\"" + filter(CSS_IMPORT_BRACKET) + "\"", CSS_IMPORT_BRACKETC, filter(CSS_IMPORT_BRACKET)); + assertEquals("key=" + CSS_LATE_IMPORT + " value=\"" + filter(CSS_LATE_IMPORT) + "\"", CSS_LATE_IMPORTC, filter(CSS_LATE_IMPORT)); + assertEquals("key=" + CSS_LATE_IMPORT2 + " value=\"" + filter(CSS_LATE_IMPORT2) + "\"", CSS_LATE_IMPORT2C, filter(CSS_LATE_IMPORT2)); + assertEquals("key=" + CSS_LATE_IMPORT3 + " value=\"" + filter(CSS_LATE_IMPORT3) + "\"", CSS_LATE_IMPORT3C, filter(CSS_LATE_IMPORT3)); + assertEquals("key=" + CSS_BOGUS_AT_RULE + " value=\"" + filter(CSS_BOGUS_AT_RULE) + "\"", CSS_BOGUS_AT_RULEC, filter(CSS_BOGUS_AT_RULE)); + assertEquals("key=" + PRESERVE_CDO_CDC + " value=\"" + filter(PRESERVE_CDO_CDC) + "\"", PRESERVE_CDO_CDCC, filter(PRESERVE_CDO_CDC)); + assertEquals("key=" + BROKEN_BEFORE_IMPORT + " value=\"" + filter(BROKEN_BEFORE_IMPORT) + "\"", BROKEN_BEFORE_IMPORTC, filter(BROKEN_BEFORE_IMPORT)); + assertEquals("key=" + BROKEN_BEFORE_MEDIA + " value=\"" + filter(BROKEN_BEFORE_MEDIA) + "\"", BROKEN_BEFORE_MEDIAC, filter(BROKEN_BEFORE_MEDIA)); } @Test public void testEscape() throws IOException, URISyntaxException { - assertTrue("key="+CSS_ESCAPED_LINK+" value=\""+filter(CSS_ESCAPED_LINK)+"\"", CSS_ESCAPED_LINKC.equals(filter(CSS_ESCAPED_LINK))); - assertTrue("key="+CSS_ESCAPED_LINK2+" value=\""+filter(CSS_ESCAPED_LINK2)+"\"", CSS_ESCAPED_LINK2C.equals(filter(CSS_ESCAPED_LINK2))); + assertEquals("key=" + CSS_ESCAPED_LINK + " value=\"" + filter(CSS_ESCAPED_LINK) + "\"", CSS_ESCAPED_LINKC, filter(CSS_ESCAPED_LINK)); + assertEquals("key=" + CSS_ESCAPED_LINK2 + " value=\"" + filter(CSS_ESCAPED_LINK2) + "\"", CSS_ESCAPED_LINK2C, filter(CSS_ESCAPED_LINK2)); } @Test @@ -1023,7 +1005,7 @@ public void testProperties() throws IOException, URISyntaxException { for(Entry entry : propertyTests.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - assertTrue("key=\""+key+"\" encoded=\""+filter(key)+"\" should be \""+value+"\"", value.equals(filter(key))); + assertEquals("key=\"" + key + "\" encoded=\"" + filter(key) + "\" should be \"" + value + "\"", value, filter(key)); } } @@ -1039,23 +1021,23 @@ private String filter(String css) throws IOException, URISyntaxException { public void testCharset() throws IOException, URISyntaxException { // Test whether @charset is passed through when it is correct. String test = "@charset \"UTF-8\";\nh2 { color: red;}"; - assertTrue("key=\""+test+"\" value=\""+filter(test)+"\"", filter(test).equals(test)); + assertEquals("key=\"" + test + "\" value=\"" + filter(test) + "\"", filter(test), test); // No quote marks String testUnquoted = "@charset UTF-8;\nh2 { color: red;}"; - assertTrue("key=\""+test+"\" value=\""+filter(test)+"\"", filter(testUnquoted).equals(test)); + assertEquals("key=\"" + test + "\" value=\"" + filter(test) + "\"", filter(testUnquoted), test); // Test whether the parse fails when @charset is not correct. String testFail = "@charset ISO-8859-1;\nh2 { color: red;};"; - try { - filter(test).equals(""); - assertFalse("Bogus @charset should have been deleted, but result is \""+filter(testFail)+"\"", false); - } catch (IOException e) { - // Ok. - } + assertThrows( + "Bogus @charset should have been deleted", + IOException.class, + ()-> filter(testFail) + ); + // Test charset extraction getCharsetTest("UTF-8"); getCharsetTest("UTF-16BE"); getCharsetTest("UTF-16LE"); - // not availiable in java 1.5.0_22 + // FIXME: these next two are not supported or produce errors // getCharsetTest("UTF-32BE"); // getCharsetTest("UTF-32LE"); @@ -1083,37 +1065,33 @@ public void testCharset() throws IOException, URISyntaxException { charsetTestUnsupported("IBM01149"); // Late charset is invalid - assertTrue("key="+LATE_CHARSET+" value=\""+filter(LATE_CHARSET)+"\"", LATE_CHARSETC.equals(filter(LATE_CHARSET))); + assertEquals("key=" + LATE_CHARSET + " value=\"" + filter(LATE_CHARSET) + "\"", LATE_CHARSETC, filter(LATE_CHARSET)); try { String output = filter(WRONG_CHARSET); - assertFalse("Should complain that detected charset differs from real charset, but returned \""+output+"\"", true); + fail("Should complain that detected charset differs from real charset, but returned \"" + output + "\""); } catch (IOException e) { // Ok. // FIXME should have a dedicated exception. } try { String output = filter(NONSENSE_CHARSET); - assertFalse("wrong charset output is \""+output+"\" but it should throw!", true); + fail("wrong charset output is \"" + output + "\" but it should throw!"); } catch (UnsupportedCharsetInFilterException e) { // Ok. } - assertTrue(BOM.equals(filter(BOM))); - assertTrue("output=\""+filter(LATE_BOM)+"\"",LATE_BOMC.equals(filter(LATE_BOM))); + assertEquals(BOM, filter(BOM)); + assertEquals("output=\"" + filter(LATE_BOM) + "\"", LATE_BOMC, filter(LATE_BOM)); } - private void getCharsetTest(String charset) throws DataFilterException, IOException, URISyntaxException { + private void getCharsetTest(String charset) throws IOException, URISyntaxException { getCharsetTest(charset, null); } - private void getCharsetTest(String charset, String family) throws DataFilterException, IOException, URISyntaxException { + private void getCharsetTest(String charset, String family) throws IOException, URISyntaxException { String original = "@charset \""+charset+"\";\nh2 { color: red;}"; byte[] bytes = original.getBytes(charset); CSSReadFilter filter = new CSSReadFilter(); - ArrayBucket inputBucket = new ArrayBucket(bytes); - ArrayBucket outputBucket = new ArrayBucket(); - InputStream inputStream = inputBucket.getInputStream(); - OutputStream outputStream = outputBucket.getOutputStream(); // Detect with original charset. String detectedCharset = filter.getCharset(bytes, bytes.length, charset); assertTrue("Charset detected \""+detectedCharset+"\" should be \""+charset+"\" even when parsing with correct charset", charset.equalsIgnoreCase(detectedCharset)); @@ -1122,83 +1100,111 @@ private void getCharsetTest(String charset, String family) throws DataFilterExce assertTrue("Charset detected \""+detectedCharset+"\" should be \""+charset+"\" or \""+family+"\" from getCharsetByBOM", detectedCharset == null || charset.equalsIgnoreCase(detectedCharset) || (family != null && family.equalsIgnoreCase(detectedCharset))); detectedCharset = ContentFilter.detectCharset(bytes, bytes.length, cssMIMEType, null); assertTrue("Charset detected \""+detectedCharset+"\" should be \""+charset+"\" from ContentFilter.detectCharset bom=\""+bomCharset+"\"", charset.equalsIgnoreCase(detectedCharset)); - FilterStatus filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4oobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), - null, null, null, null); - inputStream.close(); - outputStream.close(); + + ArrayBucket inputBucket = new ArrayBucket(bytes); + ArrayBucket outputBucket = new ArrayBucket(); + FilterStatus filterStatus; + try ( + InputStream inputStream = inputBucket.getInputStream(); + OutputStream outputStream = outputBucket.getOutputStream() + ) { + filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4oobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), + null, null, null, null + ); + } assertEquals("text/css", filterStatus.mimeType); assertEquals(charset, filterStatus.charset); String filtered = new String(BucketTools.toByteArray(outputBucket), charset); - assertTrue("ContentFilter.filter() returns \""+filtered+"\" not original \""+original+"\" for charset \""+charset+"\"", original.equals(filtered)); + assertEquals( + "ContentFilter.filter() returns \"" + filtered + "\" not original \"" + original + "\" for charset \"" + charset + "\"", + original, + filtered + ); } - private void charsetTestUnsupported(String charset) throws DataFilterException, IOException, URISyntaxException { + private void charsetTestUnsupported(String charset) throws IOException, URISyntaxException { String original = "@charset \""+charset+"\";\nh2 { color: red;}"; byte[] bytes = original.getBytes(charset); CSSReadFilter filter = new CSSReadFilter(); - SimpleReadOnlyArrayBucket inputBucket = new SimpleReadOnlyArrayBucket(bytes); - Bucket outputBucket = new ArrayBucket(); - InputStream inputStream = inputBucket.getInputStream(); - OutputStream outputStream = outputBucket.getOutputStream(); String detectedCharset; BOMDetection bom = filter.getCharsetByBOM(bytes, bytes.length); String bomCharset = detectedCharset = bom == null ? null : bom.charset; - assertTrue("Charset detected \""+detectedCharset+"\" should be unknown testing unsupported charset \""+charset+"\" from getCharsetByBOM", detectedCharset == null); + assertNull( + "Charset detected \"" + detectedCharset + "\" should be unknown testing unsupported charset \"" + charset + "\" from getCharsetByBOM", + detectedCharset + ); detectedCharset = ContentFilter.detectCharset(bytes, bytes.length, cssMIMEType, null); - assertTrue("Charset detected \""+detectedCharset+"\" should be unknown testing unsupported charset \""+charset+"\" from ContentFilter.detectCharset bom=\""+bomCharset+"\"", charset == null || "utf-8".equalsIgnoreCase(detectedCharset)); - try { - FilterStatus filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), - null,null, null, null); - // It is safe to return utf-8, as long as we clobber the actual content; utf-8 is the default, but other stuff decoded to it is unlikely to be coherent... - assertTrue("ContentFilter.filter() returned charset \""+filterStatus.charset+"\" should be unknown testing unsupported charset \""+charset+"\"", filterStatus.charset.equalsIgnoreCase(charset) || filterStatus.charset.equalsIgnoreCase("utf-8"));//If we switch to JUnit 4, this may be replaced with an assertThat - assertEquals("text/css", filterStatus.mimeType); - String filtered = new String(BucketTools.toByteArray(outputBucket), charset); - assertTrue("ContentFilter.filter() returns something: \""+filtered+"\" should be empty as unsupported charset, original: \""+original+"\" for charset \""+charset+"\"", filtered.equals("")); - } catch (UnsupportedCharsetInFilterException e) { - // Ok. - } catch (IOException e) { - // Ok. - } - finally { - inputStream.close(); - outputStream.close(); + assertTrue( + "Charset detected \""+detectedCharset+"\" should be unknown testing unsupported charset \""+charset+"\" from ContentFilter.detectCharset bom=\""+bomCharset+"\"", + "utf-8".equalsIgnoreCase(detectedCharset) + ); + SimpleReadOnlyArrayBucket inputBucket = new SimpleReadOnlyArrayBucket(bytes); + Bucket outputBucket = new ArrayBucket(); + FilterStatus filterStatus; + try ( + InputStream inputStream = inputBucket.getInputStream(); + OutputStream outputStream = outputBucket.getOutputStream() + ) { + filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), + null, null, null, null); } + // It is safe to return utf-8, as long as we clobber the actual content; utf-8 is the default, but other stuff decoded to it is unlikely to be coherent... + MatcherAssert.assertThat( + "ContentFilter.filter() returned charset \""+filterStatus.charset+"\" should be unknown testing unsupported charset \""+charset+"\"", + filterStatus.charset, + anyOf( + equalToIgnoringCase(charset), + equalToIgnoringCase("utf-8") + ) + ); + assertEquals("text/css", filterStatus.mimeType); + String filtered = new String(BucketTools.toByteArray(outputBucket), charset); + assertEquals( + "ContentFilter.filter() returns something: \"" + filtered + "\" should be empty as unsupported charset, original: \"" + original + "\" for charset \"" + charset + "\"", + "", + filtered + ); } @Test - public void testMaybeCharset() throws UnsafeContentTypeException, URISyntaxException, IOException { + public void testMaybeCharset() throws URISyntaxException, IOException { testUseMaybeCharset("UTF-8"); testUseMaybeCharset("UTF-16"); - // not availiable in java 1.5.0_22 - // testUseMaybeCharset("UTF-32LE"); + testUseMaybeCharset("UTF-32LE"); testUseMaybeCharset("IBM01140"); } - private void testUseMaybeCharset(String charset) throws URISyntaxException, UnsafeContentTypeException, IOException { + private void testUseMaybeCharset(String charset) throws URISyntaxException, IOException { String original = "h2 { color: red;}"; byte[] bytes = original.getBytes(charset); SimpleReadOnlyArrayBucket inputBucket = new SimpleReadOnlyArrayBucket(bytes); Bucket outputBucket = new ArrayBucket(); - InputStream inputStream = inputBucket.getInputStream(); - OutputStream outputStream = outputBucket.getOutputStream(); - FilterStatus filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), - null,null, null, charset); - inputStream.close(); - outputStream.close(); + FilterStatus filterStatus; + try ( + InputStream inputStream = inputBucket.getInputStream(); + OutputStream outputStream = outputBucket.getOutputStream() + ) { + filterStatus = ContentFilter.filter(inputStream, outputStream, "text/css", new URI("/CHK@OR904t6ylZOwoobMJRmSn7HsPGefHSP7zAjoLyenSPw,x2EzszO4Kqot8akqmKYXJbkD-fSj6noOVGB-K2YisZ4,AAIC--8/1-works.html"), + null, null, null, charset); + } assertEquals(charset, filterStatus.charset); assertEquals("text/css", filterStatus.mimeType); String filtered = new String(BucketTools.toByteArray(outputBucket), charset); - assertTrue("ContentFilter.filter() returns \""+filtered+"\" not original \""+original+"\" with maybeCharset \""+charset+"\"", original.equals(filtered)); + assertEquals( + "ContentFilter.filter() returns \"" + filtered + "\" not original \"" + original + "\" with maybeCharset \"" + charset + "\"", + original, + filtered + ); } @Test public void testComment() throws IOException, URISyntaxException { - assertTrue("value=\""+filter(COMMENT)+"\"",COMMENTC.equals(filter(COMMENT))); + assertEquals("value=\"" + filter(COMMENT) + "\"", COMMENTC, filter(COMMENT)); } @Test public void testWhitespace() throws IOException, URISyntaxException { - assertTrue("value=\""+filter(CSS_COMMA_WHITESPACE)+"\"", CSS_COMMA_WHITESPACE.equals(filter(CSS_COMMA_WHITESPACE))); + assertEquals("value=\"" + filter(CSS_COMMA_WHITESPACE) + "\"", CSS_COMMA_WHITESPACE, filter(CSS_COMMA_WHITESPACE)); } @Test diff --git a/test/freenet/client/filter/ContentFilterTest.java b/test/freenet/client/filter/ContentFilterTest.java index e25e5eb50a7..9da9cfe2505 100644 --- a/test/freenet/client/filter/ContentFilterTest.java +++ b/test/freenet/client/filter/ContentFilterTest.java @@ -3,15 +3,20 @@ * http://www.gnu.org/ for further details of the GPL. */ package freenet.client.filter; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.*; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.hamcrest.MatcherAssert; import org.junit.Test; import freenet.client.filter.ContentFilter.FilterStatus; @@ -27,360 +32,363 @@ * @author Florent Daignière <nextgens@freenetproject.org> */ public class ContentFilterTest { - private static final String BASE_URI_PROTOCOL = "http"; - private static final String BASE_URI_CONTENT = "localhost:8888"; - private static final String BASE_KEY = "USK@0I8gctpUE32CM0iQhXaYpCMvtPPGfT4pjXm01oid5Zc,3dAcn4fX2LyxO6uCnWFTx-2HKZ89uruurcKwLSCxbZ4,AQACAAE/Ultimate-Freenet-Index/55/"; - private static final String BASE_URI = BASE_URI_PROTOCOL+"://"+BASE_URI_CONTENT+'/'; - private static final String ALT_BASE_URI = BASE_URI_PROTOCOL+"://"+BASE_URI_CONTENT+'/'+BASE_KEY; - - private static final String EXTERNAL_LINK = "www.evilwebsite.gov"; - private static final String EXTERNAL_LINK_OK = ""; - // check that external links are not allowed - private static final String EXTERNAL_LINK_CHECK1 = ""; - private static final String EXTERNAL_LINK_CHECK2 = ""; - private static final String EXTERNAL_LINK_CHECK3 = ""; - - private static final String INTERNAL_RELATIVE_LINK = ""; - private static final String INTERNAL_ABSOLUTE_LINK = ""; - - private static final String INTERNAL_RELATIVE_LINK1 = ""; - - // @see bug #710 - private static final String ANCHOR_TEST = ""; - private static final String ANCHOR_TEST_EMPTY = ""; - private static final String ANCHOR_TEST_SPECIAL = ""; // RFC3986 / RFC 2396 - private static final String ANCHOR_TEST_SPECIAL2 = ""; - private static final String ANCHOR_TEST_SPECIAL2_RESULT = ""; - - // @see bug #2496 - private static final String ANCHOR_RELATIVE1 = ""; - private static final String ANCHOR_RELATIVE2 = ""; - private static final String ANCHOR_FALSE_POS1 = ""; // yes, this is valid - private static final String ANCHOR_FALSE_POS2 = ""; // yes, this is valid too - - // evil hack for #2496 + #2451, <#> give <%23> - private static final String ANCHOR_MIXED = ""; - private static final String ANCHOR_MIXED_RESULT = ""; - - // @see bug #2451 - private static final String POUNT_CHARACTER_ENCODING_TEST = ""; - private static final String POUNT_CHARACTER_ENCODING_TEST_RESULT = ""; - // @see bug #2297 - private static final String PREVENT_FPROXY_ACCESS = ""; - // @see bug #2921 - private static final String PREVENT_EXTERNAL_ACCESS_CSS_SIMPLE = ""; - private static final String PREVENT_EXTERNAL_ACCESS_CSS_CASE = ""; - private static final String PREVENT_EXTERNAL_ACCESS_CSS_ESCAPE = ""; - private static final String WHITELIST_STATIC_CONTENT = ""; - private static final String XHTML_VOIDELEMENT="
"; - private static final String XHTML_VOIDELEMENTC="
"; - private static final String XHTML_INCOMPLETEDOCUMENT="

helloworld

helloworld"; - private static final String XHTML_INCOMPLETEDOCUMENTC="

helloworld

helloworld

"; - private static final String XHTML_IMPROPERNESTING="helloworld"; - private static final String XHTML_IMPROPERNESTINGC="helloworld"; - - private static final String CSS_STRING_NEWLINES = ""; - private static final String CSS_STRING_NEWLINESC = ""; - - private static final String HTML_STYLESHEET_MAYBECHARSET = ""; - private static final String HTML_STYLESHEET_MAYBECHARSETC = ""; - - private static final String HTML_STYLESHEET_CHARSET = ""; - private static final String HTML_STYLESHEET_CHARSETC = ""; - - private static final String HTML_STYLESHEET_CHARSET_BAD = ""; - private static final String HTML_STYLESHEET_CHARSET_BADC = ""; - - private static final String HTML_STYLESHEET_CHARSET_BAD1 = ""; - private static final String HTML_STYLESHEET_CHARSET_BAD1C = ""; - - private static final String HTML_STYLESHEET_WITH_MEDIA = ""; - private static final String HTML_STYLESHEET_WITH_MEDIAC = ""; - - private static final String FRAME_SRC_CHARSET = ""; - private static final String FRAME_SRC_CHARSETC = ""; - - private static final String FRAME_SRC_CHARSET_BAD = ""; - private static final String FRAME_SRC_CHARSET_BADC = ""; - - private static final String FRAME_SRC_CHARSET_BAD1 = ""; - private static final String FRAME_SRC_CHARSET_BAD1C = ""; - - private static final String SPAN_WITH_STYLE = ""; - - private static final String BASE_HREF = ""; - private static final String BAD_BASE_HREF = ""; - private static final String BAD_BASE_HREF2 = ""; - private static final String BAD_BASE_HREF3 = ""; - private static final String BAD_BASE_HREF4 = ""; - private static final String BAD_BASE_HREF5 = ""; - private static final String DELETED_BASE_HREF = ""; - - // From CSS spec - - private static final String CSS_SPEC_EXAMPLE1 = "\n\n \n Bach's home page\n \n \n \n

Bach's home page

\n

Johann Sebastian Bach was a prolific composer.\n \n"; - - @Test - public void testHTMLFilter() throws Exception { - if (TestProperty.VERBOSE) { - Logger.setupStdoutLogging(LogLevel.MINOR, "freenet.client.filter.Generic:DEBUG"); - } - - // General sanity checks - // is "relativization" working? - assertEquals(INTERNAL_RELATIVE_LINK, HTMLFilter(INTERNAL_RELATIVE_LINK)); - assertEquals(INTERNAL_RELATIVE_LINK, HTMLFilter(INTERNAL_RELATIVE_LINK, true)); - assertEquals(INTERNAL_RELATIVE_LINK1, HTMLFilter(INTERNAL_RELATIVE_LINK1, true)); - assertEquals(INTERNAL_RELATIVE_LINK, HTMLFilter(INTERNAL_ABSOLUTE_LINK)); - // are external links stripped out ? - assertTrue(HTMLFilter(EXTERNAL_LINK_CHECK1).startsWith(EXTERNAL_LINK_OK)); - assertTrue(HTMLFilter(EXTERNAL_LINK_CHECK2).contains(ExternalLinkToadlet.PATH)); - assertTrue(HTMLFilter(EXTERNAL_LINK_CHECK3).startsWith(EXTERNAL_LINK_OK)); - - // regression testing - // bug #710 - assertEquals(ANCHOR_TEST, HTMLFilter(ANCHOR_TEST)); - assertEquals(ANCHOR_TEST_EMPTY, HTMLFilter(ANCHOR_TEST_EMPTY)); - assertEquals(ANCHOR_TEST_SPECIAL, HTMLFilter(ANCHOR_TEST_SPECIAL)); - assertEquals(ANCHOR_TEST_SPECIAL2_RESULT, HTMLFilter(ANCHOR_TEST_SPECIAL2)); - // bug #2496 - assertEquals(ANCHOR_RELATIVE1, HTMLFilter(ANCHOR_RELATIVE1)); - assertEquals(ANCHOR_RELATIVE2, HTMLFilter(ANCHOR_RELATIVE2)); - assertEquals(ANCHOR_FALSE_POS1, HTMLFilter(ANCHOR_FALSE_POS1)); - assertEquals(ANCHOR_FALSE_POS2, HTMLFilter(ANCHOR_FALSE_POS2)); - // EVIL HACK TEST for #2496 + #2451 - assertEquals(ANCHOR_MIXED_RESULT, HTMLFilter(ANCHOR_MIXED)); - // bug #2451 - assertEquals(POUNT_CHARACTER_ENCODING_TEST_RESULT, HTMLFilter(POUNT_CHARACTER_ENCODING_TEST)); - // bug #2297 - assertTrue(HTMLFilter(PREVENT_FPROXY_ACCESS).contains(ExternalLinkToadlet.PATH)); - // bug #2921 - assertTrue(HTMLFilter(PREVENT_EXTERNAL_ACCESS_CSS_SIMPLE).contains("div { }")); - assertTrue(HTMLFilter(PREVENT_EXTERNAL_ACCESS_CSS_ESCAPE).contains("div { }")); - assertTrue(HTMLFilter(PREVENT_EXTERNAL_ACCESS_CSS_CASE).contains("div { }")); - assertEquals(WHITELIST_STATIC_CONTENT, HTMLFilter(WHITELIST_STATIC_CONTENT)); - assertEquals(XHTML_VOIDELEMENTC,HTMLFilter(XHTML_VOIDELEMENT)); - assertEquals(XHTML_INCOMPLETEDOCUMENTC,HTMLFilter(XHTML_INCOMPLETEDOCUMENT)); - assertEquals(XHTML_IMPROPERNESTINGC,HTMLFilter(XHTML_IMPROPERNESTING)); - - assertEquals(CSS_STRING_NEWLINESC,HTMLFilter(CSS_STRING_NEWLINES)); - - assertEquals(HTML_STYLESHEET_MAYBECHARSETC, HTMLFilter(HTML_STYLESHEET_MAYBECHARSET, true)); - assertEquals(HTML_STYLESHEET_CHARSETC, HTMLFilter(HTML_STYLESHEET_CHARSET, true)); - assertEquals(HTML_STYLESHEET_CHARSET_BADC, HTMLFilter(HTML_STYLESHEET_CHARSET_BAD, true)); - assertEquals(HTML_STYLESHEET_CHARSET_BAD1C, HTMLFilter(HTML_STYLESHEET_CHARSET_BAD1, true)); - assertEquals(HTML_STYLESHEET_WITH_MEDIAC, HTMLFilter(HTML_STYLESHEET_WITH_MEDIA, true)); - - assertEquals(FRAME_SRC_CHARSETC, HTMLFilter(FRAME_SRC_CHARSET, true)); - assertEquals(FRAME_SRC_CHARSET_BADC, HTMLFilter(FRAME_SRC_CHARSET_BAD, true)); - assertEquals(FRAME_SRC_CHARSET_BAD1C, HTMLFilter(FRAME_SRC_CHARSET_BAD1, true)); - - assertEquals(CSS_SPEC_EXAMPLE1, HTMLFilter(CSS_SPEC_EXAMPLE1)); - - assertEquals(SPAN_WITH_STYLE, HTMLFilter(SPAN_WITH_STYLE)); - - assertEquals(BASE_HREF, HTMLFilter(BASE_HREF)); - assertEquals(DELETED_BASE_HREF, HTMLFilter(BAD_BASE_HREF)); - assertEquals(DELETED_BASE_HREF, HTMLFilter(BAD_BASE_HREF2)); - assertEquals(DELETED_BASE_HREF, HTMLFilter(BAD_BASE_HREF3)); - assertEquals(DELETED_BASE_HREF, HTMLFilter(BAD_BASE_HREF4)); - assertEquals(DELETED_BASE_HREF, HTMLFilter(BAD_BASE_HREF5)); - } - - private static final String META_TIME_ONLY = ""; - private static final String META_TIME_ONLY_WRONG_CASE = ""; - private static final String META_TIME_ONLY_TOO_SHORT = ""; - private static final String META_TIME_ONLY_NEGATIVE = ""; - - private static final String META_TIME_ONLY_BADNUM1 = ""; - private static final String META_TIME_ONLY_BADNUM2 = ""; - private static final String META_TIME_ONLY_BADNUM_OUT = ""; - - private static final String META_CHARSET = ""; - private static final String META_CHARSET_LOWER = "\n" - + "\n" - + "\n" - + "\n" - + "\n" - + "Some Title"; - private static final String META_CHARSET_LOWER_RES = "\n" - + "\n" - + "\n" - + "\n" // comment has additional spaces after content filter - + "\n" - + "Some Title"; - - private static final String META_VALID_REDIRECT = ""; - private static final String META_VALID_REDIRECT_NOSPACE = ""; - - private static final String META_BOGUS_REDIRECT1 = ""; - private static final String META_BOGUS_REDIRECT2 = ""; - private static final String META_BOGUS_REDIRECT3 = ""; - private static final String META_BOGUS_REDIRECT4 = ""; - private static final String META_BOGUS_REDIRECT5 = ""; - private static final String META_BOGUS_REDIRECT6 = ""; - private static final String META_BOGUS_REDIRECT1_OUT = ""; - private static final String META_BOGUS_REDIRECT3_OUT = ""; - private static final String META_BOGUS_REDIRECT4_OUT = ""; - private static final String META_BOGUS_REDIRECT_NO_URL = ""; - - @Test - public void testMetaRefresh() throws Exception { - HTMLFilter.metaRefreshSamePageMinInterval = 5; - HTMLFilter.metaRefreshRedirectMinInterval = 30; - assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY)); - assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY_WRONG_CASE)); - assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY_TOO_SHORT)); - assertEquals("", headFilter(META_TIME_ONLY_NEGATIVE)); - assertEquals(META_TIME_ONLY_BADNUM_OUT, headFilter(META_TIME_ONLY_BADNUM1)); - assertEquals(META_TIME_ONLY_BADNUM_OUT, headFilter(META_TIME_ONLY_BADNUM2)); - assertEquals(META_VALID_REDIRECT, headFilter(META_VALID_REDIRECT)); - assertEquals(META_VALID_REDIRECT, headFilter(META_VALID_REDIRECT_NOSPACE)); - assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT1)); - assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT2)); - assertEquals(META_BOGUS_REDIRECT3_OUT, headFilter(META_BOGUS_REDIRECT3)); - assertEquals(META_BOGUS_REDIRECT4_OUT, headFilter(META_BOGUS_REDIRECT4)); - assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT5)); - assertEquals(META_BOGUS_REDIRECT_NO_URL, headFilter(META_BOGUS_REDIRECT6)); - HTMLFilter.metaRefreshSamePageMinInterval = -1; - HTMLFilter.metaRefreshRedirectMinInterval = -1; - assertEquals("", headFilter(META_TIME_ONLY)); - assertEquals("", headFilter(META_TIME_ONLY_WRONG_CASE)); - assertEquals("", headFilter(META_TIME_ONLY_TOO_SHORT)); - assertEquals("", headFilter(META_TIME_ONLY_NEGATIVE)); - assertEquals("", headFilter(META_TIME_ONLY_BADNUM1)); - assertEquals("", headFilter(META_TIME_ONLY_BADNUM2)); - assertEquals("", headFilter(META_VALID_REDIRECT)); - assertEquals("", headFilter(META_VALID_REDIRECT_NOSPACE)); - assertEquals("", headFilter(META_BOGUS_REDIRECT1)); - assertEquals("", headFilter(META_BOGUS_REDIRECT2)); - assertEquals("", headFilter(META_BOGUS_REDIRECT3)); - assertEquals("", headFilter(META_BOGUS_REDIRECT4)); - assertEquals("", headFilter(META_BOGUS_REDIRECT5)); - assertEquals("", headFilter(META_BOGUS_REDIRECT6)); - } - - private String headFilter(String data) throws Exception { - String s = HTMLFilter(""+data+""); - if(s == null) return s; - if(!s.startsWith("")) - assertTrue("Head deleted???: "+s, false); - s = s.substring("".length()); - if(!s.endsWith("")) - assertTrue("Head close deleted???: "+s, false); - s = s.substring(0, s.length() - "".length()); - return s; - } - - @Test - public void testThatHtml5MetaCharsetIsPreserved() throws Exception { - assertEquals(META_CHARSET, HTMLFilter(META_CHARSET)); - assertEquals(META_CHARSET_LOWER_RES, HTMLFilter(META_CHARSET_LOWER)); - } - - @Test - public void testEvilCharset() throws IOException { - // This is why we need to disallow characters before !! - String s = "Blah"; - String end = ""; - String alt = "Blah"; - if((s.length()+end.length()) % 2 == 1) - s += " "; - s = s+end; - byte[] buf; - try { - buf = s.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new Error(e); - } - byte[] utf16bom = new byte[] { (byte)0xFE, (byte)0xFF }; - byte[] bufUTF16 = alt.getBytes("UTF-16"); - byte[] total = new byte[buf.length+utf16bom.length+bufUTF16.length]; - System.arraycopy(utf16bom, 0, total, 0, utf16bom.length); - System.arraycopy(buf, 0, total, utf16bom.length, buf.length); - System.arraycopy(bufUTF16, 0, total, utf16bom.length+buf.length, bufUTF16.length); - HTMLFilter filter = new HTMLFilter(); - boolean failed = false; - FileOutputStream fos; - try { - ArrayBucket out = new ArrayBucket(); - filter.readFilter(new ArrayBucket(total).getInputStream(), out.getOutputStream(), "UTF-16", null, null, null); - fos = new FileOutputStream("output.utf16"); - fos.write(out.toByteArray()); - fos.close(); - failed = true; - assertFalse("Filter accepted dangerous UTF8 text with BOM as UTF16! (HTMLFilter)", true); - } catch (DataFilterException e) { - System.out.println("Failure: "+e); - e.printStackTrace(); - if(e.getCause() != null) { - e.getCause().printStackTrace(); - } - // Ok. - } - try { - ArrayBucket out = new ArrayBucket(); - FilterStatus fo = ContentFilter.filter(new ArrayBucket(total).getInputStream(), out.getOutputStream(), "text/html", null, null, null); - fos = new FileOutputStream("output.filtered"); - fos.write(out.toByteArray()); - fos.close(); - failed = true; - assertFalse("Filter accepted dangerous UTF8 text with BOM as UTF16! (ContentFilter) - Detected charset: "+fo.charset, true); - } catch (DataFilterException e) { - System.out.println("Failure: "+e); - e.printStackTrace(); - if(e.getCause() != null) { - e.getCause().printStackTrace(); - } - // Ok. - } - - if(failed) { - fos = new FileOutputStream("unfiltered"); - fos.write(total); - fos.close(); - } - } - - public static String HTMLFilter(String data) throws Exception { - if(data.startsWith(""+data+"", false); - assertTrue(s.startsWith("")); - s = s.substring("".length()); - assertTrue("s = \""+s+"\"", s.endsWith("")); - s = s.substring(0, s.length() - "".length()); - return s; - } - - public static String HTMLFilter(String data, boolean alt) throws Exception { - String returnValue; - String typeName = "text/html"; - URI baseURI = new URI(alt ? ALT_BASE_URI : BASE_URI); - byte[] dataToFilter = data.getBytes("UTF-8"); - ArrayBucket input = new ArrayBucket(dataToFilter); - ArrayBucket output = new ArrayBucket(); - InputStream inputStream = input.getInputStream(); - OutputStream outputStream = output.getOutputStream(); - ContentFilter.filter(inputStream, outputStream, typeName, baseURI,null,null, null, null); - inputStream.close(); - outputStream.close(); - returnValue = output.toString(); - output.free(); - input.free(); - return returnValue; - } - - @Test - public void testLowerCaseExtensions() { - for(FilterMIMEType type : ContentFilter.mimeTypesByName.values()) { - String ext = type.primaryExtension; - if(ext != null) - assertEquals(ext, ext.toLowerCase()); - String[] exts = type.alternateExtensions; - if(ext != null) - for(String s : exts) - assertEquals(s, s.toLowerCase()); - } - } + private static final String BASE_URI_PROTOCOL = "http"; + private static final String BASE_URI_CONTENT = "localhost:8888"; + private static final String BASE_KEY = "USK@0I8gctpUE32CM0iQhXaYpCMvtPPGfT4pjXm01oid5Zc,3dAcn4fX2LyxO6uCnWFTx-2HKZ89uruurcKwLSCxbZ4,AQACAAE/Ultimate-Freenet-Index/55/"; + private static final String BASE_URI = BASE_URI_PROTOCOL + "://" + BASE_URI_CONTENT + '/'; + private static final String ALT_BASE_URI = BASE_URI_PROTOCOL + "://" + BASE_URI_CONTENT + '/' + BASE_KEY; + + private static final String EXTERNAL_LINK = "www.evilwebsite.gov"; + private static final String EXTERNAL_LINK_OK = ""; + // check that external links are not allowed + private static final String EXTERNAL_LINK_CHECK1 = ""; + private static final String EXTERNAL_LINK_CHECK2 = ""; + private static final String EXTERNAL_LINK_CHECK3 = ""; + + private static final String INTERNAL_RELATIVE_LINK = ""; + private static final String INTERNAL_ABSOLUTE_LINK = ""; + + private static final String INTERNAL_RELATIVE_LINK1 = ""; + + // @see bug #710 + private static final String ANCHOR_TEST = ""; + private static final String ANCHOR_TEST_EMPTY = ""; + private static final String ANCHOR_TEST_SPECIAL = ""; // RFC3986 / RFC 2396 + private static final String ANCHOR_TEST_SPECIAL2 = ""; + private static final String ANCHOR_TEST_SPECIAL2_RESULT = ""; + + // @see bug #2496 + private static final String ANCHOR_RELATIVE1 = ""; + private static final String ANCHOR_RELATIVE2 = ""; + private static final String ANCHOR_FALSE_POS1 = ""; // yes, this is valid + private static final String ANCHOR_FALSE_POS2 = ""; // yes, this is valid too + + // evil hack for #2496 + #2451, <#> give <%23> + private static final String ANCHOR_MIXED = ""; + private static final String ANCHOR_MIXED_RESULT = ""; + + // @see bug #2451 + private static final String POUNT_CHARACTER_ENCODING_TEST = ""; + private static final String POUNT_CHARACTER_ENCODING_TEST_RESULT = ""; + // @see bug #2297 + private static final String PREVENT_FPROXY_ACCESS = ""; + // @see bug #2921 + private static final String PREVENT_EXTERNAL_ACCESS_CSS_SIMPLE = ""; + private static final String PREVENT_EXTERNAL_ACCESS_CSS_CASE = ""; + private static final String PREVENT_EXTERNAL_ACCESS_CSS_ESCAPE = ""; + private static final String WHITELIST_STATIC_CONTENT = ""; + private static final String XHTML_VOIDELEMENT = "

"; + private static final String XHTML_VOIDELEMENTC = "
"; + private static final String XHTML_INCOMPLETEDOCUMENT = "

helloworld

helloworld"; + private static final String XHTML_INCOMPLETEDOCUMENTC = "

helloworld

helloworld

"; + private static final String XHTML_IMPROPERNESTING = "helloworld"; + private static final String XHTML_IMPROPERNESTINGC = "helloworld"; + + private static final String CSS_STRING_NEWLINES = ""; + private static final String CSS_STRING_NEWLINESC = ""; + + private static final String HTML_STYLESHEET_MAYBECHARSET = ""; + private static final String HTML_STYLESHEET_MAYBECHARSETC = ""; + + private static final String HTML_STYLESHEET_CHARSET = ""; + private static final String HTML_STYLESHEET_CHARSETC = ""; + + private static final String HTML_STYLESHEET_CHARSET_BAD = ""; + private static final String HTML_STYLESHEET_CHARSET_BADC = ""; + + private static final String HTML_STYLESHEET_CHARSET_BAD1 = ""; + private static final String HTML_STYLESHEET_CHARSET_BAD1C = ""; + + private static final String HTML_STYLESHEET_WITH_MEDIA = ""; + private static final String HTML_STYLESHEET_WITH_MEDIAC = ""; + + private static final String FRAME_SRC_CHARSET = ""; + private static final String FRAME_SRC_CHARSETC = ""; + + private static final String FRAME_SRC_CHARSET_BAD = ""; + private static final String FRAME_SRC_CHARSET_BADC = ""; + + private static final String FRAME_SRC_CHARSET_BAD1 = ""; + private static final String FRAME_SRC_CHARSET_BAD1C = ""; + + private static final String SPAN_WITH_STYLE = ""; + + private static final String BASE_HREF = ""; + private static final String BAD_BASE_HREF = ""; + private static final String BAD_BASE_HREF2 = ""; + private static final String BAD_BASE_HREF3 = ""; + private static final String BAD_BASE_HREF4 = ""; + private static final String BAD_BASE_HREF5 = ""; + private static final String DELETED_BASE_HREF = ""; + + // From CSS spec + + private static final String CSS_SPEC_EXAMPLE1 = "\n\n \n Bach's home page\n \n \n \n

Bach's home page

\n

Johann Sebastian Bach was a prolific composer.\n \n"; + + @Test + public void testHTMLFilter() throws Exception { + if (TestProperty.VERBOSE) { + Logger.setupStdoutLogging(LogLevel.MINOR, "freenet.client.filter.Generic:DEBUG"); + } + + // General sanity checks + // is "relativization" working? + assertEquals(INTERNAL_RELATIVE_LINK, htmlFilter(INTERNAL_RELATIVE_LINK)); + assertEquals(INTERNAL_RELATIVE_LINK, htmlFilter(INTERNAL_RELATIVE_LINK, true)); + assertEquals(INTERNAL_RELATIVE_LINK1, htmlFilter(INTERNAL_RELATIVE_LINK1, true)); + assertEquals(INTERNAL_RELATIVE_LINK, htmlFilter(INTERNAL_ABSOLUTE_LINK)); + // are external links stripped out ? + assertTrue(htmlFilter(EXTERNAL_LINK_CHECK1).startsWith(EXTERNAL_LINK_OK)); + assertTrue(htmlFilter(EXTERNAL_LINK_CHECK2).contains(ExternalLinkToadlet.PATH)); + assertTrue(htmlFilter(EXTERNAL_LINK_CHECK3).startsWith(EXTERNAL_LINK_OK)); + + // regression testing + // bug #710 + assertEquals(ANCHOR_TEST, htmlFilter(ANCHOR_TEST)); + assertEquals(ANCHOR_TEST_EMPTY, htmlFilter(ANCHOR_TEST_EMPTY)); + assertEquals(ANCHOR_TEST_SPECIAL, htmlFilter(ANCHOR_TEST_SPECIAL)); + assertEquals(ANCHOR_TEST_SPECIAL2_RESULT, htmlFilter(ANCHOR_TEST_SPECIAL2)); + // bug #2496 + assertEquals(ANCHOR_RELATIVE1, htmlFilter(ANCHOR_RELATIVE1)); + assertEquals(ANCHOR_RELATIVE2, htmlFilter(ANCHOR_RELATIVE2)); + assertEquals(ANCHOR_FALSE_POS1, htmlFilter(ANCHOR_FALSE_POS1)); + assertEquals(ANCHOR_FALSE_POS2, htmlFilter(ANCHOR_FALSE_POS2)); + // EVIL HACK TEST for #2496 + #2451 + assertEquals(ANCHOR_MIXED_RESULT, htmlFilter(ANCHOR_MIXED)); + // bug #2451 + assertEquals(POUNT_CHARACTER_ENCODING_TEST_RESULT, htmlFilter(POUNT_CHARACTER_ENCODING_TEST)); + // bug #2297 + assertTrue(htmlFilter(PREVENT_FPROXY_ACCESS).contains(ExternalLinkToadlet.PATH)); + // bug #2921 + assertTrue(htmlFilter(PREVENT_EXTERNAL_ACCESS_CSS_SIMPLE).contains("div { }")); + assertTrue(htmlFilter(PREVENT_EXTERNAL_ACCESS_CSS_ESCAPE).contains("div { }")); + assertTrue(htmlFilter(PREVENT_EXTERNAL_ACCESS_CSS_CASE).contains("div { }")); + assertEquals(WHITELIST_STATIC_CONTENT, htmlFilter(WHITELIST_STATIC_CONTENT)); + assertEquals(XHTML_VOIDELEMENTC, htmlFilter(XHTML_VOIDELEMENT)); + assertEquals(XHTML_INCOMPLETEDOCUMENTC, htmlFilter(XHTML_INCOMPLETEDOCUMENT)); + assertEquals(XHTML_IMPROPERNESTINGC, htmlFilter(XHTML_IMPROPERNESTING)); + + assertEquals(CSS_STRING_NEWLINESC, htmlFilter(CSS_STRING_NEWLINES)); + + assertEquals(HTML_STYLESHEET_MAYBECHARSETC, htmlFilter(HTML_STYLESHEET_MAYBECHARSET, true)); + assertEquals(HTML_STYLESHEET_CHARSETC, htmlFilter(HTML_STYLESHEET_CHARSET, true)); + assertEquals(HTML_STYLESHEET_CHARSET_BADC, htmlFilter(HTML_STYLESHEET_CHARSET_BAD, true)); + assertEquals(HTML_STYLESHEET_CHARSET_BAD1C, htmlFilter(HTML_STYLESHEET_CHARSET_BAD1, true)); + assertEquals(HTML_STYLESHEET_WITH_MEDIAC, htmlFilter(HTML_STYLESHEET_WITH_MEDIA, true)); + + assertEquals(FRAME_SRC_CHARSETC, htmlFilter(FRAME_SRC_CHARSET, true)); + assertEquals(FRAME_SRC_CHARSET_BADC, htmlFilter(FRAME_SRC_CHARSET_BAD, true)); + assertEquals(FRAME_SRC_CHARSET_BAD1C, htmlFilter(FRAME_SRC_CHARSET_BAD1, true)); + + assertEquals(CSS_SPEC_EXAMPLE1, htmlFilter(CSS_SPEC_EXAMPLE1)); + + assertEquals(SPAN_WITH_STYLE, htmlFilter(SPAN_WITH_STYLE)); + + assertEquals(BASE_HREF, htmlFilter(BASE_HREF)); + assertEquals(DELETED_BASE_HREF, htmlFilter(BAD_BASE_HREF)); + assertEquals(DELETED_BASE_HREF, htmlFilter(BAD_BASE_HREF2)); + assertEquals(DELETED_BASE_HREF, htmlFilter(BAD_BASE_HREF3)); + assertEquals(DELETED_BASE_HREF, htmlFilter(BAD_BASE_HREF4)); + assertEquals(DELETED_BASE_HREF, htmlFilter(BAD_BASE_HREF5)); + } + + private static final String META_TIME_ONLY = ""; + private static final String META_TIME_ONLY_WRONG_CASE = ""; + private static final String META_TIME_ONLY_TOO_SHORT = ""; + private static final String META_TIME_ONLY_NEGATIVE = ""; + + private static final String META_TIME_ONLY_BADNUM1 = ""; + private static final String META_TIME_ONLY_BADNUM2 = ""; + private static final String META_TIME_ONLY_BADNUM_OUT = ""; + + private static final String META_CHARSET = ""; + private static final String META_CHARSET_LOWER = "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "Some Title"; + private static final String META_CHARSET_LOWER_RES = "\n" + + "\n" + + "\n" + + "\n" // comment has additional spaces after content filter + + "\n" + + "Some Title"; + + private static final String META_VALID_REDIRECT = ""; + private static final String META_VALID_REDIRECT_NOSPACE = ""; + + private static final String META_BOGUS_REDIRECT1 = ""; + private static final String META_BOGUS_REDIRECT2 = ""; + private static final String META_BOGUS_REDIRECT3 = ""; + private static final String META_BOGUS_REDIRECT4 = ""; + private static final String META_BOGUS_REDIRECT5 = ""; + private static final String META_BOGUS_REDIRECT6 = ""; + private static final String META_BOGUS_REDIRECT1_OUT = ""; + private static final String META_BOGUS_REDIRECT3_OUT = ""; + private static final String META_BOGUS_REDIRECT4_OUT = ""; + private static final String META_BOGUS_REDIRECT_NO_URL = ""; + + @Test + public void testMetaRefresh() throws Exception { + HTMLFilter.metaRefreshSamePageMinInterval = 5; + HTMLFilter.metaRefreshRedirectMinInterval = 30; + assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY)); + assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY_WRONG_CASE)); + assertEquals(META_TIME_ONLY, headFilter(META_TIME_ONLY_TOO_SHORT)); + assertEquals("", headFilter(META_TIME_ONLY_NEGATIVE)); + assertEquals(META_TIME_ONLY_BADNUM_OUT, headFilter(META_TIME_ONLY_BADNUM1)); + assertEquals(META_TIME_ONLY_BADNUM_OUT, headFilter(META_TIME_ONLY_BADNUM2)); + assertEquals(META_VALID_REDIRECT, headFilter(META_VALID_REDIRECT)); + assertEquals(META_VALID_REDIRECT, headFilter(META_VALID_REDIRECT_NOSPACE)); + assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT1)); + assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT2)); + assertEquals(META_BOGUS_REDIRECT3_OUT, headFilter(META_BOGUS_REDIRECT3)); + assertEquals(META_BOGUS_REDIRECT4_OUT, headFilter(META_BOGUS_REDIRECT4)); + assertEquals(META_BOGUS_REDIRECT1_OUT, headFilter(META_BOGUS_REDIRECT5)); + assertEquals(META_BOGUS_REDIRECT_NO_URL, headFilter(META_BOGUS_REDIRECT6)); + HTMLFilter.metaRefreshSamePageMinInterval = -1; + HTMLFilter.metaRefreshRedirectMinInterval = -1; + assertEquals("", headFilter(META_TIME_ONLY)); + assertEquals("", headFilter(META_TIME_ONLY_WRONG_CASE)); + assertEquals("", headFilter(META_TIME_ONLY_TOO_SHORT)); + assertEquals("", headFilter(META_TIME_ONLY_NEGATIVE)); + assertEquals("", headFilter(META_TIME_ONLY_BADNUM1)); + assertEquals("", headFilter(META_TIME_ONLY_BADNUM2)); + assertEquals("", headFilter(META_VALID_REDIRECT)); + assertEquals("", headFilter(META_VALID_REDIRECT_NOSPACE)); + assertEquals("", headFilter(META_BOGUS_REDIRECT1)); + assertEquals("", headFilter(META_BOGUS_REDIRECT2)); + assertEquals("", headFilter(META_BOGUS_REDIRECT3)); + assertEquals("", headFilter(META_BOGUS_REDIRECT4)); + assertEquals("", headFilter(META_BOGUS_REDIRECT5)); + assertEquals("", headFilter(META_BOGUS_REDIRECT6)); + } + + private String headFilter(String data) throws Exception { + String s = htmlFilter("" + data + ""); + if (s == null) { + return s; + } + + MatcherAssert.assertThat(s, startsWith("")); + MatcherAssert.assertThat(s, endsWith("")); + + s = s.substring("".length()); + s = s.substring(0, s.length() - "".length()); + return s; + } + + @Test + public void testThatHtml5MetaCharsetIsPreserved() throws Exception { + assertEquals(META_CHARSET, htmlFilter(META_CHARSET)); + assertEquals(META_CHARSET_LOWER_RES, htmlFilter(META_CHARSET_LOWER)); + } + + @Test + public void testEvilCharset() throws IOException { + // This is why we need to disallow characters before !! + String s = "Blah"; + String end = ""; + String alt = "Blah"; + if ((s.length() + end.length()) % 2 == 1) { + s += " "; + } + s = s + end; + byte[] buf = s.getBytes(StandardCharsets.UTF_8); + byte[] utf16bom = new byte[]{(byte) 0xFE, (byte) 0xFF}; + byte[] bufUTF16 = alt.getBytes(StandardCharsets.UTF_16); + byte[] total = new byte[buf.length + utf16bom.length + bufUTF16.length]; + System.arraycopy(utf16bom, 0, total, 0, utf16bom.length); + System.arraycopy(buf, 0, total, utf16bom.length, buf.length); + System.arraycopy(bufUTF16, 0, total, utf16bom.length + buf.length, bufUTF16.length); + HTMLFilter filter = new HTMLFilter(); + boolean failed = false; + List failures = new ArrayList<>(); + try ( + FileOutputStream fos = new FileOutputStream("output.utf16") + ){ + ArrayBucket out = new ArrayBucket(); + filter.readFilter(new ArrayBucket(total).getInputStream(), out.getOutputStream(), "UTF-16", null, null, null); + fos.write(out.toByteArray()); + failed = true; + failures.add(new RuntimeException("Filter accepted dangerous UTF8 text with BOM as UTF16! (HTMLFilter)")); + } catch (DataFilterException e) { + System.out.println("Failure: " + e); + e.printStackTrace(); + if (e.getCause() != null) { + e.getCause().printStackTrace(); + } + // Ok. + } + try ( + FileOutputStream fos = new FileOutputStream("output.filtered") + ){ + ArrayBucket out = new ArrayBucket(); + FilterStatus fo = ContentFilter.filter(new ArrayBucket(total).getInputStream(), out.getOutputStream(), "text/html", null, null, null); + fos.write(out.toByteArray()); + failed = true; + failures.add(new RuntimeException("Filter accepted dangerous UTF8 text with BOM as UTF16! (ContentFilter) - Detected charset: " + fo.charset)); + } catch (DataFilterException e) { + System.out.println("Failure: " + e); + e.printStackTrace(); + if (e.getCause() != null) { + e.getCause().printStackTrace(); + } + // Ok. + } + + if (failed) { + try (FileOutputStream fos = new FileOutputStream("unfiltered")) { + fos.write(total); + } + throw failures.get(0); + } + } + + public static String htmlFilter(String data) throws Exception { + if (data.startsWith("" + data + "", false); + assertTrue(s.startsWith("")); + s = s.substring("".length()); + assertTrue("s = \"" + s + "\"", s.endsWith("")); + s = s.substring(0, s.length() - "".length()); + return s; + } + + public static String htmlFilter(String data, boolean alt) throws Exception { + String typeName = "text/html"; + URI baseURI = new URI(alt ? ALT_BASE_URI : BASE_URI); + byte[] dataToFilter = data.getBytes(StandardCharsets.UTF_8); + ArrayBucket input = new ArrayBucket(dataToFilter); + ArrayBucket output = new ArrayBucket(); + try ( + OutputStream outputStream = output.getOutputStream(); + InputStream inputStream = input.getInputStream() + ) { + ContentFilter.filter(inputStream, outputStream, typeName, baseURI, null, null, null, null); + } finally { + input.free(); + } + String returnValue = output.toString(); + output.free(); + return returnValue; + } + + @Test + public void testLowerCaseExtensions() { + for (FilterMIMEType type : ContentFilter.mimeTypesByName.values()) { + String ext = type.primaryExtension; + if (ext != null) { + assertEquals(ext, ext.toLowerCase()); + } + String[] exts = type.alternateExtensions; + if (ext != null) { + for (String s : exts) { + assertEquals(s, s.toLowerCase()); + } + } + } + } } diff --git a/test/freenet/client/filter/FilterUtilsTest.java b/test/freenet/client/filter/FilterUtilsTest.java index fff69705ba5..14177c329fa 100755 --- a/test/freenet/client/filter/FilterUtilsTest.java +++ b/test/freenet/client/filter/FilterUtilsTest.java @@ -6,7 +6,7 @@ public class FilterUtilsTest { @Test - public void testValidLenthUnits() { + public void testValidLengthUnits() { // Test all valid length units for CSS and valid values assertTrue(FilterUtils.isLength("1em", false)); assertTrue(FilterUtils.isLength("1.12em", false)); diff --git a/test/freenet/client/filter/GIFFilterTest.java b/test/freenet/client/filter/GIFFilterTest.java index 0394803eee7..57b6ad1a792 100644 --- a/test/freenet/client/filter/GIFFilterTest.java +++ b/test/freenet/client/filter/GIFFilterTest.java @@ -117,17 +117,16 @@ public void testFilterPairs() throws IOException { @Test public void testReject() throws IOException { + ContentDataFilter filter = new GIFFilter(); for (String reject : REJECT) { try (InputStream inStream = resourceToBucket(reject).getInputStream(); NullOutputStream outStream = new NullOutputStream()) { - ContentDataFilter filter = new GIFFilter(); - try { - filter.readFilter(inStream, outStream, "", null, null, null); - fail("Filter did not fail on reject sample " + reject); - } catch (DataFilterException e) { - // Expected. - } + assertThrows( + "Filter did not fail on reject sample " + reject, + DataFilterException.class, + () -> filter.readFilter(inStream, outStream, "", null, null, null) + ); } } } @@ -180,10 +179,6 @@ private static Bucket filterGIF(Bucket input) throws IOException { * @throws AssertionError on failure */ private static Bucket resourceToBucket(String filename) throws IOException { - InputStream is = GIFFilterTest.class.getResourceAsStream(RESOURCE_PATH + filename); - Bucket ab = new ArrayBucket(); - BucketTools.copyFrom(ab, is, Long.MAX_VALUE); - - return ab; + return ResourceFileUtil.resourceToBucket(RESOURCE_PATH + filename); } } diff --git a/test/freenet/client/filter/JPEGFilterTest.java b/test/freenet/client/filter/JPEGFilterTest.java index 7521660f946..452ac2b4d37 100644 --- a/test/freenet/client/filter/JPEGFilterTest.java +++ b/test/freenet/client/filter/JPEGFilterTest.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Arrays; import java.util.HashMap; import org.junit.Test; @@ -27,14 +26,14 @@ public void testThatAThumbnailExtensionCodeIsPreserved() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); jpegFilter.readFilter(inputStream, outputStream, "UTF-8", new HashMap<>(), null, new NullFilterCallback()); byte[] filteredJpegFile = outputStream.toByteArray(); - assertTrue(Arrays.equals(jpegFile, filteredJpegFile)); + assertArrayEquals(jpegFile, filteredJpegFile); } private byte[] createValidJpegFileWithThumbnail() throws IOException { ByteArrayOutputStream jpegFile = new ByteArrayOutputStream(); writeStartOfImageMarker(jpegFile); - writeAppMarker(jpegFile, 0, new byte[]{0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); - writeAppMarker(jpegFile, 0, new byte[]{0x4a, 0x46, 0x58, 0x58, 0x00, 0x13, 0x01, 0x01, 0x00, 0x7f, 0x00}); + writeAppMarker(jpegFile, new byte[]{0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); + writeAppMarker(jpegFile, new byte[]{0x4a, 0x46, 0x58, 0x58, 0x00, 0x13, 0x01, 0x01, 0x00, 0x7f, 0x00}); writeEndOfImageMarker(jpegFile); return jpegFile.toByteArray(); } @@ -43,8 +42,8 @@ private void writeStartOfImageMarker(OutputStream outputStream) throws IOExcepti outputStream.write(new byte[]{(byte) 0xff, (byte) 0xd8}); } - private void writeAppMarker(OutputStream outputStream, int app, byte[] payload) throws IOException { - outputStream.write(new byte[]{(byte) 0xff, (byte) (0xe0 + app)}); + private void writeAppMarker(OutputStream outputStream, byte[] payload) throws IOException { + outputStream.write(new byte[]{(byte) 0xff, (byte) (0xe0)}); int payloadLengthIncludingLength = payload.length + 2; outputStream.write(new byte[]{(byte) ((payloadLengthIncludingLength >> 8) & 0xff), (byte) (payloadLengthIncludingLength & 0xff)}); outputStream.write(payload); diff --git a/test/freenet/client/filter/M3UFilterTest.java b/test/freenet/client/filter/M3UFilterTest.java index 453d748f4a2..6a857b6ef2a 100644 --- a/test/freenet/client/filter/M3UFilterTest.java +++ b/test/freenet/client/filter/M3UFilterTest.java @@ -3,17 +3,13 @@ import static org.junit.Assert.*; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; import org.junit.Test; import freenet.support.api.Bucket; import freenet.support.io.ArrayBucket; -import freenet.support.io.BucketTools; public class M3UFilterTest { protected static String[][] testPlaylists = { @@ -34,32 +30,28 @@ public void testSuiteTest() throws IOException { String correct = test[1]; Bucket ibo; Bucket ibprocessed = new ArrayBucket(); - Bucket ibc; - ibo = resourceToBucket(original); - ibc = resourceToBucket(correct); + ArrayBucket ibc; + ibo = ResourceFileUtil.resourceToBucket(original); + ibc = ResourceFileUtil.resourceToBucket(correct); try { filter.readFilter(ibo.getInputStream(), ibprocessed.getOutputStream(), "UTF-8", null, SCHEME_HOST_PORT, new GenericReadFilterCallback(new URI(BASE_URI), null, null, null)); String result = ibprocessed.toString(); - assertTrue(original + " should be filtered as " + correct + " but was filtered as\n" + result + "\ninstead of the correct\n" + bucketToString((ArrayBucket)ibc), result.equals(bucketToString((ArrayBucket)ibc))); + assertEquals( + original + " should be filtered as " + correct + " but was filtered as\n" + result + "\ninstead of the correct\n" + bucketToString(ibc), + bucketToString(ibc), + result + ); } catch (DataFilterException dfe) { - assertTrue("Filtering " + original + " failed", false); + fail("Filtering " + original + " failed"); } catch (URISyntaxException use) { - assertTrue("Creating URI from BASE_URI " + BASE_URI + " failed", false); + fail("Creating URI from BASE_URI " + BASE_URI + " failed"); } } } - protected ArrayBucket resourceToBucket(String filename) throws IOException { - InputStream is = getClass().getResourceAsStream(filename); - if (is == null) throw new java.io.FileNotFoundException(filename); - ArrayBucket ab = new ArrayBucket(); - BucketTools.copyFrom(ab, is, Long.MAX_VALUE); - return ab; - } - protected String bucketToString(ArrayBucket bucket) throws IOException { return new String(bucket.toByteArray()); } diff --git a/test/freenet/client/filter/MP3FilterTest.java b/test/freenet/client/filter/MP3FilterTest.java index 3db95d1bde1..ed9bffd1f71 100644 --- a/test/freenet/client/filter/MP3FilterTest.java +++ b/test/freenet/client/filter/MP3FilterTest.java @@ -106,28 +106,15 @@ private static Bucket filterMP3(Bucket input) { ContentDataFilter filter = new MP3Filter(); Bucket output = new ArrayBucket(); - InputStream inStream; - OutputStream outStream; - try { - inStream = input.getInputStream(); - outStream = output.getOutputStream(); - } catch (IOException e) { - throw new AssertionError(e); - } - - try { + try ( + InputStream inStream = input.getInputStream(); + OutputStream outStream = output.getOutputStream(); + ) { filter.readFilter(inStream, outStream, "", null, null, null); } catch (Exception e) { throw new AssertionError("Unexpected exception in the content filter.", e); } - try { - inStream.close(); - outStream.close(); - } catch (IOException e) { - throw new AssertionError(e); - } - return output; } @@ -137,16 +124,10 @@ private static Bucket filterMP3(Bucket input) { * @throws AssertionError on failure */ private static Bucket resourceToBucket(String filename) { - InputStream is = MP3FilterTest.class.getResourceAsStream(RESOURCE_PATH + filename); - if (is == null) { - throw new AssertionError("Test resource could not be opened: " + filename); - } - Bucket ab = new ArrayBucket(); - try { - BucketTools.copyFrom(ab, is, Long.MAX_VALUE); - } catch (Exception e) { + try { + return ResourceFileUtil.resourceToBucket(RESOURCE_PATH + filename); + } catch (IOException e) { throw new AssertionError(e); } - return ab; } } diff --git a/test/freenet/client/filter/OggBitStreamFilterTest.java b/test/freenet/client/filter/OggBitStreamFilterTest.java index 6e1d843ffcb..da355ca5dfe 100644 --- a/test/freenet/client/filter/OggBitStreamFilterTest.java +++ b/test/freenet/client/filter/OggBitStreamFilterTest.java @@ -1,10 +1,9 @@ package freenet.client.filter; +import static freenet.client.filter.ResourceFileUtil.testResourceFile; import static org.junit.Assert.*; -import java.io.DataInputStream; import java.io.IOException; -import java.io.InputStream; import org.junit.Test; @@ -12,44 +11,42 @@ public class OggBitStreamFilterTest { @Test public void testGetVorbisBitstreamFilter() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/vorbis_header.ogg")); - OggPage page = OggPage.readPage(input); - assertEquals(VorbisBitstreamFilter.class, getFilterClass(page)); - input.close(); + testResourceFile("./ogg/vorbis_header.ogg", (input) -> { + OggPage page = OggPage.readPage(input); + assertEquals(VorbisBitstreamFilter.class, getFilterClass(page)); + }); } @Test - public void testGetTheoraBitStreamFilter() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/theora_header.ogg")); - OggPage page = OggPage.readPage(input); - assertEquals(TheoraBitstreamFilter.class, getFilterClass(page)); - input.close(); + public void testGetTheoraBitStreamFilter() throws IOException { + testResourceFile("./ogg/theora_header.ogg", (input) -> { + OggPage page = OggPage.readPage(input); + assertEquals(TheoraBitstreamFilter.class, getFilterClass(page)); + }); } @Test public void testGetFilterForInvalidFormat() throws IOException { - InputStream input = getClass().getResourceAsStream("./ogg/invalid_header.ogg"); - DataInputStream dis = new DataInputStream(input); - OggPage page = OggPage.readPage(dis); - assertEquals(null, getFilterClass(page)); - input.close(); + testResourceFile("./ogg/invalid_header.ogg", (input) -> { + OggPage page = OggPage.readPage(input); + assertNull(getFilterClass(page)); + }); } @Test public void testPagesOutOfOrderCausesException() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/pages_out_of_order.ogg")); - OggPage page = OggPage.readPage(input); - OggBitstreamFilter filter = new OggBitstreamFilter(page); - page = OggPage.readPage(input); - try { - filter.parse(page); - fail("Expected exception not caught"); - } catch(DataFilterException e) {} - input.close(); + testResourceFile("./ogg/pages_out_of_order.ogg", (input) -> { + OggPage filterPage = OggPage.readPage(input); + OggBitstreamFilter filter = new OggBitstreamFilter(filterPage); + OggPage page = OggPage.readPage(input); + assertThrows(DataFilterException.class, ()-> filter.parse(page)); + }); } private Class getFilterClass(OggPage page) { OggBitstreamFilter filter = OggBitstreamFilter.getBitstreamFilter(page); - if(filter != null) return filter.getClass(); + if(filter != null) { + return filter.getClass(); + } return null; } } diff --git a/test/freenet/client/filter/OggFilterTest.java b/test/freenet/client/filter/OggFilterTest.java index 3f5ef1404e7..317660f0a40 100644 --- a/test/freenet/client/filter/OggFilterTest.java +++ b/test/freenet/client/filter/OggFilterTest.java @@ -1,14 +1,12 @@ package freenet.client.filter; +import static freenet.client.filter.ResourceFileUtil.testResourceFile; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.net.URL; -import java.util.Arrays; import org.apache.commons.compress.utils.IOUtils; import org.junit.Before; @@ -24,49 +22,60 @@ public void setUp() { @Test public void testEmptyOutputRaisesException() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/invalid_header.ogg")); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - try { - filter.readFilter(input, output, null, null, null, null); - fail("Expected Exception not caught. Output size: "+output.toByteArray().length); - } catch(DataFilterException e) {} + testResourceFile("./ogg/invalid_header.ogg", (input) -> { + assertThrows( + DataFilterException.class, + () -> filter.readFilter(input, new ByteArrayOutputStream(), null, null, null, null) + ); + }); } @Test - public void testValidSubPageStripped() throws IOException, DataFilterException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/contains_subpages.ogg")); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - try { - filter.readFilter(input, output, null, null, null, null); - } catch(DataFilterException e) {} - assertTrue(Arrays.equals(new byte[]{}, output.toByteArray())); - input.close(); - output.close(); + public void testValidSubPageStripped() throws IOException { + testResourceFile("./ogg/contains_subpages.ogg", (input) -> { + try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { + assertThrows( + DataFilterException.class, + () -> filter.readFilter(input, output, null, null, null, null) + ); + assertArrayEquals(new byte[]{}, output.toByteArray()); + } + }); } /** the purpose of this test is to create the testoutputFile so you can check it with a video player. */ @Test - public void testFilterFfmpegEncodedVideoSegment() throws IOException, DataFilterException { - String testoutputFile = getClass().getResource( - "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered-testoutput.ogv") - .getFile(); - DataInputStream inputFileUnchanged = new DataInputStream(getClass().getResourceAsStream( - "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered.ogv")); - ByteArrayOutputStream unchangedData = new ByteArrayOutputStream(); - IOUtils.copy(inputFileUnchanged, unchangedData); - DataInputStream inputFileToParse = new DataInputStream(getClass().getResourceAsStream( - "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--orig.ogv")); - DataInputStream input = inputFileToParse; - ByteArrayOutputStream output = new ByteArrayOutputStream(); - try { - filter.readFilter(input, output, null, null, null, null); - FileOutputStream newFileStream = new FileOutputStream(testoutputFile); - System.out.println(testoutputFile); + public void testFilterFfmpegEncodedVideoSegment() throws IOException { + try ( + ByteArrayOutputStream unchangedData = new ByteArrayOutputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream() + ) { + testResourceFile( + "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered.ogv", + (inputFileUnchanged) -> IOUtils.copy(inputFileUnchanged, unchangedData) + ); + testResourceFile( + "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--orig.ogv", + (input) -> { + filter.readFilter(input, output, null, null, null, null); + writeToTestOutputFile(output); + } + ); + assertArrayEquals(unchangedData.toByteArray(), output.toByteArray()); + } + } + + private void writeToTestOutputFile(ByteArrayOutputStream output) throws IOException { + URL resource = getClass().getResource( + "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered-testoutput.ogv" + ); + if (resource == null) { + throw new RuntimeException("Test file is not found"); + } + String testOutputFile = resource.getFile(); + System.out.println(testOutputFile); + try (FileOutputStream newFileStream = new FileOutputStream(testOutputFile)) { output.writeTo(newFileStream); - newFileStream.close(); - } catch(DataFilterException e) {} - assertTrue(Arrays.equals(unchangedData.toByteArray(), output.toByteArray())); - input.close(); - output.close(); + } } } diff --git a/test/freenet/client/filter/OggPageTest.java b/test/freenet/client/filter/OggPageTest.java index 393830bc135..72844204f14 100644 --- a/test/freenet/client/filter/OggPageTest.java +++ b/test/freenet/client/filter/OggPageTest.java @@ -1,59 +1,57 @@ package freenet.client.filter; +import static freenet.client.filter.ResourceFileUtil.testResourceFile; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; import org.junit.Test; public class OggPageTest { @Test public void testStripNonsenseInterruption() throws IOException { - InputStream badData = getClass().getResourceAsStream("./ogg/nonsensical_interruption.ogg"); - ByteArrayOutputStream filteredDataStream = new ByteArrayOutputStream(); - DataOutputStream output = new DataOutputStream(filteredDataStream); - DataInputStream input = new DataInputStream(badData); - OggPage page = OggPage.readPage(input); - if(page.headerValid()) output.write(page.toArray()); - page = OggPage.readPage(input); - if(page.headerValid()) output.write(page.toArray()); - byte[] filteredData = filteredDataStream.toByteArray(); - output.close(); - input.close(); + try ( + ByteArrayOutputStream actualDataStream = new ByteArrayOutputStream(); + ByteArrayOutputStream expectedDataStream = new ByteArrayOutputStream() + ) { + testResourceFile( + "./ogg/nonsensical_interruption_filtered.ogg", + (input) -> readPages(expectedDataStream, input) + ); + testResourceFile( + "./ogg/nonsensical_interruption.ogg", + (input) -> readPages(actualDataStream, input) + ); + assertArrayEquals(expectedDataStream.toByteArray(), actualDataStream.toByteArray()); + } + } - InputStream goodData = getClass().getResourceAsStream("./ogg/nonsensical_interruption_filtered.ogg"); - input = new DataInputStream(goodData); - ByteArrayOutputStream expectedDataStream = new ByteArrayOutputStream(); - output = new DataOutputStream(expectedDataStream); - page = OggPage.readPage(input); - if(page.headerValid()) output.write(page.toArray()); + private static void readPages(ByteArrayOutputStream output, DataInputStream input) throws IOException { + OggPage page = OggPage.readPage(input); + if (page.headerValid()) { + output.write(page.toArray()); + } page = OggPage.readPage(input); - if(page.headerValid()) output.write(page.toArray()); - byte[] expectedData = expectedDataStream.toByteArray(); - output.close(); - input.close(); - - assertTrue(Arrays.equals(filteredData, expectedData)); + if (page.headerValid()) { + output.write(page.toArray()); + } } @Test public void testChecksum() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/valid_checksum.ogg")); - OggPage page = OggPage.readPage(input); - assertTrue(page.headerValid()); - input.close(); + testResourceFile("./ogg/valid_checksum.ogg", (input) -> { + OggPage page = OggPage.readPage(input); + assertTrue(page.headerValid()); + }); } @Test public void testInvalidChecksumInvalidates() throws IOException { - DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/invalid_checksum.ogg")); - OggPage page = OggPage.readPage(input); - assertFalse(page.headerValid()); - input.close(); + testResourceFile("./ogg/invalid_checksum.ogg", (input) -> { + OggPage page = OggPage.readPage(input); + assertFalse(page.headerValid()); + }); } } diff --git a/test/freenet/client/filter/PNGFilterTest.java b/test/freenet/client/filter/PNGFilterTest.java index 2ecaa0c87c6..2fa6777e460 100644 --- a/test/freenet/client/filter/PNGFilterTest.java +++ b/test/freenet/client/filter/PNGFilterTest.java @@ -1,16 +1,14 @@ package freenet.client.filter; -import static org.junit.Assert.*; +import freenet.support.api.Bucket; +import freenet.support.io.NullBucket; +import org.junit.Test; import java.io.IOException; -import java.io.InputStream; -import org.junit.Test; - -import freenet.support.api.Bucket; -import freenet.support.io.ArrayBucket; -import freenet.support.io.BucketTools; -import freenet.support.io.NullBucket; +import static freenet.client.filter.ResourceFileUtil.resourceToBucket; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class PNGFilterTest { protected static Object[][] testImages = { @@ -124,12 +122,4 @@ public void testSuiteTest() throws IOException { } } } - - protected Bucket resourceToBucket(String filename) throws IOException { - InputStream is = getClass().getResourceAsStream(filename); - if (is == null) throw new java.io.FileNotFoundException(); - ArrayBucket ab = new ArrayBucket(); - BucketTools.copyFrom(ab, is, Long.MAX_VALUE); - return ab; - } } diff --git a/test/freenet/client/filter/ResourceFileUtil.java b/test/freenet/client/filter/ResourceFileUtil.java new file mode 100644 index 00000000000..360fcdc1f1a --- /dev/null +++ b/test/freenet/client/filter/ResourceFileUtil.java @@ -0,0 +1,41 @@ +package freenet.client.filter; + +import freenet.support.io.ArrayBucket; +import freenet.support.io.BucketTools; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +class ResourceFileUtil { + + @FunctionalInterface + interface ResourceFileStreamConsumer { + void processDataInputStream(DataInputStream dataInputStream) throws IOException; + } + + static void testResourceFile(String fileName, ResourceFileStreamConsumer inputStreamConsumer) throws IOException { + try ( + InputStream resourceAsStream = ResourceFileUtil.class.getResourceAsStream(fileName); + ) { + if (resourceAsStream == null) { + throw new RuntimeException("File should exist: " + fileName); + } + try(DataInputStream input = new DataInputStream(resourceAsStream)) { + inputStreamConsumer.processDataInputStream(input); + } + } + } + + static ArrayBucket resourceToBucket(String filename) throws IOException { + ArrayBucket ab; + try (InputStream is = ResourceFileUtil.class.getResourceAsStream(filename)) { + if (is == null) { + throw new java.io.FileNotFoundException(filename); + } + ab = new ArrayBucket(); + BucketTools.copyFrom(ab, is, Long.MAX_VALUE); + } + return ab; + } +} diff --git a/test/freenet/client/filter/TagVerifierTest.java b/test/freenet/client/filter/TagVerifierTest.java index d6276f2954e..e431b8b4a66 100644 --- a/test/freenet/client/filter/TagVerifierTest.java +++ b/test/freenet/client/filter/TagVerifierTest.java @@ -21,9 +21,9 @@ public class TagVerifierTest { private static final String BASE_KEY = "USK@0I8gctpUE32CM0iQhXaYpCMvtPPGfT4pjXm01oid5Zc,3dAcn4fX2LyxO6uCnWFTx-2HKZ89uruurcKwLSCxbZ4,AQACAAE/Ultimate-Freenet-Index/55/"; private static final String ALT_BASE_URI = BASE_URI_PROTOCOL+"://"+BASE_URI_CONTENT+'/'+BASE_KEY; - static String tagname; + String tagname; LinkedHashMap attributes; - ParsedTag HTMLTag; + ParsedTag htmlTag; TagVerifier verifier; HTMLFilter filter; HTMLFilter.HTMLParseContext pc; @@ -31,7 +31,7 @@ public class TagVerifierTest { @Before public void setUp() throws Exception { filter = new HTMLFilter(); - attributes = new LinkedHashMap(); + attributes = new LinkedHashMap<>(); pc = filter.new HTMLParseContext(null, null, "utf-8", new GenericReadFilterCallback(new URI(ALT_BASE_URI), null, null, null), false); } @@ -42,11 +42,11 @@ public void tearDown() { pc = null; tagname = null; verifier = null; - HTMLTag = null; + htmlTag = null; } @Test - public void testHTMLTagWithInvalidNS() throws DataFilterException{ + public void testHTMLTagWithInvalidNS() throws DataFilterException { tagname = "html"; verifier = HTMLFilter.allowedTagsVerifiers.get(tagname); @@ -55,10 +55,10 @@ public void testHTMLTagWithInvalidNS() throws DataFilterException{ //Place a unparsed attribute into the tag attributes.put("version", "-//W3C//DTD HTML 4.01 Transitional//EN"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); final String HTML_INVALID_XMLNS = ""; - assertEquals("HTML tag containing an invalid xmlns", HTML_INVALID_XMLNS, verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("HTML tag containing an invalid xmlns", HTML_INVALID_XMLNS, verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -72,11 +72,11 @@ public void testLinkTag() throws DataFilterException { attributes.put("media", "print, handheld"); attributes.put("href", "foo.css"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); final String LINK_STYLESHEET = ""; - assertEquals("Link tag importing CSS", LINK_STYLESHEET, verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Link tag importing CSS", LINK_STYLESHEET, verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -86,9 +86,9 @@ public void testMetaTagHTMLContentType() throws DataFilterException { attributes.put("http-equiv","Content-type"); attributes.put("content","text/html; charset=UTF-8"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); - assertEquals("Meta tag describing HTML content-type", HTMLTag.toString(), verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Meta tag describing HTML content-type", htmlTag.toString(), verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -98,26 +98,25 @@ public void testMetaTagXHTMLContentType() throws DataFilterException { attributes.put("http-equiv","Content-type"); attributes.put("content","application/xhtml+xml; charset=UTF-8"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); - assertEquals("Meta tag describing XHTML content-type", HTMLTag.toString(), verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Meta tag describing XHTML content-type", htmlTag.toString(), verifier.sanitize(htmlTag, pc).toString()); } @Test - public void testMetaTagUnknownContentType() throws DataFilterException { + public void testMetaTagUnknownContentType() { tagname = "meta"; verifier = HTMLFilter.allowedTagsVerifiers.get(tagname); attributes.put("http-equiv","Content-type"); attributes.put("content","want/fishsticks; charset=UTF-8"); - HTMLTag = new ParsedTag(tagname, attributes); - - try { - verifier.sanitize(HTMLTag, pc); - assertTrue("Meta tag describing an unknown content-type: should throw an error", false); - } catch (DataFilterException e) { - // Ok. - } + htmlTag = new ParsedTag(tagname, attributes); + + assertThrows( + "Meta tag describing an unknown content-type: should throw an error", + DataFilterException.class, + () -> verifier.sanitize(htmlTag, pc) + ); } @Test @@ -129,11 +128,11 @@ public void testBodyTag() throws DataFilterException { //Let's pretend the following is malicious JavaScript attributes.put("onload", "evil_scripting_magic"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); final String BODY_TAG = ""; - assertEquals("Body tag", BODY_TAG, verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Body tag", BODY_TAG, verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -146,10 +145,10 @@ public void testFormTag() throws DataFilterException { attributes.put("accept-charset", "iso-8859-1"); attributes.put("action", "/library/"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); final String FORM_TAG = "

"; - assertEquals("Form tag", FORM_TAG, verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Form tag", FORM_TAG, verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -160,9 +159,9 @@ public void testInvalidFormMethod() throws DataFilterException { attributes.put("method", "INVALID_METHOD"); attributes.put("action", "/library/"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); - assertNull("Form tag with an invalid method", verifier.sanitize(HTMLTag, pc)); + assertNull("Form tag with an invalid method", verifier.sanitize(htmlTag, pc)); } @Test @@ -172,9 +171,9 @@ public void testValidInputTag() throws DataFilterException { attributes.put("type", "text"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); - assertEquals("Input tag with a valid type", HTMLTag.toString(), verifier.sanitize(HTMLTag, pc).toString()); + assertEquals("Input tag with a valid type", htmlTag.toString(), verifier.sanitize(htmlTag, pc).toString()); } @Test @@ -184,8 +183,8 @@ public void testInvalidInputTag() throws DataFilterException { attributes.put("type", "INVALID_TYPE"); - HTMLTag = new ParsedTag(tagname, attributes); + htmlTag = new ParsedTag(tagname, attributes); - assertNull("Input tag with an invalid type", verifier.sanitize(HTMLTag, pc)); + assertNull("Input tag with an invalid type", verifier.sanitize(htmlTag, pc)); } } diff --git a/test/freenet/client/filter/TheoraBitstreamFilterTest.java b/test/freenet/client/filter/TheoraBitstreamFilterTest.java index 888007154ed..fe58a619dd3 100644 --- a/test/freenet/client/filter/TheoraBitstreamFilterTest.java +++ b/test/freenet/client/filter/TheoraBitstreamFilterTest.java @@ -2,27 +2,28 @@ import org.junit.Test; -import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; -import static org.junit.Assert.*; +import static freenet.client.filter.ResourceFileUtil.testResourceFile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class TheoraBitstreamFilterTest { @Test public void parseIdentificationHeaderTest() throws IOException { - try (DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/theora_header.ogg"))) { + testResourceFile("./ogg/theora_header.ogg", (input) -> { OggPage page = OggPage.readPage(input); TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); assertEquals(page.asPackets(), theoraBitstreamFilter.parse(page).asPackets()); - } + }); } @Test public void parseTest() throws IOException { - try (DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/Infinite_Hands-2008-Thusnelda-2009-09-18.ogv"))) { + testResourceFile("./ogg/Infinite_Hands-2008-Thusnelda-2009-09-18.ogv", (input) -> { OggPage page = OggPage.readPage(input); int pageSerial = page.getSerial(); TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); @@ -38,16 +39,15 @@ public void parseTest() throws IOException { break; } } - } + }); } - @Test(expected = UnknownContentTypeException.class) + @Test public void parseInvalidHeaderTest() throws IOException { - try (DataInputStream input = new DataInputStream(getClass().getResourceAsStream("./ogg/invalid_header.ogg"))) { + testResourceFile("./ogg/invalid_header.ogg", (input) -> { OggPage page = OggPage.readPage(input); - TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); - theoraBitstreamFilter.parse(page); - } + assertThrows(UnknownContentTypeException.class, () -> theoraBitstreamFilter.parse(page)); + }); } } From 9904a91105b291b8ae89e5b89599c4d3222982b3 Mon Sep 17 00:00:00 2001 From: Veniamin Fernandes Date: Sun, 29 Jan 2023 18:31:22 +0200 Subject: [PATCH 5/5] Minor improvements and code formatting according to #826 review * split try-with-resouces block in FailureCodeTrackerTest.getStoredLength * simplify ResourceFileUtil usage * improve code formatting Signed-off-by: Veniamin Fernandes --- .../client/FailureCodeTrackerTest.java | 8 +- test/freenet/client/FetchContextTest.java | 2 +- test/freenet/client/filter/CSSParserTest.java | 8 +- test/freenet/client/filter/GIFFilterTest.java | 2 +- .../client/filter/OggBitStreamFilterTest.java | 79 +++++++++---------- test/freenet/client/filter/OggFilterTest.java | 51 ++++++------ test/freenet/client/filter/OggPageTest.java | 36 ++++----- .../client/filter/ResourceFileUtil.java | 29 +++---- .../filter/TheoraBitstreamFilterTest.java | 25 +++--- 9 files changed, 110 insertions(+), 130 deletions(-) diff --git a/test/freenet/client/FailureCodeTrackerTest.java b/test/freenet/client/FailureCodeTrackerTest.java index cd8690c3144..eb1ac2eefa9 100644 --- a/test/freenet/client/FailureCodeTrackerTest.java +++ b/test/freenet/client/FailureCodeTrackerTest.java @@ -32,13 +32,11 @@ public void testSize(boolean insert) throws IOException { } private int getStoredLength(FailureCodeTracker f) throws IOException { - try ( - CountedOutputStream os = new CountedOutputStream(new NullOutputStream()); - DataOutputStream dos = new DataOutputStream(os) - ) { + CountedOutputStream os = new CountedOutputStream(new NullOutputStream()); + try (DataOutputStream dos = new DataOutputStream(os)) { f.writeFixedLengthTo(dos); - return (int) os.written(); } + return (int) os.written(); } } diff --git a/test/freenet/client/FetchContextTest.java b/test/freenet/client/FetchContextTest.java index 2c4ac174f67..1828489cab9 100644 --- a/test/freenet/client/FetchContextTest.java +++ b/test/freenet/client/FetchContextTest.java @@ -23,7 +23,7 @@ public void testPersistence() throws IOException, StorageFormatException { new ArrayBucketFactory(), new SimpleEventProducer() ); - final ArrayBucket bucket = new ArrayBucket(); + ArrayBucket bucket = new ArrayBucket(); try { try (DataOutputStream dos = new DataOutputStream(bucket.getOutputStream())) { context.writeTo(dos); diff --git a/test/freenet/client/filter/CSSParserTest.java b/test/freenet/client/filter/CSSParserTest.java index b199c8b5c1e..46e480b24fa 100644 --- a/test/freenet/client/filter/CSSParserTest.java +++ b/test/freenet/client/filter/CSSParserTest.java @@ -11,8 +11,12 @@ import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.hamcrest.MatcherAssert; import org.junit.Before; @@ -927,7 +931,7 @@ private void testCssSelectorFiltering(Map cssSelectorMap) throws } private void testBadSelectorFiltering(Set badSelectorSet) throws IOException, URISyntaxException { - for(String key : badSelectorSet) { + for (String key : badSelectorSet) { assertEquals( "Bad selector filtering should produce empty string: '" + key + "'", "", diff --git a/test/freenet/client/filter/GIFFilterTest.java b/test/freenet/client/filter/GIFFilterTest.java index 57b6ad1a792..5aff6b485e3 100644 --- a/test/freenet/client/filter/GIFFilterTest.java +++ b/test/freenet/client/filter/GIFFilterTest.java @@ -117,11 +117,11 @@ public void testFilterPairs() throws IOException { @Test public void testReject() throws IOException { - ContentDataFilter filter = new GIFFilter(); for (String reject : REJECT) { try (InputStream inStream = resourceToBucket(reject).getInputStream(); NullOutputStream outStream = new NullOutputStream()) { + ContentDataFilter filter = new GIFFilter(); assertThrows( "Filter did not fail on reject sample " + reject, DataFilterException.class, diff --git a/test/freenet/client/filter/OggBitStreamFilterTest.java b/test/freenet/client/filter/OggBitStreamFilterTest.java index da355ca5dfe..43ad28320c6 100644 --- a/test/freenet/client/filter/OggBitStreamFilterTest.java +++ b/test/freenet/client/filter/OggBitStreamFilterTest.java @@ -1,52 +1,49 @@ package freenet.client.filter; -import static freenet.client.filter.ResourceFileUtil.testResourceFile; +import static freenet.client.filter.ResourceFileUtil.resourceToDataInputStream; +import static freenet.client.filter.ResourceFileUtil.resourceToOggPage; import static org.junit.Assert.*; +import java.io.DataInputStream; import java.io.IOException; import org.junit.Test; public class OggBitStreamFilterTest { - @Test - public void testGetVorbisBitstreamFilter() throws IOException { - testResourceFile("./ogg/vorbis_header.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - assertEquals(VorbisBitstreamFilter.class, getFilterClass(page)); - }); - } - - @Test - public void testGetTheoraBitStreamFilter() throws IOException { - testResourceFile("./ogg/theora_header.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - assertEquals(TheoraBitstreamFilter.class, getFilterClass(page)); - }); - } - @Test - public void testGetFilterForInvalidFormat() throws IOException { - testResourceFile("./ogg/invalid_header.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - assertNull(getFilterClass(page)); - }); - } - - @Test - public void testPagesOutOfOrderCausesException() throws IOException { - testResourceFile("./ogg/pages_out_of_order.ogg", (input) -> { - OggPage filterPage = OggPage.readPage(input); - OggBitstreamFilter filter = new OggBitstreamFilter(filterPage); - OggPage page = OggPage.readPage(input); - assertThrows(DataFilterException.class, ()-> filter.parse(page)); - }); - } - - private Class getFilterClass(OggPage page) { - OggBitstreamFilter filter = OggBitstreamFilter.getBitstreamFilter(page); - if(filter != null) { - return filter.getClass(); - } - return null; - } + @Test + public void testGetVorbisBitstreamFilter() throws IOException { + OggPage page = resourceToOggPage("./ogg/vorbis_header.ogg"); + assertEquals(VorbisBitstreamFilter.class, getFilterClass(page)); + } + + @Test + public void testGetTheoraBitStreamFilter() throws IOException { + OggPage page = resourceToOggPage("./ogg/theora_header.ogg"); + assertEquals(TheoraBitstreamFilter.class, getFilterClass(page)); + } + + @Test + public void testGetFilterForInvalidFormat() throws IOException { + OggPage page = resourceToOggPage("./ogg/invalid_header.ogg"); + assertNull(getFilterClass(page)); + } + + @Test + public void testPagesOutOfOrderCausesException() throws IOException { + try (DataInputStream input = resourceToDataInputStream("./ogg/pages_out_of_order.ogg")) { + OggPage filterPage = OggPage.readPage(input); + OggBitstreamFilter filter = new OggBitstreamFilter(filterPage); + OggPage page = OggPage.readPage(input); + assertThrows(DataFilterException.class, () -> filter.parse(page)); + } + } + + private Class getFilterClass(OggPage page) { + OggBitstreamFilter filter = OggBitstreamFilter.getBitstreamFilter(page); + if (filter != null) { + return filter.getClass(); + } + return null; + } } diff --git a/test/freenet/client/filter/OggFilterTest.java b/test/freenet/client/filter/OggFilterTest.java index 317660f0a40..e4eda7227d5 100644 --- a/test/freenet/client/filter/OggFilterTest.java +++ b/test/freenet/client/filter/OggFilterTest.java @@ -1,9 +1,10 @@ package freenet.client.filter; -import static freenet.client.filter.ResourceFileUtil.testResourceFile; +import static freenet.client.filter.ResourceFileUtil.resourceToDataInputStream; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; @@ -22,47 +23,41 @@ public void setUp() { @Test public void testEmptyOutputRaisesException() throws IOException { - testResourceFile("./ogg/invalid_header.ogg", (input) -> { + try (DataInputStream input = resourceToDataInputStream("./ogg/invalid_header.ogg")) { assertThrows( DataFilterException.class, () -> filter.readFilter(input, new ByteArrayOutputStream(), null, null, null, null) ); - }); + } } @Test public void testValidSubPageStripped() throws IOException { - testResourceFile("./ogg/contains_subpages.ogg", (input) -> { - try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { - assertThrows( - DataFilterException.class, - () -> filter.readFilter(input, output, null, null, null, null) - ); - assertArrayEquals(new byte[]{}, output.toByteArray()); - } - }); + try ( + DataInputStream input = resourceToDataInputStream("./ogg/contains_subpages.ogg"); + ByteArrayOutputStream output = new ByteArrayOutputStream() + ) { + assertThrows( + DataFilterException.class, + () -> filter.readFilter(input, output, null, null, null, null) + ); + assertArrayEquals(new byte[]{}, output.toByteArray()); + } } /** the purpose of this test is to create the testoutputFile so you can check it with a video player. */ @Test public void testFilterFfmpegEncodedVideoSegment() throws IOException { - try ( - ByteArrayOutputStream unchangedData = new ByteArrayOutputStream(); - ByteArrayOutputStream output = new ByteArrayOutputStream() - ) { - testResourceFile( - "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered.ogv", - (inputFileUnchanged) -> IOUtils.copy(inputFileUnchanged, unchangedData) - ); - testResourceFile( - "./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--orig.ogv", - (input) -> { - filter.readFilter(input, output, null, null, null, null); - writeToTestOutputFile(output); - } - ); - assertArrayEquals(unchangedData.toByteArray(), output.toByteArray()); + ByteArrayOutputStream expectedData = new ByteArrayOutputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (DataInputStream input = resourceToDataInputStream("./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--filtered.ogv")) { + IOUtils.copy(input, expectedData); + } + try (DataInputStream input = resourceToDataInputStream("./ogg/36C3_-_opening--cc-by--c3voc--fem-ags-opensuse--ccc--orig.ogv")) { + filter.readFilter(input, output, null, null, null, null); + writeToTestOutputFile(output); } + assertArrayEquals(expectedData.toByteArray(), output.toByteArray()); } private void writeToTestOutputFile(ByteArrayOutputStream output) throws IOException { diff --git a/test/freenet/client/filter/OggPageTest.java b/test/freenet/client/filter/OggPageTest.java index 72844204f14..1c48aa2901d 100644 --- a/test/freenet/client/filter/OggPageTest.java +++ b/test/freenet/client/filter/OggPageTest.java @@ -1,6 +1,7 @@ package freenet.client.filter; -import static freenet.client.filter.ResourceFileUtil.testResourceFile; +import static freenet.client.filter.ResourceFileUtil.resourceToDataInputStream; +import static freenet.client.filter.ResourceFileUtil.resourceToOggPage; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; @@ -12,20 +13,15 @@ public class OggPageTest { @Test public void testStripNonsenseInterruption() throws IOException { - try ( - ByteArrayOutputStream actualDataStream = new ByteArrayOutputStream(); - ByteArrayOutputStream expectedDataStream = new ByteArrayOutputStream() - ) { - testResourceFile( - "./ogg/nonsensical_interruption_filtered.ogg", - (input) -> readPages(expectedDataStream, input) - ); - testResourceFile( - "./ogg/nonsensical_interruption.ogg", - (input) -> readPages(actualDataStream, input) - ); - assertArrayEquals(expectedDataStream.toByteArray(), actualDataStream.toByteArray()); + ByteArrayOutputStream actualDataStream = new ByteArrayOutputStream(); + ByteArrayOutputStream expectedDataStream = new ByteArrayOutputStream(); + try (DataInputStream input = resourceToDataInputStream("./ogg/nonsensical_interruption_filtered.ogg")) { + readPages(expectedDataStream, input); } + try (DataInputStream input = resourceToDataInputStream("./ogg/nonsensical_interruption.ogg")) { + readPages(actualDataStream, input); + } + assertArrayEquals(expectedDataStream.toByteArray(), actualDataStream.toByteArray()); } private static void readPages(ByteArrayOutputStream output, DataInputStream input) throws IOException { @@ -41,17 +37,13 @@ private static void readPages(ByteArrayOutputStream output, DataInputStream inpu @Test public void testChecksum() throws IOException { - testResourceFile("./ogg/valid_checksum.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - assertTrue(page.headerValid()); - }); + OggPage page = resourceToOggPage("./ogg/valid_checksum.ogg"); + assertTrue(page.headerValid()); } @Test public void testInvalidChecksumInvalidates() throws IOException { - testResourceFile("./ogg/invalid_checksum.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - assertFalse(page.headerValid()); - }); + OggPage page = resourceToOggPage("./ogg/invalid_checksum.ogg"); + assertFalse(page.headerValid()); } } diff --git a/test/freenet/client/filter/ResourceFileUtil.java b/test/freenet/client/filter/ResourceFileUtil.java index 360fcdc1f1a..651028863da 100644 --- a/test/freenet/client/filter/ResourceFileUtil.java +++ b/test/freenet/client/filter/ResourceFileUtil.java @@ -4,34 +4,31 @@ import freenet.support.io.BucketTools; import java.io.DataInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; class ResourceFileUtil { - @FunctionalInterface - interface ResourceFileStreamConsumer { - void processDataInputStream(DataInputStream dataInputStream) throws IOException; + static DataInputStream resourceToDataInputStream(String fileName) throws IOException { + InputStream resourceAsStream = ResourceFileUtil.class.getResourceAsStream(fileName); + if (resourceAsStream == null) { + throw new FileNotFoundException(fileName); + } + return new DataInputStream(resourceAsStream); } - static void testResourceFile(String fileName, ResourceFileStreamConsumer inputStreamConsumer) throws IOException { - try ( - InputStream resourceAsStream = ResourceFileUtil.class.getResourceAsStream(fileName); - ) { - if (resourceAsStream == null) { - throw new RuntimeException("File should exist: " + fileName); - } - try(DataInputStream input = new DataInputStream(resourceAsStream)) { - inputStreamConsumer.processDataInputStream(input); - } + static OggPage resourceToOggPage(String fileName) throws IOException { + try (DataInputStream input = resourceToDataInputStream(fileName)) { + return OggPage.readPage(input); } } - static ArrayBucket resourceToBucket(String filename) throws IOException { + static ArrayBucket resourceToBucket(String fileName) throws IOException { ArrayBucket ab; - try (InputStream is = ResourceFileUtil.class.getResourceAsStream(filename)) { + try (InputStream is = ResourceFileUtil.class.getResourceAsStream(fileName)) { if (is == null) { - throw new java.io.FileNotFoundException(filename); + throw new FileNotFoundException(fileName); } ab = new ArrayBucket(); BucketTools.copyFrom(ab, is, Long.MAX_VALUE); diff --git a/test/freenet/client/filter/TheoraBitstreamFilterTest.java b/test/freenet/client/filter/TheoraBitstreamFilterTest.java index fe58a619dd3..24c396ead7e 100644 --- a/test/freenet/client/filter/TheoraBitstreamFilterTest.java +++ b/test/freenet/client/filter/TheoraBitstreamFilterTest.java @@ -2,10 +2,12 @@ import org.junit.Test; +import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; -import static freenet.client.filter.ResourceFileUtil.testResourceFile; +import static freenet.client.filter.ResourceFileUtil.resourceToDataInputStream; +import static freenet.client.filter.ResourceFileUtil.resourceToOggPage; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -13,17 +15,14 @@ public class TheoraBitstreamFilterTest { @Test public void parseIdentificationHeaderTest() throws IOException { - testResourceFile("./ogg/theora_header.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - - TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); - assertEquals(page.asPackets(), theoraBitstreamFilter.parse(page).asPackets()); - }); + OggPage page = resourceToOggPage("./ogg/theora_header.ogg"); + TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); + assertEquals(page.asPackets(), theoraBitstreamFilter.parse(page).asPackets()); } @Test public void parseTest() throws IOException { - testResourceFile("./ogg/Infinite_Hands-2008-Thusnelda-2009-09-18.ogv", (input) -> { + try (DataInputStream input = resourceToDataInputStream("./ogg/Infinite_Hands-2008-Thusnelda-2009-09-18.ogv")) { OggPage page = OggPage.readPage(input); int pageSerial = page.getSerial(); TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); @@ -39,15 +38,13 @@ public void parseTest() throws IOException { break; } } - }); + } } @Test public void parseInvalidHeaderTest() throws IOException { - testResourceFile("./ogg/invalid_header.ogg", (input) -> { - OggPage page = OggPage.readPage(input); - TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); - assertThrows(UnknownContentTypeException.class, () -> theoraBitstreamFilter.parse(page)); - }); + OggPage page = resourceToOggPage("./ogg/invalid_header.ogg"); + TheoraBitstreamFilter theoraBitstreamFilter = new TheoraBitstreamFilter(page); + assertThrows(UnknownContentTypeException.class, () -> theoraBitstreamFilter.parse(page)); } }