diff --git a/ext/berkeleylm b/ext/berkeleylm
--- a/ext/berkeleylm
+++ b/ext/berkeleylm
@@ -1 +1 @@
-Subproject commit c431057d7512d897146ebccdf0f446d387397702
+Subproject commit c431057d7512d897146ebccdf0f446d387397702-dirty
diff --git a/pom.xml b/pom.xml
index 0940023..996899d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,6 +30,7 @@
6.0.6-SNAPSHOT
src
+ test
maven-compiler-plugin
diff --git a/src/joshua/corpus/Vocabulary.java b/src/joshua/corpus/Vocabulary.java
index d79170d..9c4832a 100644
--- a/src/joshua/corpus/Vocabulary.java
+++ b/src/joshua/corpus/Vocabulary.java
@@ -47,7 +47,7 @@ public class Vocabulary {
private final static ArrayList LMs = new ArrayList<>();
- private static List idToString;
+ protected static List idToString;
private static Map stringToId;
private static final StampedLock lock = new StampedLock();
diff --git a/test/joshua/corpus/SpanTest.java b/test/joshua/corpus/SpanTest.java
index 24b4b5b..7062a1f 100644
--- a/test/joshua/corpus/SpanTest.java
+++ b/test/joshua/corpus/SpanTest.java
@@ -16,8 +16,6 @@
*/
package joshua.corpus;
-import joshua.corpus.Span;
-
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/test/joshua/decoder/DecoderThreadTest.java b/test/joshua/decoder/DecoderThreadTest.java
index 78e46bd..d087393 100644
--- a/test/joshua/decoder/DecoderThreadTest.java
+++ b/test/joshua/decoder/DecoderThreadTest.java
@@ -24,12 +24,8 @@ import java.util.Date;
import java.util.Scanner;
import joshua.corpus.Corpus;
-import joshua.corpus.alignment.AlignmentGrids;
-import joshua.corpus.suffix_array.Compile;
-import joshua.corpus.suffix_array.SuffixArrayFactory;
-import joshua.corpus.vocab.Vocabulary;
-import joshua.prefix_tree.ExtractRules;
+import joshua.corpus.Vocabulary;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/test/joshua/decoder/TranslationsTest.java b/test/joshua/decoder/TranslationsTest.java
index 50ede9b..49174ce 100644
--- a/test/joshua/decoder/TranslationsTest.java
+++ b/test/joshua/decoder/TranslationsTest.java
@@ -2,10 +2,11 @@ package joshua.decoder;
import static org.testng.Assert.*;
+import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
-import joshua.decoder.io.TranslationRequest;
-
+import joshua.decoder.io.TranslationRequestStream;
import org.testng.annotations.Test;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.AfterTest;
@@ -26,9 +27,6 @@ public class TranslationsTest {
throw new RuntimeException("Test not implemented");
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
- */
@Test(enabled = false)
public void testNext() {
fail("Not yet implemented");
@@ -44,7 +42,8 @@ public class TranslationsTest {
public void next() {
byte[] data = "1\n2\n".getBytes();
ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequest request = new TranslationRequest(input, joshuaConfiguration);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
Translations translations = new Translations(request);
assertEquals(translations.next().getSourceSentence().source(), "1");
// Remove the next two.
diff --git a/test/joshua/decoder/io/DeNormalizeTest.java b/test/joshua/decoder/io/DeNormalizeTest.java
index 9f3a404..ea773fa 100644
--- a/test/joshua/decoder/io/DeNormalizeTest.java
+++ b/test/joshua/decoder/io/DeNormalizeTest.java
@@ -234,9 +234,6 @@ public class DeNormalizeTest {
assertEquals(actual, expected);
}
- /**
- * Test method for {@link joshua.decoder.io.DeNormalize#detokenizeBracketTokens(java.lang.String)}
- */
@Test
public void testDetokenizeBracketTokens() throws Exception {
String expected, actual;
diff --git a/test/joshua/decoder/io/TranslationRequestTest.java b/test/joshua/decoder/io/TranslationRequestTest.java
index 5a3aacd..1a03aea 100644
--- a/test/joshua/decoder/io/TranslationRequestTest.java
+++ b/test/joshua/decoder/io/TranslationRequestTest.java
@@ -1,13 +1,17 @@
package joshua.decoder.io;
+import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
-import java.io.InputStream;
+import java.io.InputStreamReader;
import joshua.decoder.JoshuaConfiguration;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-import org.testng.annotations.*;
-import static org.testng.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
/**
* This class verifies the following behaviors:
@@ -41,83 +45,57 @@ public class TranslationRequestTest {
protected void tearDown() throws Exception {
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#TranslationRequest(java.io.InputStream)}.
- */
@Test(enabled = false)
public void testTranslationRequest() {
fail("Not yet implemented");
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- */
@Test(enabled = true)
public void testSize_uponConstruction() {
- InputStream in = mock(InputStream.class);
- TranslationRequest request = new TranslationRequest(in, joshuaConfiguration);
+ BufferedReader reader = mock(BufferedReader.class);
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
assertEquals(request.size(), 0);
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
@Test(enabled = true)
public void testSize_1() throws Exception {
byte[] data = "1".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequest request = new TranslationRequest(input, joshuaConfiguration);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(data)));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
request.next();
assertEquals(request.size(), 1);
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
@Test(enabled = true)
public void testSize_newline() throws Exception {
byte[] data = "\n".getBytes();
ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequest request = new TranslationRequest(input, joshuaConfiguration);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
request.next();
assertEquals(request.size(), 1);
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
@Test(enabled = true)
public void testSize_2newlines() throws Exception {
byte[] data = "\n\n".getBytes();
ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequest request = new TranslationRequest(input, joshuaConfiguration);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
request.next();
request.next();
assertEquals(request.size(), 2);
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
- * @throws Exception
- */
@Test(enabled = true)
public void testNext_2Newlines() throws Exception {
byte[] data = "\n\n".getBytes();
ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequest request = new TranslationRequest(input, joshuaConfiguration);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
assertEquals(request.next().source(), "");
assertEquals(request.next().source(), "");
}
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#remove()}.
- */
- @Test(enabled = false)
- public void testRemove() {
- fail("Not yet implemented");
- }
}
diff --git a/test/joshua/lattice/ArcTest.java b/test/joshua/lattice/ArcTest.java
index 51b3bb8..1887a9d 100644
--- a/test/joshua/lattice/ArcTest.java
+++ b/test/joshua/lattice/ArcTest.java
@@ -31,7 +31,7 @@ public class ArcTest {
private final Node head = new Node(1);
private final Node tail = new Node(2);
- private final double cost = Math.PI;
+ private final float cost = (float) Math.PI;
private final String label = "pi";
private Arc arc;
@@ -40,12 +40,12 @@ public class ArcTest {
//@Test(dependsOnGroups = {"lattice_node" })
public void constructArc() {
- arc = new Arc(head, tail, cost, label);
+ arc = new Arc(tail, head, cost, label);
- Assert.assertEquals(arc.head, head);
- Assert.assertEquals(arc.tail, tail);
- Assert.assertEquals(arc.cost, cost);
- Assert.assertEquals(arc.label, label);
+ Assert.assertEquals(arc.getHead(), head);
+ Assert.assertEquals(arc.getTail(), tail);
+ Assert.assertEquals(arc.getCost(), cost);
+ Assert.assertEquals(arc.getLabel(), label);
}
diff --git a/test/joshua/lattice/LatticeTest.java b/test/joshua/lattice/LatticeTest.java
index d0957b7..f336cf7 100644
--- a/test/joshua/lattice/LatticeTest.java
+++ b/test/joshua/lattice/LatticeTest.java
@@ -19,6 +19,7 @@ package joshua.lattice;
import java.util.ArrayList;
import java.util.List;
+import joshua.decoder.JoshuaConfiguration;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -40,13 +41,13 @@ public class LatticeTest {
nodes.add(new Node(i));
}
- nodes.get(0).addArc(nodes.get(1), 1.0, "x");
- nodes.get(1).addArc(nodes.get(2), 1.0, "y");
- nodes.get(0).addArc(nodes.get(2), 1.5, "a");
- nodes.get(2).addArc(nodes.get(3), 3.0, "b");
- nodes.get(2).addArc(nodes.get(3), 5.0, "c");
+ nodes.get(0).addArc(nodes.get(1), 1.0f, "x");
+ nodes.get(1).addArc(nodes.get(2), 1.0f, "y");
+ nodes.get(0).addArc(nodes.get(2), 1.5f, "a");
+ nodes.get(2).addArc(nodes.get(3), 3.0f, "b");
+ nodes.get(2).addArc(nodes.get(3), 5.0f, "c");
- Lattice graph = new Lattice(nodes);
+ Lattice graph = new Lattice(nodes, new JoshuaConfiguration());
Assert.assertEquals(graph.getShortestPath(0, 1), 1.0);
Assert.assertEquals(graph.getShortestPath(0, 2), 1.0);
@@ -102,7 +103,7 @@ public class LatticeTest {
")"; // End of lattice
- Lattice lattice = Lattice.createFromString(data);
+ Lattice lattice = Lattice.createStringLatticeFromString(data, new JoshuaConfiguration());
int numberOfNodes = 7;
@@ -127,25 +128,25 @@ public class LatticeTest {
// Node 0 outgoing arcs
- Arc arcA_0_5 = node0.outgoingArcs.get(0);
+ Arc arcA_0_5 = node0.getOutgoingArcs().get(0);
Assert.assertEquals(arcA_0_5.getLabel(), "A");
Assert.assertEquals(arcA_0_5.getHead(), node0);
Assert.assertEquals(arcA_0_5.getTail(), node5);
Assert.assertEquals(arcA_0_5.getCost(), 1.0);
- Arc arcB_0_2 = node0.outgoingArcs.get(1);
+ Arc arcB_0_2 = node0.getOutgoingArcs().get(1);
Assert.assertEquals(arcB_0_2.getLabel(), "B");
Assert.assertEquals(arcB_0_2.getHead(), node0);
Assert.assertEquals(arcB_0_2.getTail(), node2);
Assert.assertEquals(arcB_0_2.getCost(), 1.0);
- Arc arcC_0_3 = node0.outgoingArcs.get(2);
+ Arc arcC_0_3 = node0.getOutgoingArcs().get(2);
Assert.assertEquals(arcC_0_3.getLabel(), "C");
Assert.assertEquals(arcC_0_3.getHead(), node0);
Assert.assertEquals(arcC_0_3.getTail(), node3);
Assert.assertEquals(arcC_0_3.getCost(), 1.0);
- Arc arcD_0_1 = node0.outgoingArcs.get(3);
+ Arc arcD_0_1 = node0.getOutgoingArcs().get(3);
Assert.assertEquals(arcD_0_1.getLabel(), "D");
Assert.assertEquals(arcD_0_1.getHead(), node0);
Assert.assertEquals(arcD_0_1.getTail(), node1);
@@ -153,7 +154,7 @@ public class LatticeTest {
// Node 1 outgoing arcs
- Arc arcE_1_5 = node1.outgoingArcs.get(0);
+ Arc arcE_1_5 = node1.getOutgoingArcs().get(0);
Assert.assertEquals(arcE_1_5.getLabel(), "E");
Assert.assertEquals(arcE_1_5.getHead(), node1);
Assert.assertEquals(arcE_1_5.getTail(), node5);
@@ -161,7 +162,7 @@ public class LatticeTest {
// Node 2 outgoing arcs
- Arc arcC_2_5 = node2.outgoingArcs.get(0);
+ Arc arcC_2_5 = node2.getOutgoingArcs().get(0);
Assert.assertEquals(arcC_2_5.getLabel(), "C");
Assert.assertEquals(arcC_2_5.getHead(), node2);
Assert.assertEquals(arcC_2_5.getTail(), node5);
@@ -169,7 +170,7 @@ public class LatticeTest {
// Node 3 outgoing arcs
- Arc arcD_3_4 = node3.outgoingArcs.get(0);
+ Arc arcD_3_4 = node3.getOutgoingArcs().get(0);
Assert.assertEquals(arcD_3_4.getLabel(), "D");
Assert.assertEquals(arcD_3_4.getHead(), node3);
Assert.assertEquals(arcD_3_4.getTail(), node4);
@@ -177,7 +178,7 @@ public class LatticeTest {
// Node 4 outgoing arcs
- Arc arcE_4_5 = node4.outgoingArcs.get(0);
+ Arc arcE_4_5 = node4.getOutgoingArcs().get(0);
Assert.assertEquals(arcE_4_5.getLabel(), "E");
Assert.assertEquals(arcE_4_5.getHead(), node4);
Assert.assertEquals(arcE_4_5.getTail(), node5);
@@ -185,7 +186,7 @@ public class LatticeTest {
// Node 5 outgoing arcs
- Arc arcX_5_6 = node5.outgoingArcs.get(0);
+ Arc arcX_5_6 = node5.getOutgoingArcs().get(0);
Assert.assertEquals(arcX_5_6.getLabel(), "X");
Assert.assertEquals(arcX_5_6.getHead(), node5);
Assert.assertEquals(arcX_5_6.getTail(), node6);
diff --git a/test/joshua/lattice/NodeTest.java b/test/joshua/lattice/NodeTest.java
index 147c7fe..9a5b841 100644
--- a/test/joshua/lattice/NodeTest.java
+++ b/test/joshua/lattice/NodeTest.java
@@ -38,8 +38,8 @@ public class NodeTest {
node = new Node(id);
- Assert.assertEquals((int) node.id, (int) id);
- Assert.assertTrue(node.outgoingArcs.isEmpty());
+ Assert.assertEquals((int) node.id(), (int) id);
+ Assert.assertTrue(node.getOutgoingArcs().isEmpty());
Assert.assertEquals(node.size(), 0);
}
@@ -65,42 +65,42 @@ public class NodeTest {
public void addArc() {
Node n2 = new Node(2);
- double w2 = 0.123;
+ float w2 = 0.123f;
String l2 = "somthing cool";
Node n3 = new Node(3);
- double w3 = 124.78;
+ float w3 = 124.78f;
String l3 = "hurray!";
Node n4 = new Node(4);
- double w4 = Double.POSITIVE_INFINITY;
+ float w4 = Float.POSITIVE_INFINITY;
String l4 = "\u0000";
Assert.assertEquals(node.size(), 0);
node.addArc(n2, w2, l2);
Assert.assertEquals(node.size(), 1);
- Arc a2 = node.outgoingArcs.get(0);
- Assert.assertEquals(a2.head, node);
- Assert.assertEquals(a2.tail, n2);
- Assert.assertEquals(a2.cost, w2);
- Assert.assertEquals(a2.label, l2);
+ Arc a2 = node.getOutgoingArcs().get(0);
+ Assert.assertEquals(a2.getHead(), node);
+ Assert.assertEquals(a2.getTail(), n2);
+ Assert.assertEquals(a2.getCost(), w2);
+ Assert.assertEquals(a2.getLabel(), l2);
node.addArc(n3, w3, l3);
Assert.assertEquals(node.size(), 2);
- Arc a3 = node.outgoingArcs.get(1);
- Assert.assertEquals(a3.head, node);
- Assert.assertEquals(a3.tail, n3);
- Assert.assertEquals(a3.cost, w3);
- Assert.assertEquals(a3.label, l3);
+ Arc a3 = node.getOutgoingArcs().get(1);
+ Assert.assertEquals(a3.getHead(), node);
+ Assert.assertEquals(a3.getTail(), n3);
+ Assert.assertEquals(a3.getCost(), w3);
+ Assert.assertEquals(a3.getLabel(), l3);
node.addArc(n4, w4, l4);
Assert.assertEquals(node.size(), 3);
- Arc a4 = node.outgoingArcs.get(2);
- Assert.assertEquals(a4.head, node);
- Assert.assertEquals(a4.tail, n4);
- Assert.assertEquals(a4.cost, w4);
- Assert.assertEquals(a4.label, l4);
+ Arc a4 = node.getOutgoingArcs().get(2);
+ Assert.assertEquals(a4.getHead(), node);
+ Assert.assertEquals(a4.getTail(), n4);
+ Assert.assertEquals(a4.getCost(), w4);
+ Assert.assertEquals(a4.getLabel(), l4);
}
}
diff --git a/test/joshua/util/io/BinaryTest.java b/test/joshua/util/io/BinaryTest.java
index cda8aba..71d83ba 100644
--- a/test/joshua/util/io/BinaryTest.java
+++ b/test/joshua/util/io/BinaryTest.java
@@ -8,8 +8,7 @@ import java.io.ObjectOutput;
import java.util.HashSet;
import java.util.Set;
-import joshua.corpus.vocab.Vocabulary;
-
+import joshua.corpus.Vocabulary;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -28,7 +27,8 @@ public class BinaryTest {
}
}
- Vocabulary vocab = new Vocabulary(words);
+ Vocabulary vocab = new Vocabulary();
+ vocab.addAll(words.toArray(new String[words.size()]));
try {