apache · kottmann · Jul 6, 2017 · Aug 2, 2017
diff --git a/opennlp-dl/pom.xml b/opennlp-dl/pom.xml
@@ -26,7 +26,7 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <nd4j.version>0.7.2</nd4j.version>
+    <nd4j.version>0.8.1-SNAPSHOT</nd4j.version>
   </properties>
 
   <dependencies>
@@ -36,18 +36,25 @@
           <version>1.7.2</version>
       </dependency>
 
+      <dependency>
+          <groupId>org.nd4j</groupId>
+          <!-- artifactId>nd4j-native-platform</artifactId -->
+          <artifactId>nd4j-cuda-8.0-platform</artifactId>
+          <version>${nd4j.version}</version>
+      </dependency>
+
       <dependency>
           <groupId>org.deeplearning4j</groupId>
           <artifactId>deeplearning4j-core</artifactId>
           <version>${nd4j.version}</version>
       </dependency>
 
-
       <dependency>
           <groupId>org.deeplearning4j</groupId>
           <artifactId>deeplearning4j-nlp</artifactId>
           <version>${nd4j.version}</version>
       </dependency>
+
       <dependency>
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-simple</artifactId>

diff --git a/opennlp-dl/src/main/java/opennlp/tools/dl/NameFinderDL.java b/opennlp-dl/src/main/java/opennlp/tools/dl/NameFinderDL.java
@@ -15,17 +15,21 @@
  * limitations under the License.
  */
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
 import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
 import org.deeplearning4j.nn.api.OptimizationAlgorithm;
 import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
@@ -58,39 +62,40 @@
 public class NameFinderDL implements TokenNameFinder {
 
   private final MultiLayerNetwork network;
-  private final WordVectors wordVectors;
+  private final Map<String, double[]> wordVectors;
   private int windowSize;
   private String[] labels;
 
-  public NameFinderDL(MultiLayerNetwork network, WordVectors wordVectors, int windowSize,
+  public NameFinderDL(MultiLayerNetwork network, Map<String, double[]> wordVectors, int windowSize,
                       String[] labels) {
     this.network = network;
     this.wordVectors = wordVectors;
     this.windowSize = windowSize;
     this.labels = labels;
   }
 
-  static List<INDArray> mapToFeatureMatrices(WordVectors wordVectors, String[] tokens, int windowSize) {
+  static List<INDArray> mapToFeatureMatrices(Map<String, double[]> wordVectors, String[] tokens, int windowSize) {
 
     List<INDArray> matrices = new ArrayList<>();
 
-    // TODO: Dont' hard code word vector dimension ...
-
     for (int i = 0; i < tokens.length; i++) {
-      INDArray features = Nd4j.create(1, 300, windowSize);
+      double[][] vectorMatrix = new double[5][300];
       for (int vectorIndex = 0; vectorIndex < windowSize; vectorIndex++) {
         int tokenIndex = i + vectorIndex - ((windowSize - 1) / 2);
         if (tokenIndex >= 0 && tokenIndex < tokens.length) {
           String token = tokens[tokenIndex];
-          double[] wv = wordVectors.getWordVector(token);
+          double[] wv = wordVectors.get(token);
           if (wv != null) {
-            INDArray vector = wordVectors.getWordVectorMatrix(token);
-            features.put(new INDArrayIndex[]{NDArrayIndex.point(0), NDArrayIndex.all(),
-                NDArrayIndex.point(vectorIndex)}, vector);
+            vectorMatrix[vectorIndex] = wv;
           }
         }
       }
-      matrices.add(features);
+
+      INDArray features2 = Nd4j.create(1, 300, windowSize);
+      features2.put(new INDArrayIndex[]{NDArrayIndex.point(0), NDArrayIndex.all()},
+          Nd4j.create(vectorMatrix).transpose());
+
+      matrices.add(features2);
     }
 
     return matrices;
@@ -153,7 +158,7 @@ public Span[] find(String[] tokens) {
   public void clearAdaptiveData() {
   }
 
-  public static MultiLayerNetwork train(WordVectors wordVectors, ObjectStream<NameSample> samples,
+  public static MultiLayerNetwork train(Map<String, double[]> wordVectors, ObjectStream<NameSample> samples,
                                         int epochs, int windowSize, String[] labels) throws IOException {
     int vectorSize = 300;
     int layerSize = 256;
@@ -191,6 +196,27 @@ public static MultiLayerNetwork train(WordVectors wordVectors, ObjectStream<Name
     return net;
   }
 
+  public static Map<String, double[]> loadGloveVectors(InputStream in) throws IOException {
+    Map<String, double[]> wordVectors = new HashMap<>();
+
+    BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+
+    String line;
+    while ((line = reader.readLine()) != null) {
+      String[] parts = line.split(" ");
+
+      double[] vector = new double[parts.length - 1];
+
+      for (int i = 1; i < parts.length; i++) {
+        vector[i - 1] = Double.parseDouble(parts[i]);
+      }
+
+      wordVectors.put(parts[0], vector);
+    }
+
+    return wordVectors;
+  }
+
   public static void main(String[] args) throws Exception {
     if (args.length != 3) {
       System.out.println("Usage: trainFile testFile gloveTxt");
@@ -202,8 +228,9 @@ public static void main(String[] args) throws Exception {
     };
 
     System.out.print("Loading vectors ... ");
-    WordVectors wordVectors = WordVectorSerializer.loadTxtVectors(
-        new File(args[2]));
+    Map<String, double[]> wordVectors = loadGloveVectors(new FileInputStream(
+        new File(args[2])));
+
     System.out.println("Done");
 
     int windowSize = 5;

diff --git a/opennlp-dl/src/main/java/opennlp/tools/dl/NameSampleDataSetIterator.java b/opennlp-dl/src/main/java/opennlp/tools/dl/NameSampleDataSetIterator.java
@@ -21,9 +21,9 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
 
-import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.dataset.api.DataSetPreProcessor;
@@ -40,13 +40,14 @@ public class NameSampleDataSetIterator implements DataSetIterator {
 
   private static class NameSampleToDataSetStream extends FilterObjectStream<NameSample, DataSet> {
 
-    private final WordVectors wordVectors;
+    private final Map<String, double[]> wordVectors;
     private final String[] labels;
     private int windowSize;
 
     private Iterator<DataSet> dataSets = Collections.emptyListIterator();
 
-    NameSampleToDataSetStream(ObjectStream<NameSample> samples, WordVectors wordVectors, int windowSize, String[] labels) {
+    NameSampleToDataSetStream(ObjectStream<NameSample> samples, Map<String, double[]> wordVectors,
+                              int windowSize, String[] labels) {
       super(samples);
       this.wordVectors = wordVectors;
       this.windowSize = windowSize;
@@ -101,17 +102,15 @@ public final DataSet read() throws IOException {
 
   private final ObjectStream<DataSet> samples;
 
-  NameSampleDataSetIterator(ObjectStream<NameSample> samples, WordVectors wordVectors, int windowSize,
+  NameSampleDataSetIterator(ObjectStream<NameSample> samples, Map<String, double[]> wordVectors, int windowSize,
                             String labels[]) throws IOException {
     this.windowSize = windowSize;
     this.labels = labels;
 
     this.samples = new NameSampleToDataSetStream(samples, wordVectors, windowSize, labels);
 
     int total = 0;
-
-    DataSet sample;
-    while ((sample = this.samples.read()) != null) {
+    while (samples.read() != null) {
       total++;
     }
 

diff --git a/opennlp-mxnet/NOTICE b/opennlp-mxnet/NOTICE
@@ -0,0 +1 @@
+Includes example code from Apache MXNet
diff --git a/opennlp-mxnet/pom.xml b/opennlp-mxnet/pom.xml
@@ -0,0 +1,87 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.opennlp</groupId>
+  <artifactId>opennlp-mxnet</artifactId>
+  <version>0.1-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <nd4j.version>0.8.1-SNAPSHOT</nd4j.version>
+  </properties>
+
+    <repositories>
+        <repository>
+            <id>scala-tools.org</id>
+            <name>Scala-tools Maven2 Repository</name>
+            <url>http://scala-tools.org/repo-releases</url>
+            <snapshots>
+                <enabled>true</enabled>
+                <updatePolicy>never</updatePolicy>
+            </snapshots>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>scala-tools.org</id>
+            <name>Scala-tools Maven2 Repository</name>
+            <url>http://scala-tools.org/repo-releases</url>
+        </pluginRepository>
+    </pluginRepositories>
+
+  <dependencies>
+      <dependency>
+          <groupId>org.apache.opennlp</groupId>
+          <artifactId>opennlp-tools</artifactId>
+          <version>1.8.1</version>
+      </dependency>
+
+      <dependency>
+          <groupId>ml.dmlc.mxnet</groupId>
+          <artifactId>mxnet-full_2.11-linux-x86_64-gpu</artifactId>
+          <version>0.10.1-SNAPSHOT</version>
+      </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+        <plugin>
+            <groupId>org.scala-tools</groupId>
+            <artifactId>maven-scala-plugin</artifactId>
+            <executions>
+                <execution>
+                    <goals>
+                        <goal>compile</goal>
+                        <goal>testCompile</goal>
+                    </goals>
+                </execution>
+            </executions>
+        </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/opennlp-mxnet/src/main/java/main/GloveUtil.java b/opennlp-mxnet/src/main/java/main/GloveUtil.java
@@ -0,0 +1,34 @@
+package main;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by blue on 7/19/17.
+ */
+public class GloveUtil {
+  public static Map<String, float[]> loadGloveVectors(InputStream in) throws IOException {
+    Map<String, float[]> wordVectors = new HashMap<String, float[]>();
+
+    BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+
+    String line;
+    while ((line = reader.readLine()) != null) {
+      String[] parts = line.split(" ");
+
+      float[] vector = new float[parts.length - 1];
+
+      for (int i = 1; i < parts.length; i++) {
+        vector[i - 1] = Float.parseFloat(parts[i]);
+      }
+
+      wordVectors.put(parts[0], vector);
+    }
+
+    return wordVectors;
+  }
+}