Library maintenance (#130)

* Move solutions and old models out of main package
* Remove/disable unused dependencies
* Update setup.py and installation procedure
* Update documentation
* Improve CI

Author: Peter Izsak

.jenkins/Jenkinsfile

@@ -2,6 +2,16 @@
 * NLP Architect Jenkinsfile
 */
 
+def virt_env = '.nlp_env'
+
+def createVirtualEnv(String name) {
+    sh "python3 -m venv ${name}"
+}
+ 
+def executeIn(String environment, String script) {
+    sh ". ${environment}/bin/activate && " + script
+}
+
 pipeline {
     agent {
         node {
@@ -20,67 +30,59 @@ pipeline {
         }
         stage('Build environment') {
             steps {
-                sh 'python3 -m venv .nlp_architect_env'
-                sh """
-                . .nlp_architect_env/bin/activate
-                pip3 install -U pip setuptools h5py
-                pip3 install -r requirements.txt
-                pip3 install -r dev-requirements.txt
-                pip3 install -e .
-                python -m spacy download en
-                """
+                createVirtualEnv virt_env
+                executeIn virt_env, 'pip3 install -U pip setuptools'
+                executeIn virt_env, 'pip3 install -U -e .[all,dev]'
+                executeIn virt_env, 'python -m spacy download en'
             }
         }
-        stage('Validation') {
-            parallel {
-                stage('Tests') {
-                    steps {
-                        sh """
-                        . .nlp_architect_env/bin/activate
-                        export LC_ALL=en_US.UTF-8
-                        export LANG=en_US.UTF-8
-                        pytest ./tests -rs -vv --cov=nlp_architect --junit-xml=pytest_unit.xml
-                        """
-                    }
-                    post {
-                        always {
-                            junit 'pytest_unit.xml'
-                        }
-                        failure {
-                            script {
-                                currentBuild.result = 'FAILURE'
-                            }
-                        }
-                    }
+        stage('Tests') {
+            steps {
+                executeIn virt_env, 'pytest ./ -rs -v -n 20 --dist=loadfile --cov=nlp_architect --junit-xml=pytest_unit.xml'
+            }
+            post {
+                always {
+                    junit 'pytest_unit.xml'
                 }
-                stage('flake8') {
-                    steps {
-                        sh """
-                        . .nlp_architect_env/bin/activate
-                        ./scripts/check_flake.sh
-                        """
+                failure {
+                    script {
+                        currentBuild.result = 'FAILURE'
                     }
-                    post {
-                        failure {
-                            script {
-                                currentBuild.result = 'FAILURE'
-                            }
-                        }
+                }
+            }
+        }
+        stage('flake8') {
+            steps {
+                executeIn virt_env, 'flake8 examples nlp_architect tests solutions --config setup.cfg'
+            }
+            post {
+                failure {
+                    script {
+                        currentBuild.result = 'FAILURE'
                     }
                 }
-                stage('pylint') {
-                    steps {
-                        sh """
-                        . .nlp_architect_env/bin/activate
-                        ./scripts/check_pylint.sh
-                        """
+            }
+        }
+        stage('pylint') {
+            steps {
+                executeIn virt_env, './scripts/check_pylint.sh'
+            }
+            post {
+                failure {
+                    script {
+                        currentBuild.result = 'FAILURE'
                     }
-                    post {
-                        failure {
-                            script {
-                                currentBuild.result = 'FAILURE'
-                            }
-                        }
+                }
+            }
+        }
+        stage('Code Style') {
+            steps {
+                executeIn virt_env, 'black --check --line-length 100 --target-version py36 examples nlp_architect solutions tests'
+            }
+            post {
+                failure {
+                    script {
+                        currentBuild.result = 'FAILURE'
                     }
                 }
             }
@@ -104,4 +106,4 @@ pipeline {
             deleteDir()
         }
     }
-}
+}

MANIFEST.in

@@ -1,10 +1,4 @@
 global-exclude .DS_Store .editorconfig *.py[cod] __pycache__
-include nlp_architect/server/services.json
 graft nlp_architect/utils/resources
-graft nlp_architect/solutions/set_expansion/ui
-graft nlp_architect/solutions/trend_analysis
-graft nlp_architect/solutions/absa_solution
-graft nlp_architect/solutions/absa_solution/style
-graft nlp_architect/server/angular-ui
 graft nlp_architect/models/absa/inference/lexicons
 graft nlp_architect/models/absa/train/lexicons

README.md

@@ -1,6 +1,6 @@
 <p align="center">
     <br>
-    <img src="https://raw.githubusercontent.com/NervanaSystems/nlp-architect/master/assets/nlp_architect_logo.png" width="400"/>
+    <img src="https://raw.githubusercontent.com/NervanaSystems/nlp-architect/master/docs-source/source/assets/nlp_architect_logo_trans.png" width="400"/>
     <br>
 <p>
 <h2 align="center">
@@ -61,20 +61,31 @@ We recommend to install NLP Architect in a new python environment, to use python
 
 ### Install using `pip`
 
-Includes only core library (for examples, tutorials and the rest clone the repo)
+Install core library only
 
 ```sh
 pip install nlp-architect
 ```
 
 ### Install from source (Github)
 
+Includes core library, examples, solutions and tutorials:
+
 ```sh
 git clone https://github.com/NervanaSystems/nlp-architect.git
 cd nlp-architect
 pip install -e .  # install in developer mode
 ```
 
+### Running Examples and Solutions
+
+To run provided examples and solutions please install the library with `[all]` flag which will install extra packages required. (requires installation from source)
+
+```sh
+pip install .[all]
+```
+
+
 ## Models
 
 NLP models that provide best (or near) in class performance:

assets/bist-demo-small.png

@@ -12,7 +12,6 @@ RUN git clone https://github.com/NervanaSystems/nlp-architect.git
 RUN export PYTHONPATH="$PYTHONPATH:/Users/nlp_user/nlp_architect"
 RUN export JUPYTER_PATH="${JUPYTER_PATH}:/Users/nlp_user/nlp_architect"
 WORKDIR "/home/nlp_user/nlp-architect"
-RUN pip3 install --no-cache-dir -e .
-RUN pip3 install --no-cache-dir -r dev-requirements.txt
+RUN pip3 install --no-cache-dir -e .[all,dev]
 EXPOSE 8080
 EXPOSE 8888

assets/ie-demo-small.png

@@ -39,6 +39,13 @@ Solution execution
 All three stages of the solution are end-to-end executed by a dedicated UI.
 An example for running the UI is available at ``examples/absa/solution/absa_solution.py``.
 
+Install solution extra packages:
+
+.. code:: python
+
+  pip install -r solutions/absa_solution/requirements.txt
+
+
 .. code:: bash
 
     python3 examples/absa/solution/absa_solution.py

assets/nlp_architect_logo_white.png

@@ -45,8 +45,8 @@ Question-Answering and information retrieval tasks.
 
 Files
 =====
-- **nlp_architect/data/babi_dialog.py**: Data loader :py:class:`class <nlp_architect.data.babi_dialog.BABI_Dialog>` to download data if not present and perform preprocessing.
-- **nlp_architect/models/memn2n_dialogue.py**: Implementation of :py:class:`MemN2N_Dialog <nlp_architect.models.memn2n_dialogue.MemN2N_Dialog>` class for dialogue tasks.
+- **examples/memn2n_dialog/babi_dialog.py**: Data loader :py:class:`class <nlp_architect.data.babi_dialog.BABI_Dialog>` to download data if not present and perform preprocessing.
+- **examples/memn2n_dialog/memn2n_dialogue.py**: Implementation of :py:class:`MemN2N_Dialog <nlp_architect.models.memn2n_dialogue.MemN2N_Dialog>` class for dialogue tasks.
 - **examples/memn2n_dialog/train_model.py**: Training script to load dataset and train memory network.
 - **examples/memn2n_dialog/interactive.py**: Inference script to run interactive session with a trained goal oriented dialog agent.
 - **examples/memn2n_dialog/interactive_utils.py**: Utilities to support interactive mode and simulate backend database.

dev-requirements.txt

@@ -35,7 +35,7 @@ Files
 - **examples/reading_comprehension/train.py** -Implements the end to end model along with the training commands
 - **examples/reading_comprehension/utils.py**- Implements different utility functions to set up the data loader and for evaluation.
 - **examples/reading_comprehension/prepare_data.py**- Implements the pipeline to preprocess the dataset
-- **nlp_architect/models/matchlstm_ansptr.py**- Defines the end to end MatchLSTM and :py:class:`Answer_Pointer <nlp_architect.models.matchlstm_ansptr.MatchLSTM_AnswerPointe>` network for Reading Comprehension
+- **examples/reading_comprehension/matchlstm_ansptr.py**- Defines the end to end MatchLSTM and :py:class:`Answer_Pointer <nlp_architect.models.matchlstm_ansptr.MatchLSTM_AnswerPointe>` network for Reading Comprehension
 
 Dataset
 =======

docker/Dockerfile

@@ -24,5 +24,3 @@
 .. autoclass:: nlp_architect.api.ner_api.NerApi
 
 .. autoclass:: nlp_architect.api.intent_extraction.IntentExtractionApi
-
-.. autoclass:: nlp_architect.api.machine_comprehension.MachineComprehensionApi

docs-source/source/absa_solution.rst

@@ -270,7 +270,7 @@
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
-#  dir menu entry, description, category)
+#  directory menu entry, description, category)
 texinfo_documents = [('index', u'NLP Architect Documentation',
                       u'Intel Corporation'), ]
 

docs-source/source/archived/memn2n.rst

@@ -29,7 +29,7 @@ development packages:
 
 .. code:: bash
 
-    pip3 install -r dev-requirements.txt
+    pip3 install -U -e .[dev]
 
 
 .. include:: CONTRIBUTING.rst

docs-source/source/archived/reading_comprehension.rst

@@ -36,14 +36,6 @@ nlp\_architect.api.intent\_extraction\_api module
     :undoc-members:
     :show-inheritance:
 
-nlp\_architect.api.machine\_comprehension\_api module
------------------------------------------------------
-
-.. automodule:: nlp_architect.api.machine_comprehension_api
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nlp\_architect.api.ner\_api module
 ----------------------------------
 

assets/nlp_architect_logo.png docs-source/source/assets/nlp_architect_logo_trans.png

@@ -11,22 +11,6 @@ Subpackages
 Submodules
 ----------
 
-nlp\_architect.data.amazon\_reviews module
-------------------------------------------
-
-.. automodule:: nlp_architect.data.amazon_reviews
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nlp\_architect.data.babi\_dialog module
----------------------------------------
-
-.. automodule:: nlp_architect.data.babi_dialog
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nlp\_architect.data.conll module
 --------------------------------