diff --git a/.gitignore b/.gitignore index a2cddf1cdf..33241c26de 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ thirdparty/* !thirdparty/install.sh test/samples/ .idea +doc/_build/* diff --git a/CMakeLists.txt b/CMakeLists.txt index fa2c712ca8..725054fb13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,9 +19,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(singa) -SET(PACKAGE_VERSION "2.0.0") +SET(PACKAGE_VERSION "2.1.0") SET(SINGA_MAJOR_VERSION 2) # 0 - -SET(SINGA_MINOR_VERSION 0) # 0 - 9 +SET(SINGA_MINOR_VERSION 1) # 0 - 9 SET(SINGA_PATCH_VERSION 0) # 0 - 99 MATH(EXPR SINGA_VERSION "${SINGA_MAJOR_VERSION} * 1000 + ${SINGA_MINOR_VERSION} * 100 + ${SINGA_PATCH_VERSION}") diff --git a/doc/README.md b/doc/README.md deleted file mode 100644 index 6de5072984..0000000000 --- a/doc/README.md +++ /dev/null @@ -1,54 +0,0 @@ - -# How to Contribute to Documentation - - -## Website -This document gives step-by-step instructions for deploying [SINGA website](http://singa.incubator.apache.org). - -SINGA website is built by [Sphinx](http://www.sphinx-doc.org) >=1.4.4 from a source tree stored in git: https://github.com/apache/incubator-singa/tree/master/doc. - -To install Sphinx: - - $ pip install -U Sphinx - -To install the markdown support for Sphinx: - - $ pip install recommonmark - -To install the rtd theme: - - $ pip install sphinx_rtd_theme - -You can build the website by executing the following command from the doc folder: - - $ ./build.sh html - -Committers can update the [SINGA website](http://singa.apache.org/en/index.html) by following these steps: - - $ cd _build - $ svn co https://svn.apache.org/repos/asf/incubator/singa/site/trunk - $ cp -r html/* trunk - # svn add - $ svn commit -m "commit messages" --username --password - - -## CPP API - -To generate docs, run "doxygen" from the doc folder (Doxygen >= 1.8 recommended) diff --git a/doc/conf.py b/doc/conf.py index 0cdede60d2..d7a98b4fed 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -19,6 +19,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Directives for doing spell check only for python comments +# cSpell:includeRegExp #.* +# cSpell:includeRegExp ("""|''')[^]* + # This file is execfile()d with the current directory set to its # containing dir. # @@ -40,7 +44,6 @@ sys.path.append(os.path.abspath('../python/')) sys.path.append(os.path.abspath('../build/python/')) - # -- General configuration ------------------------------------------------ from recommonmark.parser import CommonMarkParser @@ -71,7 +74,7 @@ # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = 'en/index' # General information about the project. project = u'incubator-singa' @@ -139,7 +142,6 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False - # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -168,7 +170,7 @@ # The name of an image file (relative to this directory) to place at the top # of the sidebar. # -html_logo = '/singa.png' +html_logo = '_static/singa.png' # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 @@ -264,21 +266,21 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples @@ -315,30 +317,25 @@ # # latex_domain_indices = True - # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'incubator-singa', u'incubator-singa Documentation', - [author], 1) -] +man_pages = [(master_doc, 'incubator-singa', u'incubator-singa Documentation', + [author], 1)] # If true, show URL addresses after external links. # # man_show_urls = False - # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'incubator-singa', u'incubator-singa Documentation', - author, 'incubator-singa', 'One line description of project.', - 'Miscellaneous'), + (master_doc, 'incubator-singa', u'incubator-singa Documentation', author, + 'incubator-singa', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/doc/en/_templates/layout.html b/doc/en/_templates/layout.html deleted file mode 100755 index 2f9ca0dc78..0000000000 --- a/doc/en/_templates/layout.html +++ /dev/null @@ -1,56 +0,0 @@ -{# - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -#} -{% extends "!layout.html" %} - -{% block extrahead %} - -{% endblock %} - -{% block footer %} - -
- - - - - - incubator-singa - v: {{ version }} - - -
-
-
English
-
中文
- -
- -
-
- - - Fork me on GitHub - - -{{ super() }} -{% endblock %} diff --git a/doc/en/community/team-list.rst b/doc/en/community/team-list.rst index 043540c9a2..28ad4c6b78 100644 --- a/doc/en/community/team-list.rst +++ b/doc/en/community/team-list.rst @@ -46,11 +46,11 @@ The SINGA community has developers mainly from National University of Singapore, +--------------------+--------------------------------+-----------------------------------------------+ | Haibo Chen | hzchenhaibo@corp.netease.com | NetEase | +--------------------+--------------------------------+-----------------------------------------------+ -| Anh Dinh | dinhtta@apache.org | National University of Singapore | +| Anh Dinh | dinhtta@apache.org | Singapore University of Technology and Design | +--------------------+--------------------------------+-----------------------------------------------+ -| Jinyang Gao | jinyang@apache.org | National University of Singapore | +| Jinyang Gao | jinyang@apache.org | DAMO Academy, Alibaba Group | +--------------------+--------------------------------+-----------------------------------------------+ -| Xin Ji | jixin@comp.nus.edu.sg | National University of Singapore | +| Xin Ji | jixin@comp.nus.edu.sg | Visenze | +--------------------+--------------------------------+-----------------------------------------------+ | Chonho Lee | chonho@gmail.com | Osaka University | +--------------------+--------------------------------+-----------------------------------------------+ @@ -62,9 +62,9 @@ The SINGA community has developers mainly from National University of Singapore, +--------------------+--------------------------------+-----------------------------------------------+ | Anthony K. H. Tung | atung@comp.nus.edu.sg | National University of Singapore | +--------------------+--------------------------------+-----------------------------------------------+ -| Ji Wang | wangji@comp.nus.edu.sg | National University of Singapore | +| Ji Wang | wangji@comp.nus.edu.sg | Hangzhou MZH Technologies | +--------------------+--------------------------------+-----------------------------------------------+ -| Sheng Wang | wangsh@apache.org | National University of Singapore | +| Sheng Wang | wangsh@apache.org | DAMO Academy, Alibaba Group | +--------------------+--------------------------------+-----------------------------------------------+ | Wei Wang | wangwei@apache.org | National University of Singapore | +--------------------+--------------------------------+-----------------------------------------------+ @@ -72,13 +72,24 @@ The SINGA community has developers mainly from National University of Singapore, +--------------------+--------------------------------+-----------------------------------------------+ | Wenfeng Wu | wuwf@comp.nus.edu.sg | National University of Singapore | +--------------------+--------------------------------+-----------------------------------------------+ -| Zhongle Xie | zhongle@apache.org | National University of Singapore | +| Zhongle Xie | zhongle@apache.org | Hangzhou MZH Technologies | +--------------------+--------------------------------+-----------------------------------------------+ -| Meihui Zhang | meihui_zhang@sutd.edu.sg | Singapore University of Technology and Design | +| Meihui Zhang | meihui_zhang@sutd.edu.sg | Beijing Institute of Technology | +--------------------+--------------------------------+-----------------------------------------------+ | Kaiping Zheng | kaiping@apache.org | National University of Singapore | +--------------------+--------------------------------+-----------------------------------------------+ | Moaz Reyad | moaz@apache.org | University of Genoa | +--------------------+--------------------------------+-----------------------------------------------+ -| Chang Yao | yaochang2009@gmail.com | yzBigdata | +| Chang Yao | yaochang2009@gmail.com | Hangzhou MZH Technologies | +--------------------+--------------------------------+-----------------------------------------------+ + + + +How to become a SINGA committer +-------------------------------- + +SINGA committers are members that have write access to SINGA code and documentation repositories. They can contribute by themselves and accept other contributions. + +Becoming a SINGA committer requires understanding the Apache Way and being an active member in the SINGA project. An active member participates on mailing lists, answers questions, contributes code, tests and documentation, presents SINGA in talks and seminars, verifies releases and report issues or feature requests. + +New committers can be suggested by current committers or PPMC members, and voted upon by the PPMC according to Apache guidelines. You can start by talking to other committers and ask for their advice and guidance. \ No newline at end of file diff --git a/doc/en/develop/build.md b/doc/en/develop/build.md new file mode 100644 index 0000000000..6e40f3e00b --- /dev/null +++ b/doc/en/develop/build.md @@ -0,0 +1,408 @@ + + +# Build SINGA from Source + + +The source files could be downloaded either as a +[tar.gz file](https://dist.apache.org/repos/dist/dev/incubator/singa/), or as a git repo + + $ git clone https://github.com/apache/incubator-singa.git + $ cd incubator-singa/ + +If you want to contribute code to SINGA, refer to [this page]() for the steps and requirements. + +## Use Conda to build SINGA + +Conda-build is a building tool that installs the dependent libraries from anaconda cloud and +executes the building scripts. + +To install conda-build (after installing conda) + + conda install conda-build + +### Build CPU Version + +To build the CPU version of SINGA + + conda build tool/conda/singa/ + +The above commands have been tested on Ubuntu 16.04 and Mac OSX. +Refer to the [Travis-CI page](https://travis-ci.org/apache/incubator-singa) for more information. + +### Build GPU Version + +To build the GPU version of SINGA, the building machine must have Nvida GPU, and the CUDA driver (>= 384.81), CUDA toolkit (>=9) and cuDNN (>=7) must have be installed. The following two Docker images provide the building environment: + +1. apache/singa:conda-cuda9.0 +2. apache/singa:conda-cuda10.0 + +Once the building environment is ready, you need to export the CUDA version first, and then run conda command to build SINGA + + export CUDA=x.y (e.g. 9.0) + conda build tool/conda/singa/ + +### Post Processing + +The location of the generated package file (`.tar.gz`) is shown on the screen. +The generated package can be installed directly, + + conda install -c conda-forge --use-local + +or uploaded to anaconda cloud for others to download and install. You need to register an account on anaconda for [uploading the package](https://docs.anaconda.com/anaconda-cloud/user-guide/getting-started/). + + conda install anaconda-client + anaconda login + anaconda upload -l main + +After uploading the package to the cloud, you can see it on [Anaconda Cloud](https://anaconda.org/) website or via the following command + + conda search -c singa + +Each specific SINGA package is identified by the version and build string. To install a specific SINGA package, you need to provide all the information, e.g., + + conda install -c -c conda-forge singa=2.1.0.dev=cpu_py36 + +To make the installation command simple, you can create the following additional packages which depend on the latest CPU and GPU SINGA packages. + + # for singa-cpu + conda build tool/conda/cpu/ --python=3.6 + conda build tool/conda/cpu/ --python=3.7 + # for singa-gpu + conda build tool/conda/gpu/ --python=3.6 + conda build tool/conda/gpu/ --python=3.7 + +Therefore, when you run + + conda install -c -c conda-forge singa-xpu + +(`xpu` is either 'cpu' or 'gpu'), the corresponding real SINGA package is installed as the dependent library. + +## Use native tools to build SINGA on Ubuntu + +Refer to SINGA [Dockerfiles](https://github.com/apache/incubator-singa/blob/master/tool/docker/devel/ubuntu/cuda9/Dockerfile#L30) +for the instructions of installing the dependent libraries on Ubuntu 16.04. You can also create a Docker container using the [devel images]() and build SINGA inside the container. +To build SINGA with GPU, MKLDNN, Python and unit tests, run the following instructions + + mkdir build # at the root of incubator-singa folder + cd build + cmake -DENABLE_TEST=ON -DUSE_CUDA=ON -DUSE_MKLDNN=ON -DUSE_PYTHON3=ON .. + make + cd python + pip install . + +The details of the CMake options are explained in the last section of this page. +The last command install the Python package. You can also run `pip install -e .`, which creates symlinks instead of copying the Python files into the site-package folder. + +If SINGA is compiled with ENABLE_TEST=ON, you can run the unit tests by + + $ ./bin/test_singa + +You can see all the testing cases with testing results. If SINGA passes all +tests, then you have successfully installed SINGA. + + +## Use native tools to Build SINGA on Centos7 + +Building from source will be different for Centos7 as package names differ.Follow the instructions given below. + +### Installing dependencies + +Basic packages/libraries + + sudo yum install freetype-devel libXft-devel ncurses-devel openblas-devel blas-devel lapack devel atlas-devel kernel-headers unzip wget pkgconfig zip zlib-devel libcurl-devel cmake curl unzip dh-autoreconf git python-devel glog-devel protobuf-devel +For build-essential + + sudo yum group install "Development Tools" +For installing swig + + sudo yum install pcre-devel + wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz + tar xvzf swig-3.0.10.tar.gz + cd swig-3.0.10.tar.gz + ./configure --prefix=${RUN} + make + make install + For installing gfortran + + sudo yum install centos-release-scl-rh + sudo yum --enablerepo=centos-sclo-rh-testing install devtoolset-7-gcc-gfortran +For installing pip and other packages + + sudo yum install epel-release + sudo yum install python-pip + pip install matplotlib numpy pandas scikit-learn pydot + +### Installation +Follow steps 1-5 of _Use native tools to build SINGA on Ubuntu_ + +### Testing +You can run the unit tests by, + + $ ./bin/test_singa +You can see all the testing cases with testing results. If SINGA passes all +tests, then you have successfully installed SINGA. + +## Compile SINGA on Windows + +Instructions for building on Windows with Python support can be found [here](install_win.html). + +## More details about the compilation options + +### USE_MODULES (deprecated) + +If protobuf and openblas are not installed, you can compile SINGA together with them + + $ In SINGA ROOT folder + $ mkdir build + $ cd build + $ cmake -DUSE_MODULES=ON .. + $ make + +cmake would download OpenBlas and Protobuf (2.6.1) and compile them together +with SINGA. + +You can use `ccmake ..` to configure the compilation options. +If some dependent libraries are not in the system default paths, you need to export +the following environment variables + + export CMAKE_INCLUDE_PATH= + export CMAKE_LIBRARY_PATH= + +### USE_PYTHON + +Option for compiling the Python wrapper for SINGA, + + $ cmake -DUSE_PYTHON=ON .. + $ make + $ cd python + $ pip install . + + +### USE_CUDA + +Users are encouraged to install the CUDA and +[cuDNN](https://developer.nvidia.com/cudnn) for running SINGA on GPUs to +get better performance. + +SINGA has been tested over CUDA 9/10, and cuDNN 7. If cuDNN is +installed into non-system folder, e.g. /home/bob/local/cudnn/, the following +commands should be executed for cmake and the runtime to find it + + $ export CMAKE_INCLUDE_PATH=/home/bob/local/cudnn/include:$CMAKE_INCLUDE_PATH + $ export CMAKE_LIBRARY_PATH=/home/bob/local/cudnn/lib64:$CMAKE_LIBRARY_PATH + $ export LD_LIBRARY_PATH=/home/bob/local/cudnn/lib64:$LD_LIBRARY_PATH + +The cmake options for CUDA and cuDNN should be switched on + + # Dependent libs are install already + $ cmake -DUSE_CUDA=ON .. + $ make + +### USE_MKLDNN + +User can enable MKL-DNN to enhance the performance of CPU computation. + +Installation guide of MKL-DNN could be found [here](https://github.com/intel/mkl-dnn#installation). + +SINGA has been tested over MKL-DNN v0.17.2. + +To build SINGA with MKL-DNN support: + + # Dependent libs are installed already + $ cmake -DUSE_MKLDNN=ON .. + $ make + + + +### USE_OPENCL + +SINGA uses opencl-headers and viennacl (version 1.7.1 or newer) for OpenCL support, which +can be installed using via + + # On Ubuntu 16.04 + $ sudo apt-get install opencl-headers, libviennacl-dev + # On Fedora + $ sudo yum install opencl-headers, viennacl + +Additionally, you will need the OpenCL Installable Client Driver (ICD) for the platforms that you want to run OpenCL on. + +* For AMD and nVidia GPUs, the driver package should also install the correct OpenCL ICD. +* For Intel CPUs and/or GPUs, get the driver from the [Intel website.](https://software.intel.com/en-us/articles/opencl-drivers) Note that the drivers provided on that website only supports recent CPUs and Iris GPUs. +* For older Intel CPUs, you can use the `beignet-opencl-icd` package. + +Note that running OpenCL on CPUs is not currently recommended because it is slow. +Memory transfer is on the order of whole seconds (1000's of ms on CPUs as compared to 1's of ms on GPUs). + +More information on setting up a working OpenCL environment may be found [here](https://wiki.tiker.net/OpenCLHowTo). + +If the package version of ViennaCL is not at least 1.7.1, you will need to build it from source: + +Clone [the repository from here](https://github.com/viennacl/viennacl-dev), checkout the `release-1.7.1` tag and build it. +Remember to add its directory to `PATH` and the built libraries to `LD_LIBRARY_PATH`. + +To build SINGA with OpenCL support (tested on SINGA 1.1): + + $ cmake -DUSE_OPENCL=ON .. + $ make + + +### PACKAGE + +This setting is used to build the Debian package. Set PACKAGE=ON and build the package with make command like this: + + $ cmake -DPACKAGE=ON + $ make package + +## FAQ + +* Q: Error from 'import singa' + + A: Please check the detailed error from `python -c "from singa import _singa_wrap"`. Sometimes it is caused by the dependent libraries, e.g. there are multiple versions of protobuf, missing of cudnn, numpy version mismatch. Following steps show the solutions for different cases + 1. Check the cudnn and cuda. If cudnn is missing or not match with the wheel version, you can download the correct version of cudnn into ~/local/cudnn/ and + + $ echo "export LD_LIBRARY_PATH=/home//local/cudnn/lib64:$LD_LIBRARY_PATH" >> ~/.bashrc + + 2. If it is the problem related to protobuf. You can install protobuf (3.6.1) from source into a local folder, say ~/local/; Decompress the tar file, and then + + $ ./configure --prefix=/home/local + $ make && make install + $ echo "export LD_LIBRARY_PATH=/home//local/lib:$LD_LIBRARY_PATH" >> ~/.bashrc + $ source ~/.bashrc + + 3. If it cannot find other libs including python, then create virtual env using pip or conda; + + 4. If it is not caused by the above reasons, go to the folder of `_singa_wrap.so`, + + $ python + >> import importlib + >> importlib.import_module('_singa_wrap') + + Check the error message. For example, if the numpy version mismatches, the error message would be, + + RuntimeError: module compiled against API version 0xb but this version of numpy is 0xa + + Then you need to upgrade the numpy. + + +* Q: Error from running `cmake ..`, which cannot find the dependent libraries. + + A: If you haven't installed the libraries, install them. If you installed + the libraries in a folder that is outside of the system folder, e.g. /usr/local, + you need to export the following variables + + $ export CMAKE_INCLUDE_PATH= + $ export CMAKE_LIBRARY_PATH= + + +* Q: Error from `make`, e.g. the linking phase + + A: If your libraries are in other folders than system default paths, you need + to export the following varaibles + + $ export LIBRARY_PATH= + $ export LD_LIBRARY_PATH= + + +* Q: Error from header files, e.g. 'cblas.h no such file or directory exists' + + A: You need to include the folder of the cblas.h into CPLUS_INCLUDE_PATH, + e.g., + + $ export CPLUS_INCLUDE_PATH=/opt/OpenBLAS/include:$CPLUS_INCLUDE_PATH + +* Q:While compiling SINGA, I get error `SSE2 instruction set not enabled` + + A:You can try following command: + + $ make CFLAGS='-msse2' CXXFLAGS='-msse2' + +* Q:I get `ImportError: cannot import name enum_type_wrapper` from google.protobuf.internal when I try to import .py files. + + A: You need to install the python binding of protobuf, which could be installed via + + $ sudo apt-get install protobuf + + or from source + + $ cd /PROTOBUF/SOURCE/FOLDER + $ cd python + $ python setup.py build + $ python setup.py install + +* Q: When I build OpenBLAS from source, I am told that I need a Fortran compiler. + + A: You can compile OpenBLAS by + + $ make ONLY_CBLAS=1 + + or install it using + + $ sudo apt-get install libopenblas-dev + +* Q: When I build protocol buffer, it reports that GLIBC++_3.4.20 not found in /usr/lib64/libstdc++.so.6. + + A: This means the linker found libstdc++.so.6 but that library + belongs to an older version of GCC than was used to compile and link the + program. The program depends on code defined in + the newer libstdc++ that belongs to the newer version of GCC, so the linker + must be told how to find the newer libstdc++ shared library. + The simplest way to fix this is to find the correct libstdc++ and export it to + LD_LIBRARY_PATH. For example, if GLIBC++_3.4.20 is listed in the output of the + following command, + + $ strings /usr/local/lib64/libstdc++.so.6|grep GLIBC++ + + then you just set your environment variable as + + $ export LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH + +* Q: When I build glog, it reports that "src/logging_unittest.cc:83:20: error: ‘gflags’ is not a namespace-name" + + A: It maybe that you have installed gflags with a different namespace such as "google". so glog can't find 'gflags' namespace. + Because it is not necessary to have gflags to build glog. So you can change the configure.ac file to ignore gflags. + + 1. cd to glog src directory + 2. change line 125 of configure.ac to "AC_CHECK_LIB(gflags, main, ac_cv_have_libgflags=0, ac_cv_have_libgflags=0)" + 3. autoreconf + + After this, you can build glog again. + +* Q: When using virtual environment, every time I run pip install, it would reinstall numpy. However, the numpy would not be used when I `import numpy` + + A: It could be caused by the `PYTHONPATH` which should be set to empty when you are using virtual environment to avoid the conflicts with the path of + the virtual environment. + +* Q: When compiling PySINGA from source, there is a compilation error due to the missing of + + A: Please install numpy and export the path of numpy header files as + + $ export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH + +* Q: When I run SINGA in Mac OS X, I got the error "Fatal Python error: PyThreadState_Get: no current thread Abort trap: 6" + + A: This error happens typically when you have multiple version of Python on your system and you installed SINGA via pip (this problem is resolved for installation via conda), + e.g, the one comes with the OS and the one installed by Homebrew. The Python linked by PySINGA must be the same as the Python interpreter. + You can check your interpreter by `which python` and check the Python linked by PySINGA via `otool -L `. + To fix this error, compile SINGA with the correct version of Python. + In particular, if you build PySINGA from source, you need to specify the paths when invoking [cmake](http://stackoverflow.com/questions/15291500/i-have-2-versions-of-python-installed-but-cmake-is-using-older-version-how-do) + + $ cmake -DPYTHON_LIBRARY=`python-config --prefix`/lib/libpython2.7.dylib -DPYTHON_INCLUDE_DIR=`python-config --prefix`/include/python2.7/ .. + + If installed PySINGA from binary packages, e.g. debian or wheel, then you need to change the python interpreter, e.g., reset the $PATH to put the correct path of Python at the front position. \ No newline at end of file diff --git a/doc/en/develop/contribute-code.md b/doc/en/develop/contribute-code.md index bbb22de0ec..cc7a4bbdb0 100644 --- a/doc/en/develop/contribute-code.md +++ b/doc/en/develop/contribute-code.md @@ -16,19 +16,32 @@ specific language governing permissions and limitations under the License. --> -## How to Contribute Code +# How to Contribute Code +## Coding Style -### Coding Style +The SINGA codebase follows the Google Style for both [CPP](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml) and [Python](http://google.github.io/styleguide/pyguide.html) code. -The SINGA codebase follows the [Google C++ Style Guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml). +A simple way to enforce the Google coding styles is to use the linting and formating tools in the Visual Studio Code editor: -To check if your code follows the style, you can use the provided cpplint tool: + * [C/C++ extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cpptools) + * [Python extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python) - $ ./tool/cpplint.py YOUR_FILE +Once the extensions are installed, edit the settings.json file. + "editor.formatOnSave": true, + "python.formatting.provider": "yapf", + "python.formatting.yapfArgs": [ + "--style", + "{based_on_style: google}" + ], + "python.linting.enabled": true, + "python.linting.lintOnSave": true, + "C_Cpp.clang_format_style": "Google" -### JIRA format +You need to fix the format errors before submitting the pull requests. + +## JIRA format Like other Apache projects, SINGA uses JIRA to track bugs, improvements and other high-level discussions (e.g., system design and features). Github pull requests are @@ -40,38 +53,54 @@ used for implementation discussions, e.g., code review and code merge. document. * Set [required fields](https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark#ContributingtoSpark-JIRA) -### Pull Request - -The work flow is - -* Fork the [SINGA Github repository](https://github.com/apache/incubator-singa) to -your own Github account. - -* Clone your fork, create a new branch (e.g., feature-foo or fixbug-foo), - work on it. After finishing your job, - [rebase](https://git-scm.com/book/en/v2/Git-Branching-Rebasing) it to the - current latest master and push commits to your own Github account (the new - branch). - -* Open a pull request against the master branch of apache/incubator-singa. -The PR title should be of the form SINGA-xxxx Title, where -SINGA-xxxx is the relevant JIRA number, and Title may be the JIRA's title or a -more specific title describing the PR itself, for example, "SINGA-6 Implement thread-safe singleton". Detailed description can be copied from the JIRA. -Consider identifying committers or other contributors who have worked on the -code being changed. Find the file(s) in Github and click "Blame" to see a -line-by-line annotation of who changed the code last. You can add @username in -the PR description to ping them immediately. -Please state that the contribution is your original work and that you license -the work to the project under the project's open source license. Further commits (e.g., bug fix) -to your new branch will be added to this pull request automatically by Github. - -* Wait for one committer to review the patch. If no conflicts, the committers will merge it with -the master branch. The merge should a) not use rebase b) disable fast forward merge c) check the -commit message format and test the code/feature. - -* If there are too many small commit messages, you will be told to squash your commits into fewer meaningful -commits. If your commit message does not follow the format (i.e., SINGA-xxxx), you will be told to -reword your commit message. Both changes can be done using interactive git rebase. Once you -get the commits corrected, push them to you own github again. Your pull request -will be automatically updated. For details, please refer to -[Rebase Pull Requests](https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request). +## Git Workflow + + +1. Fork the [SINGA Github repository](https://github.com/apache/incubator-singa) to your own Github account. + +2. Clone the **repo** (short for repository) from your Github + + git clone https://github.com//incubator-singa.git + git remote add apache https://github.com/apache/incubator-singa.git + +3. Create a new branch (e.g., `feature-foo` or `fixbug-foo`), work on it and commit your code. + + git checkout -b feature-foo + # write your code + git add + git commit + + The commit message should have a **title which consists of the JIRA ticket No (SINGA-xxx) and title**. A brief description of the commit should be added in the commit message. + + If your branch has many small commits, you need to clean those commits via + + git rebase -i + + You can [squash and reword](https://help.github.com/en/articles/about-git-rebase) the commits. + +4. When you are working on the code, the `master` of SINGA may have been updated by others; In this case, you need to pull the latest master + + git checkout master + git pull apache master:master + git checkout feature-foo + + +5. [Rebase](https://git-scm.com/book/en/v2/Git-Branching-Rebasing) `feature-foo` onto the `master` branch and push commits to your own Github account (the new branch). + + git rebase master + git push origin feature-foo:feature-foo + +6. Open a pull request (PR) against the master branch of apache/incubator-singa on Github website. The PR title should be the JIRA ticket title. If you want to inform other contributors who worked on the same files, you can find the file(s) on Github and click "Blame" to see a line-by-line annotation of who changed the code last. Then, you can add @username in the PR description to ping them immediately. Please state that the contribution is your original work and that you license the work to the project under the project's open source license. Further commits (e.g., bug fix) to your new branch will be added to this pull request automatically by Github. + +7. Wait for committers to review the PR. If no conflicts and errors, the committers will merge it with the master branch. The merge should **a) not use rebase b) disable fast forward merge c) check the commit message format and test the code/feature**. During this time, the master of SINGA may have been updated by others, and then you need to [merge the latest master](https://docs.fast.ai/dev/git.html#how-to-keep-your-feature-branch-up-to-date) to resolve conflicts. Some people [rebase the PR onto the latest master](https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request) instead of merging. However, if other developers fetch this PR to add new features and then send PR, the rebase operation would introduce **duplicate commits** (with different hash) in the future PR. See [The Golden Rule of Rebasing](https://www.atlassian.com/git/tutorials/merging-vs-rebasing) for the details of when to avoid using rebase. Another simple solution to update the PR (to fix conflicts or commit errors) is to checkout a new branch from the latest master branch of Apache SINGAS repo; copy and paste the updated/added code; commit and send a new PR. + + +## Developing Environment + +Visual Studio Code is recommended as the editor. Extensions like Python, C/C++, Code Spell Checker, autoDocstring, vim, Remote Development could be installed. A reference configuration (i.e., `settings.json`) of these extensions is [here](https://gist.github.com/nudles/3d23cfb6ffb30ca7636c45fe60278c55). + +If you update the CPP code, you need to recompile SINGA [from source](./build.md). It is recommended to use the native building tools in the `*-devel` Docker images or `conda build`. + +If you only update the Python code, you can install SINGAS once, and then copy the updated Python files to replace those in the Python installation folder, + + cp python/singa/xx.py /lib/python3.7/site-packages/singa/ \ No newline at end of file diff --git a/doc/en/develop/contribute-docs.md b/doc/en/develop/contribute-docs.md new file mode 100644 index 0000000000..a69e72bed8 --- /dev/null +++ b/doc/en/develop/contribute-docs.md @@ -0,0 +1,103 @@ + + +# How to Contribute to Documentation + +## Website + +This document gives step-by-step instructions for deploying [SINGA website](http://singa.incubator.apache.org). + +SINGA website is built by [Sphinx](http://www.sphinx-doc.org) from a source tree stored in the [git repo](https://github.com/apache/incubator-singa/tree/master/doc). + +To install Sphinx: + + pip install -U Sphinx==1.5.6 + +To install the markdown support for Sphinx: + + pip install recommonmark==0.5.0 + +To install the rtd theme: + + pip install sphinx_rtd_theme==0.4.3 + +You can build the website by executing the following command from the doc folder: + + ./build.sh html + +Committers can update the [SINGA website](http://singa.apache.org/en/index.html) by copying the updated files to the [website repo](https://github.com/apache/incubator-singa-site) (suppose the site repo is ~/incubator-singa-sit) + + cd _build + rsync --checksum -rvh html/ ~/incubator-singa-site/ + cd ~/incubator-singa-site + git commit -m "update xxxx" + git push + +We fix the versions of the libs in order to generate the same (checksum) html file if the source file is not changed. Otherwise, everytime we build the documentation, the html file of the same source file could be different. As a result, many html files in the site repo need updating. + +## Python API + +## CPP API + +To generate docs, run "doxygen" from the doc folder (Doxygen >= 1.8 recommended) + +## Using Visual Studio Code (vscode) + +### Preview + +The document files (rst and md files) can be previewed in vscode via the [reStructuredText Extension](https://docs.restructuredtext.net/). + +1. Install the extension in vscode. +2. Install the dependent libs. All libs required to build the website should be installed (see the above instructions). In addition, there are two more libs to be installed. + + pip install sphinx-autobuild=0.7.1 + pip install doc8=0.8.0 +3. Configure the conf path for `restructuredtext.confPath` to the [conf.py](./conf.py) + +### Docstring Snippet + +[autoDocstring](https://marketplace.visualstudio.com/items?itemName=njpwerner.autodocstring) generates the docstring of functions, classes, etc. Choose the DocString Format to `google`. + +### Spell Check + +[Code Spell Checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker) can be configured to check the comments of the code, or .md and .rst files. + +To do spell check only for comments of Python code, add the following snippet via `File - Preferences - User Snippets - python.json` + + "cspell check" : { + "prefix": "cspell", + "body": [ + "# Directives for doing spell check only for python and c/cpp comments", + "# cSpell:includeRegExp #.* ", + "# cSpell:includeRegExp (\"\"\"|''')[^\1]*\1", + "# cSpell: CStyleComment", + ], + "description": "# spell check only for python comments" + } + +To do spell check only for comments of Cpp code, add the following snippet via `File - Preferences - User Snippets - cpp.json` + + "cspell check" : { + "prefix": "cspell", + "body": [ + "// Directive for doing spell check only for cpp comments", + "// cSpell:includeRegExp CStyleComment", + ], + "description": "# spell check only for cpp comments" + } diff --git a/doc/en/develop/how-contribute.md b/doc/en/develop/how-contribute.md index 399a9545f3..c0d5b72726 100644 --- a/doc/en/develop/how-contribute.md +++ b/doc/en/develop/how-contribute.md @@ -21,16 +21,5 @@ As with any open source project, there are several ways you can help: * Join the [mailing list](http://singa.apache.org/en/community/mail-lists.html) and answer other user's questions. -* [Build SINGA](http://singa.apache.org/en/docs/installation.html) by yourself. -* Report bugs, feature requests and other issues in the [issue tracking](http://singa.apache.org/en/community/issue-tracking.html) application. -* Check SINGA's [development schedule](http://singa.apache.org/en/develop/schedule.html) and [contribute code](http://singa.apache.org/en/develop/contribute-code.html) by providing patches. * [Help with the documentation](http://singa.apache.org/en/develop/contribute-docs.html) by updating webpages that are lacking or unclear. - - -# How to become a SINGA committer - -SINGA committers are members that have write access to SINGA code and documentation repositories. They can contribute by themselves and accept other contributions. - -Becoming a SINGA committer requires understanding the Apache Way and being an active member in the SINGA project. An active member participates on mailing lists, answers questions, contributes code, tests and documentation, presents SINGA in talks and seminars, verifies releases and report issues or feature requests. - -New committers can be suggested by current committers or PPMC members, and voted upon by the PPMC according to Apache guidelines. You can start by talking to other committers and ask for their advice and guidance. +* [Contribute code to SINGA](http://singa.apache.org/en/develop/contribute-code.html) by fixing errors or adding new features. [All issues are tracked](http://singa.apache.org/en/community/issue-tracking.html) on the JIRA system. \ No newline at end of file diff --git a/doc/en/docs/index.rst b/doc/en/docs/index.rst index 76ece28434..015f4623a4 100644 --- a/doc/en/docs/index.rst +++ b/doc/en/docs/index.rst @@ -23,21 +23,9 @@ Documentation installation software_stack - benchmark device tensor - layer - net - initializer - loss - metric - optimizer autograd - data - image_tool - snapshot - converter - utils - model_zoo/index - security - + onnx + benchmark + model_zoo/index \ No newline at end of file diff --git a/doc/en/docs/installation.md b/doc/en/docs/installation.md index 03be085486..6864698d57 100644 --- a/doc/en/docs/installation.md +++ b/doc/en/docs/installation.md @@ -22,397 +22,98 @@ Conda is a package manager for Python, CPP and other packages. -Currently, SINGA has conda packages (Python 2.7 and Python 3.6) for Linux and MacOSX. +Currently, SINGA has conda packages for Linux and MacOSX. [Miniconda3](https://conda.io/miniconda.html) is recommended to use with SINGA. After installing miniconda, execute the one of the following commands to install SINGA. -1. CPU only +1. CPU only - conda install -c nusdbsystem singa-cpu + conda install -c nusdbsystem -c conda-forge singa-cpu + [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1Ntkhi-Z6XTR8WYPXiLwujHd2dOm0772V) -2. GPU with CUDA and cuDNN +2. GPU with CUDA and cuDNN (CUDA driver >=384.81 is required) - conda install -c nusdbsystem singa-gpu + conda install -c nusdbsystem -c conda-forge singa-gpu + [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1do_TLJe18IthLOnBOsHCEe-FFPGk1sPJ) - CUDA driver (for CUDA >=9.0) must be installed before executing the above command. Singa - packages for other CUDA versions are also available. The following instruction - lists all the available Singa packages. - conda search -c nusdbsystem singa +3. Install a specific version of SINGA. The following command lists all the available SINGA packages. -If there is no error message from - - python -c "from singa import tensor" - -then SINGA is installed successfully. - - -## From source - -The source files could be downloaded either as a -[tar.gz file](https://dist.apache.org/repos/dist/dev/incubator/singa/), or as a git repo - - $ git clone https://github.com/apache/incubator-singa.git - $ cd incubator-singa/ - -### Use Conda to build SINGA - -Conda-build is a building tool that installs the dependent libraries from anaconda cloud and -executes the building scripts. The generated package can be uploaded to anaconda -cloud for others to download and install. - -To install conda-build (after installing miniconda) - - conda install conda-build - -To build the CPU version of SINGA - - conda build tool/conda/singa/ --python 3.6 -The above commands have been tested on Ubuntu 16.04 and Mac OSX. -Refer to the [Travis-CI page](https://travis-ci.org/apache/incubator-singa) for more information. - - -To build the GPU version of SINGA - - export CUDA=x.y (e.g. 9.0) - conda build tool/conda/singa/ --python 3.6 - -The commands for building on GPU platforms have been tested on Ubuntu 16.04 (cuDNN>=7 and CUDA>=9). -[Nvidia's Docker image](https://hub.docker.com/r/nvidia/cuda/) provides the building -environment with cuDNN and CUDA. - -The location of the generated package file is shown on the screen. -Refer to [conda install](https://conda.io/docs/commands/conda-install.html) for -the instructions of installing the package from the local file. - - -### Use native tools to build SINGA on Ubuntu - -The following libraries are required to compile and run SINGA. -Refer to SINGA [Dockerfiles](https://github.com/apache/incubator-singa/blob/master/tool/docker/) -for the instructions of installing them on Ubuntu 16.04. - -* cmake (>=2.8) -* gcc (>=4.8.1) -* google protobuf (>=2.5) -* blas (tested with openblas >=0.2.10) -* swig(>=3.0.10) for compiling PySINGA -* numpy(>=1.11.0) for compiling PySINGA - - -1. create a `build` folder inside incubator-singa and go into that folder -2. run `cmake [options] ..` - by default all options are OFF except `USE_PYTHON` - - * `USE_MODULES=ON`, used if protobuf and blas are not installed a prior - * `USE_CUDA=ON`, used if CUDA and cuDNN is available - * `USE_PYTHON3=ON`, used for compiling with Python 3 support. (The default is Python 2) - * `USE_OPENCL=ON`, used for compiling with OpenCL support - * `USE_MKLDNN=ON`, used for compiling with Intel MKL-dnn support - * `PACKAGE=ON`, used for building the Debian package - * `ENABLE_TEST`, used for compiling unit test cases - -3. compile the code, `make` -4. goto python folder -5. run `pip install .` or `pip install -e .` The second command creates symlinks instead of copying files into python site-package folder. - -Execute step 4 and 5 are to install PySINGA when USE_PYTHON=ON. + conda search -c nusdbsystem singa + + Loading channels: done + # Name Version Build Channel + singa 2.1.0.dev cpu_py36 nusdbsystem + singa 2.1.0.dev cpu_py37 nusdbsystem -After compiling SINGA with ENABLE_TEST=ON, you can run the unit tests by + The following command install a specific version of SINGA, - $ ./bin/test_singa + conda install -c nusdbsystem -c conda-forge singa=2.1.0.dev=cpu_py37 -You can see all the testing cases with testing results. If SINGA passes all -tests, then you have successfully installed SINGA. -### Use native tools to Build SINGA on Centos7 -Building from source will be different for Centos7 as package names differ.Follow the instructions given below. - -#### Installing dependencies -Basic packages/libraries - - sudo yum install freetype-devel libXft-devel ncurses-devel openblas-devel blas-devel lapack devel atlas-devel kernel-headers unzip wget pkgconfig zip zlib-devel libcurl-devel cmake curl unzip dh-autoreconf git python-devel glog-devel protobuf-devel -For build-essential - - sudo yum group install "Development Tools" -For installing swig - - sudo yum install pcre-devel - wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz - tar xvzf swig-3.0.10.tar.gz - cd swig-3.0.10.tar.gz - ./configure --prefix=${RUN} - make - make install - For installing gfortran - - sudo yum install centos-release-scl-rh - sudo yum --enablerepo=centos-sclo-rh-testing install devtoolset-7-gcc-gfortran -For installing pip and other packages - - sudo yum install epel-release - sudo yum install python-pip - pip install matplotlib numpy pandas scikit-learn pydot - -#### Installation -Follow steps 1-5 of _Use native tools to build SINGA on Ubuntu_ - -#### Testing -You can run the unit tests by, - - $ ./bin/test_singa -You can see all the testing cases with testing results. If SINGA passes all -tests, then you have successfully installed SINGA. -### Compile SINGA on Windows - -Instructions for building on Windows with Python support can be found [here](install_win.html). - -### More details about the compilation options - -### USE_MODULES - -If protobuf and openblas are not installed, you can compile SINGA together with them - - $ In SINGA ROOT folder - $ mkdir build - $ cd build - $ cmake -DUSE_MODULES=ON .. - $ make - -cmake would download OpenBlas and Protobuf (2.6.1) and compile them together -with SINGA. - -You can use `ccmake ..` to configure the compilation options. -If some dependent libraries are not in the system default paths, you need to export -the following environment variables - - export CMAKE_INCLUDE_PATH= - export CMAKE_LIBRARY_PATH= - -#### USE_PYTHON - -Similar to compile CPP code, PySINGA is compiled by - - $ cmake -DUSE_PYTHON=ON .. - $ make - $ cd python - $ pip install . - - -#### USE_CUDA - -Users are encouraged to install the CUDA and -[cuDNN](https://developer.nvidia.com/cudnn) for running SINGA on GPUs to -get better performance. - -SINGA has been tested over CUDA 9, and cuDNN 7. If cuDNN is -installed into non-system folder, e.g. /home/bob/local/cudnn/, the following -commands should be executed for cmake and the runtime to find it - - $ export CMAKE_INCLUDE_PATH=/home/bob/local/cudnn/include:$CMAKE_INCLUDE_PATH - $ export CMAKE_LIBRARY_PATH=/home/bob/local/cudnn/lib64:$CMAKE_LIBRARY_PATH - $ export LD_LIBRARY_PATH=/home/bob/local/cudnn/lib64:$LD_LIBRARY_PATH - -The cmake options for CUDA and cuDNN should be switched on - - # Dependent libs are install already - $ cmake -DUSE_CUDA=ON .. - $ make - -#### USE_OPENCL - -SINGA uses opencl-headers and viennacl (version 1.7.1 or newer) for OpenCL support, which -can be installed using via +If there is no error message from - # On Ubuntu 16.04 - $ sudo apt-get install opencl-headers, libviennacl-dev - # On Fedora - $ sudo yum install opencl-headers, viennacl + python -c "from singa import tensor" -Additionally, you will need the OpenCL Installable Client Driver (ICD) for the platforms that you want to run OpenCL on. +then SINGA is installed successfully. -* For AMD and nVidia GPUs, the driver package should also install the correct OpenCL ICD. -* For Intel CPUs and/or GPUs, get the driver from the [Intel website.](https://software.intel.com/en-us/articles/opencl-drivers) Note that the drivers provided on that website only supports recent CPUs and Iris GPUs. -* For older Intel CPUs, you can use the `beignet-opencl-icd` package. +## Using Docker -Note that running OpenCL on CPUs is not currently recommended because it is slow. -Memory transfer is on the order of whole seconds (1000's of ms on CPUs as compared to 1's of ms on GPUs). +Install Docker on your local host machine following the [instructions](https://docs.docker.com/install/). Add your user into the [docker group](https://docs.docker.com/install/linux/linux-postinstall/) to run docker commands without `sudo`. -More information on setting up a working OpenCL environment may be found [here](https://wiki.tiker.net/OpenCLHowTo). +1. CPU-only. -If the package version of ViennaCL is not at least 1.7.1, you will need to build it from source: + docker run -it apache/singa:nightly-cpu /bin/bash -Clone [the repository from here](https://github.com/viennacl/viennacl-dev), checkout the `release-1.7.1` tag and build it. -Remember to add its directory to `PATH` and the built libraries to `LD_LIBRARY_PATH`. +2. With GPU enabled. Install [Nvidia-Docker](https://github.com/NVIDIA/nvidia-docker) after install Docker. -To build SINGA with OpenCL support (tested on SINGA 1.1): + nvidia-docker run -it apache/singa:nightly-gpu /bin/bash - $ cmake -DUSE_OPENCL=ON .. - $ make +3. For the complete list of SINGA Docker images (tags), visit the [docker hub site](https://hub.docker.com/r/apache/singa/). For each docker image, the tag is named as + + version-(cpu|gpu)[-devel] -#### USE_MKLDNN - -User can enable MKL-DNN to enhance the performance of CPU computation. - -Installation guide of MKL-DNN could be found [here](https://github.com/intel/mkl-dnn#installation). - -SINGA has been tested over MKL-DNN v0.17.2. - -To build SINGA with MKL-DNN support: - - # Dependent libs are installed already - $ cmake -DUSE_MKLDNN=ON .. - $ make - + | Tag | Description| Example value| + | --- | --- | --- | + | `version`| SINGA version | 'nightly', '2.0.0', '1.2.0'| + | `cpu` | the image cannot run on GPUs | 'cpu' | + | `gpu` | the image can run on Nvidia GPUs| 'gpu', or 'cudax.x-cudnnx.x' e.g., 'cuda10.0-cudnn7.3'| + | `devel`| indicator for development|if absent SINGA Python package is installed for runtime only; if present, the building environment is also created, you can recompile SINGA from source at '/root/incubator-singa' + | -#### PACKAGE - -This setting is used to build the Debian package. Set PACKAGE=ON and build the package with make command like this: - - $ cmake -DPACKAGE=ON - $ make package +## From source +You can [build and install SINGA](../develop/build.md) from the source code using native building tools or conda-build, on local host OS or in a Docker container. ## FAQ -* Q: Error from 'import singa' using PySINGA installed from wheel. - - A: Please check the detailed error from `python -c "from singa import _singa_wrap"`. Sometimes it is - caused by the dependent libraries, e.g. there are multiple versions of protobuf, missing of cudnn, numpy version mismatch. Following - steps show the solutions for different cases - 1. Check the cudnn and cuda and gcc versions, cudnn5 and cuda7.5 and gcc4.8/4.9 are preferred. if gcc is 5.0, then downgrade it. - If cudnn is missing or not match with the wheel version, you can download the correct version of cudnn into ~/local/cudnn/ and - - $ echo "export LD_LIBRARY_PATH=/home//local/cudnn/lib64:$LD_LIBRARY_PATH" >> ~/.bashrc - - 2. If it is the problem related to protobuf, then download the newest whl files which have [compiled protobuf and openblas into the whl](https://issues.apache.org/jira/browse/SINGA-255) file of PySINGA. - Or you can install protobuf from source into a local folder, say ~/local/; - Decompress the tar file, and then - - $ ./configure --prefix=/home/local - $ make && make install - $ echo "export LD_LIBRARY_PATH=/home//local/lib:$LD_LIBRARY_PATH" >> ~/.bashrc - $ source ~/.bashrc - - 3. If it cannot find other libs including python, then create virtual env using pip or conda; - - 4. If it is not caused by the above reasons, go to the folder of `_singa_wrap.so`, - - $ python - >> import importlib - >> importlib.import_module('_singa_wrap') - - Check the error message. For example, if the numpy version mismatches, the error message would be, - - RuntimeError: module compiled against API version 0xb but this version of numpy is 0xa - - Then you need to upgrade the numpy. - - -* Q: Error from running `cmake ..`, which cannot find the dependent libraries. - - A: If you haven't installed the libraries, install them. If you installed - the libraries in a folder that is outside of the system folder, e.g. /usr/local, - you need to export the following variables - - $ export CMAKE_INCLUDE_PATH= - $ export CMAKE_LIBRARY_PATH= - - -* Q: Error from `make`, e.g. the linking phase - - A: If your libraries are in other folders than system default paths, you need - to export the following varaibles - - $ export LIBRARY_PATH= - $ export LD_LIBRARY_PATH= - +* Q: Error from `from singa import tensor` -* Q: Error from header files, e.g. 'cblas.h no such file or directory exists' - - A: You need to include the folder of the cblas.h into CPLUS_INCLUDE_PATH, - e.g., - - $ export CPLUS_INCLUDE_PATH=/opt/OpenBLAS/include:$CPLUS_INCLUDE_PATH - -* Q:While compiling SINGA, I get error `SSE2 instruction set not enabled` - - A:You can try following command: - - $ make CFLAGS='-msse2' CXXFLAGS='-msse2' - -* Q:I get `ImportError: cannot import name enum_type_wrapper` from google.protobuf.internal when I try to import .py files. - - A: You need to install the python binding of protobuf, which could be installed via - - $ sudo apt-get install protobuf - - or from source - - $ cd /PROTOBUF/SOURCE/FOLDER - $ cd python - $ python setup.py build - $ python setup.py install - -* Q: When I build OpenBLAS from source, I am told that I need a Fortran compiler. - - A: You can compile OpenBLAS by - - $ make ONLY_CBLAS=1 - - or install it using - - $ sudo apt-get install libopenblas-dev - -* Q: When I build protocol buffer, it reports that GLIBC++_3.4.20 not found in /usr/lib64/libstdc++.so.6. - - A: This means the linker found libstdc++.so.6 but that library - belongs to an older version of GCC than was used to compile and link the - program. The program depends on code defined in - the newer libstdc++ that belongs to the newer version of GCC, so the linker - must be told how to find the newer libstdc++ shared library. - The simplest way to fix this is to find the correct libstdc++ and export it to - LD_LIBRARY_PATH. For example, if GLIBC++_3.4.20 is listed in the output of the - following command, - - $ strings /usr/local/lib64/libstdc++.so.6|grep GLIBC++ - - then you just set your environment variable as - - $ export LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH - -* Q: When I build glog, it reports that "src/logging_unittest.cc:83:20: error: ‘gflags’ is not a namespace-name" - - A: It maybe that you have installed gflags with a different namespace such as "google". so glog can't find 'gflags' namespace. - Because it is not necessary to have gflags to build glog. So you can change the configure.ac file to ignore gflags. - - 1. cd to glog src directory - 2. change line 125 of configure.ac to "AC_CHECK_LIB(gflags, main, ac_cv_have_libgflags=0, ac_cv_have_libgflags=0)" - 3. autoreconf - - After this, you can build glog again. - -* Q: When using virtual environment, everytime I run pip install, it would reinstall numpy. However, the numpy would not be used when I `import numpy` - - A: It could be caused by the `PYTHONPATH` which should be set to empty when you are using virtual environment to avoid the conflicts with the path of - the virtual environment. - -* Q: When compiling PySINGA from source, there is a compilation error due to the missing of + A: Check the detailed error from + + python -c "from singa import _singa_wrap" + # go to the folder of _singa_wrap.so + ldd path to _singa_wrap.so + python + >> import importlib + >> importlib.import_module('_singa_wrap') + + + The folder of `_singa_wrap.so` is like ' ~/miniconda3/lib/python3.7/site-packages/singa'. + Normally, the error is caused by the mismatch or missing of dependent libraries, e.g. cuDNN or protobuf. The solution is to create a new virtual environment and install SINGA in that environment, e.g., - A: Please install numpy and export the path of numpy header files as + conda create -n singa + conda activate singa + conda install -c nusdbsystem -c conda-forge singa-cpu - $ export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH -* Q: When I run PySINGA in Mac OS X, I got the error "Fatal Python error: PyThreadState_Get: no current thread Abort trap: 6" +* Q: When using virtual environment, every time I install SINGA, numpy would be reinstalled. However, the numpy is not used when I run `import numpy` - A: This error happens typically when you have multiple version of Python on your system and you installed SINGA via pip (this problem is resolved for installation via conda), - e.g, the one comes with the OS and the one installed by Homebrew. The Python linked by PySINGA must be the same as the Python interpreter. - You can check your interpreter by `which python` and check the Python linked by PySINGA via `otool -L `. - To fix this error, compile SINGA with the correct version of Python. - In particular, if you build PySINGA from source, you need to specify the paths when invoking [cmake](http://stackoverflow.com/questions/15291500/i-have-2-versions-of-python-installed-but-cmake-is-using-older-version-how-do) + A: It could be caused by the `PYTHONPATH` environment variable which should be set to empty when you are using virtual environment to avoid the conflicts with the path of the virtual environment. - $ cmake -DPYTHON_LIBRARY=`python-config --prefix`/lib/libpython2.7.dylib -DPYTHON_INCLUDE_DIR=`python-config --prefix`/include/python2.7/ .. +* Q: When I run SINGA in Mac OS X, I got the error "Fatal Python error: PyThreadState_Get: no current thread Abort trap: 6" - If installed PySINGA from binary packages, e.g. debian or wheel, then you need to change the python interpreter, e.g., reset the $PATH to put the correct path of Python at the front position. + A: This error happens typically when you have multiple versions of Python in your system, e.g, the one comes with the OS and the one installed by Homebrew. The Python linked by SINGA must be the same as the Python interpreter. You can check your interpreter by `which python` and check the Python linked by SINGA via `otool -L `. + This problem should be resolved if SINGA is installation via conda. \ No newline at end of file diff --git a/doc/en/docs.rst b/doc/en/docs/onnx.rst similarity index 93% rename from doc/en/docs.rst rename to doc/en/docs/onnx.rst index 1b94d02b47..f180f2adfd 100644 --- a/doc/en/docs.rst +++ b/doc/en/docs/onnx.rst @@ -16,8 +16,9 @@ under the License. -Documentation -============= +ONNX +==== -.. toctree:: - docs/index +(under construction) + +ONNX .... \ No newline at end of file diff --git a/doc/en/downloads.md b/doc/en/downloads.md index 29df9314d8..e45bdec60b 100644 --- a/doc/en/downloads.md +++ b/doc/en/downloads.md @@ -16,7 +16,8 @@ specific language governing permissions and limitations under the License. --> -## Download SINGA + +# Download SINGA * To verify the downloaded tar.gz file, download the [KEYS](https://www.apache.org/dist/incubator/singa/KEYS) and ASC files and then execute the following commands diff --git a/doc/en/index.rst b/doc/en/index.rst index dc61f29a73..2502776ba8 100644 --- a/doc/en/index.rst +++ b/doc/en/index.rst @@ -94,13 +94,12 @@ Documentation How to contribute ---------------------- -* Please subscribe to our development mailing list dev-subscribe@singa.incubator.apache.org. +As with any open source project, there are several ways you can help: -* If you find any issues using SINGA, please report it to the `Issue Tracker `_. +* Join the [mailing list](http://singa.apache.org/en/community/mail-lists.html) and answer other user's questions. +* [Help with the documentation](http://singa.apache.org/en/develop/contribute-docs.html) by updating webpages that are lacking or unclear. +* [Contribute code to SINGA](http://singa.apache.org/en/develop/contribute-code.html) by fixing errors or adding new features. [All issues are tracked](http://singa.apache.org/en/community/issue-tracking.html) on the JIRA system. -* You can also contact with `SINGA committers `_ directly. - -More details on contributing to SINGA is described `here `_ . History ------------ @@ -118,6 +117,7 @@ Rafiki is a sub module of SINGA. Please cite the following paper if you use Rafi Companies like `NetEase `_, `yzBigData `_, `Shentilium `_, `Foodlg `_ and `Medilot `_ are using SINGA for their applications. + .. toctree:: :hidden: @@ -130,8 +130,6 @@ Companies like `NetEase `_ :maxdepth: 2 :caption: Development - develop/schedule - develop/how-contribute develop/contribute-code develop/contribute-docs develop/how-to-release @@ -146,8 +144,6 @@ Companies like `NetEase `_ community/issue-tracking community/team-list - - License ---------- SINGA is released under `Apache License Version 2.0 `_. @@ -155,5 +151,4 @@ SINGA is released under `Apache License Version 2.0 + +# Security + +Users can report security vulnerabilities to `SINGA Security Team Mail List `_. \ No newline at end of file diff --git a/doc/en/security.rst b/doc/en/security.rst deleted file mode 100644 index 2a848fa8d9..0000000000 --- a/doc/en/security.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Security -======== - -Users can report security vulnerabilities to `SINGA Security Team Mail List `_. - - diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6399332c8a..22a7c7d0f4 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -142,7 +142,7 @@ ENDIF() TARGET_INCLUDE_DIRECTORIES(_singa_wrap PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) -#SETUP +# substitute ${var} in setup.py.in to generate setup.py SET(SETUP_PY_IN "setup.py.in") SET(SETUP_PY "${CMAKE_BINARY_DIR}/python/setup.py") CONFIGURE_FILE(${SETUP_PY_IN} ${SETUP_PY}) diff --git a/python/setup.py.in b/python/setup.py.in index 543f42b2f7..a0d4fa5333 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -22,7 +22,8 @@ from setuptools import setup setup( name='singa', - + + # PACKAGE_VERSION will be substituted by cmake with the real version version='${PACKAGE_VERSION}', description='A General Deep Learning System', diff --git a/tool/conda/cpu/meta.yaml b/tool/conda/cpu/meta.yaml index 77e9414bcb..5f0b1d9553 100644 --- a/tool/conda/cpu/meta.yaml +++ b/tool/conda/cpu/meta.yaml @@ -17,16 +17,21 @@ # under the License. # +{% set version = "2.1.0.dev" %} package: name: singa-cpu - version: 2.0.0 + version: {{ version }} requirements: run: - - singa 2.0.0 cpu + - singa {{ version }} cpu_py{{ py }} +build: + number: 0 + string: py{{ py }} + about: home: http://singa.apache.org/ license: Apache V2 - summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers + summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers \ No newline at end of file diff --git a/tool/conda/gpu/meta.yaml b/tool/conda/gpu/meta.yaml index f6424b89e5..9ed7bae3d4 100644 --- a/tool/conda/gpu/meta.yaml +++ b/tool/conda/gpu/meta.yaml @@ -17,15 +17,22 @@ # under the License. # +{% set version = "2.1.0.dev" %} + package: name: singa-gpu - version: 2.0.0 + version: {{ version }} requirements: run: - - singa 2.0.0 cudnn7.3.1_cuda10.0_py37 + - singa {{ version }} cudnn7.3.1_cuda10.0_py{{ py }} + +build: + number: 0 + string: py{{ py }} + about: home: http://singa.apache.org/ license: Apache V2 - summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers + summary: SINGA is an Apache Incubating project for providing distributed deep learning. Apache disclaimers http://singa.apache.org/en/index.html#disclaimers \ No newline at end of file diff --git a/tool/conda/singa/conda_build_config.yaml b/tool/conda/singa/conda_build_config.yaml index 5c68cbc065..faa7a69f4d 100644 --- a/tool/conda/singa/conda_build_config.yaml +++ b/tool/conda/singa/conda_build_config.yaml @@ -33,6 +33,6 @@ python: - 3.6 - 3.7 build_str: - - "cudnn7.3.1_cuda10.0" # [environ.get("CUDA")=="10.0"] - - "cudnn7.3.1_cuda9.0" # [environ.get("CUDA")=="9.0"] - - "cpu" # [environ.get("CUDA", "")== ""] \ No newline at end of file + - "cudnn7.3.1_cuda10.0" # [environ.get("CUDA")=="10.0"] + - "cudnn7.3.1_cuda9.0" # [environ.get("CUDA")=="9.0"] + - "cpu" # [environ.get("CUDA", "")== ""] \ No newline at end of file diff --git a/tool/conda/singa/meta.yaml b/tool/conda/singa/meta.yaml index efeb541971..d36ad45297 100644 --- a/tool/conda/singa/meta.yaml +++ b/tool/conda/singa/meta.yaml @@ -17,9 +17,14 @@ # under the License. # +# https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#templating-with-jinja +# {% set data = load_setup_py_data(setup_file='../../../python/singa/setup.py', from_recipe_dir=True) %} + +{% set version = "2.1.0.dev" %} + package: name: singa - version: "{{ GIT_DESCRIBE_TAG }}" + version: {{ version }} source: # path: ../../../ @@ -28,9 +33,8 @@ source: build: number: 0 script_env: - - CUDA # E.g., export CUDA=9.0 + - CUDA # run `export CUDA=9.0` in the terminal string: {{ build_str }}_py{{ py }} - # py{{py}}_{{ environ.get('BUILD_STR', 'cpu') }} requirements: build: diff --git a/tool/travis/build.sh b/tool/travis/build.sh index 12285c34c5..101f5ae5d2 100644 --- a/tool/travis/build.sh +++ b/tool/travis/build.sh @@ -31,6 +31,7 @@ export CONDA_BLD_PATH=~/conda-bld-$suffix mkdir $CONDA_BLD_PATH conda build tool/conda/singa --python 3.6 +conda build tool/conda/singa --python 3.7 # conda install --use-local singa # cd test/python # $HOME/miniconda/bin/python run.py