AdolfVonKleist · danijel3 · Feb 8, 2021 · Nov 26, 2024 · Nov 26, 2024 · Nov 26, 2024
diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,16 @@
-FROM python:3 as build
+FROM python:3 AS build
 
 WORKDIR /build
 
 RUN apt-get -y update && apt-get -y install git g++ autoconf-archive make libtool gfortran tar gawk
 
-RUN wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.2.tar.gz && \
-    tar -xvzf openfst-1.6.2.tar.gz && \
-    cd openfst-1.6.2 && \
-    ./configure --enable-static --enable-shared --enable-far --enable-ngram-fsts && \
-    make -j $(nproc) && \
-    make install && \
-    ldconfig
+RUN wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.8.4.tar.gz && \
+	tar -xvzf openfst-1.8.4.tar.gz && \
+	cd openfst-1.8.4 && \
+	./configure --enable-static --enable-shared --enable-far --enable-ngram-fsts --enable-lookahead-fsts --with-pic && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
 
 RUN git clone https://github.com/mitlm/mitlm && \
 	cd mitlm && \
@@ -23,15 +23,16 @@ WORKDIR /build/phonetisaurus
 
 COPY . ./
 
-RUN pip3 install pybindgen
+RUN pip3 install pybindgen setuptools
 
 RUN ./configure --enable-python && \
-    make -j $(nproc) && \
-    make install 
+	make -j $(nproc) && \
+	make install 
 
 FROM python:3-slim
 
-RUN apt-get -y update && apt-get -y install gfortran && apt-get -y clean && apt-get -y autoclean
+RUN apt-get -y update && apt-get -y install gfortran && apt-get -y clean && apt-get -y autoclean && \
+	pip3 install setuptools
 
 WORKDIR /setup
 

diff --git a/Makefile.am b/Makefile.am
@@ -71,22 +71,22 @@ bin_PROGRAMS =                     \
 	phonetisaurus-g2prnn       \
 	rnnlm
 
-AM_CPPFLAGS = -I$(top_srcdir)/src -DGIT_REVISION=\"$(GIT_REVISION)\" -std=c++0x -Wall -Wno-sign-compare -Wno-unused-local-typedefs
+AM_CPPFLAGS = -I$(top_srcdir)/src/3rdparty/utfcpp -I$(top_srcdir)/src -DGIT_REVISION=\"$(GIT_REVISION)\" -std=c++17 -Wall -Wno-sign-compare -Wno-unused-local-typedefs
 
 phonetisaurus_align_SOURCES = src/bin/phonetisaurus-align.cc src/include/PhonetisaurusRex.h src/lib/util.cc src/include/util.h src/lib/LatticePruner.cc src/include/LatticePruner.h src/lib/M2MFstAligner.cc src/include/M2MFstAligner.h
 phonetisaurus_align_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS)
 phonetisaurus_align_LDADD = $(OPENFST_LDFLAGS)
 
 phonetisaurus_arpa2wfst_SOURCES = src/bin/phonetisaurus-arpa2wfst.cc src/include/ARPA2WFST.h src/lib/util.cc src/include/util.h
-phonetisaurus_arpa2wfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS)
+phonetisaurus_arpa2wfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) 
 phonetisaurus_arpa2wfst_LDADD = $(OPENFST_LDFLAGS)
 
 phonetisaurus_g2pfst_SOURCES = src/bin/phonetisaurus-g2pfst.cc src/include/PhonetisaurusScript.h src/include/PhonetisaurusRex.h src/lib/util.cc src/include/util.h
-phonetisaurus_g2pfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -funroll-loops -ffast-math
+phonetisaurus_g2pfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -funroll-loops
 phonetisaurus_g2pfst_LDADD = $(OPENFST_LDFLAGS)
 
 phonetisaurus_g2prnn_SOURCES = src/bin/phonetisaurus-g2prnn.cc src/include/LegacyRnnLMHash.h src/include/LegacyRnnLMDecodable.h src/include/LegacyRnnLMReader.h src/include/RnnLMDecoder.h src/lib/util.cc src/include/util.h src/3rdparty/rnnlm/rnnlmlib.cpp src/3rdparty/rnnlm/rnnlmlib.h
-phonetisaurus_g2prnn_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -I$(top_srcdir)/src/3rdparty/rnnlm -funroll-loops -ffast-math
+phonetisaurus_g2prnn_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -I$(top_srcdir)/src/3rdparty/rnnlm -funroll-loops
 phonetisaurus_g2prnn_LDADD = $(OPENFST_LDFLAGS)
 if WANT_OPENMP
 if OPENMP

diff --git a/Makefile.in b/Makefile.in
@@ -532,23 +532,23 @@ dist_bin_SCRIPTS = \
 	src/scripts/phonetisaurus-apply     \
 	src/scripts/phonetisaurus-train
 
-AM_CPPFLAGS = -I$(top_srcdir)/src -DGIT_REVISION=\"$(GIT_REVISION)\" -std=c++0x -Wall -Wno-sign-compare -Wno-unused-local-typedefs
+AM_CPPFLAGS = -I$(top_srcdir)/src/3rdparty/utfcpp -I$(top_srcdir)/src -DGIT_REVISION=\"$(GIT_REVISION)\" -std=c++17 -Wall -Wno-sign-compare -Wno-unused-local-typedefs
 phonetisaurus_align_SOURCES = src/bin/phonetisaurus-align.cc src/include/PhonetisaurusRex.h src/lib/util.cc src/include/util.h src/lib/LatticePruner.cc src/include/LatticePruner.h src/lib/M2MFstAligner.cc src/include/M2MFstAligner.h
 phonetisaurus_align_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS)
 phonetisaurus_align_LDADD = $(OPENFST_LDFLAGS)
 phonetisaurus_arpa2wfst_SOURCES = src/bin/phonetisaurus-arpa2wfst.cc src/include/ARPA2WFST.h src/lib/util.cc src/include/util.h
 phonetisaurus_arpa2wfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS)
 phonetisaurus_arpa2wfst_LDADD = $(OPENFST_LDFLAGS)
 phonetisaurus_g2pfst_SOURCES = src/bin/phonetisaurus-g2pfst.cc src/include/PhonetisaurusScript.h src/include/PhonetisaurusRex.h src/lib/util.cc src/include/util.h
-phonetisaurus_g2pfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -funroll-loops -ffast-math
+phonetisaurus_g2pfst_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) -funroll-loops
 phonetisaurus_g2pfst_LDADD = $(OPENFST_LDFLAGS)
 phonetisaurus_g2prnn_SOURCES = src/bin/phonetisaurus-g2prnn.cc src/include/LegacyRnnLMHash.h src/include/LegacyRnnLMDecodable.h src/include/LegacyRnnLMReader.h src/include/RnnLMDecoder.h src/lib/util.cc src/include/util.h src/3rdparty/rnnlm/rnnlmlib.cpp src/3rdparty/rnnlm/rnnlmlib.h
 phonetisaurus_g2prnn_CXXFLAGS = $(OPENFST_CXXFLAGS) $(UTFCPP_CXXFLAGS) \
-	-I$(top_srcdir)/src/3rdparty/rnnlm -funroll-loops -ffast-math \
+	-I$(top_srcdir)/src/3rdparty/rnnlm \
 	$(am__append_1)
 phonetisaurus_g2prnn_LDADD = $(OPENFST_LDFLAGS) $(am__append_2)
 rnnlm_SOURCES = src/bin/rnnlm.cc src/3rdparty/rnnlm/rnnlmlib.cpp src/3rdparty/rnnlm/rnnlmlib.h
-rnnlm_CXXFLAGS = $(OPENFST_CXXFLAGS) -I$(top_srcdir)/src/3rdparty/rnnlm -funroll-loops -ffast-math
+rnnlm_CXXFLAGS = $(OPENFST_CXXFLAGS) -I$(top_srcdir)/src/3rdparty/rnnlm -funroll-loops
 rnnlm_LDADD = $(OPENFST_LDFLAGS)
 @HAVE_PYTHON_TRUE@@WANT_PYTHON_TRUE@CLEANFILES = python/Phonetisaurus-binding.cc
 @HAVE_PYTHON_DEV_TRUE@@HAVE_PYTHON_TRUE@@WANT_PYTHON_TRUE@pyexec_LTLIBRARIES = Phonetisaurus.la

diff --git a/README.md b/README.md
@@ -1,26 +1,27 @@
 ## Phonetisaurus G2P ##
-[![Build Status](https://travis-ci.org/AdolfVonKleist/Phonetisaurus.svg?branch=master)](https://travis-ci.org/AdolfVonKleist/Phonetisaurus)
+
+### Note on this fork
+
+The original project hasn't been updated in a while, so this fork was created. The main motivation for
+this fork is to keep it synchronized with the latest versions of OpenFST as they are being updated
+in Kaldi. Another reason is to make it compile-able in modern versions of GCC. This for was created
+in late 2024 and is being updated ever since.
+
+### Original Description
 
 This repository contains scripts suitable for training, evaluating and using grapheme-to-phoneme
 models for speech recognition using the OpenFst framework.  The current build requires OpenFst
-version 1.6.0 or later, and the examples below use version 1.7.2.
+version 1.6.0 or later, and the examples below use version 1.8.4.
 
 The repository includes C++ binaries suitable for training, compiling, and evaluating G2P models.
 It also some simple python bindings which may be used to extract individual
 multigram scores, alignments, and to dump the raw lattices in .fst format for each word.
 
-The python scripts and bindings were tested most recently with python v3.8.5.
+The python scripts and bindings were tested most recently with python v3.13.
 
-Standalone distributions related to previous INTERSPEECH papers, as well as the complete, exported
-final version of the old google-code repository are available via ```git-lfs``` in a separate
-repository:
-  * https://github.com/AdolfVonKleist/phonetisaurus-downloads
+#### Scratch Build for OpenFst v1.8.4 ####
 
-#### Contact: ####
-  * [email protected]
-
-#### Scratch Build for OpenFst v1.7.2 and Ubuntu 20.04 ####
-This build was tested via AWS EC2 with a fresh Ubuntu 20.04 base, and m4.large instance.
+This build was tested in Docker using the Dockerfile included with the project.
 
 ```
 $ sudo apt-get update
@@ -38,49 +39,45 @@ $ mkdir g2p
 $ cd g2p/
 ```
 
-Next grab and install OpenFst-1.7.2:
+Next grab and install OpenFst-1.8.4:
 ```
-$ wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.7.2.tar.gz
-$ tar -xvzf openfst-1.7.2.tar.gz
-$ cd openfst-1.7.2
+$ wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.8.4.tar.gz
+$ tar -xvzf openfst-1.8.4.tar.gz
+$ cd openfst-1.8.4
 # Minimal configure, compatible with current defaults for Kaldi
 $ ./configure --enable-static --enable-shared --enable-far --enable-ngram-fsts
 $ make -j 
 # Now wait a while...
 $ sudo make install
-# Extend your LD_LIBRARY_PATH .bashrc (assumes OpenFst installed to default location):
-$ echo 'export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib/fst' \
-     >> ~/.bashrc
-$ source ~/.bashrc
 $ cd ..
 ```
 
 Checkout the latest Phonetisaurus from master and compile without bindings:
 ```
-$ git clone https://github.com/AdolfVonKleist/Phonetisaurus.git
+$ git clone https://github.com/danijel3/Phonetisaurus.git
 $ cd Phonetisaurus
 # if OpenFst is installed in the default location:
 $ ./configure
 # if OpenFst is installed in a special location:
 $ ./configure \
-      --with-openfst-includes=${OFST_PATH}/openfst-1.7.2/include \
-      --with-openfst-libs=${OFST_PATH}/openfst-1.7.2/lib
+      --with-openfst-includes=${OFST_PATH}/openfst-1.8.4/include \
+      --with-openfst-libs=${OFST_PATH}/openfst-1.8.4/lib
 $ make
 $ sudo make install
 $ cd ..
 ```
 
 Checkout the latest Phonetisaurus from master and compile with python3 bindings:
 ```
-$ git clone https://github.com/AdolfVonKleist/Phonetisaurus.git
+$ git clone https://github.com/danijel3/Phonetisaurus.git
 $ cd Phonetisaurus
-$ sudo pip3 install pybindgen
+$ sudo pip3 install pybindgen setuptools
 # if OpenFst is installed in the default location:
 $ PYTHON=python3 ./configure --enable-python
 # if OpenFst is installed in a special location:
 $ PYTHON=python3 ./configure \
-      --with-openfst-includes=${OFST_PATH}/openfst-1.7.2/include \
-      --with-openfst-libs=${OFST_PATH}/openfst-1.7.2/lib \
+      --with-openfst-includes=${OFST_PATH}/openfst-1.8.4/include \
+      --with-openfst-libs=${OFST_PATH}/openfst-1.8.4/lib \
       --enable-python
 $ make
 $ sudo make install
@@ -228,22 +225,6 @@ junkify JH AH1 NG K AH0 F AY2
 junkify JH AH1 NG K IH0 F AY2
 ```
 
-Use a special location for OpenFst, parallel build with 2 cores
-```
- $ ./configure --with-openfst-libs=/home/ubuntu/openfst-1.6.2/lib \
-          --with-openfst-includes=/home/ubuntu/openfst-1.6.2/include
- $ make -j 2 all
-```
-
-Use custom g++ under OSX (Note: OpenFst must also be compiled with this
-custom g++ alternative [untested with v1.6.2])
-```
- $ ./configure --with-openfst-libs=/home/osx/openfst-1.6.2gcc/lib \
-          --with-openfst-includes=/home/osx/openfst-1.6.2gcc/include \
-          CXX=g++-4.9
- $ make -j 2 all
-```
-
 #### Rebuild configure ####
 If you need to rebuild the configure script you can do so:
 ```
@@ -280,25 +261,18 @@ If you need to rebuild the configure script you can do so:
 
 ### Docker: ###
 
-Docker images are hosted on: https://hub.docker.com/r/phonetisaurus/phonetisaurus
+You can build a docker image using the command `docker build -t phonetisaurus .`.
 
 The images can be used in one of 3 ways:
 
   * directly, to process files on your computer without needing to install/compile anything (apart from docker)
   * as a base image for another project (using the `FROM` statement)
   * to copy portions of the binaries or libraries into a new image (using the `COPY --from=` statement) - most of the files are in `/usr/local/bin` and `/usr/local/lib`
 
-To use the program directly, you need to mount the local folder with the required files (eg. models, word lists, etc) into the Docker container under the `/work` path, as this is the default workdir in the image. Then you can call the programs directly after the name of the image, for example:
+To use the program directly, you need to mount a local folder with the required files (eg. models, word lists, etc) into the Docker container under the `/work` path, as this is the default workdir in the image. Then you can call the programs directly after the name of the image, for example:
 ```
-docker run --rm -it -v $PWD:/work phonetisaurus/phonetisaurus "phonetisaurus-apply -m model.fst -wl test.wlist"
+docker run --rm -it -v $PWD:/work phonetisaurus "phonetisaurus-apply -m model.fst -wl test.wlist"
 ```
 
 You can also use the `bash` program to simply enter the interactive shell and run everything from there.
 
-### Misc: ###
-cpplint command:
-```
- $ ./cpplint.py --filter=-whitespace/parens,-whitespace/braces,\
-      -legal/copyright,-build/namespaces,-runtime/references\
-      src/include/util.h
-```
diff --git a/src/3rdparty/utfcpp/utf8.h b/src/3rdparty/utfcpp/utf8.h
@@ -1,34 +1,46 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "utf8/checked.h"
-#include "utf8/unchecked.h"
-
-#endif // header guard
+// Copyright 2006 Nemanja Trifunovic
+
+/*
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+*/
+
+
+#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
+#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
+
+/*
+To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro
+and set it to one of the values used by the __cplusplus predefined macro.
+
+For instance,
+    #define UTF_CPP_CPLUSPLUS 199711L
+will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard.
+Some library features will be disabled.
+
+If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus.
+*/
+
+#include "utf8/checked.h"
+#include "utf8/unchecked.h"
+
+#endif // header guard