Effective mmt queue (#3425)

* WIP tests

* Resumed +60% of test set

* Removed obsolete tests

* Fixed database sql

* Cleaning old code and tests

* WIP

* Updated dependencies

* WIP, more tests

* Workin on

* More tests - Almost done!

* Restored the last test

* Fixed wrong file path

* Moved new suite in a different folder

* Fixed test database default values for datetime

* Added tests to CI

* Reduced scope build in docker-compose for tests

* Updated docker for CI

* Depends on build do not works, build explicitly images

* Update composer for ci

* Try to test outside the container

* Try build again

* Tests won't work on travis

* Injected task_runner in configuration

* Removed sql mode from command line

* Fixed mysql 8 override sql_mode

* Fixed wrong command line mysql

* Try bionic builder

* Try to fix mysql 5.7 again

* Workingb in local with mysql 8

* Exit on mysql schema import

* Explicit autoincrement on test.

* Specify php 7.4.33 in tests

* Fix comment json representation (#3412)

* Change shortcuts lib (#3347)

* Change shortcuts lib

* fix init error

* Fix: concordance shortcut

* Fix: fix translate shortcut with guess tags active

* Update submodule

* Update shortcuts

* Analyze: update page (#3270)

* Analyze: update job table

* Analyze: css wip

* Analyze: job header

* Analyze: job header

* Analyze: new details

* Analyze: css wip

* Analyze: fix outsource label

* Analyze: fix outsource label

* Fix css z-index label with tooltip component

* Analyze: css improvements

* Analyze: css improvements

* Volume analysis: css fix

* Volume analysis: css fix

* Analyze: css fix

---------

Co-authored-by: pierluigi.dicianni <[email protected]>

* Try mysql on local docker instead of travis one

* Detach docker

* Fixed local mysql host

* Fixed issue in tests

* Used docker image to run tests

* Docker compose up

* Added mysql and redis containers for test in docker compose

* Fixed exit code from test container

* Prepend tests for every build

* Fixed tests to work with new code

* Try to reduce logs

* Fixed jest pollution

* Removed error to dev/null

* Silent builds

* Fixed TMAnalysisWorker and Engines_Results_MyMemory_TMS

* All tests now use AbstractTest helper class

Signed-off-by: domenico <[email protected]>

* Fixed some wrong signatures

* Implemented new P4 queue

- Refactory toward PHP 7.4

* Updated submodule

---------

Signed-off-by: domenico <[email protected]>
Co-authored-by: Mauro Cassani <[email protected]>
Co-authored-by: Federico Ricciuti <[email protected]>
Co-authored-by: pierluigi.dicianni <[email protected]>

Author: 4 people

.gitignore

@@ -64,8 +64,6 @@ phinx.yml
 .lastjobprocessed_jpeer
 .maxJobToProcess_jpeer
 
-inc/dqf/dqf_test_secret.ini
-
 mmt_fallback_key.ini
 
 inc/google_translate/mmt_fallback_credentials.json

.travis.yml

@@ -6,7 +6,7 @@ php:
 
 branches:
   only:
-    #   - develop
+    - develop
     - /^v?(\d+\.)?(\d+\.)?(\d+)(-[a-z\d]+)?$/
     - freddy
     #  - master
@@ -27,6 +27,20 @@ before_install:
 
 script:
   - |
+    
+    echo "Load services"
+    docker-compose -f docker/docker-compose-ci.yml build mysql redis > /dev/null 2>&1
+    docker-compose -f docker/docker-compose-ci.yml create mysql redis > /dev/null 2>&1
+    
+    # keep logs for the base node
+    docker-compose -f docker/docker-compose-ci.yml build base-node
+    
+    # silent for test node
+    echo "Build test node"
+    docker-compose -f docker/docker-compose-ci.yml build test-node > /dev/null 2>&1
+    
+    # Run tests
+    docker-compose -f docker/docker-compose-ci.yml up test-node --exit-code-from test-node || exit 1
 
     if [[ ! -z "${TRAVIS_TAG}" ]] ; then
 

composer.json

@@ -1,6 +1,6 @@
 {
     "require-dev": {
-        "phpunit/phpunit": "^5.0",
+        "phpunit/phpunit": "^5.7",
         "robmorgan/phinx": "^0.4.6"
     },
     "require": {

docker

@@ -18,14 +18,20 @@ class Bootstrap {
      */
     private $autoLoadedFeatureSet;
 
-    public static function start() {
-        new self();
+    public static function start( SplFileInfo $config_file = null, SplFileInfo $task_runner_config_file = null ) {
+        new self( $config_file, $task_runner_config_file );
     }
 
-    private function __construct() {
+    private function __construct( SplFileInfo $config_file = null, SplFileInfo $task_runner_config_file = null ) {
 
         self::$_ROOT  = realpath( dirname( __FILE__ ) . '/../' );
-        self::$CONFIG = parse_ini_file( self::$_ROOT . DIRECTORY_SEPARATOR . 'inc/config.ini', true );
+
+        if( $config_file != null ){
+            self::$CONFIG = parse_ini_file( $config_file->getRealPath(), true );
+        } else {
+            self::$CONFIG = parse_ini_file( self::$_ROOT . DIRECTORY_SEPARATOR . 'inc/config.ini', true );
+        }
+
         $OAUTH_CONFIG = @parse_ini_file( self::$_ROOT . DIRECTORY_SEPARATOR . 'inc/oauth_config.ini', true );
 
         register_shutdown_function( [ 'Bootstrap', 'shutdownFunctionHandler' ] );
@@ -52,6 +58,12 @@ private function __construct() {
         //get the environment configuration
         self::initConfig();
 
+        if( $task_runner_config_file != null ){
+            INIT::$TASK_RUNNER_CONFIG = parse_ini_file( $task_runner_config_file->getRealPath(), true );
+        } else {
+            INIT::$TASK_RUNNER_CONFIG = parse_ini_file( self::$_ROOT . DIRECTORY_SEPARATOR . 'inc/task_manager_config.ini', true );
+        }
+
         ini_set( 'display_errors', false );
         if ( INIT::$PRINT_ERRORS ) {
             ini_set( 'display_errors', true );
@@ -80,8 +92,6 @@ private function __construct() {
         INIT::$CONTROLLER_ROOT                 = INIT::$ROOT . '/lib/Controller';
         INIT::$UTILS_ROOT                      = INIT::$ROOT . '/lib/Utils';
 
-        INIT::$TASK_RUNNER_CONFIG = parse_ini_file( self::$_ROOT . DIRECTORY_SEPARATOR . 'inc/task_manager_config.ini', true );
-
         try {
             Log::$uniqID = ( isset( $_COOKIE[ INIT::$PHP_SESSION_NAME ] ) ? substr( $_COOKIE[ INIT::$PHP_SESSION_NAME ], 0, 13 ) : uniqid() );
             WorkerClient::init();

inc/Bootstrap.php

@@ -627,7 +627,6 @@ public function doAction() {
         $projectStructure[ 'mt_engine' ]            = $this->postInput[ 'mt_engine' ];
         $projectStructure[ 'tms_engine' ]           = $this->postInput[ 'tms_engine' ];
         $projectStructure[ 'status' ]               = Constants_ProjectStatus::STATUS_NOT_READY_FOR_ANALYSIS;
-        $projectStructure[ 'skip_lang_validation' ] = true;
         $projectStructure[ 'owner' ]                = $this->user->email;
         $projectStructure[ 'metadata' ]             = $this->metadata;
         $projectStructure[ 'pretranslate_100' ]     = (int)!!$this->postInput[ 'pretranslate_100' ]; // Force pretranslate_100 to be 0 or 1

lib/Controller/API/NewController.php

@@ -123,7 +123,7 @@ protected function _saveActivity(){
     /**
      * @Override
      */
-    public function finalize() {
+    public function finalize($forceXliff = false) {
 
         $buffer = ob_get_contents();
         ob_get_clean();

lib/Controller/API/V2/ExportTMXController.php

@@ -12,7 +12,7 @@
 use API\V2\KleinController;
 use CatUtils;
 use Langs_Languages;
-use LQA\SizeRestriction;
+use LQA\SizeRestriction\SizeRestriction;
 use Matecat\SubFiltering\MateCatFilter;
 
 

lib/Controller/API/V3/CountWordController.php

@@ -318,7 +318,6 @@ public function doAction() {
         $projectStructure[ 'tms_engine' ]                   = $this->tms_engine;
         $projectStructure[ 'status' ]                       = Constants_ProjectStatus::STATUS_NOT_READY_FOR_ANALYSIS;
         $projectStructure[ 'lang_detect_files' ]            = $this->lang_detect_files;
-        $projectStructure[ 'skip_lang_validation' ]         = true;
         $projectStructure[ 'pretranslate_100' ]             = $this->pretranslate_100;
         $projectStructure[ 'pretranslate_101' ]             = $this->pretranslate_101;
         $projectStructure[ 'dialect_strict' ]               = $this->dialect_strict;

lib/Controller/ILegacyCatController.php

@@ -66,7 +66,7 @@ function doAction() {
         $analysisStatus = new XTRFStatus( $_project_data, $this->featureSet );
         $outputContent = $analysisStatus->fetchData()->getResult();
 
-        $this->outputContent = $this->composeZip( $_project_data[0][ 'pname' ], $outputContent );
+        $this->outputContent = $this->composeZip( $outputContent, $_project_data[0][ 'pname' ] );
         $this->_filename     = $_project_data[0][ 'pname' ] . ".zip";
 
         /**
@@ -91,7 +91,7 @@ function doAction() {
      *
      * @return false|string
      */
-    protected static function composeZip( $projectName , $outputContent ) {
+    protected static function composeZip( Array $outputContent, $projectName = null, $ignore = false ) {
 
         $fileName = tempnam( "/tmp", "zipmat" );
         $zip  = new ZipArchive();