ftp_utils: New configuration variables and FTP/SFTP ls method update

* Adds new configuration variables for Elsevier SFTP connection - port, sftp

* Adds an addition cleaning to file names on the list of files/folders from ls
  method in ftp_utils.py

Author: Wojciech Ziółek

harvestingkit/contrast_out.py

@@ -99,7 +99,9 @@ def connect(self):
             try:
                 self.ftp = FtpHandler(self.config.ELSEVIER.URL,
                                       self.config.ELSEVIER.LOGIN,
-                                      self.config.ELSEVIER.PASSWORD)
+                                      self.config.ELSEVIER.PASSWORD,
+                                      port = int(self.config.ELSEVIER.PORT),
+                                      sftp = self.config.ELSEVIER.SFTP)
                 self.logger.debug(('Successful connection to the '
                                    'Elsevier server'))
                 return

harvestingkit/ftp_utils.py

@@ -195,13 +195,17 @@ def ls(self, folder=''):
             contents = self._sftp_client.listdir()
             files = filter(lambda a: str(self._sftp_client.lstat(a)).split()[0].startswith('-'), contents)
             folders = filter(lambda a: str(self._sftp_client.lstat(a)).split()[0].startswith('d'), contents)
+            files = map(lambda a: ' '.join(a.split()[8:]), files)
+            folders = map(lambda a: ' '.join(a.split()[8:]), folders)
             self._sftp_client.chdir(current_folder)
         else:
             current_folder = self._ftp.pwd()
             self.cd(folder)
             self._ftp.retrlines('LIST', lambda a: contents.append(a))
             files = filter(lambda a: a.split()[0].startswith('-'), contents)
             folders = filter(lambda a: a.split()[0].startswith('d'), contents)
+            files = map(lambda a: ' '.join(a.split()[8:]), files)
+            folders = map(lambda a: ' '.join(a.split()[8:]), folders)
             self._ftp.cwd(current_folder)
         return files, folders
 

user_config.cfg

@@ -2,6 +2,8 @@
 login = empty
 password = empty
 url = empty
+port = 21
+sftp = True
 
 [OXFORD]
 login = empty