fix setup file bug + update readme + db data start

Author: deckbsd

README.md

@@ -1,17 +1,27 @@
 [![Known Vulnerabilities](https://snyk.io/test/github/deckbsd/glouton-satnogs-data-downloader/badge.svg)](https://snyk.io/test/github/deckbsd/glouton-satnogs-data-downloader)
 # glouton-satnogs-data-downloader
-The console app is a downloader for the data provided by the satnogs network.
+This cli app is a downloader for the data provided by the satnogs network.
 
-command example : 
+Installation :
+-------
+```
+python ./setup.py install
+```
+
+Usage :
+-------
+
+simple command example : 
 ```
-python ./glouton.py -s 2018-01-20T00:51:54 -e 2018-01-21T00:51:54 -n 28654
+python ./bin/glouton -s 2018-01-20T00:51:54 -e 2018-01-21T00:51:54 -n 28654
 ```
 command example if you just want the payload files :
 ```
-python ./glouton.py -s 2018-01-20T00:51:54 -e 2018-01-21T00:51:54 -n 28654 --payload
+python ./bin/glouton -s 2018-01-20T00:51:54 -e 2018-01-21T00:51:54 -n 28654 --payload
 ```
+command example if you want all data type from specific transmitter type, mode and uuid and apply module processing on waterfall :
 ```
-python ./glouton.py -s 2019-05-09T00:51:54 -e 2019-05-30T00:51:54 -n 40069 --waterfallm TestModule,CSV --tuuid 8oBdHqMqgmMiWvRru6fWMn --ttype Transmitter --tmode LRPT
+python ./bin/glouton -s 2019-05-09T00:51:54 -e 2019-05-30T00:51:54 -n 40069 --waterfallm TestModule,CSV --tuuid 8oBdHqMqgmMiWvRru6fWMn --ttype Transmitter --tmode LRPT
 ```
 
 Actual features :
@@ -24,13 +34,19 @@ Actual features :
         * end date
         * stations
         * observation status
+        * transmitter mode
+        * transmitter uuid
+        * transmitter type
+        * vetted status
+        * vetted user
     * working directory selection
     * Docker container
     * modules
 
 Future :
 -------
     * automatic mode for downloading automatically the new observations of one or more satellites.
+    * download data from satnogs db
 
 Modules :
 -------

bin/glouton

@@ -32,15 +32,16 @@ if __name__ == "__main__":
     try:
         glouton()
         parser = argparse.ArgumentParser(description='Execute get request.')
+        # Network API parameters :
         parser.add_argument('--norad', '-n', dest='norad_id', required='--last' not in sys.argv,
                             help='the norad satellite id')
         parser.add_argument('--gsid', '-g', dest='ground_station_id',
                             help='the ground station id')
         parser.add_argument('--status', '-t', dest='status',
                             help='the Observation status (good, bad, unknown, failed')
-        parser.add_argument('--sdate', '-s', dest='start_date', required='--last' not in sys.argv,
+        parser.add_argument('--sdate', '-s', dest='start_date', required='--last' not in sys.argv and '--db' not in sys.argv,
                             help='start date (ex: 2018-01-20T00:51:54)')
-        parser.add_argument('--edate', '-e', dest='end_date', required='--last' not in sys.argv,
+        parser.add_argument('--edate', '-e', dest='end_date', required='--last' not in sys.argv and '--db' not in sys.argv,
                             help='end date (ex: 2018-01-21T00:51:54)')
         parser.add_argument('--user', '-u', dest='user_id',
                             help='the user id')
@@ -68,6 +69,9 @@ if __name__ == "__main__":
                             help='list of the modules to use while downloading waterfall separated by a ,')
         parser.add_argument('--last', dest='last', action='store_true',
                             help='restart a download from the last command line.')
+        # DB API parameters :
+        parser.add_argument('--db', dest='db', action='store_true', default=False,
+                            help='download data from satnogs db api')
         args = parser.parse_args()
         session = SessionService()
         if args.last is True:
@@ -102,8 +106,10 @@ if __name__ == "__main__":
             args.transmitter_type)
             session.save_program_parameters(cmd)
 
-        obs = ObservationsService(cmd)
-        obs.extract()
+        if args.db is False:
+            obs = ObservationsService(cmd)
+            obs.extract()
+
         logger.Info("\n\nall jobs are finished\t(   ^ o^)\m/")
     except KeyboardInterrupt:
         print("Exit...")

glouton/commands/download/downloadCommand.py

@@ -1,12 +1,12 @@
-from glouton.infrastructure.satnogClient import SatnogClient
+from glouton.infrastructure.satnogNetworkClient import SatnogNetworkClient
 from glouton.commands.module.moduleCommandParams import ModuleCommandParams
 from glouton.commands.module.moduleCommand import ModuleCommand
 import os
 
 class DownloadCommand:
     def __init__(self, params, observation, modules_commands):
         self.observation = observation
-        self.client = SatnogClient()
+        self.client = SatnogNetworkClient()
         self.full_path = os.path.join(params.working_dir, params.sub_folder)
         self.modules = params.modules
         self.modules_commands = modules_commands

glouton/commands/module/moduleCommand.py

@@ -1,4 +1,3 @@
-from glouton.infrastructure.satnogClient import SatnogClient
 import os
 
 class ModuleCommand:

glouton/config.json

@@ -1,13 +1,15 @@
 {
     "DEFAULT": {
-        "API_URL": "https://network.satnogs.org/api/",
+        "NETWORK_API_URL": "https://network.satnogs.org/api/",
+        "DB_API_URL": "https://db.satnogs.org/api/",
         "HTTPS_PROXY": "",
         "HTTP_PROXY": ""
     },
     "MODULES": {
         "PAYLOAD": [],
         "WATERFALL": [],
         "DEMODDATA": [],
+        "FRAME": [],
         "FORALL": []
     },
     "LOGFILE": "gl.log"

glouton/infrastructure/satnogClient.py

@@ -5,9 +5,8 @@
 
 class SatnogClient:
     def __init__(self):
-        self._config = _config = config.read()
-        self._url = _config['DEFAULT']['API_URL']
-        self._proxies = self._set_proxy()
+        self.config  = config.read()
+        self.proxies = self._set_proxy()
 
     def _set_proxy(self):
         proxies = None
@@ -17,16 +16,16 @@ def _set_proxy(self):
                 'https': os.environ['https_proxy']
             }
             return proxies
-        if self._config['DEFAULT']['HTTP_PROXY'] != "" or self._config['DEFAULT']['HTTPS_PROXY'] != "":
+        if self.config['DEFAULT']['HTTP_PROXY'] != "" or self.config['DEFAULT']['HTTPS_PROXY'] != "":
             proxies = {
-                'http': self._config['DEFAULT']['HTTP_PROXY'],
-                'https': self._config['DEFAULT']['HTTPS_PROXY']
+                'http': self.config['DEFAULT']['HTTP_PROXY'],
+                'https': self.config['DEFAULT']['HTTPS_PROXY']
             }
 
         return proxies
 
     def get(self, url, params=None):
-        return requests.get(url, params=params, proxies=self._proxies)
+        raise NotImplementedError()
 
     def get_from_base(self, url, params=None):
-        return requests.get(self._url + url, params=params, proxies=self._proxies)
+        raise NotImplementedError()

glouton/infrastructure/satnogDbClient.py

@@ -0,0 +1,15 @@
+from glouton.infrastructure.satnogClient import SatnogClient
+import os
+import requests
+
+
+class SatnogDbClient(SatnogClient):
+    def __init__(self):
+        SatnogClient.__init__(self)
+        self._url = self.config['DEFAULT']['DB_API_URL']
+
+    def get(self, url, params=None):
+        return requests.get(url, params=params, proxies=self.proxies)
+
+    def get_from_base(self, url, params=None):
+        return requests.get(self._url + url, params=params, proxies=self.proxies)

glouton/infrastructure/satnogNetworkClient.py

@@ -0,0 +1,15 @@
+from glouton.infrastructure.satnogClient import SatnogClient
+import os
+import requests
+
+
+class SatnogNetworkClient(SatnogClient):
+    def __init__(self):
+        SatnogClient.__init__(self)
+        self._url = self.config['DEFAULT']['NETWORK_API_URL']
+
+    def get(self, url, params=None):
+        return requests.get(url, params=params, proxies=self.proxies)
+
+    def get_from_base(self, url, params=None):
+        return requests.get(self._url + url, params=params, proxies=self.proxies)

glouton/modules/moduleBase.py

@@ -1,4 +1,3 @@
-from glouton.infrastructure.satnogClient import SatnogClient
 import os
 
 class ModuleBase:

glouton/repositories/observation/observationsRepo.py

@@ -1,12 +1,12 @@
 from queue import Queue
-from glouton.infrastructure.satnogClient import SatnogClient
+from glouton.infrastructure.satnogNetworkClient import SatnogNetworkClient
 from glouton.shared.logger import logger
 
 
 class ObservationRepo:
     def __init__(self, cmd, repos):
         self.OBSERVATION_URL = 'observations/'
-        self.__client = SatnogClient()
+        self.__client = SatnogNetworkClient()
         self.__repos = repos
         self.__cmd = cmd
         self.__threads = []