digitalmethodsinitiative
diff --git a/‎.dockerignore‎
Lines changed: 5 additions & 0 deletions b/‎.dockerignore‎
Lines changed: 5 additions & 0 deletions
diff --git a/‎.env‎
Lines changed: 9 additions & 1 deletion b/‎.env‎
Lines changed: 9 additions & 1 deletion
diff --git a/‎.gitignore‎
Lines changed: 1 addition & 0 deletions b/‎.gitignore‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎.zenodo.json‎
Lines changed: 1 addition & 1 deletion b/‎.zenodo.json‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎4cat-daemon.py‎
Lines changed: 4 additions & 3 deletions b/‎4cat-daemon.py‎
Lines changed: 4 additions & 3 deletions
diff --git a/‎LICENSE-3DPARTY‎
Lines changed: 35 additions & 1 deletion b/‎LICENSE-3DPARTY‎
Lines changed: 35 additions & 1 deletion
diff --git a/‎README.md‎
Lines changed: 23 additions & 15 deletions b/‎README.md‎
Lines changed: 23 additions & 15 deletions
diff --git a/‎VERSION‎
Lines changed: 1 addition & 1 deletion b/‎VERSION‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎backend/bootstrap.py‎
Lines changed: 3 additions & 3 deletions b/‎backend/bootstrap.py‎
Lines changed: 3 additions & 3 deletions
diff --git a/‎backend/database.sql‎
Lines changed: 7 additions & 4 deletions b/‎backend/database.sql‎
Lines changed: 7 additions & 4 deletions
@@ -0,0 +1,5 @@
+data/
+.github/
+.ipynb_checkpoints/
+.gitignore
+.idea/
@@ -1,6 +1,7 @@
 # 4CAT Version: Update with latest release tag or 'latest'
 # https://hub.docker.com/repository/docker/digitalmethodsinitiative/4cat/tags?page=1&ordering=last_updated
 DOCKER_TAG=stable
+# You can select Postrgres Docker image tags here to suit your needs: https://hub.docker.com/_/postgres
 POSTGRES_TAG=latest
 
 # Database setup
@@ -29,5 +30,12 @@ TELEGRAM_PORT=443
 # Docker Volume Names
 DOCKER_DB_VOL=4cat_4cat_db
 DOCKER_DATA_VOL=4cat_4cat_data
-DOCKER_CONFIG_VOL=4cat_4cat_share
+DOCKER_CONFIG_VOL=4cat_4cat_config
 DOCKER_LOGS_VOL=4cat_4cat_logs
+
+# Gunicorn settings
+worker_tmp_dir=/dev/shm
+workers=4
+threads=4
+worker_class=gthread
+log_level=debug
@@ -54,6 +54,7 @@ venv/
 
 # generated by 4CAT
 webtool/static/css/colours.css
+webtool/static/img/favicon/favicon.ico
 
 # data files
 results
 
@@ -3,7 +3,7 @@
   "license": "MPL-2.0",
   "title": "4CAT Capture and Analysis Toolkit",
   "upload_type": "software",
-  "version": "v1.38",
+  "version": "v1.46",
   "keywords": [
     "webmining",
     "scraping",
 
@@ -10,6 +10,7 @@
 cli = argparse.ArgumentParser()
 cli.add_argument("--interactive", "-i", default=False, help="Run 4CAT in interactive mode (not in the background).",
                  action="store_true")
+cli.add_argument("--log-level", "-l", default="INFO", help="Set log level (\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\", \"FATAL\").")
 cli.add_argument("--no-version-check", "-n", default=False,
                  help="Skip version check that may prompt the user to migrate first.", action="store_true")
 cli.add_argument("command")
@@ -81,14 +82,14 @@
     print("Running backend in interactive mode instead.")
     import backend.bootstrap as bootstrap
 
-    bootstrap.run(as_daemon=False)
+    bootstrap.run(as_daemon=False, log_level=args.log_level)
     sys.exit(0)
 
 if args.interactive:
     print("Running backend in interactive mode.")
     import backend.bootstrap as bootstrap
 
-    bootstrap.run(as_daemon=False)
+    bootstrap.run(as_daemon=False, log_level=args.log_level)
     sys.exit(0)
 else:
     # if so, import necessary modules
@@ -129,7 +130,7 @@ def start():
                 detach_process=True
         ) as context:
             import backend.bootstrap as bootstrap
-            bootstrap.run(as_daemon=True)
+            bootstrap.run(as_daemon=True, log_level=args.log_level)
 
         sys.exit(0)
 
 
@@ -802,4 +802,38 @@ Incorporates the Graphology graph manipulation library
     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-    THE SOFTWARE.
+    THE SOFTWARE.
+
+-------------------------------------------------------------------------------
+Incorporates the zip.js library
+    - at /webtool/static/js/zip.min.js
+    - from https://github.com/gildas-lormeau/zip.js
+
+    BSD 3-Clause License
+
+    Copyright (c) 2023, Gildas Lormeau
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, this
+       list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+    3. Neither the name of the copyright holder nor the names of its
+       contributors may be used to endorse or promote products derived from
+       this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+    AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -26,39 +26,47 @@ such as the generation and visualisation of word embedding models.
 platforms that are part of the tool, but you can also [add additional data
 sources](https://github.com/digitalmethodsinitiative/4cat/wiki/How-to-make-a-data-source)
 using 4CAT's Python API. The following data sources are currently supported
-actively and can be used to collect data with 4CAT:
+actively and can be used to collect data with 4CAT directly:
 
 * 4chan and 8kun
-* BitChute
-* Reddit
 * Telegram
 * Tumblr
-* Twitter API v2 (Academic and regular tracks)
 
-The following platforms are supported through other tools, with which you can
-collect data to import data into 4CAT for analysis:
+The following platforms are supported through 
+[Zeeschuimer](https://github.com/digitalmethodsinitiative/zeeschuimer), with 
+which you can collect data to import into 4CAT for analysis:
 
-* Instagram, TikTok, 9gag, Imgur, LinkedIn, Parler, Douyin and Twitter (via
-  [Zeeschuimer](https://github.com/digitalmethodsinitiative/zeeschuimer))
-* Facebook and Instagram (via [CrowdTangle](https://www.crowdtangle.com) exports)
+* Instagram (posts)
+* TikTok (posts and comments)
+* 9gag
+* Imgur
+* LinkedIn
+* Gab
+* Douyin
+* X/Twitter
+
+It is also possible to upload data collected with other tools as CSV files. The
+following tools are explicitly supported but other data can also be uploaded as
+long as it is formatted as CSV:
+
+* Facebook and Instagram (via [CrowdTangle](https://www.crowdtangle.com) or [Facepager](https://github.com/strohne/Facepager) exports)
+* YouTube videos and comments (via the [YouTube Data Tools](https://ytdt.digitalmethods.net/))
+* Weibo (via [Bazhuayu](https://www.bazhuayu.com/))
 
 A number of other platforms have built-in support that is untested, or requires
 e.g. special API access. You can view the [data sources in our wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/Available-data-sources) or review [the data
 sources' code](https://github.com/digitalmethodsinitiative/4cat/tree/master/datasources)
-in the GitHub repository. It is also possible to import your own CSV files into 
-4CAT for analysis.
+in the GitHub repository.
 
 ## Installation
-You can install 4CAT locally or on a server via Docker or manually. For easiest installation, we reccomend copying our [`docker-compose.yml file`](https://raw.githubusercontent.com/digitalmethodsinitiative/4cat/master/docker-compose.yml), [`.env`](https://raw.githubusercontent.com/digitalmethodsinitiative/4cat/master/.env) file, and running this terminal command in the folder where those files have been saved:
+You can install 4CAT locally or on a server via Docker or manually. For easiest installation, we recommend copying our [`docker-compose.yml file`](https://raw.githubusercontent.com/digitalmethodsinitiative/4cat/master/docker-compose.yml), [`.env`](https://raw.githubusercontent.com/digitalmethodsinitiative/4cat/master/.env) file, and running this terminal command in the folder where those files have been saved:
 
 ```
 docker-compose up -d
 ```
 
 In depth instructions on both Docker installation and manual installation can be found [in our
-wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/Installing-4CAT).
-
-A video walkthrough installing 4CAT via Docker can be found on [YouTube here](https://youtu.be/oWsB7bvNfOY).
+wiki](https://github.com/digitalmethodsinitiative/4cat/wiki/Installing-4CAT). A video walkthrough installing 4CAT via Docker can be found on [YouTube here](https://youtu.be/oWsB7bvNfOY).
 
 Currently scraping of 4chan, 8chan, and 8kun require additional steps; please see the wiki.
 
 
@@ -1,4 +1,4 @@
-1.38
+1.46
 
 This file should not be modified. It is used by 4CAT to determine whether it
 needs to run migration scripts to e.g. update the database structure to a more
 
@@ -13,7 +13,7 @@
 
 from common.config_manager import config
 
-def run(as_daemon=True):
+def run(as_daemon=True, log_level="INFO"):
 	pidfile = Path(config.get('PATH_ROOT'), config.get('PATH_LOCKFILE'), "4cat.pid")
 
 	if as_daemon:
@@ -49,9 +49,9 @@ def run(as_daemon=True):
 	if config.get("USING_DOCKER"):
 		as_daemon = True
 		# Rename log if Docker setup
-		log = Logger(output=True, filename='backend_4cat.log')
+		log = Logger(output=True, filename='backend_4cat.log', log_level=log_level)
 	else:
-		log = Logger(output=not as_daemon)
+		log = Logger(output=not as_daemon, filename='4cat.log', log_level=log_level)
 
 	log.info("4CAT Backend started, logger initialised")
 	db = Database(logger=log, appname="main",
 
@@ -17,15 +17,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS unique_setting
 
 -- jobs table
 CREATE TABLE IF NOT EXISTS jobs (
-  id                     SERIAL PRIMARY KEY,
+  id                     BIGSERIAL PRIMARY KEY,
   jobtype                text    DEFAULT 'misc',
   remote_id              text,
   details                text,
   timestamp              integer,
   timestamp_after        integer DEFAULT 0,
   timestamp_lastclaimed  integer DEFAULT 0,
   timestamp_claimed      integer DEFAULT 0,
-  status                 text,
   attempts               integer DEFAULT 0,
   interval               integer DEFAULT 0
 );
@@ -46,7 +45,7 @@ CREATE TABLE IF NOT EXISTS datasets (
   key_parent        text DEFAULT '' NOT NULL,
   creator           VARCHAR DEFAULT 'anonymous',
   query             text,
-  job               integer DEFAULT 0,
+  job               BIGINT DEFAULT 0,
   parameters        text,
   result_file       text DEFAULT '',
   timestamp         integer,
@@ -57,6 +56,7 @@ CREATE TABLE IF NOT EXISTS datasets (
   is_private        boolean DEFAULT TRUE,
   software_version  text,
   software_file     text DEFAULT '',
+  software_source   text DEFAULT '',
   annotation_fields text DEFAULT ''
 );
 
@@ -81,9 +81,12 @@ CREATE TABLE IF NOT EXISTS metrics (
   datasource         text,
   board              text,
   date               text,
-  count              integer
+  count              BIGINT
 );
 
+CREATE UNIQUE INDEX IF NOT EXISTS unique_metrics
+    ON metrics (metric, datasource, board, date);
+
 -- users
 CREATE TABLE IF NOT EXISTS users (
   name               TEXT UNIQUE PRIMARY KEY,
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-1.38`
	`1`	`+1.46`
`2`	`2`
`3`	`3`	`This file should not be modified. It is used by 4CAT to determine whether it`
`4`	`4`	`needs to run migration scripts to e.g. update the database structure to a more`