Merge pull request #136 from metakgp/import-script

Improvements to import script and other changes

Author: dipamsen

.gitignore

@@ -13,4 +13,5 @@ go.work.sum
 
 frontend/public/pdf.worker.min.mjs
 *.log
-*.patch
+*.patch
+log

README.md

@@ -126,10 +126,12 @@ A user is considered as an admin if they are a part of the team `GH_ORG_TEAM_SLU
 
 ### Crawler
 
+The crawler is a go script which crawls and downloads papers from [peqp](http://10.18.24.75/peqp/) (only accessible over campus network) and spits an archive which can be imported into the database.
+
 1. Change directory to `crawler/` and run `go mod tidy`.
-2. Run the crawler by running `go run crawler.go`. (Make sure you are connected to the campus network)
+2. Run the crawler by running `go run crawler.go [flags]`. (Make sure you are connected to the campus network) (Run `go run crawler.go -h` to see flags that can be set.)
 3. This will generate a `qp.tar.gz` file. Transfer this file to the server's `backend/` folder.
-4. (Development): In the backend, run `cargo run --bin import-papers` to import the data into the database. (Make sure the database is set up and running)\
+4. (Development): In the backend, run `cargo run --bin import-papers qp.tar.gz` to import the data into the database. (Make sure the database is set up and running)\
     (Production): In the backend, run `./import_papers.sh ./qp.tar.gz` to run the import script in the docker container running the application.
 
 ## Deployment

backend/import_papers.sh

@@ -10,10 +10,18 @@ ARCHIVE_PATH="$1"
 SERVICE="iqps-backend"
 DEST_PATH="/app/qp.tar.gz"
 
+if [[ ! -f "$ARCHIVE_PATH" ]]; then
+    echo "Error: File '$ARCHIVE_PATH' not found."
+    exit 1
+fi
+
 echo "Copying '$ARCHIVE_PATH' to '$SERVICE'..."
 docker compose cp "$ARCHIVE_PATH" "$SERVICE":"$DEST_PATH"
 
 echo "Running import-papers..."
 docker compose exec "$SERVICE" ./import-papers
 
+echo "Deleting copied file from container..."
+docker compose exec "$SERVICE" rm -f "$DEST_PATH"
+
 echo "Done!"

backend/src/bin/import-papers.rs

@@ -13,22 +13,41 @@ use tempfile::tempdir;
 use tracing::{info, warn};
 use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt};
 
+#[derive(Parser, Debug)]
+#[command(
+    name = "import-papers",
+    about = "Imports papers into the database from an archive.",
+    version,
+    author
+)]
+struct Args {
+    /// Path to the .tar.gz file containing papers (e.g., qp.tar.gz)
+    file: String,
+}
+
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     if dotenvy::dotenv().is_ok() {
         println!("Loaded an existing .env file.");
     }
-    let env_vars = env::EnvVars::parse()
+
+    let env_vars = env::EnvVars::parse()?
         .process()
         .expect("Failed to parse environment variables");
 
+	let args = Args::parse();
+	if !Path::new(&args.file).exists() {
+		eprintln!("Error: file '{}' not found.", args.file);
+		std::process::exit(1);
+	}
+	
     let database = db::Database::new(&env_vars)
         .await
         .expect("Failed to connect to database");
 
     let dir = tempdir()?;
     let dir_path = dir.path();
-    extract_tar_gz("qp.tar.gz", dir_path)?;
+    extract_tar_gz(&args.file, dir_path)?;
 
     let file = fs::File::open(dir_path.join("qp.json")).expect("Failed to open JSON file");
     let reader = BufReader::new(file);
@@ -142,9 +161,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     println!("Finished uploading papers to database.");
     dir.close()?;
 
+    let total_count = database.get_unapproved_papers_count().await?;
+
     let message = format!(
-      "{} papers have been imported into IQPS!",
-      count,
+      "💥 {count} papers have been imported into IQPS!\n\n<https://qp.metakgp.org/admin|Review> | Total Unapproved papers: *{total_count}*",
     );
 
     let _ = slack::send_slack_message(

backend/src/env.rs

@@ -4,95 +4,135 @@
 
 use std::path::PathBuf;
 
-use clap::Parser;
 use hmac::{digest::InvalidLength, Hmac, Mac};
 use sha2::Sha256;
 
 use crate::pathutils::Paths;
 
-#[derive(Parser, Clone)]
+#[derive(Clone)]
 pub struct EnvVars {
     // Database
-    #[arg(env)]
     /// Database name
     pub db_name: String,
-    #[arg(env)]
     /// Database hostname
     pub db_host: String,
-    #[arg(env)]
     /// Database port
     pub db_port: String,
-    #[arg(env)]
     /// Database username
     pub db_user: String,
-    #[arg(env)]
     /// Database password
     pub db_password: String,
 
     // Auth
-    #[arg(env)]
     /// OAuth app client id (public token)
     pub gh_client_id: String,
-    #[arg(env)]
     /// An org admin's Github token (with the `read:org` permission)
     pub gh_org_admin_token: String,
-    #[arg(env)]
     /// JWT encryption secret (make it a long, randomized string)
     jwt_secret: String,
-    #[arg(env)]
     /// OAuth app client secret
     pub gh_client_secret: String,
-    #[arg(env, default_value = "")]
     /// Github organization name
     pub gh_org_name: String,
-    #[arg(env, default_value = "")]
     /// Github organization team slug (this team has access to admin dashboard)
     pub gh_org_team_slug: String,
-    #[arg(env, default_value = "")]
     /// The usernames of the admins (additional to org team members, comma separated)
     pub gh_admin_usernames: String,
-    #[arg(env, default_value = "")]
     /// URL of Slack webhook for sending notifications
     pub slack_webhook_url: String, 
 
     // Other configs
-    #[arg(env, default_value = "10")]
     /// Maximum number of papers that can be uploaded at a time
     pub max_upload_limit: usize,
-    #[arg(env, default_value = "./log/application.log")]
     /// Location where logs are stored
     pub log_location: PathBuf,
 
     // Paths
-    #[arg(env, default_value = "https://static.metakgp.org")]
     /// The URL of the static files server (odin's vault)
     static_files_url: String,
-    #[arg(env, default_value = "/srv/static")]
     /// The path where static files are served from
     static_file_storage_location: PathBuf,
-    #[arg(env, default_value = "/iqps/uploaded")]
     /// The path where uploaded papers are stored temporarily, relative to the `static_file_storage_location`
     uploaded_qps_path: PathBuf,
-    #[arg(env, default_value = "/peqp/qp")]
     /// The path where library papers (scrapped) are stored, relative to the `static_file_storage_location`
     library_qps_path: PathBuf,
 
     // Server
-    #[arg(env, default_value = "8080")]
     /// The port the server listens on
     pub server_port: i32,
 
     // CORS
-    #[arg(env, default_value = "https://qp.metakgp.org,http://localhost:5173")]
     /// List of origins allowed (as a list of values separated by commas `origin1, origin2`)
     pub cors_allowed_origins: String,
 
-    #[arg(skip)]
     /// All paths must be handled using this
     pub paths: Paths,
 }
 
-impl EnvVars {
+impl EnvVars {    
+    /// Parses the environment variables into the struct
+    pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
+        let db_name = std::env::var("DB_NAME")?;
+        let db_host = std::env::var("DB_HOST")?;
+        let db_port = std::env::var("DB_PORT")?;
+        let db_user = std::env::var("DB_USER")?;
+        let db_password = std::env::var("DB_PASSWORD")?;
+        let gh_client_id = std::env::var("GH_CLIENT_ID")?;
+        let gh_org_admin_token = std::env::var("GH_ORG_ADMIN_TOKEN")?;
+        let jwt_secret = std::env::var("JWT_SECRET")?;
+        let gh_client_secret = std::env::var("GH_CLIENT_SECRET")?;
+        let gh_org_name = std::env::var("GH_ORG_NAME").unwrap_or_default();
+        let gh_org_team_slug = std::env::var("GH_ORG_TEAM_SLUG").unwrap_or_default();
+        let gh_admin_usernames = std::env::var("GH_ADMIN_USERNAMES").unwrap_or_default();
+        let slack_webhook_url = std::env::var("SLACK_WEBHOOK_URL").unwrap_or_default();
+        let max_upload_limit = std::env::var("MAX_UPLOAD_LIMIT")
+            .unwrap_or_else(|_| "10".to_string())
+            .parse::<usize>()?;
+        let log_location = std::env::var("LOG_LOCATION")
+            .unwrap_or_else(|_| "./log/application.log".to_string())
+            .into();
+        let static_files_url = std::env::var("STATIC_FILES_URL")
+            .unwrap_or_else(|_| "https://static.metakgp.org".to_string());
+        let static_file_storage_location = std::env::var("STATIC_FILE_STORAGE_LOCATION")
+            .unwrap_or_else(|_| "/srv/static".to_string())
+            .into();
+        let uploaded_qps_path = std::env::var("UPLOADED_QPS_PATH")
+            .unwrap_or_else(|_| "/iqps/uploaded".to_string())
+            .into();
+        let library_qps_path = std::env::var("LIBRARY_QPS_PATH")
+            .unwrap_or_else(|_| "/peqp/qp".to_string())
+            .into();
+        let server_port = std::env::var("SERVER_PORT")
+            .unwrap_or_else(|_| "8080".to_string())
+            .parse::<i32>()?;
+        let cors_allowed_origins = std::env::var("CORS_ALLOWED_ORIGINS")
+            .unwrap_or_else(|_| "https://qp.metakgp.org,http://localhost:5173".to_string());
+        Ok(Self {
+            db_name,
+            db_host,
+            db_port,
+            db_user,
+            db_password,
+            gh_client_id,
+            gh_org_admin_token,
+            jwt_secret,
+            gh_client_secret,
+            gh_org_name,
+            gh_org_team_slug,
+            gh_admin_usernames,
+            slack_webhook_url,
+            max_upload_limit,
+            log_location,
+            static_files_url,
+            static_file_storage_location,
+            uploaded_qps_path,
+            library_qps_path,
+            server_port,
+            cors_allowed_origins,
+            paths: Paths::default(),
+        })
+    }
+    
     /// Processes the environment variables after reading.
     pub fn process(mut self) -> Result<Self, Box<dyn std::error::Error>> {
         self.paths = Paths::new(

backend/src/main.rs

@@ -2,7 +2,6 @@
 //!
 //! The backend is divided into multiple modules. The [`routing`] module contains all the route handlers and the [`db`] module contains all database queries and models. Other modules are utilities used throughout the backend.
 
-use clap::Parser;
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
 use tracing_subscriber::prelude::*;
 
@@ -22,7 +21,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     // Read environment variables
-    let env_vars = env::EnvVars::parse().process()?;
+    let env_vars = env::EnvVars::parse()?.process()?;
 
     // Initialize logger
     let (append_writer, _guard) = tracing_appender::non_blocking(

frontend/src/components/Common/Common.tsx

@@ -4,7 +4,10 @@ import './styles/common_styles.scss';
 import { IconType } from 'react-icons';
 
 export function Footer() {
-	return <h3 className="meta-footer">Made with ❤️ and {"</>"} by <a href="https://github.com/metakgp/iqps-go" target="_blank">MetaKGP</a></h3>;
+	return <h3 className="meta-footer">
+        Contribute on <a href="https://github.com/metakgp/iqps-go" target="_blank">GitHub</a> | 
+        Made with ❤️ and {"</>"} by <a href="https://github.com/metakgp" target="_blank">MetaKGP</a>
+    </h3>;
 }
 
 interface ILinkCommonProps {
@@ -46,4 +49,4 @@ export function Header(props: IHeaderProps) {
 
 export function Navbar() {
 	return <></>;
-}
+}