Story/cite 163 - - We need a bulk upload for files.

citesphere-importer/pom.xml

@@ -15,7 +15,8 @@
         <spring-data.version>Lovelace-RELEASE</spring-data.version>
         <thymeleaf.version>3.0.11.RELEASE</thymeleaf.version>
         <spring.kafka.version>2.2.6.RELEASE</spring.kafka.version>
-        <citesphere.messages.version>0.2</citesphere.messages.version>
+        <citesphere.messages.version>0.6</citesphere.messages.version>
+        <citesphere.model.version>1.24</citesphere.model.version>
 
         <admin.password>$2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i</admin.password>
 
@@ -69,7 +70,11 @@
             <artifactId>citesphere-messages</artifactId>
             <version>${citesphere.messages.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>edu.asu.diging</groupId>
+            <artifactId>citesphere-model</artifactId>
+            <version>${citesphere.model.version}</version>
+        </dependency>
         <!-- Spring -->
         <dependency>
             <groupId>org.springframework</groupId>
@@ -249,8 +254,12 @@
             <version>2.2.11</version>
         </dependency>
 
-
-
+		<!-- Dependency for parsing bib file -->
+		<dependency>
+			<groupId>org.jbibtex</groupId>
+			<artifactId>jbibtex</artifactId>
+			<version>1.0.20</version>
+		</dependency>
 
         <!-- Apache Kafka -->
         <dependency>
@@ -302,7 +311,7 @@
             <version>1.10.19</version>
             <scope>test</scope>
         </dependency>
-
+        
         <dependency>
             <groupId>com.opencsv</groupId>
             <artifactId>opencsv</artifactId>

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java

@@ -5,6 +5,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.kafka.annotation.KafkaListener;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -18,7 +19,8 @@ public class ReferenceImportListener {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Autowired
-    private IImportProcessor processor;
+    @Qualifier("importProcessor")
+    private IImportProcessor fileProcessor;
 
     @KafkaListener(topics = KafkaTopics.REFERENCES_IMPORT_TOPIC)
     public void receiveMessage(String message) {
@@ -32,6 +34,21 @@ public void receiveMessage(String message) {
             return;
         }
 
-        processor.process(msg);
+        fileProcessor.process(msg);
     }
+
+    @KafkaListener(topics = KafkaTopics.COLLECTION_IMPORT_TOPIC)
+    public void receiveCollectionImportMessage(String message) {
+        ObjectMapper mapper = new ObjectMapper();
+        KafkaJobMessage msg = null;
+        try {
+            msg = mapper.readValue(message, KafkaJobMessage.class);
+        } catch (IOException e) {
+            logger.error("Could not unmarshall message.", e);
+            return;
+        }
+
+        fileProcessor.process(msg);
+    }
+
 }

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java

@@ -3,6 +3,8 @@
 public class Affiliation {
 
     private String name;
+    private String uri;
+    private String localAuthorityId;
 
     public String getName() {
         return name;
@@ -11,4 +13,22 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
+
+    public String getUri() {
+        return uri;
+    }
+
+    public void setUri(String uri) {
+        this.uri = uri;
+    }
+
+    public String getLocalAuthorityId() {
+        return localAuthorityId;
+    }
+
+    public void setLocalAuthorityId(String localAuthorityId) {
+        this.localAuthorityId = localAuthorityId;
+    }
+
+
 }

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java

@@ -3,10 +3,16 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import edu.asu.diging.citesphere.model.bib.IGilesUpload;
+
 public class ArticleMeta {
 
+    private List<String> collections;
     private List<ArticleId> articleIds;
     private String articleTitle;
+    private String articleShortTitle;
     private List<ArticleCategoryGroup> categoryGroups = new ArrayList<ArticleCategoryGroup>();
     private List<Contributor> contributors;
     private String authorNotesCorrespondence;
@@ -25,10 +31,13 @@ public class ArticleMeta {
     private String copyrightYear;
     private String copyrightHolder;
     private String selfUri;
+    @JsonProperty("DOI")
+    private String doi;
     private String articleAbstract;
     private String language;
     private ReviewInfo reviewInfo;
     private String documentType;
+    private String filePath;
     private String conferenceTitle;
     private String conferenceDate;
     private String conferenceLocation;
@@ -43,7 +52,14 @@ public class ArticleMeta {
     private List<Reference> references;
     private String referenceCount;
     private String retrievalDate;
+    private List<IGilesUpload> gilesUpload;
 
+    public List<String> getCollections() {
+        return collections;
+    }
+    public void setCollections(List<String> collections) {
+        this.collections = collections;
+    }
     public List<ArticleId> getArticleIds() {
         return articleIds;
     }
@@ -56,6 +72,12 @@ public String getArticleTitle() {
     public void setArticleTitle(String articleTitle) {
         this.articleTitle = articleTitle;
     }
+    public String getArticleShortTitle() {
+        return articleShortTitle;
+    }
+    public void setArticleShortTitle(String articleShortTitle) {
+        this.articleShortTitle = articleShortTitle;
+    }
     public List<ArticleCategoryGroup> getCategories() {
         return categoryGroups;
     }
@@ -164,6 +186,12 @@ public String getSelfUri() {
     public void setSelfUri(String selfUri) {
         this.selfUri = selfUri;
     }
+    public String getDoi() {
+        return doi;
+    }
+    public void setDoi(String doi) {
+        this.doi = doi;
+    }
     public String getArticleAbstract() {
         return articleAbstract;
     }
@@ -188,6 +216,12 @@ public String getDocumentType() {
     public void setDocumentType(String documentType) {
         this.documentType = documentType;
     }
+    public String getFilePath() {
+        return filePath;
+    }
+    public void setFilePath(String filePath) {
+        this.filePath = filePath;
+    }
     public String getConferenceTitle() {
         return conferenceTitle;
     }
@@ -272,5 +306,11 @@ public String getRetrievalDate() {
     public void setRetrievalDate(String retrievalDate) {
         this.retrievalDate = retrievalDate;
     }
+    public List<IGilesUpload> getGilesUpload() {
+        return gilesUpload;
+    }
+    public void setGilesUpload(List<IGilesUpload> gilesUpload) {
+        this.gilesUpload = gilesUpload;
+    }
 
 }

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java

@@ -13,6 +13,7 @@ public class Contributor {
     private String fullSurname;
     private String fullName;
     private String emailAddress;
+    private String uri;
     private List<Affiliation> affiliations = new ArrayList<Affiliation>();
     private List<ContributorId> ids;
 
@@ -64,6 +65,15 @@ public String getEmailAddress() {
     public void setEmailAddress(String emailAddress) {
         this.emailAddress = emailAddress;
     }
+    public String getUri() {
+        return uri;
+    }
+    public void setUri(String uri) {
+        this.uri = uri;
+    }
+    public void setFullStandardizeName(String fullStandardizeName) {
+        this.fullStandardizeName = fullStandardizeName;
+    }
     public List<Affiliation> getAffiliations() {
         return affiliations;
     }

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java

@@ -0,0 +1,8 @@
+package edu.asu.diging.citesphere.importer.core.service;
+
+import edu.asu.diging.citesphere.model.bib.IGilesUpload;
+
+public interface IGilesConnector {
+
+    IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes);
+}

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java

@@ -0,0 +1,87 @@
+package edu.asu.diging.citesphere.importer.core.service.giles.impl;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
+
+import edu.asu.diging.citesphere.importer.core.service.IGilesConnector;
+import edu.asu.diging.citesphere.model.bib.IGilesUpload;
+import edu.asu.diging.citesphere.model.bib.impl.GilesUpload;
+
+@Service
+@PropertySource("classpath:/config.properties")
+public class GilesConnector implements IGilesConnector {
+
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
+    private RestTemplate restTemplate;
+
+    @Value("${_giles_baseurl}")
+    private String gilesBaseurl;
+
+    @Value("${_giles_upload_endpoint}")
+    private String uploadEndpoint;
+
+    @PostConstruct
+    public void init() {
+        restTemplate = new RestTemplate();
+    }
+
+    @Override
+    public IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes) {
+
+        if (restTemplate == null) {
+            restTemplate = new RestTemplate();  // Ensure restTemplate is initialized
+        }
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+        headers.setBearerAuth(token);
+
+        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+        body.add("files", new MultipartFileResource(fileBytes, filename));
+
+        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
+
+        ResponseEntity<UploadResponse> response = null;
+        try {
+            response = restTemplate.exchange(gilesBaseurl + uploadEndpoint, HttpMethod.POST, requestEntity, UploadResponse.class);
+        } catch (Exception ex) {
+            logger.error("Upload request failed", ex);
+            return null;
+        }
+        IGilesUpload upload = new GilesUpload();
+        upload.setProgressId(response.getBody().getId());
+        upload.setUploadingUser(username);
+        return upload;
+    }
+
+    public class MultipartFileResource extends ByteArrayResource {
+
+        private String filename;
+
+        public MultipartFileResource(byte[] bytearray, String filename) {
+            super(bytearray);
+            this.filename = filename;
+        }
+
+        @Override
+        public String getFilename() {
+            return this.filename;
+        }
+    }
+
+}

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java

@@ -0,0 +1,22 @@
+package edu.asu.diging.citesphere.importer.core.service.giles.impl;
+
+class UploadResponse {
+    private String id;
+    private String checkUrl;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getCheckUrl() {
+        return checkUrl;
+    }
+
+    public void setCheckUrl(String checkUrl) {
+        this.checkUrl = checkUrl;
+    }
+}

citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java

@@ -120,6 +120,8 @@ public void process(KafkaJobMessage message) {
         ObjectMapper mapper = new ObjectMapper();
         ArrayNode root = mapper.createArrayNode();
         int entryCounter = 0;
+        Map<String, String> filesMap = new HashMap<>();
+        ItemCreationResponse response = null;
         while (bibIterator.hasNext()) {
             BibEntry entry = bibIterator.next();
             if (entry.getArticleType() == null) {
@@ -129,13 +131,15 @@ public void process(KafkaJobMessage message) {
             ItemType type = itemTypeMapping.get(entry.getArticleType());
             JsonNode template = zoteroConnector.getTemplate(type);
             ObjectNode bibNode = generationService.generateJson(template, entry);
-
+            if(entry.getArticleMeta().getFilePath() != null) {
+                filesMap.put(entry.getArticleMeta().getArticleTitle(), entry.getArticleMeta().getFilePath());
+            }
             root.add(bibNode);
             entryCounter++;
 
             // we can submit max 50 entries to Zotoro
             if (entryCounter >= 50) {
-                submitEntries(root, info);
+                response = submitEntries(root, info);
                 entryCounter = 0;
                 root = mapper.createArrayNode();
             }
@@ -144,7 +148,6 @@ public void process(KafkaJobMessage message) {
 
         bibIterator.close();
 
-        ItemCreationResponse response = null;
         if (entryCounter > 0) {
             response = submitEntries(root, info);
         }