diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index 4d1ee8d..5ccb826 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -15,7 +15,8 @@ Lovelace-RELEASE 3.0.11.RELEASE 2.2.6.RELEASE - 0.2 + 0.6 + 1.24 $2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i @@ -69,7 +70,11 @@ citesphere-messages ${citesphere.messages.version} - + + edu.asu.diging + citesphere-model + ${citesphere.model.version} + org.springframework @@ -249,8 +254,12 @@ 2.2.11 - - + + + org.jbibtex + jbibtex + 1.0.20 + @@ -302,7 +311,7 @@ 1.10.19 test - + com.opencsv opencsv diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java index 6b4f845..bd9881f 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.kafka.annotation.KafkaListener; import com.fasterxml.jackson.databind.ObjectMapper; @@ -18,7 +19,8 @@ public class ReferenceImportListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired - private IImportProcessor processor; + @Qualifier("importProcessor") + private IImportProcessor fileProcessor; @KafkaListener(topics = KafkaTopics.REFERENCES_IMPORT_TOPIC) public void receiveMessage(String message) { @@ -32,6 +34,21 @@ public void receiveMessage(String message) { return; } - processor.process(msg); + fileProcessor.process(msg); } + + @KafkaListener(topics = KafkaTopics.COLLECTION_IMPORT_TOPIC) + public void receiveCollectionImportMessage(String message) { + ObjectMapper mapper = new ObjectMapper(); + KafkaJobMessage msg = null; + try { + msg = mapper.readValue(message, KafkaJobMessage.class); + } catch (IOException e) { + logger.error("Could not unmarshall message.", e); + return; + } + + fileProcessor.process(msg); + } + } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java index 120a8f8..818474e 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java @@ -3,6 +3,8 @@ public class Affiliation { private String name; + private String uri; + private String localAuthorityId; public String getName() { return name; @@ -11,4 +13,22 @@ public String getName() { public void setName(String name) { this.name = name; } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getLocalAuthorityId() { + return localAuthorityId; + } + + public void setLocalAuthorityId(String localAuthorityId) { + this.localAuthorityId = localAuthorityId; + } + + } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 1b403f5..9c354b1 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -3,10 +3,16 @@ import java.util.ArrayList; import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; + +import edu.asu.diging.citesphere.model.bib.IGilesUpload; + public class ArticleMeta { + private List collections; private List articleIds; private String articleTitle; + private String articleShortTitle; private List categoryGroups = new ArrayList(); private List contributors; private String authorNotesCorrespondence; @@ -25,10 +31,13 @@ public class ArticleMeta { private String copyrightYear; private String copyrightHolder; private String selfUri; + @JsonProperty("DOI") + private String doi; private String articleAbstract; private String language; private ReviewInfo reviewInfo; private String documentType; + private String filePath; private String conferenceTitle; private String conferenceDate; private String conferenceLocation; @@ -43,7 +52,14 @@ public class ArticleMeta { private List references; private String referenceCount; private String retrievalDate; + private List gilesUpload; + public List getCollections() { + return collections; + } + public void setCollections(List collections) { + this.collections = collections; + } public List getArticleIds() { return articleIds; } @@ -56,6 +72,12 @@ public String getArticleTitle() { public void setArticleTitle(String articleTitle) { this.articleTitle = articleTitle; } + public String getArticleShortTitle() { + return articleShortTitle; + } + public void setArticleShortTitle(String articleShortTitle) { + this.articleShortTitle = articleShortTitle; + } public List getCategories() { return categoryGroups; } @@ -164,6 +186,12 @@ public String getSelfUri() { public void setSelfUri(String selfUri) { this.selfUri = selfUri; } + public String getDoi() { + return doi; + } + public void setDoi(String doi) { + this.doi = doi; + } public String getArticleAbstract() { return articleAbstract; } @@ -188,6 +216,12 @@ public String getDocumentType() { public void setDocumentType(String documentType) { this.documentType = documentType; } + public String getFilePath() { + return filePath; + } + public void setFilePath(String filePath) { + this.filePath = filePath; + } public String getConferenceTitle() { return conferenceTitle; } @@ -272,5 +306,11 @@ public String getRetrievalDate() { public void setRetrievalDate(String retrievalDate) { this.retrievalDate = retrievalDate; } + public List getGilesUpload() { + return gilesUpload; + } + public void setGilesUpload(List gilesUpload) { + this.gilesUpload = gilesUpload; + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java index 623720c..1af7730 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java @@ -13,6 +13,7 @@ public class Contributor { private String fullSurname; private String fullName; private String emailAddress; + private String uri; private List affiliations = new ArrayList(); private List ids; @@ -64,6 +65,15 @@ public String getEmailAddress() { public void setEmailAddress(String emailAddress) { this.emailAddress = emailAddress; } + public String getUri() { + return uri; + } + public void setUri(String uri) { + this.uri = uri; + } + public void setFullStandardizeName(String fullStandardizeName) { + this.fullStandardizeName = fullStandardizeName; + } public List getAffiliations() { return affiliations; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java new file mode 100644 index 0000000..a7f7250 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java @@ -0,0 +1,8 @@ +package edu.asu.diging.citesphere.importer.core.service; + +import edu.asu.diging.citesphere.model.bib.IGilesUpload; + +public interface IGilesConnector { + + IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes); +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java new file mode 100644 index 0000000..efec994 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -0,0 +1,87 @@ +package edu.asu.diging.citesphere.importer.core.service.giles.impl; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; + +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; + +@Service +@PropertySource("classpath:/config.properties") +public class GilesConnector implements IGilesConnector { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private RestTemplate restTemplate; + + @Value("${_giles_baseurl}") + private String gilesBaseurl; + + @Value("${_giles_upload_endpoint}") + private String uploadEndpoint; + + @PostConstruct + public void init() { + restTemplate = new RestTemplate(); + } + + @Override + public IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes) { + + if (restTemplate == null) { + restTemplate = new RestTemplate(); // Ensure restTemplate is initialized + } + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.setBearerAuth(token); + + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("files", new MultipartFileResource(fileBytes, filename)); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + ResponseEntity response = null; + try { + response = restTemplate.exchange(gilesBaseurl + uploadEndpoint, HttpMethod.POST, requestEntity, UploadResponse.class); + } catch (Exception ex) { + logger.error("Upload request failed", ex); + return null; + } + IGilesUpload upload = new GilesUpload(); + upload.setProgressId(response.getBody().getId()); + upload.setUploadingUser(username); + return upload; + } + + public class MultipartFileResource extends ByteArrayResource { + + private String filename; + + public MultipartFileResource(byte[] bytearray, String filename) { + super(bytearray); + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java new file mode 100644 index 0000000..76fe9c8 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java @@ -0,0 +1,22 @@ +package edu.asu.diging.citesphere.importer.core.service.giles.impl; + +class UploadResponse { + private String id; + private String checkUrl; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCheckUrl() { + return checkUrl; + } + + public void setCheckUrl(String checkUrl) { + this.checkUrl = checkUrl; + } +} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java index b4c5a9b..966d116 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java @@ -120,6 +120,8 @@ public void process(KafkaJobMessage message) { ObjectMapper mapper = new ObjectMapper(); ArrayNode root = mapper.createArrayNode(); int entryCounter = 0; + Map filesMap = new HashMap<>(); + ItemCreationResponse response = null; while (bibIterator.hasNext()) { BibEntry entry = bibIterator.next(); if (entry.getArticleType() == null) { @@ -129,13 +131,15 @@ public void process(KafkaJobMessage message) { ItemType type = itemTypeMapping.get(entry.getArticleType()); JsonNode template = zoteroConnector.getTemplate(type); ObjectNode bibNode = generationService.generateJson(template, entry); - + if(entry.getArticleMeta().getFilePath() != null) { + filesMap.put(entry.getArticleMeta().getArticleTitle(), entry.getArticleMeta().getFilePath()); + } root.add(bibNode); entryCounter++; // we can submit max 50 entries to Zotoro if (entryCounter >= 50) { - submitEntries(root, info); + response = submitEntries(root, info); entryCounter = 0; root = mapper.createArrayNode(); } @@ -144,7 +148,6 @@ public void process(KafkaJobMessage message) { bibIterator.close(); - ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java index 8eb8026..1768227 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java @@ -8,6 +8,9 @@ public class JobInfo { private String zotero; private String zoteroId; private String groupId; + private String collectionId; + private String giles; + private String username; public String getZotero() { return zotero; @@ -27,5 +30,23 @@ public String getGroupId() { public void setGroupId(String groupId) { this.groupId = groupId; } + public String getCollectionId() { + return collectionId; + } + public void setCollectionId(String collectionId) { + this.collectionId = collectionId; + } + public String getGiles() { + return giles; + } + public void setGiles(String giles) { + this.giles = giles; + } + public String getUsername() { + return username; + } + public void setUsername(String username) { + this.username = username; + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java new file mode 100644 index 0000000..d5c998d --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -0,0 +1,36 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; +import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; +import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; +import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; +import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; +import edu.asu.diging.citesphere.importer.core.service.parse.iterators.BibFileIterator; + + +@Service +public class BibFileHandler implements FileHandler { + + @Autowired + private IGilesConnector gilesConnector; + + @Override + public boolean canHandle(String path) throws HandlerTestException { + if (path.toLowerCase().endsWith(".bib")) { + return true; + } + return false; + } + + @Override + public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) + throws IteratorCreationException { + return new BibFileIterator(path, info, gilesConnector); + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java new file mode 100644 index 0000000..e7ebff8 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -0,0 +1,362 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.iterators; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.jbibtex.BibTeXDatabase; +import org.jbibtex.BibTeXEntry; +import org.jbibtex.BibTeXParser; +import org.jbibtex.DigitStringValue; +import org.jbibtex.Key; +import org.jbibtex.ObjectResolutionException; +import org.jbibtex.ParseException; +import org.jbibtex.StringValue; +import org.jbibtex.TokenMgrException; +import org.jbibtex.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.social.zotero.api.Data; +import org.springframework.social.zotero.api.Item; + +import edu.asu.diging.citesphere.factory.impl.ExtraParser; +import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleId; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; +import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContributionType; +import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; +import edu.asu.diging.citesphere.importer.core.model.impl.Issn; +import edu.asu.diging.citesphere.importer.core.model.impl.Publication; +import edu.asu.diging.citesphere.importer.core.model.impl.Reference; +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; +import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; +import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.model.bib.IAffiliation; +import edu.asu.diging.citesphere.model.bib.ICitation; +import edu.asu.diging.citesphere.model.bib.ICreator; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.model.bib.IPerson; +import edu.asu.diging.citesphere.model.bib.IReference; +import edu.asu.diging.citesphere.model.bib.impl.Citation; +import edu.asu.diging.citesphere.model.bib.impl.Person; + +public class BibFileIterator implements BibEntryIterator { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private ExtraParser extraParser; + + private String filePath; + private String groupId; + private String collectionId; + private JobInfo info; + private Iterator bibIterator; + private Map typeMap; + private IGilesConnector gilesConnector; + + public BibFileIterator(String filePath, JobInfo info, IGilesConnector gilesConnector) { + this.filePath = filePath; + this.groupId = info.getGroupId(); + this.collectionId = info.getCollectionId(); + this.info = info; + this.gilesConnector = gilesConnector; + extraParser = new ExtraParser(); + extraParser.init(); + init(); + } + + private void init() { + typeMap = new HashMap(); + typeMap.put("article", Publication.ARTICLE); + typeMap.put("book", Publication.BOOK); + typeMap.put("book-chapter", Publication.BOOK_CHAPTER); + typeMap.put("monograph", Publication.BOOK); + typeMap.put("proceedings-article", Publication.PROCEEDINGS_PAPER); + typeMap.put("book-section", Publication.BOOK_CHAPTER); + typeMap.put("research-article", Publication.ARTICLE); + typeMap.put("book-review", Publication.REVIEW); + typeMap.put("patent", Publication.PROCEEDINGS_PAPER); + + try (FileReader reader = new FileReader(filePath)) { + BibTeXParser parser = new BibTeXParser(); + BibTeXDatabase db = parser.parse(reader); + Collection entries = db.getEntries().values(); + bibIterator = entries.iterator(); + } catch (IOException | ObjectResolutionException | TokenMgrException | ParseException e) { + logger.error("Could not parse .bib file: " + filePath, e); + bibIterator = Collections.emptyIterator(); + } + } + + @Override + public BibEntry next() { + BibTeXEntry bib = bibIterator.next(); + Publication entry = new Publication(); + + String rawType = bib.getType().getValue().toLowerCase(); + entry.setArticleType(typeMap.getOrDefault(rawType, Publication.ARTICLE)); + + Map fields = new HashMap<>(); + for (Map.Entry f : bib.getFields().entrySet()) { + String name = f.getKey().getValue().toLowerCase(); + Value val = f.getValue(); + String str = ""; + if (val instanceof StringValue) { + str = ((StringValue) val).getString(); + } else if (val instanceof DigitStringValue) { + str = ((DigitStringValue) val).getString(); + } + fields.put(name, str); + } + + entry.setJournalMeta(parseJournalMeta(fields)); + entry.setArticleMeta(parseArticleMeta(fields)); + + return entry; + } + + private ContainerMeta parseJournalMeta(Map fields) { + ContainerMeta meta = new ContainerMeta(); + + if(fields.containsKey("journal")) { + List journalAbbrev = new ArrayList<>(); + journalAbbrev.add(fields.get("journal")); + meta.setJournalAbbreviations(journalAbbrev); + } + meta.setPublisherName(fields.get("publisher")); + meta.setPublisherLocation(fields.get("place")); + List issnList = new ArrayList(); + if(fields.get("issn") != null) { + for(String issnString : fields.get("issn").split("and")) { + Issn issn = new Issn(); + issn.setPubType("issn"); + issn.setIssn(issnString.trim()); + issnList.add(issn); + } + } + meta.setIssns(issnList); + meta.setSeriesTitle(fields.get("series")); + return meta; + } + + private ArticleMeta parseArticleMeta(Map fields) { + ArticleMeta meta = new ArticleMeta(); + ICitation citation = new Citation(); + Item item = new Item(); + Data data = new Data(); + if(fields.containsKey("annote")) { + data.setNote(fields.get("annote").replaceAll("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); + } + if(fields.containsKey("note")) { + data.setExtra(fields.get("note").replace("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); + } + item.setData(data); + + Set authors = new HashSet<>(); + if (fields.containsKey("author")) { + String[] authorStringList = fields.get("author").split("and"); + for(String authorString: authorStringList) { + IPerson author = new Person(); + String[] authorParts = authorString.split(","); + author.setLastName(authorParts[0].trim()); + author.setFirstName(authorParts[1].trim()); + authors.add(author); + } + } + citation.setAuthors(authors); + + Set editors = new HashSet<>(); + if (fields.containsKey("editor")) { + String[] editorStringList = fields.get("editor").split("and"); + for(String editorString: editorStringList) { + IPerson editor = new Person(); + String[] editorParts = editorString.split(","); + editor.setLastName(editorParts[0].trim()); + editor.setFirstName(editorParts[1].trim()); + editors.add(editor); + } + } + citation.setEditors(editors); + + + Set creators = new HashSet<>(); + citation.setOtherCreators(creators); + + extraParser.parseMetaDataNote(citation, item); + extraParser.parseExtra(data, citation); + + List collectionIds = new ArrayList<>(); + if (collectionId != null && !collectionId.trim().isEmpty()) { + collectionIds.add(collectionId); + } + meta.setCollections(collectionIds); + meta.setArticleTitle(fields.get("title")); + meta.setArticleShortTitle(fields.get("shorttitle")); + + List contributors = new ArrayList<>(); + // List of authors + if(citation.getAuthors() != null) { + contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); + } + // List of editors + if(citation.getEditors() != null) { + contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); + } + // List of other creators + if(citation.getOtherCreators() != null) { + contributors.addAll(mapCreatorToContributor(citation.getOtherCreators())); + + } + meta.setContributors(contributors); + ArticlePublicationDate publicationDate = new ArticlePublicationDate(); + publicationDate.setPublicationYear(fields.get("year")); + meta.setPublicationDate(publicationDate); + meta.setVolume(fields.get("volume")); + meta.setIssue(fields.get("number")); + if(fields.containsKey("pages")) { + meta.setFirstPage(fields.get("pages").split("--")[0].trim()); + meta.setLastPage(fields.get("pages").split("--")[1].trim()); + } + meta.setSelfUri(fields.get("url")); + meta.setDoi(fields.get("doi")); + ArticleId doiId = new ArticleId(); + doiId.setPubIdType("doi"); + doiId.setId(fields.get("doi")); + ArticleId isbnId = new ArticleId(); + isbnId.setPubIdType("isbn"); + isbnId.setId(fields.get("isbn")); + List articleIds = new ArrayList<>(); + articleIds.add(doiId); + articleIds.add(isbnId); + meta.setArticleIds(articleIds); + meta.setArticleAbstract(fields.get("abstract")); + meta.setLanguage(fields.get("language")); + + if(citation.getReferences() != null) { + meta.setReferences(mapReferences(citation.getReferences())); + meta.setReferenceCount(meta.getReferences().size()+""); + } + + if(fields.containsKey("file")) { + Path path = Paths.get(filePath); + Path folderPath = path.getParent(); + String[] fileParts = fields.get("file").split(":"); + System.out.println(folderPath.toString()+"/"+fileParts[1]); + IGilesUpload upload = createGilesUpload(folderPath.toString()+"/"+fileParts[1], info); + List uploads = new ArrayList<>(); + uploads.add(upload); + meta.setGilesUpload(uploads); + } + + + return meta; + } + + private List mapPersonToContributor(Set citationContributors, String contributionType) { + List contributors = new ArrayList(); + for(IPerson person: citationContributors) { + Contributor contributor = mapSinglePerson(person, contributionType); + contributors.add(contributor); + } + return contributors; + } + + private Contributor mapSinglePerson(IPerson person, String contributionType) { + Contributor contributor = new Contributor(); + contributor.setContributionType(contributionType); + contributor.setGivenName(person.getFirstName()); + contributor.setSurname(person.getLastName()); + contributor.setFullName(person.getName()); + contributor.setUri(person.getUri()); + + List affiliations = new ArrayList<>(); + if(person.getAffiliations()!= null) { + for(IAffiliation institute: person.getAffiliations()) { + Affiliation affiliation = new Affiliation(); + affiliation.setName(institute.getName()); + affiliation.setUri(institute.getUri()); + affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); + affiliations.add(affiliation); + } + } + contributor.setAffiliations(affiliations); + return contributor; + } + + private List mapCreatorToContributor(Set creators) { + List contributors = new ArrayList(); + for(ICreator creator: creators) { + contributors.add(mapSinglePerson(creator.getPerson(), creator.getRole())); + } + return contributors; + } + + private List mapReferences(Set citationReferences) { + List references = new ArrayList(); + for(IReference citationRef: citationReferences) { + references.add(mapSingleReference(citationRef)); + } + + return references; + } + + private Reference mapSingleReference(IReference citationRef) { + Reference ref = new Reference(); + ref.setAuthorString(citationRef.getAuthorString()); + ref.setTitle(citationRef.getTitle()); + ref.setYear(citationRef.getYear()); + ref.setIdentifier(citationRef.getIdentifier()); + ref.setIdentifierType(citationRef.getIdentifierType()); + ref.setFirstPage(citationRef.getFirstPage()); + ref.setEndPage(citationRef.getEndPage()); + ref.setVolume(citationRef.getVolume()); + ref.setSource(citationRef.getSource()); + ref.setReferenceId(citationRef.getReferenceId()); + ref.setReferenceLabel(citationRef.getReferenceLabel()); + ref.setPublicationType(citationRef.getPublicationType()); + ref.setCitationId(citationRef.getCitationId()); + ref.setReferenceString(citationRef.getReferenceString()); + ref.setReferenceStringRaw(citationRef.getReferenceStringRaw()); + + return ref; + } + + private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { + File file = new File(gilesFilePath); + byte[] fileBytes = null; + try { + fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); + } catch (IOException e) { + e.printStackTrace(); + } + + IGilesUpload upload = gilesConnector.uploadFile(info.getUsername(), info.getGiles(), file.getName(), fileBytes); + return upload; + } + + @Override + public boolean hasNext() { + return bibIterator.hasNext(); + } + + @Override + public void close() { + + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java index 2d9240a..d594192 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java @@ -106,7 +106,7 @@ public JsonNode processRelations(JsonNode node, BibEntry bibEntry) { } public JsonNode processCollections(JsonNode node, BibEntry bibEntry) { - return mapper.createArrayNode(); + return mapper.valueToTree(bibEntry.getArticleMeta().getCollections()); } protected ObjectMapper getObjectMapper() { @@ -283,6 +283,7 @@ public String processExtra(JsonNode node, BibEntry article) { fieldHelper.createReferences(article, root); fieldHelper.createReferenceCount(article, root); fieldHelper.createRetrievalDate(article, root); + fieldHelper.createGilesUpload(article, root); try { return prefix + getObjectMapper().writeValueAsString(root); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java index 5247b07..118e7a4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java @@ -4,11 +4,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import javax.annotation.PostConstruct; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -355,9 +358,7 @@ public void createReferences(BibEntry article, ObjectNode root) { ArrayNode references = root.putArray("references"); for (Reference ref : article.getArticleMeta().getReferences()) { ObjectNode refNode = references.addObject(); - if (ref.getAuthorString() != null) { - refNode.put("authorString", ref.getAuthorString()); - } + if (ref.getContributors() != null) { ArrayNode contributors = refNode.putArray("contributors"); int idx = 0; @@ -367,52 +368,33 @@ public void createReferences(BibEntry article, ObjectNode root) { idx++; } } - if (ref.getTitle() != null) { - refNode.put("title", ref.getTitle()); - } - if (ref.getEndPage() != null) { - refNode.put("endPage", ref.getEndPage()); - } - if (ref.getFirstPage() != null) { - refNode.put("firstPage", ref.getFirstPage()); - } - if (ref.getIdentifier() != null) { - refNode.put("identifier", ref.getIdentifier()); - } - if (ref.getIdentifierType() != null) { - refNode.put("identifierType", ref.getIdentifierType()); - } - if (ref.getReferenceString() != null) { - refNode.put("referenceString", ref.getReferenceString()); - } - if (ref.getReferenceStringRaw() != null) { - refNode.put("referenceStringRaw", ref.getReferenceStringRaw()); - } - if (ref.getSource() != null) { - refNode.put("source", ref.getSource()); - } - if (ref.getVolume() != null) { - refNode.put("volume", ref.getVolume()); - } - if (ref.getYear() != null) { - refNode.put("year", ref.getYear()); - } - if (ref.getPublicationType() != null) { - refNode.put("publicationType", ref.getPublicationType()); - } - if (ref.getCitationId()!= null) { - refNode.put("citationId", ref.getCitationId()); - } - if (ref.getReferenceId() != null) { - refNode.put("referenceId", ref.getReferenceId()); - } - if (ref.getReferenceLabel() != null) { - refNode.put("referenceLabel", ref.getReferenceLabel()); - } + + putIfNotNull(ref, Reference::getAuthorString, "authorString", refNode); + putIfNotNull(ref, Reference::getTitle, "title", refNode); + putIfNotNull(ref, Reference::getEndPage, "endPage", refNode); + putIfNotNull(ref, Reference::getFirstPage, "firstPage", refNode); + putIfNotNull(ref, Reference::getIdentifier, "identifier", refNode); + putIfNotNull(ref, Reference::getIdentifierType, "identifierType", refNode); + putIfNotNull(ref, Reference::getReferenceString, "referenceString", refNode); + putIfNotNull(ref, Reference::getReferenceStringRaw, "referenceStringRaw", refNode); + putIfNotNull(ref, Reference::getSource, "source", refNode); + putIfNotNull(ref, Reference::getVolume, "volume", refNode); + putIfNotNull(ref, Reference::getYear, "year", refNode); + putIfNotNull(ref, Reference::getPublicationType, "publicationType", refNode); + putIfNotNull(ref, Reference::getCitationId, "citationId", refNode); + putIfNotNull(ref, Reference::getReferenceId, "referenceId", refNode); + putIfNotNull(ref, Reference::getReferenceLabel, "referenceLabel", refNode); } } } + private void putIfNotNull(Reference ref, Function getter, String key, ObjectNode refNode) { + String value = getter.apply(ref); + if (value != null) { + refNode.put(key, value); + } + } + public void createReferenceCount(BibEntry article, ObjectNode root) { if (article.getArticleMeta().getReferenceCount() != null) { root.put("referenceCount", article.getArticleMeta().getReferenceCount()); @@ -424,6 +406,14 @@ public void createRetrievalDate(BibEntry article, ObjectNode root) { root.put("retrievalDate", article.getArticleMeta().getRetrievalDate()); } } + + public void createGilesUpload(BibEntry article, ObjectNode root) { + if (article.getArticleMeta().getGilesUpload() != null) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode gilesJson = objectMapper.valueToTree(article.getArticleMeta().getGilesUpload()); + root.put("gilesUploads", gilesJson); + } + } private void fillPerson(Contributor contrib, ObjectNode creatorNode, int idx) { List nameParts = new ArrayList<>(); diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index b19b7ab..df90288 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -24,3 +24,7 @@ _consumer_group=citesphere.importer _zotero_base_url=https://api.zotero.org/ _zotero_template_api_endpoint=/items/new?itemType= _zotero_create_item_api_endpoint=/groups/%s/items + +_giles_baseurl=${giles.baseurl} +_giles_upload_endpoint=/api/v2/files/upload +