|
1 | | -const fs = require("fs"); |
2 | | -const tls = require("tls"); |
| 1 | +const Url = require("url"); |
| 2 | +const ftp = require("basic-ftp"); |
| 3 | +const { http, https } = require("follow-redirects"); |
3 | 4 |
|
4 | | -const { curly } = require("node-libcurl"); |
5 | | - |
6 | | -// important steps to get validation of https (as opposed to http) urls |
7 | | -// Get root certificates so https will work |
8 | | -// |
9 | | -// Write the certificates to a file |
10 | | -// https://stackoverflow.com/questions/63052127/protractor-node-libcurl-failed-ssl-peer-certificate-or-ssh-remote-key-was-not-o |
11 | | -// When doing sam build the file must be in /tmp because other wise it cannot be read |
12 | | -// due to ro file system in container |
13 | | -// https://stackoverflow.com/questions/53810516/getting-error-aws-lambda-erofs-read-only-file-system-open-var-task-assets |
14 | | -const certFilePath = "/tmp/cacert.pem"; |
15 | | -// https://nodejs.org/api/tls.html#tls_tls_rootcertificates |
16 | | -// An immutable array of strings representing the root certificates (in PEM format) from the bundled Mozilla CA store as supplied by current Node.js version. |
17 | | -// The bundled CA store, as supplied by Node.js, is a snapshot of Mozilla CA store that is fixed at release time. It is identical on all supported platforms. |
18 | | -const tlsData = tls.rootCertificates.join("\n"); |
19 | | -fs.writeFileSync(certFilePath, tlsData); |
| 5 | +// The Node url.parse returns an object where the protocol is lower case and contains the colon at the end |
| 6 | +const SECURE_FTP_PROTOCOL = "sftp:"; |
| 7 | +const FTP_PROTOCOL = "ftp:"; |
| 8 | +const HTTP_PROTOCOL = "http:"; |
| 9 | +const HTTPS_PROTOCOL = "https:"; |
20 | 10 |
|
21 | 11 | /** |
22 | 12 | * TODO: Change to array of URLs to parse |
@@ -50,10 +40,48 @@ async function checkUrl(url) { |
50 | 40 | } |
51 | 41 |
|
52 | 42 | async function run(url) { |
53 | | - const curlOpts = { |
54 | | - caInfo: certFilePath, |
55 | | - }; |
56 | | - return curly.head(url, curlOpts); |
| 43 | + const parsedUrl = Url.parse(url); |
| 44 | + const protocol = parsedUrl.protocol; // Url.parse() lower cases the protocol |
| 45 | + if (FTP_PROTOCOL === protocol || SECURE_FTP_PROTOCOL === protocol) { |
| 46 | + const secure = SECURE_FTP_PROTOCOL === protocol; |
| 47 | + const ftpClient = new ftp.Client(); |
| 48 | + try { |
| 49 | + let options = { |
| 50 | + host: parsedUrl.host, |
| 51 | + secure: secure, |
| 52 | + ...(parsedUrl.port && { port: parsedUrl.port }), |
| 53 | + }; |
| 54 | + await ftpClient.access(options); |
| 55 | + const size = await ftpClient.size(parsedUrl.path); |
| 56 | + return size > 0 |
| 57 | + ? Promise.resolve() |
| 58 | + : Promise.reject("Could not get size for " + url); |
| 59 | + } finally { |
| 60 | + ftpClient.close(); |
| 61 | + } |
| 62 | + } else if (HTTP_PROTOCOL === protocol) { |
| 63 | + return httpOrHttpsRequest(url, http); |
| 64 | + } else if (HTTPS_PROTOCOL === protocol) { |
| 65 | + return httpOrHttpsRequest(url, https); |
| 66 | + } |
| 67 | + return Promise.reject("Unsupported protocol: " + protocol); |
| 68 | +} |
| 69 | + |
| 70 | +function httpOrHttpsRequest(url, httpOrHttps) { |
| 71 | + return new Promise((resolve, reject) => { |
| 72 | + const req = httpOrHttps.request(url, { |
| 73 | + method: "HEAD", |
| 74 | + headers: { "user-agent": "Dockstore/1.0" }, // User-agent must be set for tests to pass, AWS (WAF?) blocks requests with no user-agent |
| 75 | + }); |
| 76 | + req.on("response", (res) => { |
| 77 | + if (res.statusCode < 300) { |
| 78 | + resolve(res.statusCode); |
| 79 | + } |
| 80 | + reject(res.statusCode); |
| 81 | + }); |
| 82 | + req.on("error", (err) => reject(err)); |
| 83 | + req.end(); |
| 84 | + }); |
57 | 85 | } |
58 | 86 |
|
59 | 87 | function returnResponse(fileFound) { |
|
0 commit comments