Merge pull request #8 from des-science/bugz

ENH fix bugs in data access on older systems and centos

README.md

@@ -4,6 +4,16 @@
 
 tools for accessing the DES data archive at FNAL
 
+## Installation
+
+Install the package directly from GitHub with `pip`
+
+```bash
+pip install git+https://github.com/des-science/des-archive-access.git
+```
+
+You will need at least Python 3.8.
+
 ## Instructions for Generating a CILogon Certificate (2023/07/05)
 
 In order to download files from thr archive, you need a CILogon certificate. Follow the instructions below to obtain one.
@@ -12,7 +22,7 @@ In order to download files from thr archive, you need a CILogon certificate. Fol
 2. Login with your FNAL services account.
 3. Click the ***Create Password-Protected Certificate*** link.
 4. Follow the instructions to download a certificate.
-5. Reformat the certificate by executing `des-archive-access-process-cert /path/to/cert`. This command will ask you for your password and to set a new password. You can reuse the same password if you'd like or set no password. Hopefully we don't have to do this in the future.
+5. Reformat the certificate by executing `des-archive-access-process-cert /path/to/cert`. This command will ask you for your password. (Hopefully we don't have to do this in the future.)
 
 The certificate will be stored in the `~/.des_archive_access/` directory in your home area. **Make the sure the permissions on this directory are `700` via `chmod 700 ~/.des_archive_access/`.** You can change this location by setting the environment variable `DES_ARCHIVE_ACCESS_DIR`.
 

des_archive_access/cli.py

@@ -3,7 +3,6 @@
 import os
 import subprocess
 import sys
-import tempfile
 
 import requests
 from tqdm import tqdm
@@ -64,7 +63,12 @@ def main_download():
         "-f",
         "--force",
         action="store_true",
-        help="force the download even if data already exists",
+        help="Force the download even if data already exists",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Run 'curl' with '-vv' to debug connection and download issues.",
     )
     args = parser.parse_args()
 
@@ -82,14 +86,21 @@ def main_download():
                 prefix=prefix,
                 desdata=desdata,
                 force=args.force,
+                debug=args.debug,
             )
         )
 
     if args.list is not None:
         with open(args.list) as fp:
             for line in fp:
                 line = line.strip()
-                download_file(line, prefix=prefix, desdata=desdata, force=args.force)
+                download_file(
+                    line,
+                    prefix=prefix,
+                    desdata=desdata,
+                    force=args.force,
+                    debug=args.debug,
+                )
 
 
 def main_download_metadata():
@@ -196,7 +207,7 @@ def main_process_cert():
     )
     args = parser.parse_args()
 
-    cloc = os.path.join(get_des_archive_access_dir(), "cert.p12")
+    cloc = os.path.join(get_des_archive_access_dir(), "cert.pem")
 
     if args.remove or args.force:
         try:
@@ -205,6 +216,7 @@ def main_process_cert():
             pass
 
     if args.remove:
+        print(f"Removed certificate at {cloc}.", flush=True)
         sys.exit(0)
 
     if args.cert is not None and (not os.path.exists(cloc) or args.force):
@@ -216,21 +228,23 @@ def main_process_cert():
             legacy = ""
 
         try:
-            with tempfile.TemporaryDirectory() as tmpdir, pushd(tmpdir):
-                subprocess.run(
-                    f"openssl pkcs12 -in {args.cert} -nodes {legacy} > temp",
-                    shell=True,
-                    check=True,
-                )
-                subprocess.run(
-                    "openssl pkcs12 -export -out %s -in temp" % cloc,
-                    shell=True,
-                    check=True,
-                )
+            subprocess.run(
+                f"openssl pkcs12 -in {args.cert} -out {cloc} -nodes {legacy}",
+                shell=True,
+                check=True,
+            )
         except (KeyboardInterrupt, Exception) as e:
             try:
                 os.remove(cloc)
             except Exception:
                 pass
 
             raise e
+    else:
+        print(
+            f"Certificate {cloc} already exists!\nRun your command "
+            "with the `--force` flag to forcibly replace the current "
+            "certificate.",
+            flush=True,
+        )
+        sys.exit(1)

des_archive_access/dbfiles.py

@@ -1,6 +1,7 @@
 import os
 import sqlite3
 import subprocess
+import sys
 from functools import lru_cache
 
 
@@ -40,7 +41,7 @@ def get_des_archive_access_db_conn():
     )
 
 
-def download_file(fname, prefix=None, desdata=None, force=False):
+def download_file(fname, prefix=None, desdata=None, force=False, debug=False):
     """Download a file FNAME from the DES FNAL archive
     possibly with an optional HTTPS `prefix` and optional `desdata` destination.
 
@@ -60,14 +61,21 @@ def download_file(fname, prefix=None, desdata=None, force=False):
         except Exception:
             pass
 
+    if debug:
+        debug_str = "-vv"
+    else:
+        debug_str = ""
+
     cmd = (
-        "curl -k -L --cert-type P12 --cert "
-        "{}:${{DES_ARCHIVE_ACCESS_PASSWORD}} -o {} -C - {}/{}"
+        "curl {} -k -L --cert {}:${{DES_ARCHIVE_ACCESS_PASSWORD}} -o {} -C - {}/{}"
     ).format(
-        os.path.join(get_des_archive_access_dir(), "cert.p12"),
-        fname,
+        debug_str,
+        os.path.join(get_des_archive_access_dir(), "cert.pem"),
+        fpth,
         prefix,
         fname,
     )
+    if debug:
+        print(cmd, file=sys.stderr)
     subprocess.run(cmd, shell=True, check=True, cwd=desdata)
     return fpth

pyproject.toml

@@ -8,6 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "des-archive-access"
+requires-python = ">=3.8"
 authors = [
     {name = "Matthew R. Becker", email = "[email protected]"},
 ]