Merge pull request #11 from des-science/tokens-oidc

ENH move to oidc tokens

.github/workflows/tests.yml

@@ -64,3 +64,11 @@ jobs:
         shell: bash -el {0}
         run: |
           pytest -vvs tests
+
+      - name: cli tests
+        shell: bash -el {0}
+        run: |
+          des-archive-access-download --help
+          des-archive-access-download-metadata --help
+          des-archive-access-make-token --help
+          des-archive-access --help

README.md

@@ -6,31 +6,46 @@ tools for accessing the DES data archive at FNAL
 
 ## Installation
 
-Install the package directly from GitHub with `pip`
+**Right now this package is in its alpha state and so all installation is directly from the `main` branch!**
+
+### PyPI
+
+Install the package and dependencies directly from GitHub with `pip`
 
 ```bash
+pip install git+https://github.com/fermitools/htgettoken.git
 pip install git+https://github.com/des-science/des-archive-access.git
 ```
 
+### conda
+
+Install the dependencies with `conda` and then install the tool with `pip`
+
+```bash
+git clone https://github.com/des-science/des-archive-access.git
+cd des-archive-access
+conda install --file requirements.txt --file requirements-dev.txt
+pip install --no-deps --no-build-isolation .
+cd -
+```
+
 You will need at least Python 3.8.
 
-## Instructions for Generating a CILogon Certificate (2023/07/05)
+## Instructions for Generating an OIDC Token for FNAL dCache (2023/08/18)
 
-In order to download files from thr archive, you need a CILogon certificate. Follow the instructions below to obtain one.
+In order to download files, you need to generate an OIDC token. To make the token, do the following.
 
-1. Go to [cilogon.org](https://cilogon.org/)
-2. Login with your FNAL services account.
-3. Click the ***Create Password-Protected Certificate*** link.
-4. Follow the instructions to download a certificate.
-5. Reformat the certificate by executing `des-archive-access-process-cert /path/to/cert`. This command will ask you for the password you entered when making the certificate. (Hopefully we don't have to do this in the future.)
+1. Run the command line tool `des-archive-access-make-token`.
+2. You will be redirected to a CILogon website.
+3. Login with your **FNAL SERVICES** account. You may be offered other identity providers, but you have to use the FNAL one!
 
-The certificate will be stored in the `~/.des_archive_access/` directory in your home area. **Make the sure the permissions on this directory are `700` via `chmod 700 ~/.des_archive_access/`.** You can change this location by setting the environment variable `DES_ARCHIVE_ACCESS_DIR`.
+The token will be stored in the `~/.des_archive_access/` directory in your home area. **Make the sure the permissions on this directory are `700` via `chmod 700 ~/.des_archive_access/`.** You can change this location by setting the environment variable `DES_ARCHIVE_ACCESS_DIR`.
 
 ## Usage
 
 ### Downloading the Archive Metadata
 
-Before you can query the file archive metadata, you need to download the metadata database (roughly 30GB!!!) via the `des-archive-access-download-metadata`
+Before you can query the file archive metadata, you need to download the metadata database (roughly 8GB) via the `des-archive-access-download-metadata`
 command. This command will put the data in your home area. If you'd like to specify a different path to the DB, set it via the environment variable `DES_ARCHIVE_ACCESS_DB` like this
 
 ```bash
@@ -111,9 +126,9 @@ You can use the `des-archive-access-download` command to download files from the
 
 ```bash
 $ des-archive-access-download --help
-usage: des-archive-access-download [-h] [-l LIST] [-a ARCHIVE] [-d DESDATA] [-f] [file]
+usage: des-archive-access-download [-h] [-l LIST] [-a ARCHIVE] [-d DESDATA] [-f] [--debug] [file]
 
-download files from the DES archive at FNAL
+Download files from the DES archive at FNAL.
 
 positional arguments:
   file                  file to download
@@ -125,12 +140,9 @@ options:
                         HTTPS address of the FNAL archive
   -d DESDATA, --desdata DESDATA
                         The destination DESDATA directory.
-  -f, --force           force the download even if data already exists
+  -f, --force           Force the download even if data already exists
+  --debug               Print the 'curl' command and stderr to help debug connection and download issues.
 $ des-archive-access-download OPS/finalcut/Y6A1/20181129-r4056/D00797980/p01/red/immask/D00797980_r_c27_r4056p01_immasked.fits.fz
-  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
-                                 Dload  Upload   Total   Spent    Left  Speed
-  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
-100 14.1M  100 14.1M    0     0  13.6M      0  0:00:01  0:00:01 --:--:-- 15.3M
 /Users/beckermr/DESDATA/OPS/finalcut/Y6A1/20181129-r4056/D00797980/p01/red/immask/D00797980_r_c27_r4056p01_immasked.fits.fz
 ```
 

des_archive_access/cli.py

@@ -18,7 +18,7 @@
 def main_download():
     parser = argparse.ArgumentParser(
         prog="des-archive-access-download",
-        description="download files from the DES archive at FNAL",
+        description="Download files from the DES archive at FNAL.",
     )
     parser.add_argument(
         "file",
@@ -57,7 +57,8 @@ def main_download():
     parser.add_argument(
         "--debug",
         action="store_true",
-        help="Run 'curl' with '-vv' to debug connection and download issues.",
+        help="Print the 'curl' command and stderr to help debug "
+        "connection and download issues.",
     )
     args = parser.parse_args()
 
@@ -95,7 +96,7 @@ def main_download():
 def main_download_metadata():
     parser = argparse.ArgumentParser(
         prog="des-archive-access-download-metadata",
-        description="download the metadata for the DES archive at FNAL",
+        description="Download the metadata for the DES archive at FNAL.",
     )
     parser.add_argument(
         "--url",
@@ -131,12 +132,18 @@ def main_download_metadata():
         else:
             os.makedirs(os.path.dirname(mloc), exist_ok=True)
 
+        url = args.url or (
+            "http://deslogin.cosmology.illinois.edu/~donaldp/"
+            "desdm-file-db-23-08-18-10-02/desdm-test.db.zst"
+        )
+
+        if url.endswith(".zstd"):
+            dest = mloc + ".zstd"
+        else:
+            dest = mloc
+
         try:
             # https://stackoverflow.com/questions/37573483/progress-bar-while-download-file-over-http-with-requests
-            url = args.url or (
-                "http://deslogin.cosmology.illinois.edu/~donaldp/"
-                "desdm-file-db-23-08-18-10-02/desdm-test.db.zst"
-            )
             response = requests.get(url, stream=True)
             total_size_in_bytes = int(response.headers.get("content-length", 0))
             block_size = 1024
@@ -147,106 +154,50 @@ def main_download_metadata():
                 ncols=80,
                 desc="downloading DB",
             ) as progress_bar:
-                with open(mloc + ".zstd", "wb") as file:
+                with open(dest, "wb") as file:
                     for data in response.iter_content(block_size):
                         progress_bar.update(len(data))
                         file.write(data)
             if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
                 raise RuntimeError("Download failed!")
 
             # decompress
-            print("decompressing...", end="", flush=True)
-            dctx = zstandard.ZstdDecompressor()
-            with open(mloc + ".zstd", "rb") as ifh, open(mloc, "wb") as ofh:
-                dctx.copy_stream(ifh, ofh)
-            print("done.", flush=True)
+            if url.endswith(".zstd"):
+                print("decompressing...", end="", flush=True)
+                dctx = zstandard.ZstdDecompressor()
+                with open(mloc + ".zstd", "rb") as ifh, open(mloc, "wb") as ofh:
+                    dctx.copy_stream(ifh, ofh)
+                print("done.", flush=True)
         except (KeyboardInterrupt, Exception) as e:
             try:
                 os.remove(mloc)
-                os.remove(mloc + ".zstd")
+                if url.endswith(".zstd"):
+                    os.remove(mloc + ".zstd")
             except Exception:
                 pass
 
             raise e
         finally:
             try:
-                os.remove(mloc + ".zstd")
+                if url.endswith(".zstd"):
+                    os.remove(mloc + ".zstd")
             except Exception:
                 pass
 
 
-def _is_openssl_v3():
-    res = subprocess.run(
-        "openssl version",
-        shell=True,
-        check=True,
-        capture_output=True,
-    )
-    version = res.stdout.decode("utf-8").split()[1]
-    return version[0] == "3"
-
-
-def main_process_cert():
+def main_make_token():
     parser = argparse.ArgumentParser(
-        prog="des-archive-access-process-cert",
-        description="process the CILogon certificate for DES archive access",
-    )
-    parser.add_argument(
-        "cert",
-        type=str,
-        help="certificate to process",
-        nargs="?",
-        default=None,
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="forcibly replace the current certificate",
-    )
-    parser.add_argument(
-        "--remove", action="store_true", help="remove existing certificate"
+        prog="des-archive-access-make-token",
+        description="Make the OIDC token for FNAL dCache. "
+        "Any extra arguemnts are passed to `htgettoken`.",
     )
-    args = parser.parse_args()
-
-    cloc = os.path.join(get_des_archive_access_dir(), "cert.pem")
+    args, unknown = parser.parse_known_args()
+    extra_args = " ".join(unknown)
 
-    if args.remove or args.force:
-        try:
-            os.remove(cloc)
-        except Exception:
-            pass
-
-    if args.remove:
-        print(f"Removed certificate at {cloc}.", flush=True)
-        sys.exit(0)
-
-    if args.cert is not None and (not os.path.exists(cloc) or args.force):
-
-        make_des_archive_access_dir()
-        if _is_openssl_v3():
-            legacy = "-legacy"
-        else:
-            legacy = ""
-
-        try:
-            subprocess.run(
-                f"openssl pkcs12 -in {args.cert} -out {cloc} -nodes {legacy}",
-                shell=True,
-                check=True,
-            )
-        except (KeyboardInterrupt, Exception) as e:
-            try:
-                os.remove(cloc)
-            except Exception:
-                pass
-
-            raise e
-    else:
-        print(
-            f"Certificate {cloc} already exists!\nRun your command "
-            "with the `--force` flag to forcibly replace the current "
-            "certificate.",
-            flush=True,
-        )
-        sys.exit(1)
+    make_des_archive_access_dir(fix_permissions=True)
+    tloc = os.path.join(get_des_archive_access_dir(), "token")
+    subprocess.run(
+        f"htgettoken {extra_args} -a htvaultprod.fnal.gov -i des -o {tloc}",
+        shell=True,
+        check=True,
+    )

des_archive_access/dbfiles.py

@@ -13,11 +13,14 @@ def get_des_archive_access_dir():
     )
 
 
-def make_des_archive_access_dir():
+def make_des_archive_access_dir(fix_permissions=False):
     """Make the DES_ARCHIVE_ACCESS_DIR and set permissions to 700."""
     daad = get_des_archive_access_dir()
     os.makedirs(daad, exist_ok=True)
     os.chmod(daad, 0o700)
+    if fix_permissions:
+        for fname in os.listdir(daad):
+            os.chmod(os.path.join(daad, fname), 0o600)
 
 
 def get_des_archive_access_db():
@@ -61,19 +64,19 @@ def download_file(fname, prefix=None, desdata=None, force=False, debug=False):
         except Exception:
             pass
 
-    if debug:
-        debug_str = "-vv"
-    else:
-        debug_str = ""
-
-    cmd = ("curl {} -k -L --cert {} -o {} -C - {}/{}").format(
-        debug_str,
-        os.path.join(get_des_archive_access_dir(), "cert.pem"),
+    cmd = ('curl -L -H "Authorization: Bearer $(<{})" -o {} -C - {}/{}').format(
+        os.path.join(get_des_archive_access_dir(), "token"),
         fpth,
         prefix,
         fname,
     )
     if debug:
         print(cmd, file=sys.stderr)
-    subprocess.run(cmd, shell=True, check=True, cwd=desdata)
+    subprocess.run(
+        cmd,
+        shell=True,
+        check=True,
+        cwd=desdata,
+        stderr=None if debug else subprocess.PIPE,
+    )
     return fpth

pyproject.toml

@@ -20,7 +20,7 @@ readme = "README.md"
 [project.scripts]
 des-archive-access-download = "des_archive_access.cli:main_download"
 des-archive-access-download-metadata = "des_archive_access.cli:main_download_metadata"
-des-archive-access-process-cert = "des_archive_access.cli:main_process_cert"
+des-archive-access-make-token = "des_archive_access.cli:main_make_token"
 des-archive-access = "des_archive_access.repl:cli"
 
 [project.urls]

requirements.txt

@@ -1,6 +1,7 @@
 click
 click-repl
 fitsio
+htgettoken
 numpy
 requests
 tqdm