Merge pull request #220 from /issues/208

Bugfix: Docs -- RTD build, RST formatting, and working-notes TOCs

.readthedocs.yml → .readthedocs.yaml

@@ -4,6 +4,12 @@
 # Required
 version: 2
 
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/source/conf.py
@@ -13,10 +19,5 @@ formats: []
 
 # Set the version of Python and requirements required to build the docs
 python:
-  version: 3.7
-  system_packages: true
   install:
     - requirements: docs/requirements.txt
-    - method: setuptools
-      path: .
-

broker/broker_utils/broker_utils/avro_schemas/load.py

@@ -1,14 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
-"""The ``avro_schemas.load`` module loads and returns Avro schemas.
-
+"""
+The ``avro_schemas.load`` module loads and returns Avro schemas.
 
-REGISTERED_SCHEMAS = [
-    "elasticc.v0_9.alert.avsc",
-    "elasticc.v0_9.brokerClassification.avsc",
-    "elasticc.v0_9_1.alert.avsc",
-    "elasticc.v0_9_1.brokerClassification.avsc",
-]
+To view a list of registered schema names, use: ``avro_schemas.load.REGISTERED_SCHEMAS``
 """
 
 from importlib.resources import path as rpath

broker/broker_utils/broker_utils/gcp_utils.py

@@ -355,24 +355,26 @@ def query_bigquery(
             Optional job config to send with the query.
 
     Example query:
-        ``
+
+    .. code-block:: python
+
         query = (
             f'SELECT * '
             f'FROM `{dataset_project_id}.{dataset}.{table}` '
             f'WHERE objectId={objectId} '
         )
-        ``
 
     Examples of working with the query_job:
-        ``
+
+    .. code-block:: python
+
         # Cast it to a DataFrame:
         query_job.to_dataframe()
 
         # Iterate row-by-row
         for r, row in enumerate(query_job):
             # row values can be accessed by field name or index
             print(f"objectId={row[0]}, candid={row['candid']}")
-        ``
     """
     if project_id is None:
         project_id = project_id_default

broker/broker_utils/broker_utils/math.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
 """The ``math`` module contains mathematical transforms for survey data."""
+
 from typing import Tuple
 
 import numpy as np

broker/broker_utils/broker_utils/testing.py

@@ -31,9 +31,9 @@ def __init__(self, alert_dir: Union[str, Path] = "ztf"):
                 Either the path to a directory containing alerts or one of the
                 following strings.
                 "ALERT_DIR" or "alert_dir": The path will be obtained from the
-                    environment variable ALERT_DIR.
+                environment variable ALERT_DIR.
                 "ztf" or "elasticc": The path will be obtained from the environment
-                    variable given by ``f"ALERT_DIR_{alert_dir.upper()}"``
+                variable given by ``f"ALERT_DIR_{alert_dir.upper()}"``
         """
         if alert_dir in ["ALERT_DIR", "alert_dir"]:
             self.alert_dir = Path(os.getenv("ALERT_DIR"))

docs/requirements.txt

@@ -1,26 +1,27 @@
-# # pgb-utils
-# apache_beam[gcp]
-# aplpy
-# astropy==3.2.1
-# astropy-healpix==0.6
-# beautifulsoup4==4.8
-# # google-auth
-# # google-api-core
-# # google-apitools
-# google-cloud-bigquery
-# google-cloud-logging
-# google-cloud-pubsub
-# google-cloud-storage
-# matplotlib
-# numpy
-# pandas
-# tabulate
-# typing
-
 # Needed for building docs
 docutils<0.17  # render lists properly
 myst-parser
 sphinx
 sphinx-copybutton
 sphinx_autodoc_typehints
 sphinx_rtd_theme
+
+# Needed to build docs for broker_utils
+# It would be better not to duplicate this but it's is an easy way to get RTD to build,
+# and broker_utils really needs to be removed from this repo anyway (#158).
+# ---- Below here copied from broker/broker_utils/requirements.txt ----
+# If you have trouble with astropy, jinja2, MarkupSafe, and/or soft_unicode
+# Try using a Conda environment with Python 3.7 e.g.,
+# $ conda create -n pgb python=3.7
+astropy
+fastavro
+# google-api-core
+google-cloud-bigquery
+google-cloud-logging
+google-cloud-storage>=1.38
+google-cloud-pubsub
+numpy
+pandas
+PyYAML>=5.3
+six>=1.15  # bigquery requires
+typing

docs/source/api/broker-utils/avro-schemas.rst

@@ -1,5 +1,5 @@
 broker_utils.avro_schemas
-========================
+=========================
 
 .. automodule:: broker.broker_utils.broker_utils.avro_schemas.load
    :members:

docs/source/broker/architecture-diagrams.md

@@ -1,6 +1,6 @@
 # Architecture Diagrams
 
-## Current
+## 2022 (now outdated)
 
 ![Current Architecture](architecture-current.svg)
 

docs/source/broker/components/broker-schedule.rst

@@ -22,8 +22,8 @@ Uptime Check Schedules
 
 .. note::
 
-	The following setup is cumbersome and has the further disadvantage that it has
-    to be scheduled separately from the VMs. Fixing it is part of 
+    The following setup is cumbersome and has the further disadvantage that it has
+    to be scheduled separately from the VMs. Fixing it is part of
     `#109 <https://github.com/mwvgroup/Pitt-Google-Broker/issues/109>`__.
 
 The Cloud Function ``check_cue_response`` checks whether instances are running or
@@ -60,6 +60,7 @@ To update the existing policy, or create a new one, see:
     <https://cloud.google.com/monitoring/alerts/using-alerting-api>`__
 -   `Managing notification channels
     <https://cloud.google.com/monitoring/support/notification-options>`__
+
 Where to look if there's a problem
 ----------------------------------
 

docs/source/broker/setup-ref.rst

@@ -45,7 +45,7 @@ Install broker-utils
 ---------------------
 
 Our broker-utils python library contains many functions that are helpful in working both alerts and GCP services.
-It can be installed using 
+It can be installed using
 (you may want to activate a Conda environment before installing):
 
 .. code-block:: bash

docs/source/index.rst

@@ -10,11 +10,6 @@ The Pitt-Google Broker is a cloud-based alert distribution service designed to p
 
 The Pitt-Google Broker runs on the `Google Cloud Platform <https://cloud.google.com/>`_ (GCP) and is currently focused on processing and serving alerts from the `Zwicky Transient Facility <https://www.ztf.caltech.edu/>`_ (ZTF), and extending broker capabilities using ZTF, the LSST Alert Simulator, and the DECam Alliance for Transients (DECAT) stream.
 
-.. .. toctree::
-..    :hidden:
-..
-..    Overview<self>
-
 .. toctree::
    :hidden:
    :maxdepth: 1
@@ -50,13 +45,3 @@ The Pitt-Google Broker runs on the `Google Cloud Platform <https://cloud.google.
 
    working-notes/early-dev/README
    working-notes/troyraen/index
-
-For the current broker-utils API reference, see the `update_broker_utils branch <https://pitt-broker.readthedocs.io/en/u-tjr-update_broker_utils/api/broker-utils/avro-schemas.html>`__.
-
-..
-.. Indices and tables
-.. ==================
-..
-.. * :ref:`genindex`
-.. * :ref:`modindex`
-.. * :ref:`search`

docs/source/working-notes/early-dev/README.rst

@@ -0,0 +1,78 @@
+Project Notes
+=============
+
+This directory contains various project documents and notes for use by
+the developers early in the project.
+
+Directory Table of Contents
+----------------------------
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+   :caption: All Notes
+
+   *
+   */*
+
+External Links and Resources
+----------------------------
+
+General
+^^^^^^^
+
+-  Project notes: `here <./notes/>`__
+-  Online LSST forum for data managment (DM):
+   `community.lsst.org/ <https://community.lsst.org/>`__
+-  LSST-DESC Broker Workshop talks
+   `drive.google.com/... <https://drive.google.com/drive/folders/1sjYXbdwTID3VnzZNAkcjLbjRfpwNaO_n?usp=sharing>`__
+
+LSST Documents
+^^^^^^^^^^^^^^
+
+-  Plans and Policies for alert distribution (how will community brokers
+   be chosen?): `ls.st/LDM-612 <https://ls.st/LDM-612>`__
+-  Call for letters ofiIntent for community alert brokers (how do I
+   apply to be a community broker?):
+   `ls.st/LDM-682 <https://ls.st/LDM-682>`__
+-  LSST alerts: key numbers (how many? how much? how often?):
+   `dmtn-102.lsst.io <https://dmtn-102.lsst.io>`__
+-  Data Products Definition Document (DPDD) (What data will LSST
+   deliver?): `ls.st/dpdd <https://ls.st/dpdd>`__
+-  Prototype schemas:
+   `github.com/lsst-dm/sample-avro-alert <https://github.com/lsst-dm/sample-avro-alert>`__
+-  Kafka-based alert stream:
+   `github.com/lsst-dm/alert_stream <https://github.com/lsst-dm/alert_stream>`__
+
+ZTF
+^^^
+
+-  Sample alert packets:
+   `ztf.uw.edu/alerts/public/ <https://ztf.uw.edu/alerts/public/>`__
+-  Alert packet tools: `zwicky.tf/4t5 <https://zwicky.tf/4t5>`__
+-  Alert schema documentation: `zwicky.tf/dm5 <https://zwicky.tf/dm5>`__
+-  Detailed pipelines documentation:
+   `zwicky.tf/ykv <https://zwicky.tf/ykv>`__
+-  PASP instrument papers: `zwicky.tf/3w9 <https://zwicky.tf/3w9>`__
+
+Internal Resources
+------------------
+
+-  Meeting minutes: `Available on Google
+   Docs <https://docs.google.com/document/d/1h6rNfFttoCKtpXUHnEfWOhzgjI6sxuSPmpTmzpz5qvw/edit#heading=h.44dh10bplhuh>`__.
+
+-  Letter of Intent (LOI): `Available on
+   Overleaf <https://www.overleaf.com/project/5c86ae8f6183b337fc5c9ec9>`__
+
+-  Competitive products survey:
+
+   -  `ANTARES.md <./competitive_product_survey/ANTARES.md>`__
+   -  `Lasair.md <./competitive_product_survey/Lasair.md>`__
+
+-  Notes from the DESC broker workshop:
+   `desc_broker_workshop.md <desc_broker_workshop.md>`__
+
+-  Cross matching: `cross_matching.md <./cross_matching.md>`__
+
+-  Tutorial to setting up Cloud Pub/Sub:
+   `pubsub-tutorial.md <./pubsub-tutorial.md>`__