diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a2ecc2f..59926a4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,12 +1,12 @@ { "name": "Pixbyt", - "initializeCommand": "git submodule update --init", "postCreateCommand": { ".meltano": "ln -s /project/.meltano .", - ".env": "cp .env.sample .env" + ".env": "cp .env.sample .env", + "apps.yml": "cp -n apps.yml.sample apps.yml", + "devices.yml": "cp -n devices.yml.sample devices.yml" }, - "build": { "context": "..", "dockerfile": "../Dockerfile", @@ -15,7 +15,6 @@ }, "cacheFrom": "type=gha" }, - "customizations": { "codespaces": { "openFiles": [ @@ -32,13 +31,12 @@ } } }, - "secrets": { "TIDBYT_DEVICE_ID": { "description": "Optional. Find your Device ID in the Tidbyt mobile app under Settings > General > Get API Key." }, - "TIDBYT_TOKEN": { - "description": "Optional. Find your API Token in the Tidbyt mobile app under Settings > General > Get API Key." + "TIDBYT_KEY": { + "description": "Optional. Find your Key in the Tidbyt mobile app under Settings > General > Get API Key." } } } diff --git a/.env.sample b/.env.sample index 6c24727..b9bafae 100644 --- a/.env.sample +++ b/.env.sample @@ -2,16 +2,25 @@ # This is used by many apps that show the (relative) date and time. TZ="America/Mexico_City" -## target-tidbyt -# Find your Device ID and API Token in the Tidbyt mobile app under Settings > General > Get API Key. +### Devices +# Find your Device ID and Key in the Tidbyt mobile app under Settings > General > Get API Key. +# Multiple Tidbyts can be defined in devices.yml. + +## Default TIDBYT_DEVICE_ID="" -TIDBYT_TOKEN="" +TIDBYT_KEY="" + +# ## +# TIDBYT__DEVICE_ID="" +# TIDBYT__KEY="" + +### Apps ## hello-world # This is used by the `hello-world` example app. Optionally, replace `world` with your own name. HELLO_WORLD_NAME="world" # ## -# For any config key the app defines under `app_config:` in its `pixbyt.yml` file, add a value for the uppercase environment variable: +# # For any config key the app defines under `app_config:` in its `pixbyt.yml` file, add a value for the uppercase environment variable: # _="" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 453167e..ca008e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,8 +40,11 @@ jobs: with: version: '2.20.2' - - name: Test / Create .env from sample - run: cp .env.sample .env + - name: Test / Create .env, apps.yml, and devices.yml from sample + run: | + cp .env.sample .env + cp -n apps.yml.sample apps.yml + cp -n devices.yml.sample devices.yml - name: Test / Render hello-world to image run: | diff --git a/README.md b/README.md index 0a1adcf..49450aa 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ Apps running on Pixbyt have none of these limitations: Pixbyt lets you realize your wildest Tidbyt dreams by making it easy to: - **build** advanced Tidbyt apps, - **install** advanced apps built by the community, -- **manage** app configurations and schedules, +- **manage** multiple Tidbyts and apps, - **package** apps together in a [Docker](https://www.docker.com/) image, and - **launch** the app server using [Docker Compose](https://docs.docker.com/compose/). @@ -178,8 +178,7 @@ Codespaces will automatically install the necessary dependencies and launch you 1. Click the green "Use this template" button at the top of this page 1. Choose "Open in a codespace" and wait for the codespace to start -1. Update `.env` with your configuration: - - `HELLO_WORLD_NAME`: Optionally, replace `world` with your own name. +1. Optionally, update the `HELLO_WORLD_NAME` environment variable in `.env` to replace `world` with your own name. 1. Render app to a WebP image file: ```bash @@ -189,9 +188,8 @@ Codespaces will automatically install the necessary dependencies and launch you The image will be created at `output/hello-world/.webp`. The exact path is also printed in the command output. 1. Render app to your Tidbyt: - 1. Update `.env` with your configuration: - - `TIDBYT_DEVICE_ID`: Find your Device ID in the Tidbyt mobile app under Settings > General > Get API Key. - - `TIDBYT_TOKEN`: Find your API Token in the Tidbyt mobile app under Settings > General > Get API Key. + 1. Find your Device ID and Key in the Tidbyt mobile app under Settings > General > Get API Key. + 1. Update the `TIDBYT_DEVICE_ID` and `TIDBYT_KEY` environment variables in `.env`. 1. Render the `hello-world` app and send it to your Tidbyt (in the foreground): ```bash @@ -224,8 +222,7 @@ If you haven't launched a codespace yet: 1. Click the green "Use this template" button at the top of this page 1. Choose "Create a new repository" -2. Create a new (private) repo - +1. Create a new (private) repo 1. Clone your new repository and enter the new directory: ```bash @@ -235,27 +232,72 @@ If you haven't launched a codespace yet: -### 2. Configure your Tidbyt - -1. If no `.env` configuration file exists yet, create one from the sample: +### 2. Configure Pixbyt +1. If the `.env` configuration file doesn't exist yet, create it from the sample: ```bash cp .env.sample .env ``` +1. Update the `TZ` environment variable in `.env` to your ["TZ" timezone identifier](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). This is used by many apps that show the (relative) date and time. + +#### Option A: Configure just one Tidbyt + +1. Find your Device ID and Key in the Tidbyt mobile app under Settings > General > Get API Key. +1. Update the `TIDBYT_DEVICE_ID` and `TIDBYT_KEY` environment variables in `.env`. + +
+ + +#### Option B: Configure multiple Tidbyts + + + +1. If `devices.yml` doesn't exists yet, create it from the sample: + ```bash + cp devices.yml.sample devices.yml + ``` +1. For each device: + 1. Add the device to `devices.yml` under `devices:`: + ```yaml + devices: + # ... + - name: + id: $TIDBYT__DEVICE_ID + key: $TIDBYT__KEY + ``` + + 1. Replace `` with the room name or another identifier. + 1. Replace `` with the uppercased version thereof. + + For example: + + ```yaml + devices: + # ... + - name: office + id: $TIDBYT_OFFICE_DEVICE_ID + key: $TIDBYT_OFFICE_KEY + ``` + 1. Find your Device ID and Key in the Tidbyt mobile app under Settings > General > Get API Key. + 1. Add the `TIDBYT__DEVICE_ID` and `TIDBYT__KEY` environment variables in `.env`. + For example: -1. Update `.env` with your configuration: - - `TZ`: Find your ["TZ" timezone identifier](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). This is used by many apps that show the (relative) date and time. - - `TIDBYT_DEVICE_ID`: Find your Device ID in the Tidbyt mobile app under Settings > General > Get API Key. - - `TIDBYT_TOKEN`: Find your API Token in the Tidbyt mobile app under Settings > General > Get API Key. + ```bash + TIDBYT_OFFICE_DEVICE_ID="foo-bar-baz-qux-abc" + TIDBYT_OFFICE_KEY="" + ``` + +
### 3. Add your apps -The most important files in your Pixbyt repo (and likely the only ones you'll want to edit) are the following, which define the apps, their configuration, and their schedules: +The most important files in your Pixbyt repo (and likely the only ones you'll want to edit) are the following, which define the apps, their configuration, and their apps: ```bash pixbyt -├─ apps.yml # Schedules ├─ .env # Configuration +├─ apps.yml # App schedules +├─ devices.yml # Optional: Devices └─ apps └─ # One directory for each app ├─ .star # Main Pixlet applet @@ -364,36 +406,41 @@ Skip ahead to step 4 to build and launch the app server. #### 3.2. Configure the app -1. Add the app's update schedule to `apps.yml` under `schedules:`: +1. Add the app and its update schedule to `apps.yml` under `apps:`: ```yaml - schedules: + apps: # ... - name: - interval: '' - job: + schedule: '' + # If you have multiple Tidbyts defined in `devices.yml`, you can optionally filter them by name: + # devices: [] ``` 1. Replace `` with the name of the app. - 1. Replace `` with an appropriate [cron schedule expression](https://crontab.guru/): + 1. Replace `` with an appropriate [cron expression](https://crontab.guru/): - Clocks should use `* * * * *` to update every minute, so that the displayed time is always as fresh as possible. - Apps that display a random entry from a list can use `*/5 * * * *` to update every 5 minutes, so that a fresh entry is shown on every app rotation. - Apps that show the latest data from some API can use `*/15 * * * *` to update every 15 minutes, or something else appropriate for your data source and the expected data freshness. - Apps that will always generate the same image can use `0 0 * * *` to update every day at midnight, just to be sure. - (A recommended schedule is typically documented in the app's `README`.) + A recommended schedule is typically documented in the app's `README`. + 1. Optionally, replace `` with the name of a device defined in `devices.yml` to only send the app to that device. By default, the app will be sent to all devices. For example: ```yaml - schedules: + apps: + # ... - name: hello-world - interval: '0 * * * *' # On the hour - job: hello-world + schedule: '0/15 * * * *' # Every 15 minutes + devices: [office] # Optional ``` -1. If the app requires configuration, update `.env`: + Note that examples in apps' `README`s sometimes use `schedules` and `interval` keys instead of `apps` and `schedules`. They are equivalent and both are supported, but the latter is preferred. + +1. If the app requires configuration, add its environment variables to `.env`: For any config key the app defines under `app_config:` in its `pixbyt.yml` file, add a value for the uppercase environment variable: @@ -401,7 +448,7 @@ Skip ahead to step 4 to build and launch the app server. _="" ``` - (The exact keys are typically documented in the app's `README`.) + The exact environment variables are typically documented in the app's `README`. For example: @@ -484,7 +531,7 @@ Note that you'll need to do this each time your apps or their schedules change a ``` Your Pixbyt app server is now running, and your apps will update on schedule! -You can find logs for your apps under `logs//`. +You can find logs for your apps under `logs/apps//`. ## How to develop apps with Pixbyt diff --git a/apps.yml b/apps.yml deleted file mode 100644 index dc3b3b7..0000000 --- a/apps.yml +++ /dev/null @@ -1,8 +0,0 @@ -schedules: -- name: hello-world - interval: '0 * * * *' # Every hour - job: hello-world - -# - name: -# interval: '*/15 * * * *' # Every 15 minutes -# job: diff --git a/apps.yml.sample b/apps.yml.sample new file mode 100644 index 0000000..125a483 --- /dev/null +++ b/apps.yml.sample @@ -0,0 +1,7 @@ +apps: +- name: hello-world + schedule: '0 * * * *' # Every hour + +# - name: +# schedule: '*/15 * * * *' # Every 15 minutes +# devices: [] # If you have multiple Tidbyts defined in devices.yml, you can optionally filter them by name. diff --git a/apps/hello-world/client.star b/apps/hello-world/client.star index 55bb8f8..5fbbb7a 100644 --- a/apps/hello-world/client.star +++ b/apps/hello-world/client.star @@ -4,4 +4,9 @@ def client.get_image(): return file.read("logo.png") def client.get_response(name): - return file.exec("hello.py", {"name": name})["response"] + input = {"name": name} + output = file.exec("hello.py", input) + + print("%s ---hello.py--> %s" % (input, output)) + + return output["response"] diff --git a/devices.yml.sample b/devices.yml.sample new file mode 100644 index 0000000..bdbf5da --- /dev/null +++ b/devices.yml.sample @@ -0,0 +1,8 @@ +devices: +- name: default + id: $TIDBYT_DEVICE_ID + key: $TIDBYT_KEY + +# - name: +# id: $TIDBYT__DEVICE_ID +# key: $TIDBYT__KEY diff --git a/plugins/airflow/airflow.cfg b/plugins/airflow/airflow.cfg index af72d19..23c2dd5 100644 --- a/plugins/airflow/airflow.cfg +++ b/plugins/airflow/airflow.cfg @@ -27,7 +27,7 @@ executor = LocalExecutor # Airflow, regardless of the worker count. Generally this value, multiplied by the number of # schedulers in your cluster, is the maximum number of task instances with the running # state in the metadata database. -parallelism = 16 +parallelism = 32 # The maximum number of task instances allowed to run concurrently in each DAG. To calculate # the number of tasks that is running concurrently for a DAG, add up the number of running @@ -76,7 +76,7 @@ dagbag_import_error_tracebacks = True dagbag_import_error_traceback_depth = 2 # How long before timing out a DagFileProcessor, which processes a dag file -dag_file_processor_timeout = 90 +dag_file_processor_timeout = 50 # The class to use for running task instances in a subprocess. # Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class @@ -926,11 +926,11 @@ scheduler_idle_sleep_time = 1 # Number of seconds after which a DAG file is parsed. The DAG file is parsed every # ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after # this interval. Keeping this number low will increase CPU usage. -min_file_process_interval = 900 +min_file_process_interval = 30 # How often (in seconds) to check for stale DAGs (DAGs which are no longer present in # the expected files) which should be deactivated. -deactivate_stale_dags_interval = 900 +deactivate_stale_dags_interval = 60 # How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. dag_dir_list_interval = 300 diff --git a/plugins/airflow/dags/pixbyt.py b/plugins/airflow/dags/pixbyt.py index 7ca6dad..e86beb2 100644 --- a/plugins/airflow/dags/pixbyt.py +++ b/plugins/airflow/dags/pixbyt.py @@ -24,7 +24,7 @@ "start_date": datetime(1970, 1, 1, 0, 0, 0), } } -DEFAULT_INTERVAL = "*/15 * * * *" +DEFAULT_SCHEDULE = "*/15 * * * *" PROJECT_ROOT = os.getenv("MELTANO_PROJECT_ROOT", os.getcwd()) MELTANO_EXECUTABLE = ".meltano/run/bin" @@ -33,24 +33,38 @@ apps_path = Path(PROJECT_ROOT).joinpath(APPS_FILENAME) apps_config = yaml.safe_load(apps_path.read_text()) -schedules = apps_config.get("schedules", []) +if isinstance(apps_config, list): + apps_config = {"apps": apps_config} # `apps:` is optional +apps = [ + *apps_config.get("apps", []), + *apps_config.get("schedules", []) # Backwards compatibility with meltano.yml format +] -for schedule in schedules: - name = schedule.get("name") +for app in apps: + name = app.get("name") if not name: logger.warning("Skipping app without a name") continue - interval = schedule.get("interval", DEFAULT_INTERVAL) - job = schedule.get("job", name) + schedule = ( + app.get("schedule") + or app.get("interval") # Backwards compatibility with meltano.yml format + or DEFAULT_SCHEDULE + ) + job = app.get("job", name) - env = schedule.get("env", {}) - env = {k: str(v) for k, v in env.items()} + env = app.get("env", {}) # Backwards compatibility with meltano.yml format + + devices = app.get("devices", []) + if devices: + env["TIDBYT_DEVICE_NAMES"] = str(devices) dag_id = name.replace("/", "--") - with DAG(dag_id, schedule=interval, **DEFAULT_DAG_OPTS) as dag: + + with DAG(dag_id, schedule=schedule, **DEFAULT_DAG_OPTS) as dag: cmd = f"{MELTANO_EXECUTABLE} run {job}" + env = {k: str(v) for k, v in env.items()} task = BashOperator( dag=dag, @@ -62,4 +76,4 @@ ) globals()[dag_id] = dag - logger.info(f"Created DAG '{dag_id}': interval='{interval}', cmd='{cmd}', env={env}") + logger.info(f"Created DAG '{dag_id}': schedule='{schedule}', cmd='{cmd}', env={env}") diff --git a/plugins/plugins.meltano.yml b/plugins/plugins.meltano.yml index 70fcb86..4b57ba8 100644 --- a/plugins/plugins.meltano.yml +++ b/plugins/plugins.meltano.yml @@ -19,12 +19,17 @@ plugins: namespace: target_tidbyt pip_url: git+https://github.com/DouweM/target-tidbyt.git settings: - - name: token - kind: password - name: device_id - config: - device_id: $TIDBYT_DEVICE_ID - token: $TIDBYT_TOKEN + value: $TIDBYT_DEVICE_ID + - name: key + kind: password + value: $TIDBYT_KEY + + - name: devices_path + value: $MELTANO_PROJECT_ROOT/devices.yml + - name: device_names + kind: array + value: $TIDBYT_DEVICE_NAMES - name: target-webp namespace: target_webp