From d233fc741b7d17158028d288acd41a6edd00aa0c Mon Sep 17 00:00:00 2001 From: "repo-mountie[bot]" <44246292+repo-mountie[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2020 19:14:43 +0000 Subject: [PATCH 01/10] Add compliance audit file Signed-off-by: repo-mountie --- COMPLIANCE.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 COMPLIANCE.yaml diff --git a/COMPLIANCE.yaml b/COMPLIANCE.yaml new file mode 100644 index 0000000..4a5be9e --- /dev/null +++ b/COMPLIANCE.yaml @@ -0,0 +1,11 @@ +name: compliance +description: | + This document is used to track a projects PIA and STRA + compliance. +spec: + - name: PIA + status: PIA_STATUS_HERE + last-updated: '2020-11-04T19:14:37.870Z' + - name: STRA + status: STRA_STATUS_HERE + last-updated: '2020-11-04T19:14:37.870Z' From 40bd0c6d50c66d1225f94521ec35362f329e420d Mon Sep 17 00:00:00 2001 From: "repo-mountie[bot]" <44246292+repo-mountie[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2020 19:15:12 +0000 Subject: [PATCH 02/10] Add Apache License 2.0 Signed-off-by: repo-mountie --- LICENSE | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 31c92557e2bd62d02113424cb525c1128d67647e Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Tue, 3 May 2022 14:23:15 -0700 Subject: [PATCH 03/10] Update service-rate-limit.yaml --- docs/samples/service-plugins/service-rate-limit.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/samples/service-plugins/service-rate-limit.yaml b/docs/samples/service-plugins/service-rate-limit.yaml index a22d055..8af8af3 100644 --- a/docs/samples/service-plugins/service-rate-limit.yaml +++ b/docs/samples/service-plugins/service-rate-limit.yaml @@ -1,4 +1,5 @@ - +# policy: local | redis +# limit_by: consumer | ip | service | header | path services: - name: MY_REST_API tags: [ _NS_ ] @@ -10,6 +11,7 @@ services: fault_tolerant: true hide_client_headers: false limit_by: consumer + policy: local minute: 10 header_name: null second: null From eadff4bd1d4f0f89a323a87e0de38953b28eb530 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Fri, 6 May 2022 14:05:47 -0700 Subject: [PATCH 04/10] Update USER-JOURNEY.md --- USER-JOURNEY.md | 513 +----------------------------------------------- 1 file changed, 1 insertion(+), 512 deletions(-) diff --git a/USER-JOURNEY.md b/USER-JOURNEY.md index 31659ce..87e4c75 100644 --- a/USER-JOURNEY.md +++ b/USER-JOURNEY.md @@ -1,514 +1,3 @@ # API Owner Flow -The following steps walk an API Owner through setting up an API on the BC Gov API Gateway in our Test instance. If you are ready to deploy to our Production instance, use the links found at the bottom of this document ([here](#production-links)). - -## 1. Register a new namespace - -A `namespace` represents a collection of Kong Gateway Services and Routes that are managed independently. - -To create a new namespace, login to the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca). - -After login, click the namespace dropdown in the top right next to your user name (it may show `No Active Namespace`), then click `Create New Namespace`. - -The namespace must be an alphanumeric string between 5 and 15 characters (RegExp reference: `^[a-z][a-z0-9-]{4,14}$`). - -You can select and manage namespaces by clicking the namespace dropdown in the top right next to your user name. - -## 2. Generate a Service Account - -Go to the `Namespaces` tab, click the `Service Accounts` link, and click the `New Service Account`. A new credential will be created - make a note of the `ID` and `Secret`. - -Before the credential can be used, you need to grant it the appropriate permissions. This can be done by going to the `API Access` tab and click the "Manage Resources" for the `Gateway Administration API`. Click the link that corresponds to your newly created namespace. Click the `Grant Service Account Access` and enter in the `ID` and the scopes you want to grant to the Service Account, then click the `Share` button to grant the permissions. - -The available Scopes are: -| Scope | Permission | -| ----- | ---------- | -| `Namespace.Manage` | Permission to update the Access Control List for controlling access to viewing metrics, service configuration and service account management (effectively a superuser for the namespace) | -| `Namespace.View` | Read-only access to the namespace | -| `GatewayConfig.Publish` | Permission to publish gateway configuration to Kong and to view the status of the upstreams | -| `Content.Publish` | Permission to update the documentation on the portal | -| `CredentialIssuer.Admin` | Permission to create Authorization Profiles so that they are available to be used when configuring Product Environments | -| `Access.Manage` | Permission to approve/reject access requests to your APIs that you make discoverable | - -## 3. Prepare configuration - -The gateway configuration can be hand-crafted or you can use a command line interface that we developed called `gwa` to convert your Openapi v3 spec to a Kong configuration. - -### 3.1. Hand-crafted (recommended if you don't have an Openapi spec) - -**Simple Example** - -```bash -export NS="my_namespace" -export NAME="a-service-for-$NS" -echo " -services: -- name: $NAME - host: httpbin.org - tags: [ ns.$NS ] - port: 443 - protocol: https - retries: 0 - routes: - - name: $NAME-route - tags: [ ns.$NS ] - hosts: - - $NAME.api.gov.bc.ca - paths: - - / - methods: - - GET - strip_path: false - https_redirect_status_code: 426 - path_handling: v0 -" > sample.yaml -``` - -> To view common plugin config go to [COMMON-CONFIG.md](https://github.com/bcgov/gwa-api/blob/dev/docs/COMMON-CONFIG.md) - -> To view some other plugin examples go [here](https://github.com/bcgov/gwa-api/blob/dev/docs/samples/service-plugins). - -> **Declarative Config** Behind the scenes, DecK is used to sync your configuration with Kong. For reference: https://docs.konghq.com/deck/overview/ - -> **Splitting Your Config:** A namespace `tag` with the format `ns.$NS` is mandatory for each service/route/plugin. But if you have separate pipelines for your environments (i.e./ dev, test and prod), you can split your configuration and update the `tags` with the qualifier. So for example, you can use a tag `ns.$NS.dev` to sync Kong configuration for `dev` Service and Routes only. - -> **Upstream Services on OCP4:** If your service is running on OCP4, you should specify the Kubernetes Service in the `Service.host`. It must have the format: `..svc`. Also make sure your `Service.port` matches your Kubernetes Service Port. Any Security Policies for egress from the Gateway will be setup automatically on the API Gateway side. -> The Aporeto Network Security Policies are being removed in favor of the Kubernetes Security Policies (KSP). You will need to create a KSP on your side looking something like this to allow the Gateway's test and prod environments to route traffic to your API: - -```yaml -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: allow-traffic-from-gateway-to-your-api -spec: - podSelector: - matchLabels: - name: my-upstream-api - ingress: - - from: - - namespaceSelector: - matchLabels: - environment: test - name: 264e6f - - from: - - namespaceSelector: - matchLabels: - environment: prod - name: 264e6f -``` - -> **Migrating from OCP3 to OCP4?** Please review the [OCP4-Migration](https://github.com/bcgov/gwa-api/blob/dev/docs/OCP4-MIGRATION.md) instructions to help with transitioning to OCP4 and the new APS Gateway. - -> **Require mTLS between the Gateway and your Upstream Service?** To support mTLS on your Upstream Service, you will need to provide client certificate details and if you want to verify the upstream endpoint then the `ca_certificates` and `tls_verify` is required as well. An example: - -```yaml -services: - - name: my-upstream-service - host: my-upstream.site - tags: [_NS_] - port: 443 - protocol: https - tls_verify: true - ca_certificates: [0a780ee0-626c-11eb-ae93-0242ac130012] - client_certificate: 8fc131ef-9752-43a4-ba70-eb10ba442d4e - routes: [...] -certificates: - - cert: "" - key: "" - tags: [_NS_] - id: 8fc131ef-9752-43a4-ba70-eb10ba442d4e -ca_certificates: - - cert: "" - tags: [_NS_] - id: 0a780ee0-626c-11eb-ae93-0242ac130012 -``` - -> NOTE: You must generate a UUID (`python -c 'import uuid; print(uuid.uuid4())'`) for each certificate and ca_certificate you create (set the `id`) and reference it in your `services` details. - -> HELPER: Python command to get a PEM file on one line: `python -c 'import sys; import json; print(json.dumps(open(sys.argv[1]).read()))' my.pem` - -### 3.2. gwa Command Line - -Run: `gwa new` and follow the prompts. - -Example: - -```bash -gwa new -o sample.yaml \ - --route-host myapi.api.gov.bc.ca \ - --service-url https://httpbin.org \ - https://bcgov.github.io/gwa-api/openapi/simple.yaml -``` - -> See below for the `gwa` CLI install instructions. - -## 4. Apply gateway configuration - -The Swagger console for the `gwa-api` can be used to publish Kong Gateway configuration, or the `gwa Command Line` can be used. - -### 4.1. gwa Command Line (recommended) - -**Install (for Linux)** - -```bash -GWA_CLI_VERSION=v1.2.0; curl -L -O https://github.com/bcgov/gwa-cli/releases/download/${GWA_CLI_VERSION}/gwa_${GWA_CLI_VERSION}_linux_x64.zip -unzip gwa_${GWA_CLI_VERSION}_linux_x64.zip -./gwa --version -``` - -> **Using MacOS or Windows?** Download here: [https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0](https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0) - -> NOTE: Version 1.2.0 introduces support for v2 of our api. To continue using v1 of the api, ensure that the API Version is set to 1 (see below) - -**Configure** - -Create a `.env` file and update the CLIENT_ID and CLIENT_SECRET with the new credentials that were generated in step #2: - -```bash -echo " -GWA_NAMESPACE=$NS -CLIENT_ID= -CLIENT_SECRET= -GWA_ENV=test -API_VERSION=2 -" > .env - -OR run: - -gwa init -T --api-version=2 --namespace=$NS --client-id= --client-secret= - -``` - -> NOTE: The `-T` indicates our Test environment. For production use `-P`. - -**Publish** - -```bash -gwa pg sample.yaml -``` - -If you want to see the expected changes but not actually apply them, you can run: - -```bash -gwa pg --dry-run sample.yaml -``` - -### 4.2. Swagger Console - -Go to [gwa-api Swagger Console](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/). - -Select the `PUT` `/namespaces/{namespace}/gateway` API. - -The Service Account uses the OAuth2 Client Credentials Grant Flow. Click the `lock` link on the right and enter in the Service Account credentials that were generated in step #2. - -For the `Parameter namespace`, enter the namespace that you created in step #1. - -Select `dryRun` to `true`. - -Select a `configFile` file. - -Send the request. - -### 4.3. Postman - -From the Postman App, click the `Import` button and go to the `Link` tab. - -Enter a URL: https://openapi-to-postman-api-gov-bc-ca.test.api.gov.bc.ca/?u=https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/v2/openapi.yaml - -After creation, go to `Collections` and right-click on the `Gateway Administration (GWA) API` collection and select `edit`. - -Go to the `Authorization` tab, enter in your `Client ID` and `Client Secret` and click `Get New Access Token`. - -You should get a successful dialog to proceed. Click `Proceed` and `Use Token`. - -You can then verify that the token works by going to the Collection `Return key information about authenticated identity` and click `Send`. - -## 5. Verify routes - -To verify that the Gateway can access the upstream services, run the command: `gwa status`. - -In our test environment, the hosts that you defined in the routes get altered; to see the actual hosts, log into the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca), go to the `Namespaces` tab, go to `Gateway Services` and select your particular service to get the routing details. - -```bash -curl https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -ab -n 20 -c 2 https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -``` - -To help with troubleshooting, you can use the GWA API to get a health check for each of the upstream services to verify the Gateway is connecting OK. - -Go to the [GWA API](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/#/Service%20Status/get_namespaces__namespace__services), enter in the new credentials that were generated in step #2, click `Try it out`, enter your namespace and click `Execute`. The results are returned in a JSON object. - -## 6. View metrics - -The following metrics can be viewed in real-time for the Services that you configure on the Gateway: - -- Request Rate : Requests / Second (by Service/Route, by HTTP Status) -- Latency : Standard deviations measured for latency inside Kong and on the Upstream Service (by Service/Route) -- Bandwidth : Ingress/egress bandwidth (by Service/Route) -- Total Requests : In 5 minute windows (by Consumer, by User Agent, by Service, by HTTP Status) - -All metrics can be viewed by an arbitrary time window - defaults to `Last 24 Hours`. - -Go to [Grafana](https://grafana-apps-gov-bc-ca.test.api.gov.bc.ca) to view metrics for your configured services. - -You can also access summarized metrics from the `API Services Portal` by going to the `Namespaces` tab and clicking the `Gateway Services` link. - -## 7. Grant access to others - -To grant access to others, you need to grant them the appropriate Scopes. This can be done from the `API Services Portal`, going to the `API Access` tab and clicking the "Manage Resources" for the `Gateway Administration API` product. Click the link that corresponds to your namespace. Click the `Grant User Access` and enter in a username (i.e./ `jsmith@idir`) and the scopes you want to grant to the User, then click the `Share` button to grant the permissions. - -## 8. Add to your CI/CD Pipeline - -Update your CI/CD pipelines to run the `gwa-cli` to keep your services updated on the gateway. - -### 8.1. Github Actions Example - -In the repository that you maintain your CI/CD Pipeline configuration, use the Service Account details from `Step 2` to set up two `Secrets`: - -- GWA_ACCT_ID - -- GWA_ACCT_SECRET - -Add a `.gwa` folder (can be called anything) that will be used to hold your gateway configuration. - -An example Github Workflow: - -```yaml -env: - NS: "" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v1 - with: - node-version: 10 - TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get GWA Command Line - run: | - curl -L -O https://github.com/bcgov/gwa-cli/releases/download/v1.2.0/gwa_v1.2.0_linux_x64.zip - unzip gwa_v1.2.0_linux_x64.zip - export PATH=`pwd`:$PATH - - - name: Apply Namespace Configuration - run: | - export PATH=`pwd`:$PATH - cd .gwa/$NS - - gwa init -T \ - --namespace=$NS \ - --client-id=${{ secrets.TEST_GWA_ACCT_ID }} \ - --client-secret=${{ secrets.TEST_GWA_ACCT_SECRET }} - - gwa pg - - gwa acl --managers acope@idir -``` - -## 9. Share your API for Discovery - -Package your APIs and make them available for discovery through the API Services Portal and BC Data Catalog. - -The `API Services Portal` Directory organizes your APIs by Products and Environments. You can manage them via an API or through the UI. - -To use the Directory API, the following scopes are required: - -- For `contents` (documentation), the service account must have the `Content.Publish` scope -- For `datasets` and `products`, the service account must have the `Namespace.Manage` scope -- For `issuers`, the service account must have the `CredentialIssuer.Admin` scope - -View the Directory API in the [Swagger Console](https://openapi-apps-gov-bc-ca.test.api.gov.bc.ca/?url=https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api/openapi.yaml) - -> NOTE: The steps below use `restish`, but we will be working on upgrading the `gwa` command line interface to support these APIs - -> Can use `y2j` to convert from YAML to JSON -> -> `echo '#!/usr/bin/env python\nimport sys,yaml,json\nprint(json.dumps(yaml.safe_load(open(sys.argv[1]).read())))' > /usr/local/bin/y2j` -> -> `chmod +x /usr/local/bin/y2j` - -**Restish Setup** - -``` -restish api configure my_api -``` - -Base URI : https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api - -`Edit Profile default` - -Select `Setup Auth` > `oauth-client-credentials` - -Enter the `client_id` and `client_secret` for your Service Account. - -`token_url` : Provided to you when you created the Service Account - -`scopes` : openid - -Select `Finished with profile` and then `Save and exit` - -To verify that restish is working, run: - -`restish my_api list` or `restish my_api get-new-id product` - -The below example works with the service created above, so we will want to reference the NS environment variable. - -``` -export NS="" -``` - -### 9.1 Setup Authorization Profiles - -If your APIS are protected with one of the OAuth2 grant types, then an Authorization profile must be created with the appropriate credentials for accessing the corresponding Identity Provider/Broker and Authorization Server. - -```yaml -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -``` -y2j issuer.yaml | restish my_api put-issuer $NS -``` - -### 9.2 Setup your Product, Environments and link your Services - -```yaml -kind: DraftDataset -name: my-draft-dataset -organization: ministry-of-health -organizationUnit: planning-and-innovation-division -title: My API -notes: Some information about this API -tags: [health, standards, openapi] -sector: Service -license_title: Access Only -view_audience: Government -security_class: LOW-PUBLIC -record_publish_date: "2021-05-27" -``` - -``` -y2j dataset.yaml | restish my_api put-dataset $NS -``` - -```yaml -kind: Product -appId: 2B04C28E08AW -name: My API -dataset: my-draft-dataset -environments: - - id: 1F7CA929 - name: dev - active: false - approval: false - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Please provide a bit more of this - services: [] - - id: 2F7CA929 - name: test - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Asking for test environment? Please provide some more info - services: [a-service-for-$NS] - - id: 3F7CA929 - name: prod - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: client-credentials - credentialIssuer: Resource Server $NS - additionalDetailsToRequest: Production? Great, please provide X, Y and Z - services: [] -``` - -``` -y2j prod.yaml | restish my_api put-product $NS -``` - -### 9.3 Update Gateway Configuration based on Flow - -In the previous section our example defines an environment that is protected using Kong's API Key and ACL plugins. To activate an environment, the corresponding plugins need to exist on the Gateway for that service or routes. The ACL `allow` corresponds to the unique `Environment ID` defined in section 9.2. - -``` - plugins: - - name: key-auth - tags: [ ns.$NS ] - protocols: [ http, https ] - config: - key_names: ["X-API-KEY"] - run_on_preflight: true - hide_credentials: true - key_in_body: false - - name: acl - tags: [ ns.$NS ] - config: - hide_groups_header: true - allow: [ ] -``` - -### 9.4 Publish Environments - -Your products will not appear on the Directory until you mark the relevant environments as Active. You can do this by either updating the Product Environment configuration above to `active: true`, or going to the API Services Portal UI and editing the Environment details. - -### 9.5 Publish Documentation - -```yaml -kind: Content -title: Getting Started with Example API -description: Getting Started with Example API -externalLink: https://github.com/bcgov/$NS/getting_started.md -order: 1 -tags: [ns.$NS] -isComplete: true -isPublic: true -publishDate: "2021-06-02T08:00:00.000-08:00" -``` - -``` -y2j content.yaml | restish my_api put-content $NS - -echo "# here is some markdown and more!" > doc.md - -restish my_api put-content $NS \ - externalLink: "https://github.com/bcgov/$NS/getting_started.md", \ - content: @doc.md -``` - -### 9.6 View your product in the API Directory - -Find your API in the [API Services Portal Directory](https://api-gov-bc-ca.test.api.gov.bc.ca/devportal/api-discovery) - -It is now ready to receive access requests from the community! - -# Production Links - -- [API Services Portal](https://api.gov.bc.ca) -- [gwa-api Swagger Console](https://gwa.api.gov.bc.ca/docs/) -- [gwa-api Postman Collection](https://openapi-to-postman.api.gov.bc.ca/?u=https://gwa.api.gov.bc.ca/docs/v2/openapi.yaml) -- [Gateway Metrics - Grafana](https://grafana.apps.gov.bc.ca) +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/guides/owner-journey From 6daa76f473043fbf359e48fbf414034d7063eae7 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Fri, 6 May 2022 14:11:56 -0700 Subject: [PATCH 05/10] upd docs --- .gitignore | 1 + README.md | 52 +- docs/AVAILABLE-PLUGINS.md | 51 +- docs/COMMON-CONFIG.md | 105 +--- docs/SSL.md | 40 +- docs/guides/intro-signed-jwt.md | 125 +---- docs/guides/intro-signed-jwt.yaml | 8 - docs/guides/tutorial-idp-client-cred-flow.md | 182 +------ .../guides/tutorial-idp-client-cred-flow.yaml | 8 - docs/guides/user-journey.md | 509 +----------------- docs/guides/user-journey.yaml | 8 - docs/static/openapi/.env | 4 + docs/static/openapi/kong-with-edits.yaml | 21 + 13 files changed, 58 insertions(+), 1056 deletions(-) create mode 100644 .gitignore delete mode 100644 docs/guides/intro-signed-jwt.yaml delete mode 100644 docs/guides/tutorial-idp-client-cred-flow.yaml delete mode 100644 docs/guides/user-journey.yaml create mode 100644 docs/static/openapi/.env create mode 100644 docs/static/openapi/kong-with-edits.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..496ee2c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index b47dfd0..32f92e9 100644 --- a/README.md +++ b/README.md @@ -6,38 +6,38 @@ For self-service of APIs, a set of microservices are used to coordinate updates by the providers of APIs. -* `Gateway` : Provides a way for API Owners to update their Kong configuration (and internally the OCP Edge Router) -* `Authz` : Provides a way for API Owners to update Keycloak for access to functionality on the API Services Portal -* `Catalog` : Provides a way for API Owners to update the API details in the BC Data Catalog +- `Gateway` : Provides a way for API Owners to update their Kong configuration (and internally the OCP Edge Router) +- `Authz` : Provides a way for API Owners to update Keycloak for access to functionality on the API Services Portal +- `Catalog` : Provides a way for API Owners to update the API details in the BC Data Catalog All APIs are protected by an OIDC JWT Token with the following claims: -* `aud` : `gwa` -* `namespace` : Identifies the namespace that the APIs belong to, used to scope what changes are allowed +- `aud` : `gwa` +- `namespace` : Identifies the namespace that the APIs belong to, used to scope what changes are allowed **Configuration:** -| Variable | Description | Example | -| -------- | ----------- | ------- | -| `PORT` | Port | `2000` | -| `LOG_LEVEL` | Log level for the application | `INFO` | -| `ENVIRONMENT` | Indicates what environment config to use (development|test|production) | `production` | -| `CONFIG_PATH` | Location of the config | `/tmp/production.json` | -| `OIDC_BASE_URL` | External base url used by the Swagger console for an externally available Auth endpoint. | `https://keycloak.domain/auth/realms/abc` -| `TOKEN_MATCH_AUD` | The `audience` that the token must match. | `gwa` -| `WORKING_FOLDER` | Temporary working folder that only exists for the duration of the POD. | `/tmp` -| `KONG_ADMIN_URL` | The Kong Admin endpoint. | `http://kong-admin-api:8001` -| `KC_SERVER_URL` | Keycloak access for administrative rights to manage groups for namespaces and for OIDC Discovery for getting the `jwks_uri` for the list of supported keys | `https://auth.domain/auth` -| `KC_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `aps` -| `KC_CLIENT_ID` | Keycloak access for administrative rights to manage groups for namespaces | `admin-cli` -| `KC_CLIENT_SECRET`| Keycloak access for administrative rights to manage groups for namespaces | `` -| `KC_USER_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `master` -| `KC_USERNAME` | Keycloak access for administrative rights to manage groups for namespaces | `kcadmin` -| `KC_PASSWORD` | Keycloak access for administrative rights to manage groups for namespaces | `xxx` -| `HOST_TRANSFORM_ENABLED` | For Dev and Test a way to transform the host for working in these environments | `false` -| `HOST_TRANSFORM_BASE_URL` | For Dev and Test a way to transform the host for working in these environments | -| `PLUGINS_RATELIMITING_REDIS_PASSWORD` | The Redis credential added to the rate-limiting Kong plugin during publish | +| Variable | Description | Example | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ----------- | ------------ | +| `PORT` | Port | `2000` | +| `LOG_LEVEL` | Log level for the application | `INFO` | +| `ENVIRONMENT` | Indicates what environment config to use (development | test | production) | `production` | +| `CONFIG_PATH` | Location of the config | `/tmp/production.json` | +| `OIDC_BASE_URL` | External base url used by the Swagger console for an externally available Auth endpoint. | `https://keycloak.domain/auth/realms/abc` | +| `TOKEN_MATCH_AUD` | The `audience` that the token must match. | `gwa` | +| `WORKING_FOLDER` | Temporary working folder that only exists for the duration of the POD. | `/tmp` | +| `KONG_ADMIN_URL` | The Kong Admin endpoint. | `http://kong-admin-api:8001` | +| `KC_SERVER_URL` | Keycloak access for administrative rights to manage groups for namespaces and for OIDC Discovery for getting the `jwks_uri` for the list of supported keys | `https://auth.domain/auth` | +| `KC_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `aps` | +| `KC_CLIENT_ID` | Keycloak access for administrative rights to manage groups for namespaces | `admin-cli` | +| `KC_CLIENT_SECRET` | Keycloak access for administrative rights to manage groups for namespaces | `` | +| `KC_USER_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `master` | +| `KC_USERNAME` | Keycloak access for administrative rights to manage groups for namespaces | `kcadmin` | +| `KC_PASSWORD` | Keycloak access for administrative rights to manage groups for namespaces | `xxx` | +| `HOST_TRANSFORM_ENABLED` | For Dev and Test a way to transform the host for working in these environments | `false` | +| `HOST_TRANSFORM_BASE_URL` | For Dev and Test a way to transform the host for working in these environments | +| `PLUGINS_RATELIMITING_REDIS_PASSWORD` | The Redis credential added to the rate-limiting Kong plugin during publish | # API Provider Flow -[See Details](USER-JOURNEY.md) +[See Details](https://bcgov.github.io/aps-infra-platform/guides/owner-journey) diff --git a/docs/AVAILABLE-PLUGINS.md b/docs/AVAILABLE-PLUGINS.md index 7c22840..cfda916 100644 --- a/docs/AVAILABLE-PLUGINS.md +++ b/docs/AVAILABLE-PLUGINS.md @@ -1,52 +1,3 @@ Current list of plugins available on the API Gateway. -> NOTE: The `httplog` is not available for individual teams to use as it is used at a global level to feed logs to other systems for audit/monitoring/etc. - -> If there are other plugins that you would like to have added to the Gateway, please reach out to us at our Rocket.Chat channel `#aps-ops`. - -```json - "available_on_server": { - "basic-auth": true, - "ip-restriction": true, - "request-transformer": true, - "response-transformer": true, - "request-size-limiting": true, - "rate-limiting": true, - "response-ratelimiting": true, - "syslog": true, - "loggly": true, - "datadog": true, - "ldap-auth": true, - "statsd": true, - "bot-detection": true, - "aws-lambda": true, - "request-termination": true, - "azure-functions": true, - "zipkin": true, - "pre-function": true, - "post-function": true, - "prometheus": true, - "proxy-cache": true, - "session": true, - "acme": true, - "grpc-web": true, - "grpc-gateway": true, - "kong-spec-expose": true, - "jwt-keycloak": true, - "referer": true, - "bcgov-gwa-endpoint": true, - "gwa-ip-anonymity": true, - "oidc": true, - "jwt": true, - "acl": true, - "correlation-id": true, - "cors": true, - "oauth2": true, - "tcp-log": true, - "udp-log": true, - "file-log": true, - "http-log": true, - "key-auth": true, - "hmac-auth": true - } - ``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/available-plugins diff --git a/docs/COMMON-CONFIG.md b/docs/COMMON-CONFIG.md index c790c81..df9987b 100644 --- a/docs/COMMON-CONFIG.md +++ b/docs/COMMON-CONFIG.md @@ -1,106 +1,3 @@ # Common Gateway Controls -The following are sample Gateway controls for common scenarios. - -## Returning an HTTP Redirect - -``` -plugins: -- name: pre-function - tags: [ _NS_ ] - config: - access: - - "kong.response.exit(307, 'site moved - redirecting...', {['Location'] = 'https://my-new-url.site'})" -``` - -## Adding Headers For Best Security Practices - -``` -plugins: -- name: response-transformer - tags: [ _NS_ ] - config: - add: - headers: - - "X-Frame-Options: DENY" - - "X-XSS-Protection: 1; mode=block" - - "X-Content-Type-Options: nosniff" - - "Strict-Transport-Security: max-age=31536000" - - "Content-Security-Policy: script-src 'self'" -``` - -> For further information on individual headers, see: https://owasp.org/www-project-secure-headers/ - -## Rate Limit - -### Option 1 - Using a Distributed Cache - -This provides the most accurate because it uses a centralized Cache that all Kong nodes use. The downside is that there is a 100-200ms latency. - -``` -plugins: -- name: rate-limiting - tags: [ _NS_ ] - config: - fault_tolerant: true - hide_client_headers: false - limit_by: ip - minute: 30000 - second: null - hour: null - day: null - month: null - year: null -``` - -### Option 2 - Node Local Caching - -This provides the fastest rate limiting option, with minimal latency (~1ms). The downside is that it is local to each node so calculating the actual load on your upstream is a function of the number of nodes. - -``` -plugins: -- name: rate-limiting - tags: [ _NS_ ] - config: - policy: local - fault_tolerant: true - hide_client_headers: false - limit_by: ip - minute: 30000 - second: null - hour: null - day: null - month: null - year: null -``` - -## Two-tiered access setup - -The `key-auth` and `jwt-keycloak` plugins support the concept of allowing "anonymous" access, which allows you to define a "free" service which might have limits around it (like only allowing 100 requests/minute), and then an "elevated" access where the Consumer would get an improved level of service, such as higher rate limits. - -There is a global "anonymous" consumer that is identified as "ce26955a-cf08-4907-9427-12d01c8bd94c" in both our Test and Production environments. - -To enable anonymous access to your API, update your plugin configuration with: - -### key-auth - -``` -- name: key-auth - tags: [ _NS_ ] - config: - ... - anonymous: ce26955a-cf08-4907-9427-12d01c8bd94c -``` - -### jwt-keycloak - -``` -- name: jwt-keycloak - tags: [ _NS_ ] - config: - ... - consumer_match: true - consumer_match_ignore_not_found: false - consumer_match_claim_custom_id: false - anonymous: ce26955a-cf08-4907-9427-12d01c8bd94c -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/common-config diff --git a/docs/SSL.md b/docs/SSL.md index d06e0ab..f6bee91 100644 --- a/docs/SSL.md +++ b/docs/SSL.md @@ -1,41 +1,3 @@ - # SSL Termination -If you would like to verify the SSL endpoint, you can run the following two commands and compare the fingerprint and serial no. - -``` -export A_HOST=httpbin-regression.api.gov.bc.ca -openssl s_client -showcerts -verify 5 -connect 142.34.194.118:443 -servername ${A_HOST} < /dev/null | awk '/BEGIN/,/END/{ if(/BEGIN/){a++}; print}' > gw.crt - -openssl x509 -in gw.crt -fingerprint -serial -dates -noout - -``` - -## *.api.gov.bc.ca - -| Issue Date | Expires | Deployed | SHA1 Fingerprint | Serial No. | -|-------------|-------------|-------------|-------------------------------------------------------------|----------------------------------| -| Oct 6 2020 | Oct 16 2021 | Oct 6 2020 | 20:7D:15:9D:42:BE:CC:BC:FD:EF:DF:13:77:C7:25:A3:A4:72:45:05 | 7876EB597E14F728C8455504177D3BC9 | -| Feb 16 2021 | Oct 16 2021 | Feb 25 2021 | 4D:EA:CE:C4:0A:73:67:D3:B4:03:F6:63:C4:E1:67:2C:47:9D:EA:82 | 3B5849D8A670251A3C20EA7859BDF996 | -| Sep 27 2021 | Oct 16 2022 | Oct 6 2021 | E3:DF:EC:89:BC:03:9B:E9:7D:57:91:EB:52:18:59:46:AA:A9:3A:15 | 1B588948FBB2945DE5D3A0034E355539 | - -You can run the above as one line: - -``` -A_HOST=httpbin-regression.api.gov.bc.ca; openssl s_client -showcerts -verify 5 -connect ${A_HOST}:443 -servername ${A_HOST} < /dev/null | awk '/BEGIN/,/END/{ if(/BEGIN/){a++}; print}' | openssl x509 -fingerprint -serial -dates -noout -``` - - -**Individual File Verification** - -``` -openssl x509 -in data-api-wildcard-2020.crt -fingerprint -serial -dates -noout -openssl x509 -in data-api-wildcard-2021.crt -fingerprint -serial -dates -noout -``` - -**Cert/Key Verification** - -``` -openssl x509 -noout -modulus -in data-api-wildcard.crt | openssl md5 -openssl rsa -noout -modulus -in data-api-wildcard.key | openssl md5 -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/ssl diff --git a/docs/guides/intro-signed-jwt.md b/docs/guides/intro-signed-jwt.md index 960989d..97cade3 100644 --- a/docs/guides/intro-signed-jwt.md +++ b/docs/guides/intro-signed-jwt.md @@ -1,126 +1,3 @@ # Signed JWT Authentication -The following example uses NodeJS code to show how to prepare for signed JWT authentication to an API on the BC Government API Gateway. - -## 1. Generate a Certificate Key Pair - -``` -npm install --save node-jose ms - -echo " -const fs = require('fs'); -const jose = require('node-jose'); - -const keyStore = jose.JWK.createKeyStore() -keyStore.generate('RSA', 2048, {alg: 'RS256', use: 'sig' }) -.then(result => { - fs.writeFileSync( - 'keys.json', - JSON.stringify(keyStore.toJSON(true), null, ' ') - ) - const [key] = keyStore.all({ use: 'sig' }) - console.log(key.toPEM(true)) - console.log(key.toPEM(false)) -}) -" | node -``` - -## 2. Publish the Public Key - -Publish the Public Key in JWKS Format at a location that is publically accessible. - -``` -echo " -const fs = require('fs') -const jose = require('node-jose'); - -const ks = fs.readFileSync('keys.json') -jose.JWK.asKeyStore(ks.toString()).then(keyStore => - console.log(JSON.stringify(keyStore.toJSON(), null, 3))); -" | node -``` - -Place the JWKS JSON file somewhere that it can be reached publically. - -``` -https://ikethecoder.github.io/temp-place/certs.json -``` - -## 3. Request Access to an API - -https://api-gov-bc-ca.test.api.gov.bc.ca/ - -Make a note of the `Client ID`, `Issuer` and `Token Endpoint`. - -``` -export CID="" -export ISS="" -export TURL="" -``` - -## 4. Request a Client JWT Token - -Requesting a Client JWT Token is a two-step process: - -a) Build a Client Assertion Token that is signed with the private key you generated earlier. - -b) Request a token from the Token Endpoint using the Client Assertion. - -The following sample performs both steps: - -``` -npm install --save njwt node-fetch - -echo " -const njwt = require('njwt'); - -const fs = require('fs') - -const jose = require('node-jose'); -const ks = fs.readFileSync('keys.json') -jose.JWK.asKeyStore(ks.toString()).then(keyStore => { - const [key] = keyStore.all({ use: 'sig' }) - - const privateKey = key.toPEM(true) // Load an RSA private key from configuration - const clientId = process.env.CID; // Or load from configuration - const now = Math.floor( new Date().getTime() / 1000 ); // seconds since epoch - const plus5Minutes = new Date( ( now + (5*60) ) * 1000); // Date object - const alg = 'RS256'; // one of RSA or ECDSA algorithms: RS256, RS384, RS512, ES256, ES384, ES512 - - const claims = { - aud: process.env.ISS - }; - - const jwt = njwt.create(claims, privateKey, alg) - .setIssuedAt(now) - .setExpiration(plus5Minutes) - .setIssuer(clientId) - .setSubject(clientId) - .compact(); - - const fetch = require('node-fetch'); - const { URLSearchParams } = require('url'); - - const params = new URLSearchParams(); - params.append('grant_type', 'client_credentials'); - params.append('client_id', process.env.CID); - params.append('scopes', 'openid'); - params.append('client_assertion_type', 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'); - params.append('client_assertion', jwt); - - fetch(process.env.TURL, { method: 'POST', body: params }) - .then(res => res.json()) - .then(json => console.log(json)); -}); -" | node -``` - -## 5. Call the API - -Call the API using the newly generated Token returned from the Identity Provider. - -``` -curl -v https://a-protected-api.test.api.gov.bc.ca/headers \ - -H "Authorization: Bearer $TOK" - -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/intro-signed-jwt.yaml b/docs/guides/intro-signed-jwt.yaml deleted file mode 100644 index 0f26ce3..0000000 --- a/docs/guides/intro-signed-jwt.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/intro-signed-jwt.md -title: "Getting Started with Signed JWT" -description: "Accessing an API with Signed JWT and Client Credential Grant" -order: 1 -tags: ["ns.platform", "tutorial"] -isComplete: true -isPublic: true -publishDate: "2021-09-27T20:00:00.000-08:00" diff --git a/docs/guides/tutorial-idp-client-cred-flow.md b/docs/guides/tutorial-idp-client-cred-flow.md index 713421b..12ad070 100644 --- a/docs/guides/tutorial-idp-client-cred-flow.md +++ b/docs/guides/tutorial-idp-client-cred-flow.md @@ -1,183 +1,3 @@ # Protecting an API with OAuth2 Client Credential Flow -Steps for protecting and calling an API using the OAuth2 Client Credential Grant. - -1. Configuring an API on the Kong Gateway -2. Granting Access to the IdP -3. Client Requesting Access -4. Provider Approving Access -5. Client Retrieving the Access Token -6. Client Calling an API -7. Gateway Proxying to Upstream - -![alt text](assets/oauth2.png "Protecting an API") - -## 1. Configuring an API on the Kong Gateway - -The API Provider User Journey document provides the steps to enable a new API on the Kong Gateway. It will walk you through the creation of a new Namespace, and a Service Account that can be used to configure the Kong Gateway and the Authorization Profile described in step 2. - -Once the API is working on the Gateway, you can then define a Product, which can be made available on the API Services Portal Directory. - -## 2. Granting Access to the IdP - -### a) Prerequisites - -Before the Portal can be configured, a new set of credentials must be created on the IdP. For this tutorial, we will include the steps when Keycloak is the IdP. - -**Create a new Client on the IdP** - -Create a new client with Access Type `confidential`. All flows except `Service Accounts` should be turned off. - -Make a note of the `Client ID` and `Client Secret` , they will be used when the Portal `Authorization Profile` is created. - -The `Full Scope Allowed` can be turned off and the `realm-management` client roles for `manage-clients` and `manage-users` should be added. - -Add the `manage-clients` and `manage-users` client roles to the `Service Account Roles`. - -### b) Setup the Authorization Profile - -A credential with `CredentialIssuer.Admin` is required to update Authorization Profiles (`CredentialIssuer`). - -Authorization Profiles can be setup either via the Portal or by using a Service Account with the Portal Directory API. - -Update the below `CredentialIssuer` to include the environment details and the Scopes and Roles setup for authorization. - -``` -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -clientMappers: - - name: audience - defaultValue: '' -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -### c) Link the Authorization Profile to the Product - -Before making the API available on the Directory, the API should be configured with a plugin for protecting access. To do this, an API Provider can edit the Product details to select `Oauth2 Client Credential Flow` and the newly created Authorization Profile. - -### d) Update your Gateway Configuration with the Plugin - -Update your Gateway Configuration to include the `jwt-keycloak` plugin. - -> HINT: When you configure the Product Environment, a `Plugin Template` will be displayed - this can be a starting point for protecting your API on the Gateway. - -Finally, from the Portal, `enable` the Environment to make it available on the API Directory. - -### e) Optional Configuration - -**Scopes** - -If you have Client Scopes that you want to have controlled by the Portal, add them to the Realm's `Client Scopes` and `Default Client Scopes` on the IdP. - -Update the `CredentialIssuer` record above to match the `availableScopes` with the ones added on the IdP. - -**Roles** - -If you have Roles that you want to have controlled by the Portal, add them to the Client's `Roles`. - -Update the `CredentialIssuer` record above to match the `clientRoles` with the ones added on the IdP. - -**Client Mappers** - -The `audience` is an optional mapper that can be added to a Client. - -The IdP needs to have a policy that allows Audience to be added as a Protocol Mapper to the client. - -In Keycloak, this is updated under the Realm's `Client Registration` -> `Client Registration Policies`. - -Edit the Authenticated Access Policies -> Allowed Protocol Mapper Types to include the `oidc-audience-mapper`. - -**UMA2 Authorization Resources** - -If you want to use the Authorization services, then set `Authorization Enabled` to `ON` for the Client on the IdP. You will also want to set the `Decision Strategy` to `Affirmative`. - -Update the following `CredentialIssuer` attributes: - -``` - resourceType: "" - resourceAccessScope: "" - resourceScopes: [] -``` - -- `resourceType`: The Resource Type of the resources that will be managed (required) -- `resourceScopes`: A list of the Authorization Scopes managed for the particular Resources (required) -- `resourceAccessScope`: Used in the case where the Resource Server owns all the resources, a user must have the `resourceAccessScope` assigned in order to be allowed to manage the access. If it is not set, then the user has to be the resource owner in order to manage access. - -> `resourceAccessScope` - The API Services Portal has not completed the implementation for the scenario where the User is the Resource Owner (`resourceAccessScope` is left blank). It uses the `Token Exchange` capability but it's an optional service available on Keycloak and has numerous caveats around it. Please contact the APS team if interested to know more. - -## 3. Client Requesting Access - -Request access to the API via the API Services Portal and generate the credentials to be used below. - -The Portal will use the credentials setup in the Authorization Profile, to create a disabled Client on the IdP (with any applicable Client Mappers) and return the credentials to the Requesting user. - -## 4. Provider Approving Access - -An Access Manager reviews the access request, sets any additional controls, grants the relevant permissions (i.e./ scopes and roles), and approves. The Portal will enable the Client and apply the permissions on the IdP. - -The Portal sends a notification to the Requester letting them know that API Access has been approved (or rejected). - -## 5. Client Retrieving the Access Token - -Using the Credentials generated in step 3, the Requester calls the Token endpoint to get a new JWT token. - -``` -export CID="" -export CSC="" -export URL="" - -curl $URL \ - -X POST -H "Content-Type: application/x-www-form-urlencoded" \ - -d client_id=$CID -d client_secret=$CSC \ - -d grant_type=client_credentials \ - -d scopes=openid -``` - -## 6. Client Calling the API - -Extract the Access Token from the response, set it in a `TOK` environment variable, and call the API. - -``` -curl -v -H "Authorization: Bearer $TOK" \ - https://myservice.api.gov.bc.ca/v1/status -``` - -The API Gateway's `jwt-keycloak` plugin will use the IdP's public keys to validate the token and depending on the plugin configuration, validate the scopes or roles. - -## 7. Gateway Proxying to Upstream - -A technical, but important step in the integration is the Gateway proxying the request to the Upstream Service API and deciding on the different options for securing that interaction. - -Options: - -- `Network Policy` : If the Services are co-located on the same Cluster as the Gateway's Data Plane, then native network policies can be used to protect the channel between the Gateway and the Upstream Service. This approach is used for Services running on the Openshift Silver cluster. - -- `Kong Upstream JWT`: This plugin adds a signed JWT to the request headers so that the Upstream Service can verify that the request came specifically from the Gateway. - -- `Client Certificates` : Client certificates (mTLS) provides a way for the Upstream Service to provide a secure channel from the Gateway and to verify that the request came specifically from the Gateway. - -- `Firewall IP Restrictions` : This provides a low-level of protection by limiting the IPs to the ones of the Gateway Data Planes. Because the Data Planes are typically on shared infrastructure, this would still allow traffic from other tenants. This might be acceptable based on the type of data delivered by or to the Upstream Service. - -## Variations - -### Signed JWT - -The Authorization Profile `clientAuthenticator` was set to `client-secret` in this tutorial, but there is an alternate setup that can be used: - -- `client-jwt-jwks-url` (Signed JWT with JWKS URL) - -In this scenario, when a Client is requesting access, they will be required to enter a public URL that holds the public key information for a key pair that will be used in step 5 when retrieving the Bearer Token. For further details, you can see the specific example `intro-signed-jwt.md`. +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/tutorial-idp-client-cred-flow.yaml b/docs/guides/tutorial-idp-client-cred-flow.yaml deleted file mode 100644 index 9573ba3..0000000 --- a/docs/guides/tutorial-idp-client-cred-flow.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/tutorial-idp-client-cred-flow.md -title: "Setting up Client Credential Protection" -description: "Configuration the Gateway with an Identity Provider and Client Credential Grant" -order: 1 -tags: ["ns.platform", "tutorial"] -isComplete: true -isPublic: true -publishDate: "2021-09-27T20:00:00.000-08:00" diff --git a/docs/guides/user-journey.md b/docs/guides/user-journey.md index 2779667..04c10c4 100644 --- a/docs/guides/user-journey.md +++ b/docs/guides/user-journey.md @@ -1,510 +1,3 @@ # API Owner Flow -The following steps walk an API Owner through setting up an API on the BC Gov API Gateway in our Test instance. If you are ready to deploy to our Production instance, use the links found at the bottom of this document ([here](#production-links)). - -## 1. Register a new namespace - -A `namespace` represents a collection of Kong Gateway Services and Routes that are managed independently. - -To create a new namespace, login to the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca). - -After login, click the namespace dropdown in the top right next to your user name (it may show `No Active Namespace`), then click `Create New Namespace`. - -The namespace must be an alphanumeric string between 5 and 15 characters (RegExp reference: `^[a-z][a-z0-9-]{4,14}$`). - -You can select and manage namespaces by clicking the namespace dropdown in the top right next to your user name. - -## 2. Generate a Service Account - -Go to the `Namespaces` tab, click the `Service Accounts` link, and click the `New Service Account` and select the `GatewayConfig.Publish` permissions for the Service Account and click `Share`. A new credential will be created - make a note of the `ID` and `Secret`. - -The available Scopes are: -| Scope | Permission | -| ----- | ---------- | -| `Namespace.Manage` | Permission to update the Access Control List for controlling access to viewing metrics, service configuration and service account management (effectively a superuser for the namespace) | -| `Namespace.View` | Read-only access to the namespace | -| `GatewayConfig.Publish` | Permission to publish gateway configuration to Kong and to view the status of the upstreams | -| `Content.Publish` | Permission to update the documentation on the portal | -| `CredentialIssuer.Admin` | Permission to create Authorization Profiles so that they are available to be used when configuring Product Environments | -| `Access.Manage` | Permission to approve/reject access requests to your APIs that you make discoverable | - -## 3. Prepare configuration - -The gateway configuration can be hand-crafted or you can use a command line interface that we developed called `gwa` to convert your Openapi v3 spec to a Kong configuration. - -### 3.1. Hand-crafted (recommended if you don't have an Openapi spec) - -**Simple Example** - -```bash -export NS="my_namespace" -export NAME="a-service-for-$NS" -echo " -services: -- name: $NAME - host: httpbin.org - tags: [ ns.$NS ] - port: 443 - protocol: https - retries: 0 - routes: - - name: $NAME-route - tags: [ ns.$NS ] - hosts: - - $NAME.api.gov.bc.ca - paths: - - / - methods: - - GET - strip_path: false - https_redirect_status_code: 426 - path_handling: v0 -" > sample.yaml -``` - -> To view common plugin config go to [COMMON-CONFIG.md](https://github.com/bcgov/gwa-api/blob/dev/docs/COMMON-CONFIG.md) - -> To view some other plugin examples go [here](https://github.com/bcgov/gwa-api/blob/dev/docs/samples/service-plugins). - -> **Declarative Config** Behind the scenes, DecK is used to sync your configuration with Kong. For reference: https://docs.konghq.com/deck/overview/ - -> **Splitting Your Config:** A namespace `tag` with the format `ns.$NS` is mandatory for each service/route/plugin. But if you have separate pipelines for your environments (i.e./ dev, test and prod), you can split your configuration and update the `tags` with the qualifier. So for example, you can use a tag `ns.$NS.dev` to sync Kong configuration for `dev` Service and Routes only. - -> **Upstream Services on OCP4:** If your service is running on OCP4, you should specify the Kubernetes Service in the `Service.host`. It must have the format: `..svc`. Also make sure your `Service.port` matches your Kubernetes Service Port. Any Security Policies for egress from the Gateway will be setup automatically on the API Gateway side. -> The Aporeto Network Security Policies are being removed in favor of the Kubernetes Security Policies (KSP). You will need to create a KSP on your side looking something like this to allow the Gateway's test and prod environments to route traffic to your API: - -```yaml -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: allow-traffic-from-gateway-to-your-api -spec: - podSelector: - matchLabels: - name: my-upstream-api - ingress: - - from: - - namespaceSelector: - matchLabels: - environment: test - name: 264e6f - - from: - - namespaceSelector: - matchLabels: - environment: prod - name: 264e6f -``` - -> **Migrating from OCP3 to OCP4?** Please review the [OCP4-Migration](https://github.com/bcgov/gwa-api/blob/dev/docs/OCP4-MIGRATION.md) instructions to help with transitioning to OCP4 and the new APS Gateway. - -> **Require mTLS between the Gateway and your Upstream Service?** To support mTLS on your Upstream Service, you will need to provide client certificate details and if you want to verify the upstream endpoint then the `ca_certificates` and `tls_verify` is required as well. An example: - -```yaml -services: - - name: my-upstream-service - host: my-upstream.site - tags: [_NS_] - port: 443 - protocol: https - tls_verify: true - ca_certificates: [0a780ee0-626c-11eb-ae93-0242ac130012] - client_certificate: 8fc131ef-9752-43a4-ba70-eb10ba442d4e - routes: [...] -certificates: - - cert: "" - key: "" - tags: [_NS_] - id: 8fc131ef-9752-43a4-ba70-eb10ba442d4e -ca_certificates: - - cert: "" - tags: [_NS_] - id: 0a780ee0-626c-11eb-ae93-0242ac130012 -``` - -> NOTE: You must generate a UUID (`python -c 'import uuid; print(uuid.uuid4())'`) for each certificate and ca_certificate you create (set the `id`) and reference it in your `services` details. - -> HELPER: Python command to get a PEM file on one line: `python -c 'import sys; import json; print(json.dumps(open(sys.argv[1]).read()))' my.pem` - -### 3.2. gwa Command Line - -Run: `gwa new` and follow the prompts. - -Example: - -```bash -gwa new -o sample.yaml \ - --route-host myapi.api.gov.bc.ca \ - --service-url https://httpbin.org \ - https://bcgov.github.io/gwa-api/openapi/simple.yaml -``` - -> See below for the `gwa` CLI install instructions. - -## 4. Apply gateway configuration - -The Swagger console for the `gwa-api` can be used to publish Kong Gateway configuration, or the `gwa Command Line` can be used. - -### 4.1. gwa Command Line (recommended) - -**Install (for Linux)** - -```bash -GWA_CLI_VERSION=v1.2.0; curl -L -O https://github.com/bcgov/gwa-cli/releases/download/${GWA_CLI_VERSION}/gwa_${GWA_CLI_VERSION}_linux_x64.zip -unzip gwa_${GWA_CLI_VERSION}_linux_x64.zip -./gwa --version -``` - -> **Using MacOS or Windows?** Download here: [https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0](https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0) - -> NOTE: Version 1.2.0 introduces support for v2 of our api. To continue using v1 of the api, ensure that the API Version is set to 1 (see below) - -**Configure** - -Create a `.env` file and update the CLIENT_ID and CLIENT_SECRET with the new credentials that were generated in step #2: - -```bash -echo " -GWA_NAMESPACE=$NS -CLIENT_ID= -CLIENT_SECRET= -GWA_ENV=test -API_VERSION=2 -" > .env - -OR run: - -gwa init -T --api-version=2 --namespace=$NS --client-id= --client-secret= - -``` - -> NOTE: The `-T` indicates our Test environment. For production use `-P`. - -**Publish** - -```bash -gwa pg sample.yaml -``` - -If you want to see the expected changes but not actually apply them, you can run: - -```bash -gwa pg --dry-run sample.yaml -``` - -### 4.2. Swagger Console - -Go to [gwa-api Swagger Console](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/). - -Select the `PUT` `/namespaces/{namespace}/gateway` API. - -The Service Account uses the OAuth2 Client Credentials Grant Flow. Click the `lock` link on the right and enter in the Service Account credentials that were generated in step #2. - -For the `Parameter namespace`, enter the namespace that you created in step #1. - -Select `dryRun` to `true`. - -Select a `configFile` file. - -Send the request. - -### 4.3. Postman - -From the Postman App, click the `Import` button and go to the `Link` tab. - -Enter a URL: https://openapi-to-postman-api-gov-bc-ca.test.api.gov.bc.ca/?u=https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/v2/openapi.yaml - -After creation, go to `Collections` and right-click on the `Gateway Administration (GWA) API` collection and select `edit`. - -Go to the `Authorization` tab, enter in your `Client ID` and `Client Secret` and click `Get New Access Token`. - -You should get a successful dialog to proceed. Click `Proceed` and `Use Token`. - -You can then verify that the token works by going to the Collection `Return key information about authenticated identity` and click `Send`. - -## 5. Verify routes - -To verify that the Gateway can access the upstream services, run the command: `gwa status`. - -In our test environment, the hosts that you defined in the routes get altered; to see the actual hosts, log into the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca), go to the `Namespaces` tab, go to `Gateway Services` and select your particular service to get the routing details. - -```bash -curl https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -ab -n 20 -c 2 https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -``` - -To help with troubleshooting, you can use the GWA API to get a health check for each of the upstream services to verify the Gateway is connecting OK. - -Go to the [GWA API](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/#/Service%20Status/get_namespaces__namespace__services), enter in the new credentials that were generated in step #2, click `Try it out`, enter your namespace and click `Execute`. The results are returned in a JSON object. - -## 6. View metrics - -The following metrics can be viewed in real-time for the Services that you configure on the Gateway: - -- Request Rate : Requests / Second (by Service/Route, by HTTP Status) -- Latency : Standard deviations measured for latency inside Kong and on the Upstream Service (by Service/Route) -- Bandwidth : Ingress/egress bandwidth (by Service/Route) -- Total Requests : In 5 minute windows (by Consumer, by User Agent, by Service, by HTTP Status) - -All metrics can be viewed by an arbitrary time window - defaults to `Last 24 Hours`. - -Go to [Grafana](https://grafana-apps-gov-bc-ca.test.api.gov.bc.ca) to view metrics for your configured services. - -You can also access summarized metrics from the `API Services Portal` by going to the `Namespaces` tab and clicking the `Gateway Services` link. - -## 7. Grant access to others - -To grant access to others, you need to grant them the appropriate Scopes. This can be done from the `API Services Portal`, selecting the relevant `Namespace` and going to the Namespaces `Namespace Access` page. From here, you are able to grant Users access to the Namespace. - -## 8. Add to your CI/CD Pipeline - -Update your CI/CD pipelines to run the `gwa-cli` to keep your services updated on the gateway. - -### 8.1. Github Actions Example - -In the repository that you maintain your CI/CD Pipeline configuration, use the Service Account details from `Step 2` to set up two `Secrets`: - -- GWA_ACCT_ID - -- GWA_ACCT_SECRET - -Add a `.gwa` folder (can be called anything) that will be used to hold your gateway configuration. - -An example Github Workflow: - -```yaml -env: - NS: "" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v1 - with: - node-version: 10 - TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get GWA Command Line - run: | - curl -L -O https://github.com/bcgov/gwa-cli/releases/download/v1.2.0/gwa_v1.2.0_linux_x64.zip - unzip gwa_v1.2.0_linux_x64.zip - export PATH=`pwd`:$PATH - - - name: Apply Namespace Configuration - run: | - export PATH=`pwd`:$PATH - cd .gwa/$NS - - gwa init -T \ - --namespace=$NS \ - --client-id=${{ secrets.TEST_GWA_ACCT_ID }} \ - --client-secret=${{ secrets.TEST_GWA_ACCT_SECRET }} - - gwa pg -``` - -## 9. Share your API for Discovery - -Package your APIs and make them available for discovery through the API Services Portal and BC Data Catalog. - -The `API Services Portal` Directory organizes your APIs by Products and Environments. You can manage them via an API or through the UI. - -To use the Directory API, the following scopes are required: - -- For `contents` (documentation), the service account must have the `Content.Publish` scope -- For `datasets` and `products`, the service account must have the `Namespace.Manage` scope -- For `issuers`, the service account must have the `CredentialIssuer.Admin` scope - -View the Directory API in the [Swagger Console](https://openapi-apps-gov-bc-ca.test.api.gov.bc.ca/?url=https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api/openapi.yaml) - -> NOTE: The steps below use `restish`, but we will be working on upgrading the `gwa` command line interface to support these APIs - -> Can use `y2j` to convert from YAML to JSON -> -> `echo -e "#"'!'"/usr/bin/env python\nimport sys,yaml,json\nprint(json.dumps(yaml.safe_load(open(sys.argv[1]).read())))" > /usr/local/bin/y2j` -> -> `chmod +x /usr/local/bin/y2j` - -**Restish Setup** - -``` -restish api configure my_api -``` - -Base URI : https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api - -`Edit Profile default` - -Select `Setup Auth` > `oauth-client-credentials` - -Enter the `client_id` and `client_secret` for your Service Account. - -`token_url` : Provided to you when you created the Service Account - -`scopes` : openid - -Select `Finished with profile` and then `Save and exit` - -To verify that restish is working, run: - -`restish my_api list` or `restish my_api get-new-id product` - -The below example works with the service created above, so we will want to reference the NS environment variable. - -``` -export NS="" -``` - -### 9.1 Setup Authorization Profiles - -If your APIS are protected with one of the OAuth2 grant types, then an Authorization profile must be created with the appropriate credentials for accessing the corresponding Identity Provider/Broker and Authorization Server. - -```yaml -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -``` -y2j issuer.yaml | restish my_api put-issuer $NS -``` - -### 9.2 Setup your Product, Environments and link your Services - -```yaml -kind: DraftDataset -name: my-draft-dataset -organization: ministry-of-health -organizationUnit: planning-and-innovation-division -title: My API -notes: Some information about this API -tags: [health, standards, openapi] -sector: Service -license_title: Access Only -view_audience: Government -security_class: LOW-PUBLIC -record_publish_date: "2021-05-27" -``` - -``` -y2j dataset.yaml | restish my_api put-dataset $NS -``` - -```yaml -kind: Product -appId: 2B04C28E08AW -name: My API -dataset: my-draft-dataset -environments: - - id: 1F7CA929 - name: dev - active: false - approval: false - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Please provide a bit more of this - services: [] - - id: 2F7CA929 - name: test - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Asking for test environment? Please provide some more info - services: [a-service-for-$NS] - - id: 3F7CA929 - name: prod - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: client-credentials - credentialIssuer: Resource Server $NS - additionalDetailsToRequest: Production? Great, please provide X, Y and Z - services: [] -``` - -``` -y2j prod.yaml | restish my_api put-product $NS -``` - -### 9.3 Update Gateway Configuration based on Flow - -In the previous section our example defines an environment that is protected using Kong's API Key and ACL plugins. To activate an environment, the corresponding plugins need to exist on the Gateway for that service or routes. The ACL `allow` corresponds to the unique `Environment ID` defined in section 9.2. - -``` - plugins: - - name: key-auth - tags: [ ns.$NS ] - protocols: [ http, https ] - config: - key_names: ["X-API-KEY"] - run_on_preflight: true - hide_credentials: true - key_in_body: false - - name: acl - tags: [ ns.$NS ] - config: - hide_groups_header: true - allow: [ ] -``` - -### 9.4 Publish Environments - -Your products will not appear on the Directory until you mark the relevant environments as Active. You can do this by either updating the Product Environment configuration above to `active: true`, or going to the API Services Portal UI and editing the Environment details. - -### 9.5 Publish Documentation - -```yaml -kind: Content -title: Getting Started with Example API -description: Getting Started with Example API -externalLink: https://github.com/bcgov/$NS/getting_started.md -order: 1 -tags: [ns.$NS] -isComplete: true -isPublic: true -publishDate: "2021-06-02T08:00:00.000-08:00" -``` - -``` -y2j content.yaml | restish my_api put-content $NS - -echo "# here is some markdown and more!" > doc.md - -restish my_api put-content $NS \ - externalLink: "https://github.com/bcgov/$NS/getting_started.md", \ - content: @doc.md -``` - -### 9.6 View your product in the API Directory - -Find your API in the [API Services Portal Directory](https://api-gov-bc-ca.test.api.gov.bc.ca/devportal/api-discovery) - -It is now ready to receive access requests from the community! - -# Production Links - -- [API Services Portal](https://api.gov.bc.ca) -- [gwa-api Swagger Console](https://gwa.api.gov.bc.ca/docs/) -- [gwa-api Postman Collection](https://openapi-to-postman.api.gov.bc.ca/?u=https://gwa.api.gov.bc.ca/docs/v2/openapi.yaml) -- [Gateway Metrics - Grafana](https://grafana.apps.gov.bc.ca) +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/user-journey.yaml b/docs/guides/user-journey.yaml deleted file mode 100644 index cac09b4..0000000 --- a/docs/guides/user-journey.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/USER-JOURNEY.md -title: "API Owner User Journey" -description: "Setup a new namespace for onboarding services on the BC Gov API Gateway." -order: 1 -tags: ["ns.platform"] -isComplete: true -isPublic: true -publishDate: "2021-06-10T20:00:00.000-08:00" diff --git a/docs/static/openapi/.env b/docs/static/openapi/.env new file mode 100644 index 0000000..da0441a --- /dev/null +++ b/docs/static/openapi/.env @@ -0,0 +1,4 @@ +GWA_NAMESPACE=openapi-tests +CLIENT_ID=sa-openapi-tests-tihw03lvo5 +CLIENT_SECRET=40f4852f-a71f-47ed-ad65-fc6f60471234 +GWA_ENV=dev diff --git a/docs/static/openapi/kong-with-edits.yaml b/docs/static/openapi/kong-with-edits.yaml new file mode 100644 index 0000000..0e24387 --- /dev/null +++ b/docs/static/openapi/kong-with-edits.yaml @@ -0,0 +1,21 @@ +_format_version: "1.1" +services: + - name: Sample_API + url: https://httpbin.org + plugins: [] + routes: + - tags: + - OAS3_import + - ns.openapi-tests.loc + name: Sample_API-path-get + methods: + - GET + hosts: + - openapi-gen-sample.api.gov.bc.ca + paths: + - /headers + strip_path: false + tags: + - OAS3_import + - ns.openapi-tests.loc + From c189a49ac991c37d0aa2d81234dc99b054b98667 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Fri, 6 May 2022 14:16:43 -0700 Subject: [PATCH 06/10] fix readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32f92e9..eec2adf 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,10 @@ All APIs are protected by an OIDC JWT Token with the following claims: **Configuration:** | Variable | Description | Example | -| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ----------- | ------------ | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | | `PORT` | Port | `2000` | | `LOG_LEVEL` | Log level for the application | `INFO` | -| `ENVIRONMENT` | Indicates what environment config to use (development | test | production) | `production` | +| `ENVIRONMENT` | Indicates what environment config to use (development , test, production) | `production` | | `CONFIG_PATH` | Location of the config | `/tmp/production.json` | | `OIDC_BASE_URL` | External base url used by the Swagger console for an externally available Auth endpoint. | `https://keycloak.domain/auth/realms/abc` | | `TOKEN_MATCH_AUD` | The `audience` that the token must match. | `gwa` | From a10354b46472c37c07ef4caf1027bd5042659928 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Fri, 15 Jul 2022 13:57:15 -0700 Subject: [PATCH 07/10] fix issue with consumer plugin calls --- microservices/gatewayApi/app.py | 8 ++++++++ microservices/gatewayApi/v2/routes/consumers.py | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/microservices/gatewayApi/app.py b/microservices/gatewayApi/app.py index 1726229..41bce0d 100644 --- a/microservices/gatewayApi/app.py +++ b/microservices/gatewayApi/app.py @@ -6,6 +6,7 @@ import os import time import config +from werkzeug.exceptions import HTTPException from authlib.jose.errors import JoseError, ExpiredTokenError from flask import Flask, g, jsonify, request, make_response, url_for, Response from flask_compress import Compress @@ -102,6 +103,12 @@ def bad_request_error(error): log.error(request.headers) return make_response(content, HTTPStatus.BAD_REQUEST) + @app.errorhandler(HTTPException) + def token_error(error): + log.error("Denied access %s - %s" % (request.remote_addr, str(error))) + content = jsonify({"error":"Invalid Token"}) + return make_response(content, HTTPStatus.UNAUTHORIZED) + @app.errorhandler(JoseError) def forbidden(error): log.error("Denied access %s - %s" % (request.remote_addr, str(error))) @@ -115,6 +122,7 @@ def expired_token(error): @app.errorhandler(Exception) def other_exception(error): + log.error("Unexpected error %s", type(error)) log.error(error) content = jsonify({"error":"Unexpected Error"}) return make_response(content, HTTPStatus.BAD_REQUEST) diff --git a/microservices/gatewayApi/v2/routes/consumers.py b/microservices/gatewayApi/v2/routes/consumers.py index 26d9779..76c82a4 100644 --- a/microservices/gatewayApi/v2/routes/consumers.py +++ b/microservices/gatewayApi/v2/routes/consumers.py @@ -12,10 +12,10 @@ consumers = Blueprint('consumers_v2', 'consumers') -@admin_jwt(None) -@uma_enforce('namespace', 'Access.Manage') @consumers.route('//plugins', methods=['POST'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def create_consumer_plugin(namespace: str, consumer_id: str) -> object: event_id = str(uuid.uuid4()) log = app.logger @@ -43,10 +43,10 @@ def create_consumer_plugin(namespace: str, consumer_id: str) -> object: return response -@admin_jwt(None) -@uma_enforce('namespace', 'Access.Manage') @consumers.route('//plugins/', methods=['PUT'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> object: event_id = str(uuid.uuid4()) log = app.logger @@ -75,10 +75,10 @@ def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> return response -@admin_jwt(None) -@uma_enforce('namespace', 'Access.Manage') @consumers.route('//plugins/', methods=['DELETE'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def delete_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> object: event_id = str(uuid.uuid4()) selectTag = "ns.%s" % namespace From 80cd8dbb6b9e6ad438e596e0c1a0b22d6d6cb28d Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Fri, 22 Jul 2022 15:54:15 -0700 Subject: [PATCH 08/10] use gunicorn to fix a concurrency issue --- microservices/gatewayApi/auth/token.py | 3 +- microservices/gatewayApi/entrypoint.sh | 2 +- microservices/gatewayApi/requirements.txt | 3 +- .../gatewayApi/v2/routes/consumers.py | 30 ++++++++++++------- .../v2/services/gateway_consumers.py | 17 +++++------ microservices/gatewayApi/wsgi.py | 6 ++-- 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/microservices/gatewayApi/auth/token.py b/microservices/gatewayApi/auth/token.py index b016bf6..2151cde 100644 --- a/microservices/gatewayApi/auth/token.py +++ b/microservices/gatewayApi/auth/token.py @@ -1,6 +1,6 @@ import requests import time -from authlib.jose import jwt +from authlib.jose import JsonWebToken from authlib.jose.errors import JoseError, ExpiredTokenError from authlib.oauth2.rfc6749 import TokenMixin from authlib.oauth2.rfc6750 import BearerTokenValidator @@ -47,6 +47,7 @@ def __init__(self, token_cls, realm=None): self.jwk = jwk_r.json() def authenticate_token(self, token_string): + jwt = JsonWebToken(['RS256']) token = jwt.decode(token_string, self.jwk) token.validate() diff --git a/microservices/gatewayApi/entrypoint.sh b/microservices/gatewayApi/entrypoint.sh index 001c87a..2c3fd12 100755 --- a/microservices/gatewayApi/entrypoint.sh +++ b/microservices/gatewayApi/entrypoint.sh @@ -58,4 +58,4 @@ cat > /tmp/deck.yaml < object: validate_tags(plugin_data, selectTag) try: + log.debug("[add_consumer_plugin] %s" % consumer_id) response = cnsr.add_consumer_plugin(consumer_id, plugin_data) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='add', namespace=namespace, message='failed to create plugin.') + action='add', namespace=namespace, message='failed to create plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'add', 'completed', namespace) return response @@ -63,13 +64,14 @@ def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> rate_limiting(plugin_data) validate_tags(plugin_data, selectTag) - verify_tags(consumer_id, plugin_id, selectTag) try: + log.debug("[update_consumer_plugin] %s" % consumer_id) + verify_tags(consumer_id, plugin_id, selectTag) response = cnsr.update_consumer_plugin(consumer_id, plugin_id, plugin_data) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='update', namespace=namespace, message='failed to update plugin.') + action='update', namespace=namespace, message='failed to update plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'update', 'completed', namespace) return response @@ -80,16 +82,22 @@ def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> @admin_jwt(None) @uma_enforce('namespace', 'Access.Manage') def delete_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> object: + log = app.logger event_id = str(uuid.uuid4()) selectTag = "ns.%s" % namespace cnsr = GatewayConsumerService() - verify_tags(consumer_id, plugin_id, selectTag) - + try: - response = cnsr.delete_consumer_plugin(consumer_id, plugin_id) + log.debug("[delete_consumer_plugin] %s %s" % (consumer_id, plugin_id)) + plugin = cnsr.get_consumer_plugin(consumer_id, plugin_id) + if plugin is None: + response = {} + else: + verify_tags(consumer_id, plugin_id, selectTag) + response = cnsr.delete_consumer_plugin(consumer_id, plugin_id) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='delete', namespace=namespace, message='failed to delete plugin.') + action='delete', namespace=namespace, message='failed to delete plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'delete', 'completed', namespace) return response @@ -98,16 +106,18 @@ def delete_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> def handle_exceptions(**kwargs): log = app.logger if not kwargs['exception'] == None: + log.error("handle_exception of %s" % kwargs['exception']) if isinstance(kwargs['exception'], HTTPException): - log.error("%s. %s" % (kwargs['message'], kwargs['exception'])) + log.error("%s. (HTTP)" % (kwargs['message'])) else: log.error("%s. %s" % (kwargs['message'], sys.exc_info()[0])) - traceback.print_exc() + # traceback.print_exc() record_custom_event(kwargs['event_id'], kwargs['type'], kwargs['action'], 'failed', kwargs['namespace'], kwargs['message']) abort(make_response(jsonify(error=kwargs['message']), 400)) - + else: + abort(make_response(jsonify(error="unknown"), 400)) def transform_tags(data, namespace, required_tag): log = app.logger diff --git a/microservices/gatewayApi/v2/services/gateway_consumers.py b/microservices/gatewayApi/v2/services/gateway_consumers.py index b652f08..a9fae5b 100644 --- a/microservices/gatewayApi/v2/services/gateway_consumers.py +++ b/microservices/gatewayApi/v2/services/gateway_consumers.py @@ -1,5 +1,5 @@ from typing import Any -from flask import make_response, jsonify, current_app as app +from flask import current_app as app import requests @@ -19,13 +19,14 @@ def delete_consumer_plugin(self, consumer_id: str, plugin_id: str): def get_consumer_plugin(self, consumer_id: str, plugin_id: str): url = app.config['kongAdminUrl'] + '/consumers/' + consumer_id + '/plugins/' + plugin_id - return requests.get(url, timeout=5).json() + res = requests.get(url, timeout=5) + if res.status_code == 404: + return None + return res.json() def make_http_request(action: str, id: str, method: str, **rqst_params): - log = app.logger message = "STARTED" - log.debug("%s %s[%s]" % (message, action, id)) mod = __import__('requests') method_to_call = getattr(mod, method) response = method_to_call(**rqst_params) @@ -33,12 +34,8 @@ def make_http_request(action: str, id: str, method: str, **rqst_params): if response.status_code in (200, 201): res = response.json() elif response.status_code == 204: - res = make_response(jsonify(message='done'), response.status_code) + res = {message:'done'} else: message = "FAILED" - try: - res = make_response(response.json(), response.status_code) - except: - res = make_response(jsonify(error='failed'), response.status_code) - log.debug("%s %s[%s]" % (message, action, id)) + raise Exception("%s %s [%s] %s" % (message, action, response.status_code, response.reason)) return res diff --git a/microservices/gatewayApi/wsgi.py b/microservices/gatewayApi/wsgi.py index 4972a46..7f06603 100644 --- a/microservices/gatewayApi/wsgi.py +++ b/microservices/gatewayApi/wsgi.py @@ -1,8 +1,8 @@ #!/usr/bin/python -from gevent import monkey +#from gevent import monkey # Patch Sockets to make requests asynchronous -monkey.patch_all() +#monkey.patch_all() import logging import sys @@ -23,7 +23,7 @@ dictConfig({ 'version': 1, 'formatters': {'default': { - 'format': '%(asctime)s %(levelname)5s %(module)-15s: %(message)s', + 'format': '%(asctime)s [%(process)3d] %(levelname)5s %(module)-15s: %(message)s', }}, 'handlers': {'wsgi': { 'class': 'logging.StreamHandler', From 71e3d7e4f7c2d8496e2e9bba57bdeaa26852270c Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Thu, 28 Jul 2022 21:43:04 -0700 Subject: [PATCH 09/10] add logging if error to kong --- microservices/gatewayApi/v2/services/gateway_consumers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/microservices/gatewayApi/v2/services/gateway_consumers.py b/microservices/gatewayApi/v2/services/gateway_consumers.py index a9fae5b..c2aa436 100644 --- a/microservices/gatewayApi/v2/services/gateway_consumers.py +++ b/microservices/gatewayApi/v2/services/gateway_consumers.py @@ -26,6 +26,8 @@ def get_consumer_plugin(self, consumer_id: str, plugin_id: str): def make_http_request(action: str, id: str, method: str, **rqst_params): + log = app.logger + message = "STARTED" mod = __import__('requests') method_to_call = getattr(mod, method) @@ -37,5 +39,6 @@ def make_http_request(action: str, id: str, method: str, **rqst_params): res = {message:'done'} else: message = "FAILED" + log.error("%s %s : %s", response.status_code, response.reason, response.text) raise Exception("%s %s [%s] %s" % (message, action, response.status_code, response.reason)) return res From 7ceb3399faad90b746c3b9c5f1a2c0c6d7c5f329 Mon Sep 17 00:00:00 2001 From: "repo-mountie[bot]" <44246292+repo-mountie[bot]@users.noreply.github.com> Date: Fri, 5 Aug 2022 20:39:04 +0000 Subject: [PATCH 10/10] Updating compliance audit file Signed-off-by: repo-mountie --- COMPLIANCE.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/COMPLIANCE.yaml b/COMPLIANCE.yaml index 4a5be9e..e0201bc 100644 --- a/COMPLIANCE.yaml +++ b/COMPLIANCE.yaml @@ -4,8 +4,8 @@ description: | compliance. spec: - name: PIA - status: PIA_STATUS_HERE - last-updated: '2020-11-04T19:14:37.870Z' + status: completed + last-updated: '2022-08-05T20:39:03.693Z' - name: STRA - status: STRA_STATUS_HERE - last-updated: '2020-11-04T19:14:37.870Z' + status: completed + last-updated: '2022-08-05T20:39:03.693Z'