From 29751808c21715efbd0092f39087a833da80fac2 Mon Sep 17 00:00:00 2001 From: ikethecoder Date: Thu, 28 Jul 2022 23:26:45 -0700 Subject: [PATCH] Improved concurrency (#69) * Update service-rate-limit.yaml * Update USER-JOURNEY.md * use gunicorn to fix a concurrency issue * add logging if error to kong --- .gitignore | 1 + README.md | 52 +- USER-JOURNEY.md | 513 +----------------- docs/AVAILABLE-PLUGINS.md | 51 +- docs/COMMON-CONFIG.md | 105 +--- docs/SSL.md | 40 +- docs/guides/intro-signed-jwt.md | 125 +---- docs/guides/intro-signed-jwt.yaml | 8 - docs/guides/tutorial-idp-client-cred-flow.md | 182 +------ .../guides/tutorial-idp-client-cred-flow.yaml | 8 - docs/guides/user-journey.md | 509 +---------------- docs/guides/user-journey.yaml | 8 - .../service-plugins/service-rate-limit.yaml | 4 +- docs/static/openapi/.env | 4 + docs/static/openapi/kong-with-edits.yaml | 21 + microservices/gatewayApi/app.py | 8 + microservices/gatewayApi/auth/token.py | 3 +- microservices/gatewayApi/entrypoint.sh | 2 +- microservices/gatewayApi/requirements.txt | 3 +- .../gatewayApi/v2/routes/consumers.py | 42 +- .../v2/services/gateway_consumers.py | 18 +- microservices/gatewayApi/wsgi.py | 6 +- 22 files changed, 113 insertions(+), 1600 deletions(-) create mode 100644 .gitignore delete mode 100644 docs/guides/intro-signed-jwt.yaml delete mode 100644 docs/guides/tutorial-idp-client-cred-flow.yaml delete mode 100644 docs/guides/user-journey.yaml create mode 100644 docs/static/openapi/.env create mode 100644 docs/static/openapi/kong-with-edits.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..496ee2c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index b47dfd0..eec2adf 100644 --- a/README.md +++ b/README.md @@ -6,38 +6,38 @@ For self-service of APIs, a set of microservices are used to coordinate updates by the providers of APIs. -* `Gateway` : Provides a way for API Owners to update their Kong configuration (and internally the OCP Edge Router) -* `Authz` : Provides a way for API Owners to update Keycloak for access to functionality on the API Services Portal -* `Catalog` : Provides a way for API Owners to update the API details in the BC Data Catalog +- `Gateway` : Provides a way for API Owners to update their Kong configuration (and internally the OCP Edge Router) +- `Authz` : Provides a way for API Owners to update Keycloak for access to functionality on the API Services Portal +- `Catalog` : Provides a way for API Owners to update the API details in the BC Data Catalog All APIs are protected by an OIDC JWT Token with the following claims: -* `aud` : `gwa` -* `namespace` : Identifies the namespace that the APIs belong to, used to scope what changes are allowed +- `aud` : `gwa` +- `namespace` : Identifies the namespace that the APIs belong to, used to scope what changes are allowed **Configuration:** -| Variable | Description | Example | -| -------- | ----------- | ------- | -| `PORT` | Port | `2000` | -| `LOG_LEVEL` | Log level for the application | `INFO` | -| `ENVIRONMENT` | Indicates what environment config to use (development|test|production) | `production` | -| `CONFIG_PATH` | Location of the config | `/tmp/production.json` | -| `OIDC_BASE_URL` | External base url used by the Swagger console for an externally available Auth endpoint. | `https://keycloak.domain/auth/realms/abc` -| `TOKEN_MATCH_AUD` | The `audience` that the token must match. | `gwa` -| `WORKING_FOLDER` | Temporary working folder that only exists for the duration of the POD. | `/tmp` -| `KONG_ADMIN_URL` | The Kong Admin endpoint. | `http://kong-admin-api:8001` -| `KC_SERVER_URL` | Keycloak access for administrative rights to manage groups for namespaces and for OIDC Discovery for getting the `jwks_uri` for the list of supported keys | `https://auth.domain/auth` -| `KC_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `aps` -| `KC_CLIENT_ID` | Keycloak access for administrative rights to manage groups for namespaces | `admin-cli` -| `KC_CLIENT_SECRET`| Keycloak access for administrative rights to manage groups for namespaces | `` -| `KC_USER_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `master` -| `KC_USERNAME` | Keycloak access for administrative rights to manage groups for namespaces | `kcadmin` -| `KC_PASSWORD` | Keycloak access for administrative rights to manage groups for namespaces | `xxx` -| `HOST_TRANSFORM_ENABLED` | For Dev and Test a way to transform the host for working in these environments | `false` -| `HOST_TRANSFORM_BASE_URL` | For Dev and Test a way to transform the host for working in these environments | -| `PLUGINS_RATELIMITING_REDIS_PASSWORD` | The Redis credential added to the rate-limiting Kong plugin during publish | +| Variable | Description | Example | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| `PORT` | Port | `2000` | +| `LOG_LEVEL` | Log level for the application | `INFO` | +| `ENVIRONMENT` | Indicates what environment config to use (development , test, production) | `production` | +| `CONFIG_PATH` | Location of the config | `/tmp/production.json` | +| `OIDC_BASE_URL` | External base url used by the Swagger console for an externally available Auth endpoint. | `https://keycloak.domain/auth/realms/abc` | +| `TOKEN_MATCH_AUD` | The `audience` that the token must match. | `gwa` | +| `WORKING_FOLDER` | Temporary working folder that only exists for the duration of the POD. | `/tmp` | +| `KONG_ADMIN_URL` | The Kong Admin endpoint. | `http://kong-admin-api:8001` | +| `KC_SERVER_URL` | Keycloak access for administrative rights to manage groups for namespaces and for OIDC Discovery for getting the `jwks_uri` for the list of supported keys | `https://auth.domain/auth` | +| `KC_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `aps` | +| `KC_CLIENT_ID` | Keycloak access for administrative rights to manage groups for namespaces | `admin-cli` | +| `KC_CLIENT_SECRET` | Keycloak access for administrative rights to manage groups for namespaces | `` | +| `KC_USER_REALM` | Keycloak access for administrative rights to manage groups for namespaces | `master` | +| `KC_USERNAME` | Keycloak access for administrative rights to manage groups for namespaces | `kcadmin` | +| `KC_PASSWORD` | Keycloak access for administrative rights to manage groups for namespaces | `xxx` | +| `HOST_TRANSFORM_ENABLED` | For Dev and Test a way to transform the host for working in these environments | `false` | +| `HOST_TRANSFORM_BASE_URL` | For Dev and Test a way to transform the host for working in these environments | +| `PLUGINS_RATELIMITING_REDIS_PASSWORD` | The Redis credential added to the rate-limiting Kong plugin during publish | # API Provider Flow -[See Details](USER-JOURNEY.md) +[See Details](https://bcgov.github.io/aps-infra-platform/guides/owner-journey) diff --git a/USER-JOURNEY.md b/USER-JOURNEY.md index 31659ce..87e4c75 100644 --- a/USER-JOURNEY.md +++ b/USER-JOURNEY.md @@ -1,514 +1,3 @@ # API Owner Flow -The following steps walk an API Owner through setting up an API on the BC Gov API Gateway in our Test instance. If you are ready to deploy to our Production instance, use the links found at the bottom of this document ([here](#production-links)). - -## 1. Register a new namespace - -A `namespace` represents a collection of Kong Gateway Services and Routes that are managed independently. - -To create a new namespace, login to the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca). - -After login, click the namespace dropdown in the top right next to your user name (it may show `No Active Namespace`), then click `Create New Namespace`. - -The namespace must be an alphanumeric string between 5 and 15 characters (RegExp reference: `^[a-z][a-z0-9-]{4,14}$`). - -You can select and manage namespaces by clicking the namespace dropdown in the top right next to your user name. - -## 2. Generate a Service Account - -Go to the `Namespaces` tab, click the `Service Accounts` link, and click the `New Service Account`. A new credential will be created - make a note of the `ID` and `Secret`. - -Before the credential can be used, you need to grant it the appropriate permissions. This can be done by going to the `API Access` tab and click the "Manage Resources" for the `Gateway Administration API`. Click the link that corresponds to your newly created namespace. Click the `Grant Service Account Access` and enter in the `ID` and the scopes you want to grant to the Service Account, then click the `Share` button to grant the permissions. - -The available Scopes are: -| Scope | Permission | -| ----- | ---------- | -| `Namespace.Manage` | Permission to update the Access Control List for controlling access to viewing metrics, service configuration and service account management (effectively a superuser for the namespace) | -| `Namespace.View` | Read-only access to the namespace | -| `GatewayConfig.Publish` | Permission to publish gateway configuration to Kong and to view the status of the upstreams | -| `Content.Publish` | Permission to update the documentation on the portal | -| `CredentialIssuer.Admin` | Permission to create Authorization Profiles so that they are available to be used when configuring Product Environments | -| `Access.Manage` | Permission to approve/reject access requests to your APIs that you make discoverable | - -## 3. Prepare configuration - -The gateway configuration can be hand-crafted or you can use a command line interface that we developed called `gwa` to convert your Openapi v3 spec to a Kong configuration. - -### 3.1. Hand-crafted (recommended if you don't have an Openapi spec) - -**Simple Example** - -```bash -export NS="my_namespace" -export NAME="a-service-for-$NS" -echo " -services: -- name: $NAME - host: httpbin.org - tags: [ ns.$NS ] - port: 443 - protocol: https - retries: 0 - routes: - - name: $NAME-route - tags: [ ns.$NS ] - hosts: - - $NAME.api.gov.bc.ca - paths: - - / - methods: - - GET - strip_path: false - https_redirect_status_code: 426 - path_handling: v0 -" > sample.yaml -``` - -> To view common plugin config go to [COMMON-CONFIG.md](https://github.com/bcgov/gwa-api/blob/dev/docs/COMMON-CONFIG.md) - -> To view some other plugin examples go [here](https://github.com/bcgov/gwa-api/blob/dev/docs/samples/service-plugins). - -> **Declarative Config** Behind the scenes, DecK is used to sync your configuration with Kong. For reference: https://docs.konghq.com/deck/overview/ - -> **Splitting Your Config:** A namespace `tag` with the format `ns.$NS` is mandatory for each service/route/plugin. But if you have separate pipelines for your environments (i.e./ dev, test and prod), you can split your configuration and update the `tags` with the qualifier. So for example, you can use a tag `ns.$NS.dev` to sync Kong configuration for `dev` Service and Routes only. - -> **Upstream Services on OCP4:** If your service is running on OCP4, you should specify the Kubernetes Service in the `Service.host`. It must have the format: `..svc`. Also make sure your `Service.port` matches your Kubernetes Service Port. Any Security Policies for egress from the Gateway will be setup automatically on the API Gateway side. -> The Aporeto Network Security Policies are being removed in favor of the Kubernetes Security Policies (KSP). You will need to create a KSP on your side looking something like this to allow the Gateway's test and prod environments to route traffic to your API: - -```yaml -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: allow-traffic-from-gateway-to-your-api -spec: - podSelector: - matchLabels: - name: my-upstream-api - ingress: - - from: - - namespaceSelector: - matchLabels: - environment: test - name: 264e6f - - from: - - namespaceSelector: - matchLabels: - environment: prod - name: 264e6f -``` - -> **Migrating from OCP3 to OCP4?** Please review the [OCP4-Migration](https://github.com/bcgov/gwa-api/blob/dev/docs/OCP4-MIGRATION.md) instructions to help with transitioning to OCP4 and the new APS Gateway. - -> **Require mTLS between the Gateway and your Upstream Service?** To support mTLS on your Upstream Service, you will need to provide client certificate details and if you want to verify the upstream endpoint then the `ca_certificates` and `tls_verify` is required as well. An example: - -```yaml -services: - - name: my-upstream-service - host: my-upstream.site - tags: [_NS_] - port: 443 - protocol: https - tls_verify: true - ca_certificates: [0a780ee0-626c-11eb-ae93-0242ac130012] - client_certificate: 8fc131ef-9752-43a4-ba70-eb10ba442d4e - routes: [...] -certificates: - - cert: "" - key: "" - tags: [_NS_] - id: 8fc131ef-9752-43a4-ba70-eb10ba442d4e -ca_certificates: - - cert: "" - tags: [_NS_] - id: 0a780ee0-626c-11eb-ae93-0242ac130012 -``` - -> NOTE: You must generate a UUID (`python -c 'import uuid; print(uuid.uuid4())'`) for each certificate and ca_certificate you create (set the `id`) and reference it in your `services` details. - -> HELPER: Python command to get a PEM file on one line: `python -c 'import sys; import json; print(json.dumps(open(sys.argv[1]).read()))' my.pem` - -### 3.2. gwa Command Line - -Run: `gwa new` and follow the prompts. - -Example: - -```bash -gwa new -o sample.yaml \ - --route-host myapi.api.gov.bc.ca \ - --service-url https://httpbin.org \ - https://bcgov.github.io/gwa-api/openapi/simple.yaml -``` - -> See below for the `gwa` CLI install instructions. - -## 4. Apply gateway configuration - -The Swagger console for the `gwa-api` can be used to publish Kong Gateway configuration, or the `gwa Command Line` can be used. - -### 4.1. gwa Command Line (recommended) - -**Install (for Linux)** - -```bash -GWA_CLI_VERSION=v1.2.0; curl -L -O https://github.com/bcgov/gwa-cli/releases/download/${GWA_CLI_VERSION}/gwa_${GWA_CLI_VERSION}_linux_x64.zip -unzip gwa_${GWA_CLI_VERSION}_linux_x64.zip -./gwa --version -``` - -> **Using MacOS or Windows?** Download here: [https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0](https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0) - -> NOTE: Version 1.2.0 introduces support for v2 of our api. To continue using v1 of the api, ensure that the API Version is set to 1 (see below) - -**Configure** - -Create a `.env` file and update the CLIENT_ID and CLIENT_SECRET with the new credentials that were generated in step #2: - -```bash -echo " -GWA_NAMESPACE=$NS -CLIENT_ID= -CLIENT_SECRET= -GWA_ENV=test -API_VERSION=2 -" > .env - -OR run: - -gwa init -T --api-version=2 --namespace=$NS --client-id= --client-secret= - -``` - -> NOTE: The `-T` indicates our Test environment. For production use `-P`. - -**Publish** - -```bash -gwa pg sample.yaml -``` - -If you want to see the expected changes but not actually apply them, you can run: - -```bash -gwa pg --dry-run sample.yaml -``` - -### 4.2. Swagger Console - -Go to [gwa-api Swagger Console](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/). - -Select the `PUT` `/namespaces/{namespace}/gateway` API. - -The Service Account uses the OAuth2 Client Credentials Grant Flow. Click the `lock` link on the right and enter in the Service Account credentials that were generated in step #2. - -For the `Parameter namespace`, enter the namespace that you created in step #1. - -Select `dryRun` to `true`. - -Select a `configFile` file. - -Send the request. - -### 4.3. Postman - -From the Postman App, click the `Import` button and go to the `Link` tab. - -Enter a URL: https://openapi-to-postman-api-gov-bc-ca.test.api.gov.bc.ca/?u=https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/v2/openapi.yaml - -After creation, go to `Collections` and right-click on the `Gateway Administration (GWA) API` collection and select `edit`. - -Go to the `Authorization` tab, enter in your `Client ID` and `Client Secret` and click `Get New Access Token`. - -You should get a successful dialog to proceed. Click `Proceed` and `Use Token`. - -You can then verify that the token works by going to the Collection `Return key information about authenticated identity` and click `Send`. - -## 5. Verify routes - -To verify that the Gateway can access the upstream services, run the command: `gwa status`. - -In our test environment, the hosts that you defined in the routes get altered; to see the actual hosts, log into the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca), go to the `Namespaces` tab, go to `Gateway Services` and select your particular service to get the routing details. - -```bash -curl https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -ab -n 20 -c 2 https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -``` - -To help with troubleshooting, you can use the GWA API to get a health check for each of the upstream services to verify the Gateway is connecting OK. - -Go to the [GWA API](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/#/Service%20Status/get_namespaces__namespace__services), enter in the new credentials that were generated in step #2, click `Try it out`, enter your namespace and click `Execute`. The results are returned in a JSON object. - -## 6. View metrics - -The following metrics can be viewed in real-time for the Services that you configure on the Gateway: - -- Request Rate : Requests / Second (by Service/Route, by HTTP Status) -- Latency : Standard deviations measured for latency inside Kong and on the Upstream Service (by Service/Route) -- Bandwidth : Ingress/egress bandwidth (by Service/Route) -- Total Requests : In 5 minute windows (by Consumer, by User Agent, by Service, by HTTP Status) - -All metrics can be viewed by an arbitrary time window - defaults to `Last 24 Hours`. - -Go to [Grafana](https://grafana-apps-gov-bc-ca.test.api.gov.bc.ca) to view metrics for your configured services. - -You can also access summarized metrics from the `API Services Portal` by going to the `Namespaces` tab and clicking the `Gateway Services` link. - -## 7. Grant access to others - -To grant access to others, you need to grant them the appropriate Scopes. This can be done from the `API Services Portal`, going to the `API Access` tab and clicking the "Manage Resources" for the `Gateway Administration API` product. Click the link that corresponds to your namespace. Click the `Grant User Access` and enter in a username (i.e./ `jsmith@idir`) and the scopes you want to grant to the User, then click the `Share` button to grant the permissions. - -## 8. Add to your CI/CD Pipeline - -Update your CI/CD pipelines to run the `gwa-cli` to keep your services updated on the gateway. - -### 8.1. Github Actions Example - -In the repository that you maintain your CI/CD Pipeline configuration, use the Service Account details from `Step 2` to set up two `Secrets`: - -- GWA_ACCT_ID - -- GWA_ACCT_SECRET - -Add a `.gwa` folder (can be called anything) that will be used to hold your gateway configuration. - -An example Github Workflow: - -```yaml -env: - NS: "" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v1 - with: - node-version: 10 - TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get GWA Command Line - run: | - curl -L -O https://github.com/bcgov/gwa-cli/releases/download/v1.2.0/gwa_v1.2.0_linux_x64.zip - unzip gwa_v1.2.0_linux_x64.zip - export PATH=`pwd`:$PATH - - - name: Apply Namespace Configuration - run: | - export PATH=`pwd`:$PATH - cd .gwa/$NS - - gwa init -T \ - --namespace=$NS \ - --client-id=${{ secrets.TEST_GWA_ACCT_ID }} \ - --client-secret=${{ secrets.TEST_GWA_ACCT_SECRET }} - - gwa pg - - gwa acl --managers acope@idir -``` - -## 9. Share your API for Discovery - -Package your APIs and make them available for discovery through the API Services Portal and BC Data Catalog. - -The `API Services Portal` Directory organizes your APIs by Products and Environments. You can manage them via an API or through the UI. - -To use the Directory API, the following scopes are required: - -- For `contents` (documentation), the service account must have the `Content.Publish` scope -- For `datasets` and `products`, the service account must have the `Namespace.Manage` scope -- For `issuers`, the service account must have the `CredentialIssuer.Admin` scope - -View the Directory API in the [Swagger Console](https://openapi-apps-gov-bc-ca.test.api.gov.bc.ca/?url=https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api/openapi.yaml) - -> NOTE: The steps below use `restish`, but we will be working on upgrading the `gwa` command line interface to support these APIs - -> Can use `y2j` to convert from YAML to JSON -> -> `echo '#!/usr/bin/env python\nimport sys,yaml,json\nprint(json.dumps(yaml.safe_load(open(sys.argv[1]).read())))' > /usr/local/bin/y2j` -> -> `chmod +x /usr/local/bin/y2j` - -**Restish Setup** - -``` -restish api configure my_api -``` - -Base URI : https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api - -`Edit Profile default` - -Select `Setup Auth` > `oauth-client-credentials` - -Enter the `client_id` and `client_secret` for your Service Account. - -`token_url` : Provided to you when you created the Service Account - -`scopes` : openid - -Select `Finished with profile` and then `Save and exit` - -To verify that restish is working, run: - -`restish my_api list` or `restish my_api get-new-id product` - -The below example works with the service created above, so we will want to reference the NS environment variable. - -``` -export NS="" -``` - -### 9.1 Setup Authorization Profiles - -If your APIS are protected with one of the OAuth2 grant types, then an Authorization profile must be created with the appropriate credentials for accessing the corresponding Identity Provider/Broker and Authorization Server. - -```yaml -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -``` -y2j issuer.yaml | restish my_api put-issuer $NS -``` - -### 9.2 Setup your Product, Environments and link your Services - -```yaml -kind: DraftDataset -name: my-draft-dataset -organization: ministry-of-health -organizationUnit: planning-and-innovation-division -title: My API -notes: Some information about this API -tags: [health, standards, openapi] -sector: Service -license_title: Access Only -view_audience: Government -security_class: LOW-PUBLIC -record_publish_date: "2021-05-27" -``` - -``` -y2j dataset.yaml | restish my_api put-dataset $NS -``` - -```yaml -kind: Product -appId: 2B04C28E08AW -name: My API -dataset: my-draft-dataset -environments: - - id: 1F7CA929 - name: dev - active: false - approval: false - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Please provide a bit more of this - services: [] - - id: 2F7CA929 - name: test - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Asking for test environment? Please provide some more info - services: [a-service-for-$NS] - - id: 3F7CA929 - name: prod - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: client-credentials - credentialIssuer: Resource Server $NS - additionalDetailsToRequest: Production? Great, please provide X, Y and Z - services: [] -``` - -``` -y2j prod.yaml | restish my_api put-product $NS -``` - -### 9.3 Update Gateway Configuration based on Flow - -In the previous section our example defines an environment that is protected using Kong's API Key and ACL plugins. To activate an environment, the corresponding plugins need to exist on the Gateway for that service or routes. The ACL `allow` corresponds to the unique `Environment ID` defined in section 9.2. - -``` - plugins: - - name: key-auth - tags: [ ns.$NS ] - protocols: [ http, https ] - config: - key_names: ["X-API-KEY"] - run_on_preflight: true - hide_credentials: true - key_in_body: false - - name: acl - tags: [ ns.$NS ] - config: - hide_groups_header: true - allow: [ ] -``` - -### 9.4 Publish Environments - -Your products will not appear on the Directory until you mark the relevant environments as Active. You can do this by either updating the Product Environment configuration above to `active: true`, or going to the API Services Portal UI and editing the Environment details. - -### 9.5 Publish Documentation - -```yaml -kind: Content -title: Getting Started with Example API -description: Getting Started with Example API -externalLink: https://github.com/bcgov/$NS/getting_started.md -order: 1 -tags: [ns.$NS] -isComplete: true -isPublic: true -publishDate: "2021-06-02T08:00:00.000-08:00" -``` - -``` -y2j content.yaml | restish my_api put-content $NS - -echo "# here is some markdown and more!" > doc.md - -restish my_api put-content $NS \ - externalLink: "https://github.com/bcgov/$NS/getting_started.md", \ - content: @doc.md -``` - -### 9.6 View your product in the API Directory - -Find your API in the [API Services Portal Directory](https://api-gov-bc-ca.test.api.gov.bc.ca/devportal/api-discovery) - -It is now ready to receive access requests from the community! - -# Production Links - -- [API Services Portal](https://api.gov.bc.ca) -- [gwa-api Swagger Console](https://gwa.api.gov.bc.ca/docs/) -- [gwa-api Postman Collection](https://openapi-to-postman.api.gov.bc.ca/?u=https://gwa.api.gov.bc.ca/docs/v2/openapi.yaml) -- [Gateway Metrics - Grafana](https://grafana.apps.gov.bc.ca) +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/guides/owner-journey diff --git a/docs/AVAILABLE-PLUGINS.md b/docs/AVAILABLE-PLUGINS.md index 7c22840..cfda916 100644 --- a/docs/AVAILABLE-PLUGINS.md +++ b/docs/AVAILABLE-PLUGINS.md @@ -1,52 +1,3 @@ Current list of plugins available on the API Gateway. -> NOTE: The `httplog` is not available for individual teams to use as it is used at a global level to feed logs to other systems for audit/monitoring/etc. - -> If there are other plugins that you would like to have added to the Gateway, please reach out to us at our Rocket.Chat channel `#aps-ops`. - -```json - "available_on_server": { - "basic-auth": true, - "ip-restriction": true, - "request-transformer": true, - "response-transformer": true, - "request-size-limiting": true, - "rate-limiting": true, - "response-ratelimiting": true, - "syslog": true, - "loggly": true, - "datadog": true, - "ldap-auth": true, - "statsd": true, - "bot-detection": true, - "aws-lambda": true, - "request-termination": true, - "azure-functions": true, - "zipkin": true, - "pre-function": true, - "post-function": true, - "prometheus": true, - "proxy-cache": true, - "session": true, - "acme": true, - "grpc-web": true, - "grpc-gateway": true, - "kong-spec-expose": true, - "jwt-keycloak": true, - "referer": true, - "bcgov-gwa-endpoint": true, - "gwa-ip-anonymity": true, - "oidc": true, - "jwt": true, - "acl": true, - "correlation-id": true, - "cors": true, - "oauth2": true, - "tcp-log": true, - "udp-log": true, - "file-log": true, - "http-log": true, - "key-auth": true, - "hmac-auth": true - } - ``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/available-plugins diff --git a/docs/COMMON-CONFIG.md b/docs/COMMON-CONFIG.md index c790c81..df9987b 100644 --- a/docs/COMMON-CONFIG.md +++ b/docs/COMMON-CONFIG.md @@ -1,106 +1,3 @@ # Common Gateway Controls -The following are sample Gateway controls for common scenarios. - -## Returning an HTTP Redirect - -``` -plugins: -- name: pre-function - tags: [ _NS_ ] - config: - access: - - "kong.response.exit(307, 'site moved - redirecting...', {['Location'] = 'https://my-new-url.site'})" -``` - -## Adding Headers For Best Security Practices - -``` -plugins: -- name: response-transformer - tags: [ _NS_ ] - config: - add: - headers: - - "X-Frame-Options: DENY" - - "X-XSS-Protection: 1; mode=block" - - "X-Content-Type-Options: nosniff" - - "Strict-Transport-Security: max-age=31536000" - - "Content-Security-Policy: script-src 'self'" -``` - -> For further information on individual headers, see: https://owasp.org/www-project-secure-headers/ - -## Rate Limit - -### Option 1 - Using a Distributed Cache - -This provides the most accurate because it uses a centralized Cache that all Kong nodes use. The downside is that there is a 100-200ms latency. - -``` -plugins: -- name: rate-limiting - tags: [ _NS_ ] - config: - fault_tolerant: true - hide_client_headers: false - limit_by: ip - minute: 30000 - second: null - hour: null - day: null - month: null - year: null -``` - -### Option 2 - Node Local Caching - -This provides the fastest rate limiting option, with minimal latency (~1ms). The downside is that it is local to each node so calculating the actual load on your upstream is a function of the number of nodes. - -``` -plugins: -- name: rate-limiting - tags: [ _NS_ ] - config: - policy: local - fault_tolerant: true - hide_client_headers: false - limit_by: ip - minute: 30000 - second: null - hour: null - day: null - month: null - year: null -``` - -## Two-tiered access setup - -The `key-auth` and `jwt-keycloak` plugins support the concept of allowing "anonymous" access, which allows you to define a "free" service which might have limits around it (like only allowing 100 requests/minute), and then an "elevated" access where the Consumer would get an improved level of service, such as higher rate limits. - -There is a global "anonymous" consumer that is identified as "ce26955a-cf08-4907-9427-12d01c8bd94c" in both our Test and Production environments. - -To enable anonymous access to your API, update your plugin configuration with: - -### key-auth - -``` -- name: key-auth - tags: [ _NS_ ] - config: - ... - anonymous: ce26955a-cf08-4907-9427-12d01c8bd94c -``` - -### jwt-keycloak - -``` -- name: jwt-keycloak - tags: [ _NS_ ] - config: - ... - consumer_match: true - consumer_match_ignore_not_found: false - consumer_match_claim_custom_id: false - anonymous: ce26955a-cf08-4907-9427-12d01c8bd94c -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/common-config diff --git a/docs/SSL.md b/docs/SSL.md index d06e0ab..f6bee91 100644 --- a/docs/SSL.md +++ b/docs/SSL.md @@ -1,41 +1,3 @@ - # SSL Termination -If you would like to verify the SSL endpoint, you can run the following two commands and compare the fingerprint and serial no. - -``` -export A_HOST=httpbin-regression.api.gov.bc.ca -openssl s_client -showcerts -verify 5 -connect 142.34.194.118:443 -servername ${A_HOST} < /dev/null | awk '/BEGIN/,/END/{ if(/BEGIN/){a++}; print}' > gw.crt - -openssl x509 -in gw.crt -fingerprint -serial -dates -noout - -``` - -## *.api.gov.bc.ca - -| Issue Date | Expires | Deployed | SHA1 Fingerprint | Serial No. | -|-------------|-------------|-------------|-------------------------------------------------------------|----------------------------------| -| Oct 6 2020 | Oct 16 2021 | Oct 6 2020 | 20:7D:15:9D:42:BE:CC:BC:FD:EF:DF:13:77:C7:25:A3:A4:72:45:05 | 7876EB597E14F728C8455504177D3BC9 | -| Feb 16 2021 | Oct 16 2021 | Feb 25 2021 | 4D:EA:CE:C4:0A:73:67:D3:B4:03:F6:63:C4:E1:67:2C:47:9D:EA:82 | 3B5849D8A670251A3C20EA7859BDF996 | -| Sep 27 2021 | Oct 16 2022 | Oct 6 2021 | E3:DF:EC:89:BC:03:9B:E9:7D:57:91:EB:52:18:59:46:AA:A9:3A:15 | 1B588948FBB2945DE5D3A0034E355539 | - -You can run the above as one line: - -``` -A_HOST=httpbin-regression.api.gov.bc.ca; openssl s_client -showcerts -verify 5 -connect ${A_HOST}:443 -servername ${A_HOST} < /dev/null | awk '/BEGIN/,/END/{ if(/BEGIN/){a++}; print}' | openssl x509 -fingerprint -serial -dates -noout -``` - - -**Individual File Verification** - -``` -openssl x509 -in data-api-wildcard-2020.crt -fingerprint -serial -dates -noout -openssl x509 -in data-api-wildcard-2021.crt -fingerprint -serial -dates -noout -``` - -**Cert/Key Verification** - -``` -openssl x509 -noout -modulus -in data-api-wildcard.crt | openssl md5 -openssl rsa -noout -modulus -in data-api-wildcard.key | openssl md5 -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform/gateway/ssl diff --git a/docs/guides/intro-signed-jwt.md b/docs/guides/intro-signed-jwt.md index 960989d..97cade3 100644 --- a/docs/guides/intro-signed-jwt.md +++ b/docs/guides/intro-signed-jwt.md @@ -1,126 +1,3 @@ # Signed JWT Authentication -The following example uses NodeJS code to show how to prepare for signed JWT authentication to an API on the BC Government API Gateway. - -## 1. Generate a Certificate Key Pair - -``` -npm install --save node-jose ms - -echo " -const fs = require('fs'); -const jose = require('node-jose'); - -const keyStore = jose.JWK.createKeyStore() -keyStore.generate('RSA', 2048, {alg: 'RS256', use: 'sig' }) -.then(result => { - fs.writeFileSync( - 'keys.json', - JSON.stringify(keyStore.toJSON(true), null, ' ') - ) - const [key] = keyStore.all({ use: 'sig' }) - console.log(key.toPEM(true)) - console.log(key.toPEM(false)) -}) -" | node -``` - -## 2. Publish the Public Key - -Publish the Public Key in JWKS Format at a location that is publically accessible. - -``` -echo " -const fs = require('fs') -const jose = require('node-jose'); - -const ks = fs.readFileSync('keys.json') -jose.JWK.asKeyStore(ks.toString()).then(keyStore => - console.log(JSON.stringify(keyStore.toJSON(), null, 3))); -" | node -``` - -Place the JWKS JSON file somewhere that it can be reached publically. - -``` -https://ikethecoder.github.io/temp-place/certs.json -``` - -## 3. Request Access to an API - -https://api-gov-bc-ca.test.api.gov.bc.ca/ - -Make a note of the `Client ID`, `Issuer` and `Token Endpoint`. - -``` -export CID="" -export ISS="" -export TURL="" -``` - -## 4. Request a Client JWT Token - -Requesting a Client JWT Token is a two-step process: - -a) Build a Client Assertion Token that is signed with the private key you generated earlier. - -b) Request a token from the Token Endpoint using the Client Assertion. - -The following sample performs both steps: - -``` -npm install --save njwt node-fetch - -echo " -const njwt = require('njwt'); - -const fs = require('fs') - -const jose = require('node-jose'); -const ks = fs.readFileSync('keys.json') -jose.JWK.asKeyStore(ks.toString()).then(keyStore => { - const [key] = keyStore.all({ use: 'sig' }) - - const privateKey = key.toPEM(true) // Load an RSA private key from configuration - const clientId = process.env.CID; // Or load from configuration - const now = Math.floor( new Date().getTime() / 1000 ); // seconds since epoch - const plus5Minutes = new Date( ( now + (5*60) ) * 1000); // Date object - const alg = 'RS256'; // one of RSA or ECDSA algorithms: RS256, RS384, RS512, ES256, ES384, ES512 - - const claims = { - aud: process.env.ISS - }; - - const jwt = njwt.create(claims, privateKey, alg) - .setIssuedAt(now) - .setExpiration(plus5Minutes) - .setIssuer(clientId) - .setSubject(clientId) - .compact(); - - const fetch = require('node-fetch'); - const { URLSearchParams } = require('url'); - - const params = new URLSearchParams(); - params.append('grant_type', 'client_credentials'); - params.append('client_id', process.env.CID); - params.append('scopes', 'openid'); - params.append('client_assertion_type', 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'); - params.append('client_assertion', jwt); - - fetch(process.env.TURL, { method: 'POST', body: params }) - .then(res => res.json()) - .then(json => console.log(json)); -}); -" | node -``` - -## 5. Call the API - -Call the API using the newly generated Token returned from the Identity Provider. - -``` -curl -v https://a-protected-api.test.api.gov.bc.ca/headers \ - -H "Authorization: Bearer $TOK" - -``` +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/intro-signed-jwt.yaml b/docs/guides/intro-signed-jwt.yaml deleted file mode 100644 index 0f26ce3..0000000 --- a/docs/guides/intro-signed-jwt.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/intro-signed-jwt.md -title: "Getting Started with Signed JWT" -description: "Accessing an API with Signed JWT and Client Credential Grant" -order: 1 -tags: ["ns.platform", "tutorial"] -isComplete: true -isPublic: true -publishDate: "2021-09-27T20:00:00.000-08:00" diff --git a/docs/guides/tutorial-idp-client-cred-flow.md b/docs/guides/tutorial-idp-client-cred-flow.md index 713421b..12ad070 100644 --- a/docs/guides/tutorial-idp-client-cred-flow.md +++ b/docs/guides/tutorial-idp-client-cred-flow.md @@ -1,183 +1,3 @@ # Protecting an API with OAuth2 Client Credential Flow -Steps for protecting and calling an API using the OAuth2 Client Credential Grant. - -1. Configuring an API on the Kong Gateway -2. Granting Access to the IdP -3. Client Requesting Access -4. Provider Approving Access -5. Client Retrieving the Access Token -6. Client Calling an API -7. Gateway Proxying to Upstream - -![alt text](assets/oauth2.png "Protecting an API") - -## 1. Configuring an API on the Kong Gateway - -The API Provider User Journey document provides the steps to enable a new API on the Kong Gateway. It will walk you through the creation of a new Namespace, and a Service Account that can be used to configure the Kong Gateway and the Authorization Profile described in step 2. - -Once the API is working on the Gateway, you can then define a Product, which can be made available on the API Services Portal Directory. - -## 2. Granting Access to the IdP - -### a) Prerequisites - -Before the Portal can be configured, a new set of credentials must be created on the IdP. For this tutorial, we will include the steps when Keycloak is the IdP. - -**Create a new Client on the IdP** - -Create a new client with Access Type `confidential`. All flows except `Service Accounts` should be turned off. - -Make a note of the `Client ID` and `Client Secret` , they will be used when the Portal `Authorization Profile` is created. - -The `Full Scope Allowed` can be turned off and the `realm-management` client roles for `manage-clients` and `manage-users` should be added. - -Add the `manage-clients` and `manage-users` client roles to the `Service Account Roles`. - -### b) Setup the Authorization Profile - -A credential with `CredentialIssuer.Admin` is required to update Authorization Profiles (`CredentialIssuer`). - -Authorization Profiles can be setup either via the Portal or by using a Service Account with the Portal Directory API. - -Update the below `CredentialIssuer` to include the environment details and the Scopes and Roles setup for authorization. - -``` -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -clientMappers: - - name: audience - defaultValue: '' -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -### c) Link the Authorization Profile to the Product - -Before making the API available on the Directory, the API should be configured with a plugin for protecting access. To do this, an API Provider can edit the Product details to select `Oauth2 Client Credential Flow` and the newly created Authorization Profile. - -### d) Update your Gateway Configuration with the Plugin - -Update your Gateway Configuration to include the `jwt-keycloak` plugin. - -> HINT: When you configure the Product Environment, a `Plugin Template` will be displayed - this can be a starting point for protecting your API on the Gateway. - -Finally, from the Portal, `enable` the Environment to make it available on the API Directory. - -### e) Optional Configuration - -**Scopes** - -If you have Client Scopes that you want to have controlled by the Portal, add them to the Realm's `Client Scopes` and `Default Client Scopes` on the IdP. - -Update the `CredentialIssuer` record above to match the `availableScopes` with the ones added on the IdP. - -**Roles** - -If you have Roles that you want to have controlled by the Portal, add them to the Client's `Roles`. - -Update the `CredentialIssuer` record above to match the `clientRoles` with the ones added on the IdP. - -**Client Mappers** - -The `audience` is an optional mapper that can be added to a Client. - -The IdP needs to have a policy that allows Audience to be added as a Protocol Mapper to the client. - -In Keycloak, this is updated under the Realm's `Client Registration` -> `Client Registration Policies`. - -Edit the Authenticated Access Policies -> Allowed Protocol Mapper Types to include the `oidc-audience-mapper`. - -**UMA2 Authorization Resources** - -If you want to use the Authorization services, then set `Authorization Enabled` to `ON` for the Client on the IdP. You will also want to set the `Decision Strategy` to `Affirmative`. - -Update the following `CredentialIssuer` attributes: - -``` - resourceType: "" - resourceAccessScope: "" - resourceScopes: [] -``` - -- `resourceType`: The Resource Type of the resources that will be managed (required) -- `resourceScopes`: A list of the Authorization Scopes managed for the particular Resources (required) -- `resourceAccessScope`: Used in the case where the Resource Server owns all the resources, a user must have the `resourceAccessScope` assigned in order to be allowed to manage the access. If it is not set, then the user has to be the resource owner in order to manage access. - -> `resourceAccessScope` - The API Services Portal has not completed the implementation for the scenario where the User is the Resource Owner (`resourceAccessScope` is left blank). It uses the `Token Exchange` capability but it's an optional service available on Keycloak and has numerous caveats around it. Please contact the APS team if interested to know more. - -## 3. Client Requesting Access - -Request access to the API via the API Services Portal and generate the credentials to be used below. - -The Portal will use the credentials setup in the Authorization Profile, to create a disabled Client on the IdP (with any applicable Client Mappers) and return the credentials to the Requesting user. - -## 4. Provider Approving Access - -An Access Manager reviews the access request, sets any additional controls, grants the relevant permissions (i.e./ scopes and roles), and approves. The Portal will enable the Client and apply the permissions on the IdP. - -The Portal sends a notification to the Requester letting them know that API Access has been approved (or rejected). - -## 5. Client Retrieving the Access Token - -Using the Credentials generated in step 3, the Requester calls the Token endpoint to get a new JWT token. - -``` -export CID="" -export CSC="" -export URL="" - -curl $URL \ - -X POST -H "Content-Type: application/x-www-form-urlencoded" \ - -d client_id=$CID -d client_secret=$CSC \ - -d grant_type=client_credentials \ - -d scopes=openid -``` - -## 6. Client Calling the API - -Extract the Access Token from the response, set it in a `TOK` environment variable, and call the API. - -``` -curl -v -H "Authorization: Bearer $TOK" \ - https://myservice.api.gov.bc.ca/v1/status -``` - -The API Gateway's `jwt-keycloak` plugin will use the IdP's public keys to validate the token and depending on the plugin configuration, validate the scopes or roles. - -## 7. Gateway Proxying to Upstream - -A technical, but important step in the integration is the Gateway proxying the request to the Upstream Service API and deciding on the different options for securing that interaction. - -Options: - -- `Network Policy` : If the Services are co-located on the same Cluster as the Gateway's Data Plane, then native network policies can be used to protect the channel between the Gateway and the Upstream Service. This approach is used for Services running on the Openshift Silver cluster. - -- `Kong Upstream JWT`: This plugin adds a signed JWT to the request headers so that the Upstream Service can verify that the request came specifically from the Gateway. - -- `Client Certificates` : Client certificates (mTLS) provides a way for the Upstream Service to provide a secure channel from the Gateway and to verify that the request came specifically from the Gateway. - -- `Firewall IP Restrictions` : This provides a low-level of protection by limiting the IPs to the ones of the Gateway Data Planes. Because the Data Planes are typically on shared infrastructure, this would still allow traffic from other tenants. This might be acceptable based on the type of data delivered by or to the Upstream Service. - -## Variations - -### Signed JWT - -The Authorization Profile `clientAuthenticator` was set to `client-secret` in this tutorial, but there is an alternate setup that can be used: - -- `client-jwt-jwks-url` (Signed JWT with JWKS URL) - -In this scenario, when a Client is requesting access, they will be required to enter a public URL that holds the public key information for a key pair that will be used in step 5 when retrieving the Bearer Token. For further details, you can see the specific example `intro-signed-jwt.md`. +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/tutorial-idp-client-cred-flow.yaml b/docs/guides/tutorial-idp-client-cred-flow.yaml deleted file mode 100644 index 9573ba3..0000000 --- a/docs/guides/tutorial-idp-client-cred-flow.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/tutorial-idp-client-cred-flow.md -title: "Setting up Client Credential Protection" -description: "Configuration the Gateway with an Identity Provider and Client Credential Grant" -order: 1 -tags: ["ns.platform", "tutorial"] -isComplete: true -isPublic: true -publishDate: "2021-09-27T20:00:00.000-08:00" diff --git a/docs/guides/user-journey.md b/docs/guides/user-journey.md index 2779667..04c10c4 100644 --- a/docs/guides/user-journey.md +++ b/docs/guides/user-journey.md @@ -1,510 +1,3 @@ # API Owner Flow -The following steps walk an API Owner through setting up an API on the BC Gov API Gateway in our Test instance. If you are ready to deploy to our Production instance, use the links found at the bottom of this document ([here](#production-links)). - -## 1. Register a new namespace - -A `namespace` represents a collection of Kong Gateway Services and Routes that are managed independently. - -To create a new namespace, login to the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca). - -After login, click the namespace dropdown in the top right next to your user name (it may show `No Active Namespace`), then click `Create New Namespace`. - -The namespace must be an alphanumeric string between 5 and 15 characters (RegExp reference: `^[a-z][a-z0-9-]{4,14}$`). - -You can select and manage namespaces by clicking the namespace dropdown in the top right next to your user name. - -## 2. Generate a Service Account - -Go to the `Namespaces` tab, click the `Service Accounts` link, and click the `New Service Account` and select the `GatewayConfig.Publish` permissions for the Service Account and click `Share`. A new credential will be created - make a note of the `ID` and `Secret`. - -The available Scopes are: -| Scope | Permission | -| ----- | ---------- | -| `Namespace.Manage` | Permission to update the Access Control List for controlling access to viewing metrics, service configuration and service account management (effectively a superuser for the namespace) | -| `Namespace.View` | Read-only access to the namespace | -| `GatewayConfig.Publish` | Permission to publish gateway configuration to Kong and to view the status of the upstreams | -| `Content.Publish` | Permission to update the documentation on the portal | -| `CredentialIssuer.Admin` | Permission to create Authorization Profiles so that they are available to be used when configuring Product Environments | -| `Access.Manage` | Permission to approve/reject access requests to your APIs that you make discoverable | - -## 3. Prepare configuration - -The gateway configuration can be hand-crafted or you can use a command line interface that we developed called `gwa` to convert your Openapi v3 spec to a Kong configuration. - -### 3.1. Hand-crafted (recommended if you don't have an Openapi spec) - -**Simple Example** - -```bash -export NS="my_namespace" -export NAME="a-service-for-$NS" -echo " -services: -- name: $NAME - host: httpbin.org - tags: [ ns.$NS ] - port: 443 - protocol: https - retries: 0 - routes: - - name: $NAME-route - tags: [ ns.$NS ] - hosts: - - $NAME.api.gov.bc.ca - paths: - - / - methods: - - GET - strip_path: false - https_redirect_status_code: 426 - path_handling: v0 -" > sample.yaml -``` - -> To view common plugin config go to [COMMON-CONFIG.md](https://github.com/bcgov/gwa-api/blob/dev/docs/COMMON-CONFIG.md) - -> To view some other plugin examples go [here](https://github.com/bcgov/gwa-api/blob/dev/docs/samples/service-plugins). - -> **Declarative Config** Behind the scenes, DecK is used to sync your configuration with Kong. For reference: https://docs.konghq.com/deck/overview/ - -> **Splitting Your Config:** A namespace `tag` with the format `ns.$NS` is mandatory for each service/route/plugin. But if you have separate pipelines for your environments (i.e./ dev, test and prod), you can split your configuration and update the `tags` with the qualifier. So for example, you can use a tag `ns.$NS.dev` to sync Kong configuration for `dev` Service and Routes only. - -> **Upstream Services on OCP4:** If your service is running on OCP4, you should specify the Kubernetes Service in the `Service.host`. It must have the format: `..svc`. Also make sure your `Service.port` matches your Kubernetes Service Port. Any Security Policies for egress from the Gateway will be setup automatically on the API Gateway side. -> The Aporeto Network Security Policies are being removed in favor of the Kubernetes Security Policies (KSP). You will need to create a KSP on your side looking something like this to allow the Gateway's test and prod environments to route traffic to your API: - -```yaml -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: allow-traffic-from-gateway-to-your-api -spec: - podSelector: - matchLabels: - name: my-upstream-api - ingress: - - from: - - namespaceSelector: - matchLabels: - environment: test - name: 264e6f - - from: - - namespaceSelector: - matchLabels: - environment: prod - name: 264e6f -``` - -> **Migrating from OCP3 to OCP4?** Please review the [OCP4-Migration](https://github.com/bcgov/gwa-api/blob/dev/docs/OCP4-MIGRATION.md) instructions to help with transitioning to OCP4 and the new APS Gateway. - -> **Require mTLS between the Gateway and your Upstream Service?** To support mTLS on your Upstream Service, you will need to provide client certificate details and if you want to verify the upstream endpoint then the `ca_certificates` and `tls_verify` is required as well. An example: - -```yaml -services: - - name: my-upstream-service - host: my-upstream.site - tags: [_NS_] - port: 443 - protocol: https - tls_verify: true - ca_certificates: [0a780ee0-626c-11eb-ae93-0242ac130012] - client_certificate: 8fc131ef-9752-43a4-ba70-eb10ba442d4e - routes: [...] -certificates: - - cert: "" - key: "" - tags: [_NS_] - id: 8fc131ef-9752-43a4-ba70-eb10ba442d4e -ca_certificates: - - cert: "" - tags: [_NS_] - id: 0a780ee0-626c-11eb-ae93-0242ac130012 -``` - -> NOTE: You must generate a UUID (`python -c 'import uuid; print(uuid.uuid4())'`) for each certificate and ca_certificate you create (set the `id`) and reference it in your `services` details. - -> HELPER: Python command to get a PEM file on one line: `python -c 'import sys; import json; print(json.dumps(open(sys.argv[1]).read()))' my.pem` - -### 3.2. gwa Command Line - -Run: `gwa new` and follow the prompts. - -Example: - -```bash -gwa new -o sample.yaml \ - --route-host myapi.api.gov.bc.ca \ - --service-url https://httpbin.org \ - https://bcgov.github.io/gwa-api/openapi/simple.yaml -``` - -> See below for the `gwa` CLI install instructions. - -## 4. Apply gateway configuration - -The Swagger console for the `gwa-api` can be used to publish Kong Gateway configuration, or the `gwa Command Line` can be used. - -### 4.1. gwa Command Line (recommended) - -**Install (for Linux)** - -```bash -GWA_CLI_VERSION=v1.2.0; curl -L -O https://github.com/bcgov/gwa-cli/releases/download/${GWA_CLI_VERSION}/gwa_${GWA_CLI_VERSION}_linux_x64.zip -unzip gwa_${GWA_CLI_VERSION}_linux_x64.zip -./gwa --version -``` - -> **Using MacOS or Windows?** Download here: [https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0](https://github.com/bcgov/gwa-cli/releases/tag/v1.2.0) - -> NOTE: Version 1.2.0 introduces support for v2 of our api. To continue using v1 of the api, ensure that the API Version is set to 1 (see below) - -**Configure** - -Create a `.env` file and update the CLIENT_ID and CLIENT_SECRET with the new credentials that were generated in step #2: - -```bash -echo " -GWA_NAMESPACE=$NS -CLIENT_ID= -CLIENT_SECRET= -GWA_ENV=test -API_VERSION=2 -" > .env - -OR run: - -gwa init -T --api-version=2 --namespace=$NS --client-id= --client-secret= - -``` - -> NOTE: The `-T` indicates our Test environment. For production use `-P`. - -**Publish** - -```bash -gwa pg sample.yaml -``` - -If you want to see the expected changes but not actually apply them, you can run: - -```bash -gwa pg --dry-run sample.yaml -``` - -### 4.2. Swagger Console - -Go to [gwa-api Swagger Console](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/). - -Select the `PUT` `/namespaces/{namespace}/gateway` API. - -The Service Account uses the OAuth2 Client Credentials Grant Flow. Click the `lock` link on the right and enter in the Service Account credentials that were generated in step #2. - -For the `Parameter namespace`, enter the namespace that you created in step #1. - -Select `dryRun` to `true`. - -Select a `configFile` file. - -Send the request. - -### 4.3. Postman - -From the Postman App, click the `Import` button and go to the `Link` tab. - -Enter a URL: https://openapi-to-postman-api-gov-bc-ca.test.api.gov.bc.ca/?u=https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/v2/openapi.yaml - -After creation, go to `Collections` and right-click on the `Gateway Administration (GWA) API` collection and select `edit`. - -Go to the `Authorization` tab, enter in your `Client ID` and `Client Secret` and click `Get New Access Token`. - -You should get a successful dialog to proceed. Click `Proceed` and `Use Token`. - -You can then verify that the token works by going to the Collection `Return key information about authenticated identity` and click `Send`. - -## 5. Verify routes - -To verify that the Gateway can access the upstream services, run the command: `gwa status`. - -In our test environment, the hosts that you defined in the routes get altered; to see the actual hosts, log into the [API Services Portal](https://api-gov-bc-ca.test.api.gov.bc.ca), go to the `Namespaces` tab, go to `Gateway Services` and select your particular service to get the routing details. - -```bash -curl https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -ab -n 20 -c 2 https://${NAME}-api-gov-bc-ca.test.api.gov.bc.ca/headers - -``` - -To help with troubleshooting, you can use the GWA API to get a health check for each of the upstream services to verify the Gateway is connecting OK. - -Go to the [GWA API](https://gwa-api-gov-bc-ca.test.api.gov.bc.ca/docs/#/Service%20Status/get_namespaces__namespace__services), enter in the new credentials that were generated in step #2, click `Try it out`, enter your namespace and click `Execute`. The results are returned in a JSON object. - -## 6. View metrics - -The following metrics can be viewed in real-time for the Services that you configure on the Gateway: - -- Request Rate : Requests / Second (by Service/Route, by HTTP Status) -- Latency : Standard deviations measured for latency inside Kong and on the Upstream Service (by Service/Route) -- Bandwidth : Ingress/egress bandwidth (by Service/Route) -- Total Requests : In 5 minute windows (by Consumer, by User Agent, by Service, by HTTP Status) - -All metrics can be viewed by an arbitrary time window - defaults to `Last 24 Hours`. - -Go to [Grafana](https://grafana-apps-gov-bc-ca.test.api.gov.bc.ca) to view metrics for your configured services. - -You can also access summarized metrics from the `API Services Portal` by going to the `Namespaces` tab and clicking the `Gateway Services` link. - -## 7. Grant access to others - -To grant access to others, you need to grant them the appropriate Scopes. This can be done from the `API Services Portal`, selecting the relevant `Namespace` and going to the Namespaces `Namespace Access` page. From here, you are able to grant Users access to the Namespace. - -## 8. Add to your CI/CD Pipeline - -Update your CI/CD pipelines to run the `gwa-cli` to keep your services updated on the gateway. - -### 8.1. Github Actions Example - -In the repository that you maintain your CI/CD Pipeline configuration, use the Service Account details from `Step 2` to set up two `Secrets`: - -- GWA_ACCT_ID - -- GWA_ACCT_SECRET - -Add a `.gwa` folder (can be called anything) that will be used to hold your gateway configuration. - -An example Github Workflow: - -```yaml -env: - NS: "" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v1 - with: - node-version: 10 - TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Get GWA Command Line - run: | - curl -L -O https://github.com/bcgov/gwa-cli/releases/download/v1.2.0/gwa_v1.2.0_linux_x64.zip - unzip gwa_v1.2.0_linux_x64.zip - export PATH=`pwd`:$PATH - - - name: Apply Namespace Configuration - run: | - export PATH=`pwd`:$PATH - cd .gwa/$NS - - gwa init -T \ - --namespace=$NS \ - --client-id=${{ secrets.TEST_GWA_ACCT_ID }} \ - --client-secret=${{ secrets.TEST_GWA_ACCT_SECRET }} - - gwa pg -``` - -## 9. Share your API for Discovery - -Package your APIs and make them available for discovery through the API Services Portal and BC Data Catalog. - -The `API Services Portal` Directory organizes your APIs by Products and Environments. You can manage them via an API or through the UI. - -To use the Directory API, the following scopes are required: - -- For `contents` (documentation), the service account must have the `Content.Publish` scope -- For `datasets` and `products`, the service account must have the `Namespace.Manage` scope -- For `issuers`, the service account must have the `CredentialIssuer.Admin` scope - -View the Directory API in the [Swagger Console](https://openapi-apps-gov-bc-ca.test.api.gov.bc.ca/?url=https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api/openapi.yaml) - -> NOTE: The steps below use `restish`, but we will be working on upgrading the `gwa` command line interface to support these APIs - -> Can use `y2j` to convert from YAML to JSON -> -> `echo -e "#"'!'"/usr/bin/env python\nimport sys,yaml,json\nprint(json.dumps(yaml.safe_load(open(sys.argv[1]).read())))" > /usr/local/bin/y2j` -> -> `chmod +x /usr/local/bin/y2j` - -**Restish Setup** - -``` -restish api configure my_api -``` - -Base URI : https://api-gov-bc-ca.test.api.gov.bc.ca/ds/api - -`Edit Profile default` - -Select `Setup Auth` > `oauth-client-credentials` - -Enter the `client_id` and `client_secret` for your Service Account. - -`token_url` : Provided to you when you created the Service Account - -`scopes` : openid - -Select `Finished with profile` and then `Save and exit` - -To verify that restish is working, run: - -`restish my_api list` or `restish my_api get-new-id product` - -The below example works with the service created above, so we will want to reference the NS environment variable. - -``` -export NS="" -``` - -### 9.1 Setup Authorization Profiles - -If your APIS are protected with one of the OAuth2 grant types, then an Authorization profile must be created with the appropriate credentials for accessing the corresponding Identity Provider/Broker and Authorization Server. - -```yaml -kind: CredentialIssuer -name: Resource Server Example -namespace: $NS -description: Authorization Profile for protecting Ministry of XYZ -flow: client-credentials -mode: auto -authPlugin: jwt-keycloak -clientAuthenticator: client-secret -clientRoles: [] -availableScopes: [Function1/read, Function2/*, Function3/write, Function3/read] -owner: -environmentDetails: - - environment: prod - issuerUrl: https://auth-issuer - clientId: testapp-client - clientRegistration: managed - clientSecret: "" -``` - -``` -y2j issuer.yaml | restish my_api put-issuer $NS -``` - -### 9.2 Setup your Product, Environments and link your Services - -```yaml -kind: DraftDataset -name: my-draft-dataset -organization: ministry-of-health -organizationUnit: planning-and-innovation-division -title: My API -notes: Some information about this API -tags: [health, standards, openapi] -sector: Service -license_title: Access Only -view_audience: Government -security_class: LOW-PUBLIC -record_publish_date: "2021-05-27" -``` - -``` -y2j dataset.yaml | restish my_api put-dataset $NS -``` - -```yaml -kind: Product -appId: 2B04C28E08AW -name: My API -dataset: my-draft-dataset -environments: - - id: 1F7CA929 - name: dev - active: false - approval: false - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Please provide a bit more of this - services: [] - - id: 2F7CA929 - name: test - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: kong-api-key-acl - additionalDetailsToRequest: Asking for test environment? Please provide some more info - services: [a-service-for-$NS] - - id: 3F7CA929 - name: prod - active: false - approval: true - legal: terms-of-use-for-api-gateway-1 - flow: client-credentials - credentialIssuer: Resource Server $NS - additionalDetailsToRequest: Production? Great, please provide X, Y and Z - services: [] -``` - -``` -y2j prod.yaml | restish my_api put-product $NS -``` - -### 9.3 Update Gateway Configuration based on Flow - -In the previous section our example defines an environment that is protected using Kong's API Key and ACL plugins. To activate an environment, the corresponding plugins need to exist on the Gateway for that service or routes. The ACL `allow` corresponds to the unique `Environment ID` defined in section 9.2. - -``` - plugins: - - name: key-auth - tags: [ ns.$NS ] - protocols: [ http, https ] - config: - key_names: ["X-API-KEY"] - run_on_preflight: true - hide_credentials: true - key_in_body: false - - name: acl - tags: [ ns.$NS ] - config: - hide_groups_header: true - allow: [ ] -``` - -### 9.4 Publish Environments - -Your products will not appear on the Directory until you mark the relevant environments as Active. You can do this by either updating the Product Environment configuration above to `active: true`, or going to the API Services Portal UI and editing the Environment details. - -### 9.5 Publish Documentation - -```yaml -kind: Content -title: Getting Started with Example API -description: Getting Started with Example API -externalLink: https://github.com/bcgov/$NS/getting_started.md -order: 1 -tags: [ns.$NS] -isComplete: true -isPublic: true -publishDate: "2021-06-02T08:00:00.000-08:00" -``` - -``` -y2j content.yaml | restish my_api put-content $NS - -echo "# here is some markdown and more!" > doc.md - -restish my_api put-content $NS \ - externalLink: "https://github.com/bcgov/$NS/getting_started.md", \ - content: @doc.md -``` - -### 9.6 View your product in the API Directory - -Find your API in the [API Services Portal Directory](https://api-gov-bc-ca.test.api.gov.bc.ca/devportal/api-discovery) - -It is now ready to receive access requests from the community! - -# Production Links - -- [API Services Portal](https://api.gov.bc.ca) -- [gwa-api Swagger Console](https://gwa.api.gov.bc.ca/docs/) -- [gwa-api Postman Collection](https://openapi-to-postman.api.gov.bc.ca/?u=https://gwa.api.gov.bc.ca/docs/v2/openapi.yaml) -- [Gateway Metrics - Grafana](https://grafana.apps.gov.bc.ca) +> :exclamation: This document has moved to the BC Gov API Services Docs site at: https://bcgov.github.io/aps-infra-platform diff --git a/docs/guides/user-journey.yaml b/docs/guides/user-journey.yaml deleted file mode 100644 index cac09b4..0000000 --- a/docs/guides/user-journey.yaml +++ /dev/null @@ -1,8 +0,0 @@ -externalLink: https://github.com/bcgov/gwa-api/blob/dev/USER-JOURNEY.md -title: "API Owner User Journey" -description: "Setup a new namespace for onboarding services on the BC Gov API Gateway." -order: 1 -tags: ["ns.platform"] -isComplete: true -isPublic: true -publishDate: "2021-06-10T20:00:00.000-08:00" diff --git a/docs/samples/service-plugins/service-rate-limit.yaml b/docs/samples/service-plugins/service-rate-limit.yaml index a22d055..8af8af3 100644 --- a/docs/samples/service-plugins/service-rate-limit.yaml +++ b/docs/samples/service-plugins/service-rate-limit.yaml @@ -1,4 +1,5 @@ - +# policy: local | redis +# limit_by: consumer | ip | service | header | path services: - name: MY_REST_API tags: [ _NS_ ] @@ -10,6 +11,7 @@ services: fault_tolerant: true hide_client_headers: false limit_by: consumer + policy: local minute: 10 header_name: null second: null diff --git a/docs/static/openapi/.env b/docs/static/openapi/.env new file mode 100644 index 0000000..da0441a --- /dev/null +++ b/docs/static/openapi/.env @@ -0,0 +1,4 @@ +GWA_NAMESPACE=openapi-tests +CLIENT_ID=sa-openapi-tests-tihw03lvo5 +CLIENT_SECRET=40f4852f-a71f-47ed-ad65-fc6f60471234 +GWA_ENV=dev diff --git a/docs/static/openapi/kong-with-edits.yaml b/docs/static/openapi/kong-with-edits.yaml new file mode 100644 index 0000000..0e24387 --- /dev/null +++ b/docs/static/openapi/kong-with-edits.yaml @@ -0,0 +1,21 @@ +_format_version: "1.1" +services: + - name: Sample_API + url: https://httpbin.org + plugins: [] + routes: + - tags: + - OAS3_import + - ns.openapi-tests.loc + name: Sample_API-path-get + methods: + - GET + hosts: + - openapi-gen-sample.api.gov.bc.ca + paths: + - /headers + strip_path: false + tags: + - OAS3_import + - ns.openapi-tests.loc + diff --git a/microservices/gatewayApi/app.py b/microservices/gatewayApi/app.py index 1726229..41bce0d 100644 --- a/microservices/gatewayApi/app.py +++ b/microservices/gatewayApi/app.py @@ -6,6 +6,7 @@ import os import time import config +from werkzeug.exceptions import HTTPException from authlib.jose.errors import JoseError, ExpiredTokenError from flask import Flask, g, jsonify, request, make_response, url_for, Response from flask_compress import Compress @@ -102,6 +103,12 @@ def bad_request_error(error): log.error(request.headers) return make_response(content, HTTPStatus.BAD_REQUEST) + @app.errorhandler(HTTPException) + def token_error(error): + log.error("Denied access %s - %s" % (request.remote_addr, str(error))) + content = jsonify({"error":"Invalid Token"}) + return make_response(content, HTTPStatus.UNAUTHORIZED) + @app.errorhandler(JoseError) def forbidden(error): log.error("Denied access %s - %s" % (request.remote_addr, str(error))) @@ -115,6 +122,7 @@ def expired_token(error): @app.errorhandler(Exception) def other_exception(error): + log.error("Unexpected error %s", type(error)) log.error(error) content = jsonify({"error":"Unexpected Error"}) return make_response(content, HTTPStatus.BAD_REQUEST) diff --git a/microservices/gatewayApi/auth/token.py b/microservices/gatewayApi/auth/token.py index b016bf6..2151cde 100644 --- a/microservices/gatewayApi/auth/token.py +++ b/microservices/gatewayApi/auth/token.py @@ -1,6 +1,6 @@ import requests import time -from authlib.jose import jwt +from authlib.jose import JsonWebToken from authlib.jose.errors import JoseError, ExpiredTokenError from authlib.oauth2.rfc6749 import TokenMixin from authlib.oauth2.rfc6750 import BearerTokenValidator @@ -47,6 +47,7 @@ def __init__(self, token_cls, realm=None): self.jwk = jwk_r.json() def authenticate_token(self, token_string): + jwt = JsonWebToken(['RS256']) token = jwt.decode(token_string, self.jwk) token.validate() diff --git a/microservices/gatewayApi/entrypoint.sh b/microservices/gatewayApi/entrypoint.sh index 001c87a..2c3fd12 100755 --- a/microservices/gatewayApi/entrypoint.sh +++ b/microservices/gatewayApi/entrypoint.sh @@ -58,4 +58,4 @@ cat > /tmp/deck.yaml </plugins', methods=['POST'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def create_consumer_plugin(namespace: str, consumer_id: str) -> object: event_id = str(uuid.uuid4()) log = app.logger @@ -34,19 +34,20 @@ def create_consumer_plugin(namespace: str, consumer_id: str) -> object: validate_tags(plugin_data, selectTag) try: + log.debug("[add_consumer_plugin] %s" % consumer_id) response = cnsr.add_consumer_plugin(consumer_id, plugin_data) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='add', namespace=namespace, message='failed to create plugin.') + action='add', namespace=namespace, message='failed to create plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'add', 'completed', namespace) return response -@admin_jwt(None) -@uma_enforce('namespace', 'Access.Manage') @consumers.route('//plugins/', methods=['PUT'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> object: event_id = str(uuid.uuid4()) log = app.logger @@ -63,33 +64,40 @@ def update_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> rate_limiting(plugin_data) validate_tags(plugin_data, selectTag) - verify_tags(consumer_id, plugin_id, selectTag) try: + log.debug("[update_consumer_plugin] %s" % consumer_id) + verify_tags(consumer_id, plugin_id, selectTag) response = cnsr.update_consumer_plugin(consumer_id, plugin_id, plugin_data) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='update', namespace=namespace, message='failed to update plugin.') + action='update', namespace=namespace, message='failed to update plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'update', 'completed', namespace) return response -@admin_jwt(None) -@uma_enforce('namespace', 'Access.Manage') @consumers.route('//plugins/', methods=['DELETE'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'Access.Manage') def delete_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> object: + log = app.logger event_id = str(uuid.uuid4()) selectTag = "ns.%s" % namespace cnsr = GatewayConsumerService() - verify_tags(consumer_id, plugin_id, selectTag) - + try: - response = cnsr.delete_consumer_plugin(consumer_id, plugin_id) + log.debug("[delete_consumer_plugin] %s %s" % (consumer_id, plugin_id)) + plugin = cnsr.get_consumer_plugin(consumer_id, plugin_id) + if plugin is None: + response = {} + else: + verify_tags(consumer_id, plugin_id, selectTag) + response = cnsr.delete_consumer_plugin(consumer_id, plugin_id) except Exception as ex: handle_exceptions(exception=ex, event_id=event_id, type='GatewayConsumerPlugin', - action='delete', namespace=namespace, message='failed to delete plugin.') + action='delete', namespace=namespace, message='failed to delete plugin') record_custom_event(event_id, 'GatewayConsumerPlugin', 'delete', 'completed', namespace) return response @@ -98,16 +106,18 @@ def delete_consumer_plugin(namespace: str, consumer_id: str, plugin_id: str) -> def handle_exceptions(**kwargs): log = app.logger if not kwargs['exception'] == None: + log.error("handle_exception of %s" % kwargs['exception']) if isinstance(kwargs['exception'], HTTPException): - log.error("%s. %s" % (kwargs['message'], kwargs['exception'])) + log.error("%s. (HTTP)" % (kwargs['message'])) else: log.error("%s. %s" % (kwargs['message'], sys.exc_info()[0])) - traceback.print_exc() + # traceback.print_exc() record_custom_event(kwargs['event_id'], kwargs['type'], kwargs['action'], 'failed', kwargs['namespace'], kwargs['message']) abort(make_response(jsonify(error=kwargs['message']), 400)) - + else: + abort(make_response(jsonify(error="unknown"), 400)) def transform_tags(data, namespace, required_tag): log = app.logger diff --git a/microservices/gatewayApi/v2/services/gateway_consumers.py b/microservices/gatewayApi/v2/services/gateway_consumers.py index b652f08..c2aa436 100644 --- a/microservices/gatewayApi/v2/services/gateway_consumers.py +++ b/microservices/gatewayApi/v2/services/gateway_consumers.py @@ -1,5 +1,5 @@ from typing import Any -from flask import make_response, jsonify, current_app as app +from flask import current_app as app import requests @@ -19,13 +19,16 @@ def delete_consumer_plugin(self, consumer_id: str, plugin_id: str): def get_consumer_plugin(self, consumer_id: str, plugin_id: str): url = app.config['kongAdminUrl'] + '/consumers/' + consumer_id + '/plugins/' + plugin_id - return requests.get(url, timeout=5).json() + res = requests.get(url, timeout=5) + if res.status_code == 404: + return None + return res.json() def make_http_request(action: str, id: str, method: str, **rqst_params): log = app.logger + message = "STARTED" - log.debug("%s %s[%s]" % (message, action, id)) mod = __import__('requests') method_to_call = getattr(mod, method) response = method_to_call(**rqst_params) @@ -33,12 +36,9 @@ def make_http_request(action: str, id: str, method: str, **rqst_params): if response.status_code in (200, 201): res = response.json() elif response.status_code == 204: - res = make_response(jsonify(message='done'), response.status_code) + res = {message:'done'} else: message = "FAILED" - try: - res = make_response(response.json(), response.status_code) - except: - res = make_response(jsonify(error='failed'), response.status_code) - log.debug("%s %s[%s]" % (message, action, id)) + log.error("%s %s : %s", response.status_code, response.reason, response.text) + raise Exception("%s %s [%s] %s" % (message, action, response.status_code, response.reason)) return res diff --git a/microservices/gatewayApi/wsgi.py b/microservices/gatewayApi/wsgi.py index 4972a46..7f06603 100644 --- a/microservices/gatewayApi/wsgi.py +++ b/microservices/gatewayApi/wsgi.py @@ -1,8 +1,8 @@ #!/usr/bin/python -from gevent import monkey +#from gevent import monkey # Patch Sockets to make requests asynchronous -monkey.patch_all() +#monkey.patch_all() import logging import sys @@ -23,7 +23,7 @@ dictConfig({ 'version': 1, 'formatters': {'default': { - 'format': '%(asctime)s %(levelname)5s %(module)-15s: %(message)s', + 'format': '%(asctime)s [%(process)3d] %(levelname)5s %(module)-15s: %(message)s', }}, 'handlers': {'wsgi': { 'class': 'logging.StreamHandler',