diff --git a/charts/tesk/service-info/service-info.yaml b/charts/tesk/service-info/service-info.yaml deleted file mode 100644 index a3dff20d..00000000 --- a/charts/tesk/service-info/service-info.yaml +++ /dev/null @@ -1,17 +0,0 @@ -id: 'uk.ac.ebi.tsc.tesk' -name: TESK -type: - group: org.ga4gh - artifact: TES - version: 1.0 -description: GA4GH TES Server implementation for Kubernetes -organization: - name: EMBL-EBI - url: https://www.ebi.ac.uk/ -contactUrl: https://github.com/EMBL-EBI-TSI/TESK/issues -documentationUrl: https://github.com/EMBL-EBI-TSI/TESK -createdAt: 2021-04-06 -environment: dev -version: 1.0 -storage: -- ftp://ftp-private.ebi.ac.uk diff --git a/charts/tesk/templates/_helpers.tpl b/charts/tesk/templates/_helpers.tpl index c6a62d3d..1e2a5d2c 100644 --- a/charts/tesk/templates/_helpers.tpl +++ b/charts/tesk/templates/_helpers.tpl @@ -57,6 +57,42 @@ secret) from its environment. {{- include "tesk.fullname" . }}-service-info {{- end -}} +{{/* +The ConfigMap with the service info that the API mounts: the operator's +(`serviceInfo.existingConfigMap`, if set, also if YAML reads it as 0 or +false) or the chart's (`tesk.serviceInfoName`, rendered only then). +*/}} +{{- define "tesk.serviceInfoConfigMapName" -}} +{{- $existing := (default dict .Values.serviceInfo).existingConfigMap -}} +{{- if include "tesk.isSet" $existing -}} +{{- toString $existing -}} +{{- else -}} +{{- include "tesk.serviceInfoName" . -}} +{{- end -}} +{{- end -}} + +{{/* +The tag of the API's image (tesk-api): `tesk.image.tag`, or the chart's +`appVersion` if it is unset or empty (not `default`, which would replace a +tag of 0). Whole numbers, which Helm reads from values files as floats, +are rendered as written (e.g., 20261110, not 2.026111e+07) up to 2^53, and +fail above, where they would be rounded (as in `tesk.yamlText`). +*/}} +{{- define "tesk.apiImageTag" -}} +{{- $tag := dig "image" "tag" "" .Values.tesk -}} +{{- if or (kindIs "invalid" $tag) (eq (toString $tag) "") -}} +{{- $tag = .Chart.AppVersion -}} +{{- end -}} +{{- if and (kindIs "float64" $tag) (eq (floor $tag) $tag) -}} +{{- if not (and (lt $tag 9007199254740992.0) (gt $tag -9007199254740992.0)) -}} +{{- fail "tesk.image.tag is a number too large to be exact: quote it in the values file or use --set-string" -}} +{{- end -}} +{{- printf "%.0f" $tag -}} +{{- else -}} +{{- toString $tag -}} +{{- end -}} +{{- end -}} + {{- define "tesk.oauthClientName" -}} {{- include "tesk.fullname" . }}-oauth-client {{- end -}} @@ -226,6 +262,14 @@ The notes that `helm install` and `helm upgrade` print (`NOTES.txt`). {{- define "tesk.notes" -}} TESK is deployed as release {{ .Release.Name }} in namespace {{ .Release.Namespace }}. See https://tesk.readthedocs.io/en/latest/deployment/ to check the installation. +{{- $serviceInfo := default dict .Values.serviceInfo -}} +{{- if and (not (include "tesk.isSet" $serviceInfo.existingConfigMap)) (eq (toString $serviceInfo.id) "uk.ac.ebi.tsc.tesk") }} + +NOTE: serviceInfo.id is the chart's default, so GET /service-info describes +no particular deployment. Set serviceInfo.id, serviceInfo.organization, +serviceInfo.contactUrl and serviceInfo.environment for yours; see +https://tesk.readthedocs.io/en/latest/configuration/#service-info-serviceinfo +{{- end }} {{- /* Any user name, also one that YAML reads as 0 or false */ -}} {{- $username := .Values.ftp.username -}} {{- $hasUsername := include "tesk.isSet" $username -}} @@ -316,6 +360,114 @@ reached only within a trusted network (e.g., a provider in the cluster). {{- end }} {{- end -}} +{{/* +The service info (`serviceInfo`) as the YAML file that tesk-api reads +(`TesServiceInfo`), or nothing with `serviceInfo.existingConfigMap`. Text +fields are quoted (`tesk.yamlText`), so that tesk-api reads them as +text whatever they look like; `createdAt` and `updatedAt` are YAML +timestamps, unquoted, as tesk-api reads them as dates. Empty optional +fields are left out; the fields that the GA4GH service-info specification +requires fail if empty. `type` (TES 1.0.0, which tesk-api implements) and +`version` (the tag of tesk-api's image) are set here. +*/}} +{{- define "tesk.serviceInfo" -}} +{{- $info := default dict .Values.serviceInfo -}} +{{- if not (include "tesk.isSet" $info.existingConfigMap) -}} +{{- $organization := default dict $info.organization -}} +{{- range $field, $value := dict "id" $info.id "name" $info.name "organization.name" $organization.name "organization.url" $organization.url -}} +{{- if not (include "tesk.isSet" $value) -}} +{{- fail (printf "serviceInfo.%s must be set: the GA4GH service-info specification requires it" $field) -}} +{{- end -}} +{{- end -}} +{{- /* YAML 1.1 timestamps, as SnakeYAML resolves them (two-digit month +and day without a time); the same pattern as `$defs/timestamp` in +values.schema.json, which checks them first */ -}} +{{- $timestamp := "^([0-9]{4}-[0-9]{2}-[0-9]{2}|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}([Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(\\.[0-9]*)?([ \t]*(Z|[-+][0-9]{1,2}(:[0-9]{2})?))?)$" -}} +{{- range $field := list "createdAt" "updatedAt" -}} +{{- $value := index $info $field -}} +{{- if include "tesk.isSet" $value -}} +{{- $text := toString $value -}} +{{- /* Dates and times that don't exist (e.g., 2021-02-30, 24:00:00), +which SnakeYAML would read as other ones, fail too: Go's parser rejects +them, and `toDate` then gives the zero time. The date part has no colons, +so the first match of the time pattern is the time */ -}} +{{- $valid := regexMatch $timestamp $text -}} +{{- if $valid -}} +{{- $valid = not (toDate "2006-1-2" (regexFind "^[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}" $text)).IsZero -}} +{{- end -}} +{{- $time := regexFind "[0-9]{1,2}:[0-9]{2}:[0-9]{2}" $text -}} +{{- if and $valid $time -}} +{{- $valid = not (toDate "15:04:05" $time).IsZero -}} +{{- end -}} +{{- if not $valid -}} +{{- fail (printf "serviceInfo.%s must be a date (e.g., 2026-11-10) or a date and time (e.g., 2026-11-10T12:00:00Z), not %q" $field $text) -}} +{{- end -}} +{{- end -}} +{{- end -}} +id: {{ include "tesk.yamlText" (dict "value" $info.id "key" "serviceInfo.id") }} +name: {{ include "tesk.yamlText" (dict "value" $info.name "key" "serviceInfo.name") }} +type: + group: "org.ga4gh" + artifact: "TES" + version: "1.0.0" +{{- if include "tesk.isSet" $info.description }} +description: {{ include "tesk.yamlText" (dict "value" $info.description "key" "serviceInfo.description") }} +{{- end }} +organization: + name: {{ include "tesk.yamlText" (dict "value" $organization.name "key" "serviceInfo.organization.name") }} + url: {{ include "tesk.yamlText" (dict "value" $organization.url "key" "serviceInfo.organization.url") }} +{{- range $field := list "contactUrl" "documentationUrl" }} +{{- $value := index $info . }} +{{- if include "tesk.isSet" $value }} +{{ $field }}: {{ include "tesk.yamlText" (dict "value" $value "key" (printf "serviceInfo.%s" $field)) }} +{{- end }} +{{- end }} +{{- range $field := list "createdAt" "updatedAt" }} +{{- $value := index $info . }} +{{- if include "tesk.isSet" $value }} +{{ $field }}: {{ toString $value }} +{{- end }} +{{- end }} +{{- if include "tesk.isSet" $info.environment }} +environment: {{ include "tesk.yamlText" (dict "value" $info.environment "key" "serviceInfo.environment") }} +{{- end }} +version: {{ include "tesk.apiImageTag" . | toJson }} +{{- with $info.storage }} +storage: +{{- range . }} +- {{ include "tesk.yamlText" (dict "value" . "key" "serviceInfo.storage") }} +{{- end }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +A value as a quoted YAML string (a JSON string, which YAML reads as a +double-quoted scalar, with DEL and NEL escaped too, which JSON leaves as +they are and Helm's YAML parser rejects). Other characters that YAML +doesn't allow and JSON doesn't escape (the other C1 controls, U+FFFE and +U+FFFF) fail with the value's key. Whole numbers, which Helm reads from values files as floats, +are rendered as written (e.g., 12345678, not 1.2345678e+07) up to 2^53, +and fail above, where they would be rounded: quote them in the values +file. Decimals lose trailing zeros (1.10 is 1.1). + +Takes the value (`value`) and its key (`key`), for the errors. +*/}} +{{- define "tesk.yamlText" -}} +{{- $value := .value -}} +{{- $text := toString $value -}} +{{- if and (kindIs "float64" $value) (eq (floor $value) $value) -}} +{{- if not (and (lt $value 9007199254740992.0) (gt $value -9007199254740992.0)) -}} +{{- fail (printf "%s is a number too large to be exact: quote it in the values file or use --set-string" .key) -}} +{{- end -}} +{{- $text = printf "%.0f" $value -}} +{{- end -}} +{{- if regexMatch "[\u0080-\u0084\u0086-\u009f\ufffe\uffff]" $text -}} +{{- fail (printf "%s has a control character that YAML doesn't allow (U+0080 to U+009F but U+0085, U+FFFE or U+FFFF)" .key) -}} +{{- end -}} +{{- toJson $text | replace "\u007f" "\\u007f" | replace "\u0085" "\\u0085" -}} +{{- end -}} + {{/* Whole seconds from a value under `tesk` (`name`), with a `minimum` (0 or 1), as the taskmaster reads them: empty if unset (null or empty); fails on diff --git a/charts/tesk/templates/common/service-info-cm.yaml b/charts/tesk/templates/common/service-info-cm.yaml index b19c49ce..a12a47de 100644 --- a/charts/tesk/templates/common/service-info-cm.yaml +++ b/charts/tesk/templates/common/service-info-cm.yaml @@ -1,8 +1,13 @@ +{{- /* The service info from `serviceInfo`, unless an existing ConfigMap +replaces it (`serviceInfo.existingConfigMap`) */ -}} +{{- with include "tesk.serviceInfo" . }} apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "tesk.serviceInfoName" . }} + name: {{ include "tesk.serviceInfoName" $ }} labels: - {{- include "tesk.labels" . | nindent 4 }} + {{- include "tesk.labels" $ | nindent 4 }} data: -{{ (.Files.Glob "service-info/*").AsConfig | indent 2 }} + service-info.yaml: | + {{- . | nindent 4 }} +{{- end }} diff --git a/charts/tesk/templates/common/tesk-deployment.yaml b/charts/tesk/templates/common/tesk-deployment.yaml index e494009c..e28053db 100644 --- a/charts/tesk/templates/common/tesk-deployment.yaml +++ b/charts/tesk/templates/common/tesk-deployment.yaml @@ -6,13 +6,11 @@ whole chart */ -}} {{- include "tesk.checkImageReleases" . -}} {{- $tesk := .Values.tesk -}} {{- $repository := dig "image" "repository" "" $tesk | default "docker.io/elixircloud/tesk-api" -}} -{{- $tag := dig "image" "tag" "" $tesk -}} -{{- /* Not `default`, which would replace a tag of 0 */ -}} -{{- if or (kindIs "invalid" $tag) (eq (toString $tag) "") -}} -{{- $tag = .Chart.AppVersion -}} -{{- end -}} +{{- /* Also the version of the service info */ -}} +{{- $tag := include "tesk.apiImageTag" . -}} {{- $ftpSecretName := include "tesk.ftpSecretName" . -}} {{- $executorSecurityContext := include "tesk.executorSecurityContext" . -}} +{{- $serviceInfo := include "tesk.serviceInfo" . -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -28,10 +26,16 @@ spec: metadata: labels: {{- include "tesk.selectorLabels" . | nindent 8 }} - {{- /* tesk-api reads the ConfigMap only at startup */}} - {{- with $executorSecurityContext }} + {{- /* tesk-api reads the executors' ConfigMap only at startup, and + the service info at startup and then every hour */}} + {{- if or $executorSecurityContext $serviceInfo }} annotations: + {{- with $executorSecurityContext }} checksum/executor-config: {{ sha256sum . }} + {{- end }} + {{- with $serviceInfo }} + checksum/service-info: {{ sha256sum . }} + {{- end }} {{- end }} spec: serviceAccountName: {{ include "tesk.taskmasterName" . }} @@ -226,7 +230,7 @@ spec: volumes: - name: service-info configMap: - name: {{ include "tesk.serviceInfoName" . }} + name: {{ include "tesk.serviceInfoConfigMapName" . | quote }} {{- with $tesk.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/tesk/values.schema.json b/charts/tesk/values.schema.json index 5b72e917..75c0c77b 100644 --- a/charts/tesk/values.schema.json +++ b/charts/tesk/values.schema.json @@ -47,6 +47,14 @@ }, "switch": { "type": "boolean" + }, + "timestamp": { + "type": [ + "string", + "null" + ], + "pattern": "^([0-9]{4}-[0-9]{2}-[0-9]{2}|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}([Tt]|[ \\t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}(\\.[0-9]*)?([ \\t]*(Z|[-+][0-9]{1,2}(:[0-9]{2})?))?)?$", + "$comment": "A YAML 1.1 timestamp, as tesk-api's SnakeYAML resolves it (two-digit month and day without a time), or empty; the same pattern as in tesk.serviceInfo (templates/_helpers.tpl)" } }, "properties": { @@ -532,6 +540,83 @@ }, "additionalProperties": false }, + "serviceInfo": { + "type": [ + "object", + "null" + ], + "description": "Service info, the answer of GET /service-info (GA4GH service-info specification); type and version are set by the chart", + "additionalProperties": false, + "properties": { + "existingConfigMap": { + "type": [ + "string", + "null" + ], + "description": "Name of a ConfigMap of your own with the key service-info.yaml, instead of the chart's; the other fields are then not used, but still checked" + }, + "id": { + "$ref": "#/$defs/scalar", + "description": "Unique ID of the deployment, in reverse domain notation (required)" + }, + "name": { + "$ref": "#/$defs/scalar", + "description": "Name of the service (required)" + }, + "description": { + "$ref": "#/$defs/scalar", + "description": "Description of the service" + }, + "organization": { + "type": [ + "object", + "null" + ], + "description": "The organisation that runs the service", + "additionalProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/scalar", + "description": "Name of the organisation (required)" + }, + "url": { + "$ref": "#/$defs/scalar", + "description": "URL of the organisation (required)" + } + } + }, + "contactUrl": { + "$ref": "#/$defs/scalar", + "description": "Where users get support, e.g., a mailto: URL" + }, + "documentationUrl": { + "$ref": "#/$defs/scalar", + "description": "Documentation for users of the service" + }, + "createdAt": { + "$ref": "#/$defs/timestamp", + "description": "When the service was created: a date (2026-11-10) or a date and time (2026-11-10T12:00:00Z)" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp", + "description": "When the service was last updated: a date or a date and time" + }, + "environment": { + "$ref": "#/$defs/scalar", + "description": "Environment of the service, e.g., test or prod" + }, + "storage": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Storage locations that tasks can use, e.g., s3://bucket" + } + } + }, "service": { "type": "object", "properties": { diff --git a/charts/tesk/values.yaml b/charts/tesk/values.yaml index bc1ed6b2..9f6a80c2 100644 --- a/charts/tesk/values.yaml +++ b/charts/tesk/values.yaml @@ -246,6 +246,42 @@ auth: # "openid:Standard openid,eduperson_entitlement:Access to groups membership" scopes: "" +# Service info, the answer of GET /service-info (GA4GH service-info +# specification), rendered into a ConfigMap. The defaults describe no +# particular deployment: set at least `id`, `organization`, `contactUrl` and +# `environment` for yours. The chart sets `type` (TES 1.0.0) and `version` +# (the tag of tesk-api's image) itself. Empty optional fields are left out +serviceInfo: + # Name of a ConfigMap of your own with the key `service-info.yaml`, in the + # release's namespace, instead of the chart's; the other fields are then + # not used, but still checked against the schema + existingConfigMap: "" + # Unique ID of this deployment, in reverse domain notation, e.g., + # org.example.tes (required). The notes remind you while it is this + # default (`tesk.notes` in templates/_helpers.tpl). Quote text that YAML + # would read as a number (e.g., "1.10") + id: uk.ac.ebi.tsc.tesk + # Name of the service (required) + name: TESK + description: GA4GH TES Server implementation for Kubernetes + # The organisation that runs the service (both required) + organization: + name: EMBL-EBI + url: https://www.ebi.ac.uk/ + # Where users get support (a URL, e.g., mailto:tes@example.org) + contactUrl: https://github.com/elixir-cloud-aai/TESK/issues + # Documentation for users of the service + documentationUrl: https://tesk.readthedocs.io/ + # When the service was created and last updated: a date with two-digit + # month and day (2026-11-10) or a date and time (2026-11-10T12:00:00Z) + createdAt: "2021-04-06" + updatedAt: "" + # e.g., test, staging or prod + environment: dev + # Storage locations that tasks can use, e.g., [s3://bucket] + storage: + - ftp://ftp-private.ebi.ac.uk + service: # Type of the API's Service: `NodePort` to expose the API directly, # `LoadBalancer` on clusters of cloud providers, or empty for `ClusterIP` diff --git a/docs/configuration.md b/docs/configuration.md index 33a63c10..c6642ba3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -139,6 +139,62 @@ The values of the provider and the group model (`auth.userInfoUri`, set: empty or null values leave tesk-api's defaults (see [Access control](tesk-api/access-control.md#configuration) of tesk-api). +### Service info (`serviceInfo`) + +`GET /service-info` answers with the service info of the deployment, in +the format of the [GA4GH service-info +specification](https://github.com/ga4gh-discovery/ga4gh-service-info), +which registries and clients read to find out what the service is and who +runs it. The chart renders it from these values into a ConfigMap (see +[Names of the objects](#names-of-the-objects)), which it mounts into the +API. The defaults describe no particular deployment, so set at least `id`, +`organization`, `contactUrl` and `environment` for yours; the notes of +`helm install` remind you while `id` is the default. + +| Value | Default | Description | +| --- | --- | --- | +| `serviceInfo.id` | `uk.ac.ebi.tsc.tesk` | Unique ID of the deployment, in reverse domain notation, e.g., `org.example.tes`. Required. | +| `serviceInfo.name` | `TESK` | Name of the service. Required. | +| `serviceInfo.description` | `GA4GH TES Server implementation for Kubernetes` | Description of the service. | +| `serviceInfo.organization.name` | `EMBL-EBI` | Name of the organisation that runs the service. Required. | +| `serviceInfo.organization.url` | `https://www.ebi.ac.uk/` | URL of the organisation. Required. | +| `serviceInfo.contactUrl` | `https://github.com/elixir-cloud-aai/TESK/issues` | Where users get support, e.g., `mailto:tes@example.org`. | +| `serviceInfo.documentationUrl` | `https://tesk.readthedocs.io/` | Documentation for users of the service. | +| `serviceInfo.createdAt` | `"2021-04-06"` | When the service was created: a date (`2026-11-10`, with two-digit month and day) or a date and time (`2026-11-10T12:00:00Z`), as YAML timestamps; other text fails the installation, as tesk-api couldn't read it. | +| `serviceInfo.updatedAt` | `""` | When the service was last updated, in the same form. | +| `serviceInfo.environment` | `dev` | Environment of the service, e.g., `test`, `staging` or `prod`. | +| `serviceInfo.storage` | `[ftp://ftp-private.ebi.ac.uk]` | Storage locations that tasks can use, e.g., `[s3://bucket]`. | +| `serviceInfo.existingConfigMap` | `""` | Name of a ConfigMap of your own, in the release's namespace, with the key `service-info.yaml`, instead of the chart's; the fields above are then not used, but still checked against the schema. | + +Empty optional fields are left out of the service info; the required ones +fail the installation if they are empty. The chart passes text on as text, +but YAML reads unquoted numbers in values files as numbers first: quote +text that looks like a number (e.g., `id: "1.10"`, which would otherwise +become `1.1`). The chart sets two fields itself: +`type` (`org.ga4gh`, `TES`, version `1.0.0`, which tesk-api implements) and +`version` (the tag of tesk-api's image, `tesk.image.tag`, by default the +chart's `appVersion`). A change of the service info restarts the API's pod +(tesk-api reads the file at startup, and then only every hour); with +`serviceInfo.existingConfigMap`, changes of your ConfigMap take effect within +an hour, or when you restart the pod. Service info is public by default, +also with `auth.mode: auth` (see `auth.serviceInfoPublic`). + +```yaml +serviceInfo: + id: org.example.tes + name: Example TES + description: TES of Example for the members of its research groups + organization: + name: Example + url: https://example.org/ + contactUrl: mailto:tes@example.org + documentationUrl: https://example.org/tes/ + createdAt: "2026-11-10" + environment: prod + storage: + - s3://example-tes +``` + ### Service (`service`) | Value | Default | Description | @@ -195,7 +251,7 @@ and a `nameOverride` with capital letters, fail the installation unless | --- | --- | --- | --- | | Deployment and Service of the API | `-api` | `tesk-api` | `prod-tesk-api` | | ServiceAccount, Role and RoleBinding of the API and the taskmasters | `-taskmaster` | `tesk-taskmaster` | `prod-tesk-taskmaster` | -| ConfigMap with the service info | `-service-info` | `tesk-service-info` | `prod-tesk-service-info` | +| ConfigMap with the service info (none with `serviceInfo.existingConfigMap`) | `-service-info` | `tesk-service-info` | `prod-tesk-service-info` | | ConfigMap with the executors' security context (`tesk.executor.securityContext`) | `-executor` | `tesk-executor` | `prod-tesk-executor` | | Secret with the OIDC client (`auth.mode: auth`) | `-oauth-client` | `tesk-oauth-client` | `prod-tesk-oauth-client` | | Secret with the FTP credentials | `ftp.classicFtpSecret`, or `-ftp` if it is null (none if it is empty or `false`) | `tesk-ftp` | `prod-tesk-ftp` | @@ -234,16 +290,6 @@ of Helm charts (`app.kubernetes.io/version`, `app.kubernetes.io/managed-by`, ## Files -### Service info - -The response of `GET /service-info` comes from -`charts/tesk/service-info/service-info.yaml`, mounted into the API. Edit it -for your deployment before installing: at least `id` (unique, in reverse -domain notation), `name`, `organization`, `contactUrl` and `environment`. -See the [TES -specification](https://github.com/ga4gh/task-execution-schemas) for its -fields. - ### Secrets | File | For | diff --git a/docs/deployment.md b/docs/deployment.md index 0bae0e73..9fb16447 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -51,10 +51,12 @@ helm upgrade --install tesk charts/tesk \ FTP account (`ftp.username`, `ftp.password`) and the OIDC client of the Swagger UI (`auth.clientId`). Keep it out of version control. It is optional: leave out `-f secrets.yaml` if you need none of these. +- Describe your deployment in the service info that `GET /service-info` + answers with (`serviceInfo` in `my-values.yaml`, see [Service + info](configuration.md#service-info-serviceinfo)): at least its `id`, + `organization`, `contactUrl` and `environment`. - Some settings are files in the chart directory rather than values: the - service info (`charts/tesk/service-info/service-info.yaml`, see [Service - info](configuration.md#service-info)), the `.netrc` file for FTP - (`charts/tesk/ftp/.netrc`) and the S3 configuration + `.netrc` file for FTP (`charts/tesk/ftp/.netrc`) and the S3 configuration (`charts/tesk/s3-config/`). Edit or add them before installing. The names of the objects start with the release name (e.g., `tesk-api` and diff --git a/docs/releases/0.3.0.md b/docs/releases/0.3.0.md index 6cee0204..e79bdd1b 100644 --- a/docs/releases/0.3.0.md +++ b/docs/releases/0.3.0.md @@ -10,6 +10,7 @@ TESK 0.3.0 is the first release of the TESK Helm chart since 0.1.1 (2024), and t - Default images tesk-api 1.2.0 and taskmaster and filer 0.11.0 (tagged without `v` from now on): the Swagger UI is off unless `tesk.swaggerUi.enabled` is `true`, and the filer sends FTP credentials only to `ftp.hosts`; the filer fails outputs with symbolic links, and the taskmaster and filer tags must be the same release (the installation fails on different releases from 0.11.0 on); see also the upgrade notes of tesk-api and tesk-core (#273, #278) - With `auth.mode: auth`, tasks created before the upgrade to tesk-api 1.2.0, which records task creators without loss in other labels, no longer belong to their creators; the new value `auth.legacyUserLabels` keeps them theirs for a transition, and the notes warn if it is on with an older tesk-api (#271, #275) - Charts are released from signed `vX.Y.Z` tags as files attached to GitHub releases; the Helm repository at `https://elixir-cloud-aai.github.io/TESK/` keeps 0.1.0 and 0.1.1 but gets no new versions (#222, #234, #240, #241) +- The service info comes from the values `serviceInfo.*`, and the file `charts/tesk/service-info/service-info.yaml` is removed: move edits of it into values, or into a ConfigMap of your own (`serviceInfo.existingConfigMap`); the defaults describe no deployment, and point `contactUrl` and `documentationUrl` to this repository and its docs (#282, #285) ## Features @@ -21,6 +22,7 @@ TESK 0.3.0 is the first release of the TESK Helm chart since 0.1.1 (2024), and t - `tesk.filerHttpTimeout` sets the timeout of the filer's HTTP(S) requests (#259, #260) - `tesk.swaggerUi.enabled` turns the Swagger UI on or off (#243, #245) - `auth.serviceInfoPublic` controls whether `GET /service-info` needs a token (#216, #217) +- `serviceInfo.*` sets `GET /service-info` when installing a packaged chart; the chart sets `type` and `version` (the tesk-api tag), and the API restarts when the service info changes (#282, #285) ## Security diff --git a/docs/upgrading.md b/docs/upgrading.md index b52ef54a..9e188686 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -436,6 +436,51 @@ too: killed (OOM), and its CPU is throttled. Tasks that relied on using more than they requested must request more. +### Service info from values + +The service info comes from the values under `serviceInfo` now, so that it +can be set when installing a packaged chart (see [Service +info](configuration.md#service-info-serviceinfo)). The file +`charts/tesk/service-info/service-info.yaml` is gone, and the chart no +longer reads it. If you edited that file, move its fields into your values +file: + +```yaml +serviceInfo: + id: org.example.tes + name: TESK + description: TES of Example + organization: + name: Example + url: https://example.org/ + contactUrl: mailto:tes@example.org + documentationUrl: https://example.org/tes/ + createdAt: "2026-11-10" + environment: prod + storage: + - s3://example-tes +``` + +To keep managing the file yourself instead, create a ConfigMap from it +(`kubectl create configmap my-service-info -n tesk +--from-file=service-info.yaml`) and set `serviceInfo.existingConfigMap: +my-service-info`. + +The defaults are those of the old file, except for these fields: + +- `contactUrl` and `documentationUrl` point to this repository's issues + and to these docs, not to the former repository. +- The chart now sets `type.version` to `1.0.0` (the TES version that + tesk-api implements; was `1.0`) and `version` to the tag of tesk-api's + image (`tesk.image.tag`, by default the chart's `appVersion`; was `1.0`). + Values can't override either. + +All other defaults are unchanged, including `id`, `organization` (EMBL-EBI), +`environment: dev` and `storage`: a release installed without your own +`serviceInfo` still describes no particular deployment. The notes of `helm +install` remind you to describe yours while `serviceInfo.id` is the default. A change of the service info now restarts +the API's pod, so that it takes effect at once. + ### Public service info The new value `auth.serviceInfoPublic` (default `true`) controls whether diff --git a/tests/test_chart.py b/tests/test_chart.py index 4cbfd255..8e14a96f 100644 --- a/tests/test_chart.py +++ b/tests/test_chart.py @@ -10,6 +10,7 @@ from __future__ import annotations import base64 +import datetime import hashlib import json import os @@ -731,7 +732,8 @@ def test_security_context_null() -> None: def test_executor_security_context_default(args: tuple[str, ...]) -> None: """Without a security context, executors run as before. - No ConfigMap, no `CONFIGMAP`, and no checksum annotation. + No ConfigMap, no `CONFIGMAP`, and no checksum annotation of the + executors. Args: args: The `helm template` arguments. @@ -739,8 +741,8 @@ def test_executor_security_context_default(args: tuple[str, ...]) -> None: manifests = render(*args) assert find(manifests, "ConfigMap", "tesk-executor") is None assert "CONFIGMAP" not in env(manifests) - annotations = deployment(manifests)["spec"]["template"]["metadata"] - assert "annotations" not in annotations + metadata = deployment(manifests)["spec"]["template"]["metadata"] + assert "checksum/executor-config" not in metadata.get("annotations", {}) @pytest.mark.parametrize( @@ -1358,10 +1360,10 @@ def test_swagger_ui_invalid(args: tuple[str, ...]) -> None: def test_service_info() -> None: - """The service info is mounted from the chart's `service-info/`.""" + """The service info is rendered from the values into a ConfigMap.""" manifests = render() config = get(manifests, "ConfigMap", "tesk-service-info") - assert "service-info.yaml" in config["data"] + assert list(config["data"]) == ["service-info.yaml"] assert container(manifests)["volumeMounts"] == [ {"name": "service-info", "mountPath": "/etc/tesk/service-info"} ] @@ -1369,6 +1371,430 @@ def test_service_info() -> None: assert volumes == [ {"name": "service-info", "configMap": {"name": "tesk-service-info"}} ] + assert ( + env(manifests)["TESK_API_SERVICE_INFO_LOCATION"] + == "file:///etc/tesk/service-info/service-info.yaml" + ) + + +def _service_info(*args: str) -> tuple[dict[str, Any], str]: + """Render the service info. + + Args: + args: More `helm template` arguments. + + Returns: + The service info, parsed, and its text. + """ + config = get(render(*args), "ConfigMap", "tesk-service-info") + text: str = config["data"]["service-info.yaml"] + info: dict[str, Any] = yaml.safe_load(text) + return info, text + + +def test_service_info_defaults() -> None: + """The defaults describe EMBL-EBI's deployment, as the file of 0.1.x did. + + Except for the contact and documentation URLs, which point to this + repository and its docs, and `type` and `version`, which the chart sets + (`version` to the tag of the API's image). + """ + manifests = render() + assert container(manifests)["image"].endswith(f":{METADATA['appVersion']}") + info, text = _service_info() + assert info == { + "id": "uk.ac.ebi.tsc.tesk", + "name": "TESK", + "type": {"group": "org.ga4gh", "artifact": "TES", "version": "1.0.0"}, + "description": "GA4GH TES Server implementation for Kubernetes", + "organization": {"name": "EMBL-EBI", "url": "https://www.ebi.ac.uk/"}, + "contactUrl": "https://github.com/elixir-cloud-aai/TESK/issues", + "documentationUrl": "https://tesk.readthedocs.io/", + "createdAt": datetime.date(2021, 4, 6), + "environment": "dev", + "version": str(METADATA["appVersion"]), + "storage": ["ftp://ftp-private.ebi.ac.uk"], + } + # A YAML timestamp, as tesk-api reads `createdAt` (a quoted string fails) + assert "\ncreatedAt: 2021-04-06\n" in text + + +@pytest.mark.parametrize("tag", ["1.2.1", "dev", "0"]) +def test_service_info_version(tag: str) -> None: + """The version of the service info is the tag of the API's image. + + Args: + tag: `tesk.image.tag`. + """ + manifests = render("--set-string", f"tesk.image.tag={tag}") + assert container(manifests)["image"].endswith(f":{tag}") + config = get(manifests, "ConfigMap", "tesk-service-info") + assert yaml.safe_load(config["data"]["service-info.yaml"])["version"] == tag + + +def test_service_info_version_number(tmp_path: Path) -> None: + """A numeric tag from a values file is the image's tag and version as written. + + Args: + tmp_path: A directory for the values file. + """ + path = tmp_path / "values.yaml" + path.write_text("tesk:\n image:\n tag: 20261110\n") + manifests = render("--values", str(path)) + assert container(manifests)["image"].endswith(":20261110") + config = get(manifests, "ConfigMap", "tesk-service-info") + info = yaml.safe_load(config["data"]["service-info.yaml"]) + assert info["version"] == "20261110" + + +def test_image_tag_number_too_large(tmp_path: Path) -> None: + """A numeric tag that a float can't hold exactly fails with its key. + + Helm would otherwise render a rounded tag, for the image and the + version of the service info. + + Args: + tmp_path: A directory for the values file. + """ + path = tmp_path / "values.yaml" + path.write_text("tesk:\n image:\n tag: 12345678901234567890\n") + with pytest.raises(subprocess.CalledProcessError) as error: + render("--values", str(path)) + assert "tesk.image.tag is a number too large" in error.value.stderr + + +@pytest.mark.parametrize("character", ["\\u0081", "\\u009f", "\\ufffe"]) +def test_service_info_control_characters(character: str) -> None: + """Characters that YAML doesn't allow, and JSON doesn't escape, fail. + + Args: + character: The character, as a JSON escape. + """ + with pytest.raises(subprocess.CalledProcessError) as error: + render("--set-json", f'serviceInfo.description="a{character}b"') + assert "serviceInfo.description has a control character" in error.value.stderr + + +def test_service_info_values(tmp_path: Path) -> None: + """All fields of the service info come from the values. + + Args: + tmp_path: A directory for the values file. + """ + service_info: dict[str, Any] = { + "id": "org.example.tes", + "name": "Example TES", + "description": 'Text with "quotes", # and: colons', + "organization": {"name": "Example", "url": "https://example.org/"}, + "contactUrl": "mailto:tes@example.org", + "documentationUrl": "https://example.org/docs", + "createdAt": "2026-11-10T12:00:00Z", + "updatedAt": "2026-11-11", + "environment": "prod", + "storage": ["s3://bucket", "ftp://ftp.example.org"], + } + values = {"tesk": {"image": {"tag": "1.2.1"}}, "serviceInfo": service_info} + path = tmp_path / "values.yaml" + path.write_text(yaml.safe_dump(values)) + info, text = _service_info("--values", str(path)) + expected = dict(service_info) + expected["createdAt"] = datetime.datetime(2026, 11, 10, 12, tzinfo=datetime.UTC) + expected["updatedAt"] = datetime.date(2026, 11, 11) + expected["type"] = {"group": "org.ga4gh", "artifact": "TES", "version": "1.0.0"} + expected["version"] = "1.2.1" + assert info == expected + assert "\ncreatedAt: 2026-11-10T12:00:00Z\n" in text + + +@pytest.mark.parametrize( + "args", + [ + ("--set", "serviceInfo.description="), + ("--set", "serviceInfo.description=null"), + ], +) +def test_service_info_optional(args: tuple[str, ...]) -> None: + """Optional fields that are empty or null are left out. + + Args: + args: More `helm template` arguments. + """ + info, _ = _service_info( + *args, + "--set", + "serviceInfo.contactUrl=", + "--set", + "serviceInfo.documentationUrl=null", + "--set", + "serviceInfo.createdAt=", + "--set", + "serviceInfo.environment=", + "--set-json", + "serviceInfo.storage=[]", + ) + for field in ( + "description", + "contactUrl", + "documentationUrl", + "createdAt", + "updatedAt", + "environment", + "storage", + ): + assert field not in info, field + assert set(info) == {"id", "name", "type", "organization", "version"} + + +@pytest.mark.parametrize( + ("field", "args", "value"), + [ + ("id", ("--set", "serviceInfo.id=12345"), "12345"), + ("name", ("--set", "serviceInfo.name=true"), "true"), + ( + "environment", + ("--set-string", "serviceInfo.environment=2026-11-10"), + "2026-11-10", + ), + ("name", ("--set-json", 'serviceInfo.name="a\\u007fb\\u0085c"'), "a\x7fb\x85c"), + ("description", ("--set-json", 'serviceInfo.description="a\\nb"'), "a\nb"), + ], +) +def test_service_info_strings(field: str, args: tuple[str, ...], value: str) -> None: + """Text fields stay strings, also if YAML would read them otherwise. + + tesk-api reads them as strings; a date would fail as one. Characters that + YAML doesn't allow as they are (DEL, NEL) are escaped. + + Args: + field: The field of the service info. + args: More `helm template` arguments. + value: The expected text. + """ + info, _ = _service_info(*args) + assert isinstance(info[field], str) + assert info[field] == value + + +def test_service_info_numbers_values_file(tmp_path: Path) -> None: + """Whole numbers from a values file are rendered as written, up to 2^53. + + Args: + tmp_path: A directory for the values files. + """ + path = tmp_path / "values.yaml" + path.write_text("serviceInfo:\n id: 12345678\n") + info, _ = _service_info("--values", str(path)) + assert info["id"] == "12345678" + path.write_text("serviceInfo:\n id: 12345678901234567890\n") + with pytest.raises(subprocess.CalledProcessError) as error: + render("--values", str(path)) + assert "serviceInfo.id is a number too large" in error.value.stderr + + +@pytest.mark.parametrize( + "field", + ["id", "name", "organization.name", "organization.url"], +) +def test_service_info_required(field: str) -> None: + """The fields that the service info specification requires must be set. + + Args: + field: The field under `serviceInfo`. + """ + with pytest.raises(subprocess.CalledProcessError) as error: + render("--set", f"serviceInfo.{field}=") + assert f"serviceInfo.{field} must be set" in error.value.stderr + + +@pytest.mark.parametrize( + "args", + [ + ("--set", "serviceInfo.createdAt=yesterday"), + ("--set", "serviceInfo.updatedAt=2026-13-01x"), + ("--set", "serviceInfo.createdAt=10.11.2026"), + # YAML 1.1 reads dates with one-digit months or days as text + ("--set", "serviceInfo.createdAt=2026-1-5"), + ("--set", "serviceInfo.updatedAt=2026-11-1"), + ("--set", "serviceInfo.existingConfigMap=false"), + ("--set-json", "serviceInfo.storage=[1]"), + ("--set-json", "serviceInfo.organization=[]"), + ("--set-json", "serviceInfo.storage={}"), + ("--set", "serviceInfo.organization.email=a@example.org"), + ("--set", "serviceInfo.type.group=org.example"), + ("--set", "serviceInfo.version=2"), + ], +) +def test_service_info_invalid(args: tuple[str, ...]) -> None: + """The schema rejects dates that aren't YAML timestamps, and unknown keys. + + tesk-api can't read the file otherwise, and answers without service + info; `type` and `version` are set by the chart. + + Args: + args: More `helm template` arguments. + """ + with pytest.raises(subprocess.CalledProcessError) as error: + render(*args) + assert "values don't meet the specifications of the schema" in error.value.stderr + + +@pytest.mark.parametrize("value", ["today", "2026-1-5"]) +def test_service_info_invalid_without_schema(value: str) -> None: + """The templates check the dates too, also without the schema. + + Args: + value: `serviceInfo.createdAt`. + """ + with pytest.raises(subprocess.CalledProcessError) as error: + render("--skip-schema-validation", "--set", f"serviceInfo.createdAt={value}") + assert "serviceInfo.createdAt must be a date" in error.value.stderr + + +@pytest.mark.parametrize( + "value", + [ + "2021-02-30", + "2021-13-01", + "2021-2-30T12:00:00Z", + "2021-04-06T24:00:00Z", + "2021-04-06T12:60:00Z", + "2021-04-06T12:00:60Z", + ], +) +def test_service_info_impossible_dates(value: str) -> None: + """Dates and times that don't exist fail, though they look like timestamps. + + SnakeYAML would read them as other dates (e.g., 2021-02-30 as + 2021-03-02). + + Args: + value: `serviceInfo.createdAt`. + """ + with pytest.raises(subprocess.CalledProcessError) as error: + render("--set-string", f"serviceInfo.createdAt={value}") + assert "serviceInfo.createdAt must be a date" in error.value.stderr + + +def test_service_info_existing_config_map_schema() -> None: + """With an existing ConfigMap, the other fields are still checked.""" + with pytest.raises(subprocess.CalledProcessError) as error: + render( + "--set", + "serviceInfo.existingConfigMap=my-service-info", + "--set", + "serviceInfo.createdAt=today", + ) + assert "values don't meet the specifications of the schema" in error.value.stderr + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("2026-11-10", datetime.date(2026, 11, 10)), + ( + "2026-1-5T1:00:00Z", + datetime.datetime(2026, 1, 5, 1, tzinfo=datetime.UTC), + ), + ( + "2026-11-10 12:00:00.5 +01:00", + datetime.datetime( + 2026, + 11, + 10, + 12, + 0, + 0, + 500000, + tzinfo=datetime.timezone(datetime.timedelta(hours=1)), + ), + ), + ], +) +def test_service_info_timestamps(value: str, expected: datetime.date) -> None: + """Dates that pass the checks are YAML 1.1 timestamps. + + PyYAML resolves timestamps as SnakeYAML, which tesk-api uses, does. + + Args: + value: `serviceInfo.createdAt`. + expected: The timestamp that YAML reads. + """ + info, _ = _service_info("--set-string", f"serviceInfo.createdAt={value}") + assert info["createdAt"] == expected + + +def test_service_info_existing_config_map_unset() -> None: + """Without the schema, `existingConfigMap: false` is set, consistently.""" + manifests = render( + "--skip-schema-validation", "--set", "serviceInfo.existingConfigMap=false" + ) + assert find(manifests, "ConfigMap", "tesk-service-info") is None + volumes = deployment(manifests)["spec"]["template"]["spec"]["volumes"] + assert volumes[0]["configMap"] == {"name": "false"} + + +def test_service_info_existing_config_map() -> None: + """An existing ConfigMap replaces the chart's.""" + manifests = render("--set", "serviceInfo.existingConfigMap=my-service-info") + assert find(manifests, "ConfigMap", "tesk-service-info") is None + volumes = deployment(manifests)["spec"]["template"]["spec"]["volumes"] + assert volumes == [ + {"name": "service-info", "configMap": {"name": "my-service-info"}} + ] + annotations = deployment(manifests)["spec"]["template"]["metadata"].get( + "annotations", {} + ) + assert "checksum/service-info" not in annotations + + +def test_service_info_checksum() -> None: + """A change of the service info restarts the API's pod. + + tesk-api reads the file at startup, and then only every hour. + """ + + def checksum(*args: str) -> str: + meta = deployment(render(*args))["spec"]["template"]["metadata"] + value: str = meta["annotations"]["checksum/service-info"] + return value + + assert checksum() == checksum() + assert checksum() != checksum("--set", "serviceInfo.environment=prod") + # Together with the executors' checksum + meta = deployment( + render("--set-json", 'tesk.executor.securityContext={"runAsUser": 1000}') + )["spec"]["template"]["metadata"] + assert set(meta["annotations"]) == { + "checksum/service-info", + "checksum/executor-config", + } + + +@pytest.mark.parametrize( + ("args", "noted"), + [ + ((), True), + (("--set", "serviceInfo.id=org.example.tes"), False), + (("--set", "serviceInfo.existingConfigMap=mine"), False), + ], +) +def test_notes_service_info(tmp_path: Path, args: tuple[str, ...], noted: bool) -> None: + """The notes remind operators to describe their deployment. + + Args: + tmp_path: A directory for a copy of the chart. + args: More `helm template` arguments. + noted: Whether the notes have the reminder. + """ + # The notes compare the ID with the default of `values.yaml` + assert VALUES["serviceInfo"]["id"] == "uk.ac.ebi.tsc.tesk" + text = notes(*args, tmp_path=tmp_path) + assert ("NOTE: serviceInfo.id is the chart's default" in text) == noted + + +def test_service_info_file_removed() -> None: + """The service info no longer comes from a file of the chart.""" + assert not (CHART / "service-info").exists() def test_ingress() -> None: