Initial draft for job configuration schema

handlebar-demo.ts

const Handlebars = require('handlebars');

const json = {
  job: {
    pids: ["10/1","10/2","10/3"],
    owner: "bliven_s",
    status: "SUCCESS"
  },
  secrets: {
    jwt_token: "8AK820="
  }
};

const source = `Dear {{job.owner}},

The following datasets finished with status {{job.status}}:
{{#each job.pids}}
- {{this}}
{{/each}}
`;

const template = Handlebars.compile(source);
const rendered = template(json);

console.log(rendered);

jobconfig.yaml

# To validate: yamale -s jobconfig_schema.yaml jobconfig_example.yaml
type: beautify
# "Template 1" - validate incoming data from the user
actions:
  create:
    - authorization: DatasetCreateOwn
      validate:
      - scope: body
        schema: >-
          {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
              "pids": {
                "type": "array",
                "items": {
                  "type": "string"
                }
              }
            },
            "required": ["pids"]
          }

      method:
          type: url
          url: localhost/beautify
          verb: post
          # Template that gets sent to external
          body: '{"pids":{{ job.pids | jsonify }}}'
  update:
    - authorization: ["admin","JobUpdateAny"]
      method:
          type: url
          url: localhost/jobid/{{jobid}}
          verb: patch

---

# Adapted from https://github.com/SciCatProject/scicat-backend-next/issues/741
type: archive
actions:
  create:
  - authorization: all
    # method:
    #   type: rabbitmq
    #   server: myrabbitmq.myinstitution.ext
    #   queue: archive_jobs
    method:
      type: url
      url: afsd
      verb: get
  update:
  - authorization: archiver_service
    validate:
      # How to specify that PIDs should be extracted from the request body?
    - type: job
      field: status
      allowed:
      - IN_PROGRESS
      - SUCCESSFUL
      - FAILED
      transitions:  # What's meant here?
        CREATED:
        - IN_PROGRESS
        IN_PROGRESS:
        - SUCCESSFUL
        - FAILED
    method:
      type: rabbitmq
      server: myrabbitmq.myinstitution.ext
      queue: archive_updates
      templates:
        # Example variable names
        status: '{{job.status}}'
        message: '{{job.message}}'
        job: '{{job.id}}'
        token: '{{secrets.jwt_token}}'
  post:
  - method:
      type: email
      smtp:
        url: myemailserver.myinstitution.ext
        user: scicatuser
      to: "{{ join job.recipients ' '}}"
      subject: 'Archive job {{job.pid}} has completed.'
      body: |
        Dear {{job.owner}},
        
        Archive job {{job.pid}} has completed. Results: {{job.status}}

  - filter:
      status: SUCCESSFUL
    method:
      type: url
      url: myscicat.myinstitution.ext/datablock/archived
      verb: post
      headers:
        AuthBearer: '{{secrets.jwt_token}}'
      body: '{ structure to create archive files }'

schema.yaml

# Yamale schema. https://github.com/23andMe/Yamale
type: str()  # Job type, eg "archive"
actions:
  # Initial job creation request
  create: subset(include('action'), allow_empty=True)
  # Update job status
  update: subset(include("action"), allow_empty=True)
  # After creation/update
  post: subset(include("action"), allow_empty=True)
---
action:
  # Only run action if criteria are met
  filter: include("filter", required=False)
  # List of roles allowed to use this action
  authorization: subset(str(), allow_empty=True)
  # Validation steps
  validate: subset(include('validator'), allow_empty=True)
  method: any(include("url_method"), include("rabbitmq_method"), include("email_method"))
---
url_method:
  type: enum("url")
  url: include('template')  # Allows templating, eg for parameters
  verb: enum("get","post","delete","put","patch")
  headers: map(str(),key=str(), required=False)
  body: include('template', required=False)
---
rabbitmq_method:
  type: enum("rabbitmq")
  server: str()
  queue: include('template')
  templates:
    status: include('template')
    message: include('template')
    job: include('template')
    token: include('template')
---
email_method:
  type: enum("email")
  smtp: any()
  to: str()
  subject: include('template')
  body: include('template')
---
validator: any()
  # scope: enum("body","param")
  # schema: str(required=False)  # json schema to apply?
---
# Handlebar templates are just strings.
template: str()
---
filter:
  status: str(required=False)

Yamale is a simple yaml-based schema language. Run it like this:

yamale -s schema.yaml jobconfig.yaml

For production I would convert to a proper json schema, but I find yaml easier to iterate with.

The handlebars demo just shows a few basic templating ideas. Run with node handlebar-demo.ts.