{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://catalog.lintel.tools/schemas/schemastore/datayoga-job/latest.json", "title": "Job", "description": "Job descriptor", "x-lintel": { "source": "https://raw.githubusercontent.com/datayoga-io/datayoga/main/schemas/job.schema.json", "sourceSha256": "08e6668afd781809578bde2e4689bc7545cd1d210927040f2ef8b1a54c92208f", "fileMatch": [ "**/jobs/**/*.dy.yaml" ], "parsers": [ "yaml" ] }, "type": "object", "properties": { "error_handling": { "default": "ignore", "description": "Error handling strategy: abort - terminate job, ignore - skip", "enum": [ "abort", "ignore" ], "type": "string" }, "input": { "$ref": "#/$defs/block" }, "steps": { "items": { "$ref": "#/$defs/block" }, "type": "array" } }, "$defs": { "block": { "allOf": [ { "if": { "properties": { "uses": { "const": "add_field", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "description": "Add fields to a record", "oneOf": [ { "required": [ "fields" ], "description": "Add multiple fields", "examples": [ { "fields": [ { "expression": "concat([name.fname, ' ', name.lname])", "field": "name.full_name", "language": "jmespath" }, { "expression": "upper(name.fname)", "field": "name.fname_upper", "language": "jmespath" } ] } ], "properties": { "fields": { "description": "Fields", "items": { "type": "object", "properties": { "expression": { "description": "Expression", "type": "string" }, "field": { "description": "Field", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "field", "expression", "language" ], "additionalProperties": false }, "type": "array" } }, "additionalProperties": false, "type": "object" }, { "required": [ "field", "expression", "language" ], "description": "Add one field", "examples": [ { "expression": "country_code || ' - ' || UPPER(country_name)", "field": "country", "language": "sql" } ], "properties": { "expression": { "description": "Expression", "type": "string" }, "field": { "description": "Field", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "additionalProperties": false, "type": "object" } ], "title": "add_field", "type": "object" } } } }, { "if": { "properties": { "uses": { "const": "azure.read_event_hub", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "description": "Read from Azure Event Hub", "properties": { "batch_size": { "default": 300, "description": "The maximum number of events to receive in each batch.", "type": "integer" }, "checkpoint_store_connection_string": { "description": "The connection string for the Azure Storage account used as the checkpoint store.", "type": "string" }, "checkpoint_store_container_name": { "description": "The name of the container within the checkpoint store to store the checkpoints.", "type": "string" }, "event_hub_connection_string": { "description": "The connection string for the Azure Event Hub namespace.", "type": "string" }, "event_hub_consumer_group_name": { "description": "The name of the consumer group to read events from.", "type": "string" }, "event_hub_name": { "description": "The name of the Azure Event Hub.", "type": "string" } }, "required": [ "event_hub_connection_string", "event_hub_consumer_group_name", "event_hub_name", "checkpoint_store_connection_string", "checkpoint_store_container_name" ], "title": "azure.read_event_hub", "type": "object" } } } }, { "if": { "properties": { "uses": { "const": "cassandra.write", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Write into a Cassandra data store", "examples": [ { "id": "load_snowflake", "properties": { "connection": "eu_datalake", "load_strategy": "APPEND", "schema": "dbo", "table": "employees" }, "type": "relational.write" } ], "properties": { "connection": { "description": "Logical connection name as defined in the connections.dy.yaml", "examples": [ "europe_db", "target", "eu_dwh" ], "title": "The connection to use for loading", "type": "string" }, "keys": { "examples": [ [ "fname", { "lname": "last_name" } ] ], "items": { "title": "name of column", "type": [ "string", "object" ] }, "title": "Business keys", "type": "array" }, "keyspace": { "description": "Keyspace", "examples": [ "employees" ], "title": "Keyspace", "type": "string" }, "mapping": { "examples": [ [ "fname", { "lname": "last_name" }, "address", "gender" ] ], "items": { "title": "name of column", "type": [ "string", "object" ] }, "title": "Fields to write", "type": "array" }, "opcode_field": { "description": "Name of the field in the payload that holds the operation (c - create, d - delete, u - update) for this record in the DB", "type": "string" }, "table": { "description": "Target table name", "examples": [ "employees" ], "title": "The target table name", "type": "string" } }, "required": [ "connection", "keyspace", "table", "keys", "mapping", "opcode_field" ], "title": "cassandra.write", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "files.read_csv", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Read data from CSV", "examples": [ { "delimiter": ";", "file": "archive.csv" } ], "properties": { "batch_size": { "default": 1000, "description": "Number of records to read per batch", "minimum": 1, "type": "number" }, "delimiter": { "default": ",", "description": "Delimiter to use for splitting the csv records", "maxLength": 1, "minLength": 1, "type": "string" }, "encoding": { "default": "utf-8", "description": "Encoding to use for reading the file", "type": "string" }, "fields": { "additionalItems": true, "default": null, "description": "List of columns to use for extract", "examples": [ [ "fname", "lname" ] ], "items": { "description": "field name", "examples": [ "fname" ], "type": "string" }, "minLength": 1, "title": "List of columns to use", "type": "array" }, "file": { "description": "Filename. Can contain a regexp or glob expression", "type": "string" }, "quotechar": { "default": "\"", "description": "A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to '", "maxLength": 1, "minLength": 1, "type": "string" }, "skip": { "default": 0, "description": "Number of lines to skip", "minimum": 0, "type": "number" } }, "required": [ "file" ], "title": "files.read_csv", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "filter", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Filter records", "examples": [ { "expression": "age>20", "language": "sql" } ], "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "title": "filter", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "http.receiver", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Receives HTTP requests and process the data.", "examples": [ { "host": "localhost", "port": 8080 } ], "properties": { "host": { "default": "0.0.0.0", "description": "Host to listen", "type": "string" }, "port": { "default": 8080, "description": "Port to listen", "type": "integer" } }, "title": "http.receiver", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "http.write", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "description": "Write data using an HTTP request", "examples": [ { "connection": "http_example", "endpoint": { "expression": "concat(['users/', id])", "language": "jmespath" }, "extra_headers": { "my_header": { "expression": "lname || '-' || fname", "language": "sql" } }, "extra_query_parameters": { "fname": { "expression": "UPPER(fname)", "language": "sql" } }, "method": "PUT", "output": { "body": "response.content", "headers": "response.headers", "status_code": "response.status_code" }, "payload": { "full_name": { "expression": "full_name", "language": "jmespath" }, "greeting": { "expression": "greeting", "language": "jmespath" } }, "timeout": 3 } ], "properties": { "connection": { "description": "Logical connection name as defined in the connections.dy.yaml", "examples": [ "api_connection", "external_service" ], "title": "The connection to use for the HTTP request", "type": "string" }, "endpoint": { "oneOf": [ { "description": "The endpoint URL for the HTTP request", "examples": [ "/users" ], "title": "API Endpoint", "type": "string" }, { "examples": [ { "expression": "concat(['/users/', id]')", "language": "jmespath" } ], "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "type": "object" } ] }, "extra_headers": { "type": "object", "description": "Extra headers to be included in the HTTP request", "title": "Additional HTTP Headers", "additionalProperties": { "oneOf": [ { "type": "string" }, { "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "type": "object" } ] } }, "extra_query_parameters": { "type": "object", "description": "Extra query parameters to be included in the HTTP request", "additionalProperties": { "oneOf": [ { "type": "string" }, { "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "type": "object" } ] } }, "method": { "description": "HTTP method to be used for the request", "enum": [ "GET", "PUT", "POST", "DELETE" ], "examples": [ "GET", "POST" ], "title": "HTTP Method", "type": "string" }, "output": { "properties": { "body": { "description": "Name of the field where the HTTP response content will be stored after the request", "title": "Body Field Name", "type": "string" }, "headers": { "description": "Name of the field where the HTTP response headers will be stored after the request", "title": "Headers Field Name", "type": "string" }, "status_code": { "description": "Name of the field where the HTTP response status code will be stored after the request", "title": "Status Code Field Name", "type": "string" } }, "type": "object" }, "payload": { "type": "object", "description": "Data to be sent in the request body", "title": "Request Payload", "additionalProperties": { "oneOf": [ { "type": "string" }, { "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "type": "object" } ] } }, "timeout": { "description": "Timeout duration for this specific HTTP request in seconds", "title": "Timeout in Seconds", "type": "integer" } }, "required": [ "connection", "endpoint", "method" ], "title": "http.write", "type": "object" } } } }, { "if": { "properties": { "uses": { "const": "jinja_template", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Apply Jinja template to a field", "examples": [ { "field": "name.full_name", "template": "{{ name.fname }} {{ name.lname }}" }, { "field": "name.fname_upper", "template": "{{ name.fname | upper }}" } ], "properties": { "field": { "description": "Field", "type": "string" }, "template": { "description": "Jinja Template", "type": "string" } }, "required": [ "field", "template" ], "title": "jinja_template", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "map", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Maps a record into a new output based on expressions", "examples": [ { "expression": { "country": "country", "first_name": "first_name", "full_name": "full_name", "greeting": "'Hello ' || CASE WHEN gender = 'F' THEN 'Ms.' WHEN gender = 'M' THEN 'Mr.' ELSE 'N/A' END || ' ' || full_name", "last_name": "last_name" }, "language": "sql" }, { "expression": "{\"CustomerId\": \"customer_id\", \"FirstName\": \"first_name\", \"LastName\": \"last_name\", \"Company\": \"company\", \"Location\": {\"Street\": \"address\", \"City\": \"city\", \"State\": \"state\", \"Country\": \"country\", \"PostalCode\": \"postal_code\"}, \"Phone\": \"phone\", \"Fax\": \"fax\", \"Email\": \"email\"}", "language": "jmespath" } ], "properties": { "expression": { "description": "Expression", "type": [ "object", "string" ] }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "title": "map", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "parquet.read", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Read data from parquet", "examples": [ { "file": "data.parquet" } ], "properties": { "file": { "description": "Filename. Can contain a regexp or glob expression", "type": "string" } }, "required": [ "file" ], "title": "parquet.read", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "parquet.write", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Write data to parquet", "examples": [ { "file": "data.parquet" } ], "properties": { "file": { "description": "Filename. Can contain a regexp or glob expression", "type": "string" } }, "required": [ "file" ], "title": "parquet.write", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "redis.lookup", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Lookup data from Redis using the given command and key", "properties": { "args": { "description": "The list of expressions produces arguments", "items": { "type": "string" }, "title": "Redis command arguments", "type": "array" }, "cmd": { "description": "The command to execute", "title": "Redis command", "type": "string" }, "connection": { "title": "Connection name", "type": "string" }, "field": { "description": "The field to write the result to", "title": "Target field", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" }, "when": { "type": "object", "description": "Optional condition to determine whether to execute the lookup for each record", "properties": { "expression": { "description": "Expression that evaluates to a boolean to determine if lookup should be executed", "title": "Condition expression", "type": "string" }, "language": { "description": "Language for the condition expression", "enum": [ "jmespath", "sql" ], "title": "Expression language", "type": "string" } }, "required": [ "expression", "language" ], "title": "Conditional execution", "additionalProperties": false } }, "required": [ "connection", "cmd", "args", "language", "field" ], "title": "redis.lookup", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "redis.read_stream", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Read from Redis stream", "properties": { "connection": { "description": "Connection name", "type": "string" }, "snapshot": { "default": false, "description": "Snapshot current entries and quit", "title": "Snapshot current entries and quit", "type": "boolean" }, "stream_name": { "description": "Source stream name", "title": "Source stream name", "type": "string" } }, "required": [ "connection", "stream_name" ], "title": "redis.read_stream", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "redis.write", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Write to a Redis data structure", "properties": { "command": { "default": "HSET", "description": "Redis command", "enum": [ "HSET", "SADD", "XADD", "RPUSH", "LPUSH", "SET", "ZADD" ], "title": "Redis command", "type": "string" }, "connection": { "title": "Connection name", "type": "string" }, "key": { "description": "Field to use as the Redis key", "properties": { "expression": { "description": "Expression", "type": "string" }, "language": { "description": "Language", "enum": [ "jmespath", "sql" ], "type": "string" } }, "required": [ "expression", "language" ], "type": "object" } }, "required": [ "connection", "key" ], "title": "redis.write", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "relational.lookup", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Lookup values in a relational table", "examples": [ { "id": "load_snowflake", "properties": { "connection": "eu_datalake", "load_strategy": "APPEND", "schema": "dbo", "table": "employees" }, "type": "relational.write" } ], "properties": { "condition": { "description": "Use any valid SQL syntax. Use the alias `lookup` for the lookup table and `incoming` for the base table", "examples": [ "lookup.account_number = incoming.i_acct_no" ], "title": "The lookup condition", "type": "string" }, "fields": { "examples": [ [ "fname", { "lname": "last_name" }, "address", "gender" ] ], "items": { "title": "name of column", "type": [ "string", "object" ] }, "title": "Columns to add to the output from the lookup table", "type": "array" }, "multiple_match_policy": { "default": "", "description": "How to handle multiple matches in the lookup table", "enum": [ "first", "last", "all" ], "type": "string" }, "order_by": { "examples": [ [ "country_name" ] ], "items": { "title": "name of column", "type": [ "string" ] }, "title": "List of keys to use for ordering. Applicable for multiple matches", "type": "array" }, "query": { "description": "Use any valid SQL syntax. Use the alias `lookup` for the lookup table and `incoming` for the base table", "examples": [ "select country_code,country_name from countries where is_active=1" ], "title": "Query string to use as an override to the built in query", "type": "string" }, "schema": { "description": "If not specified, no specific schema will be used when connecting to the database.", "examples": [ "dbo" ], "title": "The table schema of the lookup table", "type": "string" }, "table": { "description": "Lookup table name", "examples": [ "countries" ], "title": "The lookup table name", "type": "string" } }, "required": [], "title": "relational.lookup", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "relational.read", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "description": "Read a table from an SQL-compatible data store", "examples": [ { "id": "read_snowflake", "properties": { "connection": "eu_datalake", "schema": "dbo", "table": "employees" }, "type": "relational.read" } ], "properties": { "columns": { "examples": [ [ "fname", { "lname": "last_name" } ] ], "items": { "title": "name of column", "type": [ "string", "object" ] }, "title": "Optional subset of columns to load", "type": "array" }, "connection": { "description": "Logical connection name as defined in the connections.dy.yaml", "examples": [ "europe_db", "target", "eu_dwh" ], "title": "The connection to use for loading", "type": "string" }, "schema": { "description": "If left blank, the default schema of this connection will be used as defined in the connections.dy.yaml", "examples": [ "dbo" ], "title": "The table schema of the table", "type": "string" }, "table": { "description": "Table name", "examples": [ "employees" ], "title": "The table name", "type": "string" } }, "required": [ "connection", "table" ], "title": "relational.read", "additionalProperties": false } } } }, { "if": { "properties": { "uses": { "const": "relational.write", "description": "Block type", "type": "string" } }, "required": [ "uses" ], "type": "object" }, "then": { "properties": { "with": { "type": "object", "allOf": [ { "not": { "required": [ "opcode_field", "load_strategy" ] } } ], "description": "Write into a SQL-compatible data store", "examples": [ { "id": "load_snowflake", "properties": { "connection": "eu_datalake", "load_strategy": "APPEND", "schema": "dbo", "table": "employees" }, "type": "relational.write" } ], "properties": { "active_record_indicator": { "description": "Used for `TYPE2` load_strategy. An SQL expression used to identify which rows are active", "examples": [ "is_active='Y'", "deletedAt is null" ], "type": "string" }, "connection": { "description": "Logical connection name as defined in the connections.dy.yaml", "examples": [ "europe_db", "target", "eu_dwh" ], "title": "The connection to use for loading", "type": "string" }, "foreach": { "description": "Use a JMESPath expression to split a column into multiple records. The expression should be in the format column: expression.", "examples": [ "order_line: lines[]" ], "pattern": "^(?!:).*:.*(?