Data Pipelines

Data Pipelines enable you to orchestrate and visualize complex analytical processes by combining various components including ETL jobs, machine learning model inference tasks, notifications, and AWS AI services.

Below is the sample resource definition file for Data Pipeline:

{
  "rDataPipeline": {
    "Type": "DataPipeline",
    "Properties": {
      "DataPipelineName": "cicddatapipeline",
      "Description": "Data pipeline created using CICD",
      "Keywords": [
        "Owner: john"
      ],
      "DefaultExecutionProperties": {
        "from": ""
      },
      "Nodes": [
        {
          "ModuleType": "start_node",
          "NodeName": "Start"
        },
        {
          "ModuleType": "end_node",
          "NodeName": "End"
        },
        {
          "ModuleType": "email",
          "NodeName": "email",
          "Arguments": {},
          "EmailToExecutionPropertyKey": "from",
          "EmailSubjectExecutionPropertyKey": "from",
          "EmailBodyExecutionPropertyKey": "from"
        }
      ],
      "Graph": {
        "Start": {
          "success": [
            "email"
          ],
          "failure": []
        },
        "End": {
          "success": [],
          "failure": []
        },
        "email": {
          "success": [
            "End"
          ],
          "failure": []
        }
      },
      "tags": {},
      "CostTags": []
    }
  }
}