Changes:
- In the "programs" section: the ProgramId of the Program will be replaced by ProgramId of the workflow if the program is associated with a workflow. If not the Programs show as it is.
- In the "relations" section: a new field called "workflow" will be added for all the relations that could be collapsed based on workflows. The programs field will be a collection of all the programs that were collapsed to form this workflow.
- The API will request collapse=workflow for this view. If collapse=workflow is not provided then, the workflow fields are filled (if applicable) but relations are not collapsed.
curl "http://127.0.0.1:10000/v3/namespaces/default/datasets/EmpAgg/lineage?collapse=access&collapse=run&collapse=component&collapse=program&end=now&levels=1&start=now-7d" | python -m json.tool
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1006 100 1006 0 0 121k 0 --:--:-- --:--:-- --:--:-- 140k
{
"data": {
"dataset.default.EmpAgg": {
"entityId": {
"id": {
"instanceId": "EmpAgg",
"namespace": {
"id": "default"
}
},
"type": "datasetinstance"
}
},
"dataset.default.conn-0.e0591f36-9661-11e6-af4a-0000007182af": {
"entityId": {
"id": {
"instanceId": "conn-0.e0591f36-9661-11e6-af4a-0000007182af",
"namespace": {
"id": "default"
}
},
"type": "datasetinstance"
}
}
},
"end": 1476926333,
"programs": {
"<workflow name>": {
"entityId": {
"id": {
"application": {
"applicationId": "EmployeePipe_Long_copy",
"namespace": {
"id": "default"
}
},
"id": "phase-2",
"type": "Mapreduce"
},
"type": "<type workflow>"
}
}
},
"relations": [
{
"accesses": [
"read",
"write"
],
"components": [],
"data": "dataset.default.conn-0.e0591f36-9661-11e6-af4a-0000007182af",
"workflow" : "<workflow name>"
"program": [
"mapreduce.default.EmployeePipe_Long_copy.phase-2",
"mapreduce.default.EmployeePipe_Long_copy.phase-1"
]
"runs": [
"e4051038-9661-11e6-8060-000000d79ea8",
"e4051038-9661-11e6-8060-000000d79ea8"
]
},
{
"accesses": [
"write",
"read"
],
"components": [],
"data": "dataset.default.EmpAgg",
"workflow": "",
"program": "mapreduce.default.EmployeePipe_Long_copy.phase-2",
"runs": [
"e4051038-9661-11e6-8060-000000d79ea8"
]
}
],
"start": 1476321533
}