Reference > Paperspace > Legacy Tools > Gradient API, CLI, & SDK > Command Reference > workflows

Was this page helpful?

workflows

Validated on 14 Dec 2023 • Last edited on 8 Jul 2024

Note

A new Paperspace API is now available. The Gradient and Core API endpoints will become unavailable on 15 July, 2024. For more information, see the release notes.

create

Create workflow

gradient workflows create \
  --name "NewWorkflow" \
  --projectId "prjpkflqz"

curl -X POST 'https://api.paperspace.io/workflows' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
    "name": "New-Workflow",
    "projectId": "prjpkflqz"
}'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.create(
    name='New-Workflow',
    project_id='prjpkflqz'
))

Options

Name	Type	Attributes	Description
`--name`	string	required	Workflow name
`--projectId`	string	required	Project ID
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
cURL
Python

+--------------+--------------------------------------+
| Name         | ID                                   |
+--------------+--------------------------------------+
| New-Workflow | 075615ac-a036-408c-a95e-36e5b18c73aa |
+--------------+--------------------------------------+

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

get

Get workflow info

Examples

CLI
cURL
Python

gradient workflows get \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa"

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.get(
    workflow_id='075615ac-a036-408c-a95e-36e5b18c73aa'
))

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--show-runs`		optional	Fetch runs
`--run`	string	optional	Specify workload run
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
cURL
Python

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

list

List workflows

Examples

CLI
cURL
Python

gradient workflows list \
  --projectId "prjpkflqz"

curl -g -X GET 'https://api.paperspace.io/workflows?filter[where][projectId]=prjpkflqz' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.list(
    project_id='prjpkflqz'
))

Options

Name	Type	Attributes	Description
`--projectId`	string	required	Project ID
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
cURL
Python

+---------------+--------------------------------------+
| Name          | ID                                   |
+---------------+--------------------------------------+
| New-Workflow  | 075615ac-a036-408c-a95e-36e5b18c73aa |
+---------------+--------------------------------------+

[
  {
    "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
    "name": "New-Workflow",
    "dtCreated": "2022-03-22T08:10:29.677Z",
    "dtModified": "2022-03-22T08:10:29.677Z"
  }
]

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    ))
  ),
];

logs

List logs for specific workflow

Examples

CLI
cURL
Python

gradient workflows logs \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --run 1

# First query to get logId values which is used for jobId in subsequent steps

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs/1' \
-H 'x-api-key: d44808a2785d6a...'

# Subsequent queries to get logs for each job

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrj67d8451ea1bf49749925755f5e61a46e' \
-H 'x-api-key: d44808a2785d6a...'

# Another query

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrjaedc275373ec4d239f3ea1af029dfd60' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(
    workflow_client.logs(job_id='wfrj67d8451ea1bf49749925755f5e61a46e'),
    workflow_client.logs(job_id='wfrjaedc275373ec4d239f3ea1af029dfd60')
)

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--run`	string	required	Specify workload run
`--logId`	string	optional	Workflow log ID
`--line`	integer	optional	Specific line number of logs
`--limit`	integer	optional	Limit of logs to fetch
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
cURL
Python

+Workflows wfrjaedc275373ec4d239f3ea1af029dfd60 logs------------------------------------------------------------------------------------------------------------------------------------+
| LINE | MESSAGE                                                                                                                                                                        |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1    | Collecting scipy==1.3.3                                                                                                                                                        |
| 2    |   Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB) |
| 3    | Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)                                                            |
| 4    | Installing collected packages: scipy                                                                                                                                           |
| 5    | Successfully installed scipy-1.3.3                                                                                                                                             |
| 6    | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 7    | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 8    | Collecting requests==2.22.0                                                                                                                                                    |
| 9    |   Downloading https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl (57kB)           |
| 10   | Collecting chardet<3.1.0,>=3.0.2 (from requests==2.22.0)                                                                                                                       |
| 11   |   Downloading https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl (133kB)            |
| 12   | Collecting certifi>=2017.4.17 (from requests==2.22.0)                                                                                                                          |
| 13   |   Downloading https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl (149kB)        |
| 14   | Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests==2.22.0)                                                                                                     |
| 15   |   Downloading https://files.pythonhosted.org/packages/56/aa/4ef5aa67a9a62505db124a5cb5262332d1d4153462eb8fd89c9fa41e5d92/urllib3-1.25.11-py2.py3-none-any.whl (127kB)          |
| 16   | Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests==2.22.0) (2.6)                                                                  |
| 17   | Installing collected packages: chardet, certifi, urllib3, requests                                                                                                             |
| 18   | Successfully installed certifi-2021.10.8 chardet-3.0.4 requests-2.22.0 urllib3-1.25.11                                                                                         |
| 19   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 20   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 21   | Collecting Pillow==6.2.1                                                                                                                                                       |
| 22   |   Downloading https://files.pythonhosted.org/packages/10/5c/0e94e689de2476c4c5e644a3bd223a1c1b9e2bdb7c510191750be74fa786/Pillow-6.2.1-cp36-cp36m-manylinux1_x86_64.whl (2.1MB) |
| 23   | Installing collected packages: Pillow                                                                                                                                          |
| 24   | Successfully installed Pillow-6.2.1                                                                                                                                            |
| 25   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 26   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

// Response to first query
{
  "id": 1,
  "uuid": "4dbd74f4-03d1-46fc-97cb-15a76805f3a1",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUCCEEDED",
    "logId": "wfr4dbd74f403d146fc97cb15a76805f3a1",
    "started": "2022-03-22T08:28:53.000Z",
    "finished": "2022-03-22T08:31:37.833Z",
    "jobs": {
      "CloneRepo": {
        "phase": "SUCCEEDED",
        "logId": "wfrj67d8451ea1bf49749925755f5e61a46e",
        "started": "2022-03-22T08:29:07.000Z",
        "finished": "2022-03-22T08:29:22.000Z"
      },
      "StyleGan2": {
        "phase": "SUCCEEDED",
        "logId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
        "started": "2022-03-22T08:29:23.000Z",
        "finished": "2022-03-22T08:31:16.000Z",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:3w8iqvi", "isCommitted": true }
          }
        }
      }
    }
  }
}


// Response to second query

[
  {
    "jobId": "wfrj67d8451ea1bf49749925755f5e61a46e",
    "line": 1,
    "timestamp": "2022-03-22T08:29:20.836Z",
    "message": "Cloning into '/outputs/repo'...\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-3308432544",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  }
]

// Response to third query

[
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 1,
    "timestamp": "2022-03-22T08:29:42.347Z",
    "message": "Collecting scipy==1.3.3\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 2,
    "timestamp": "2022-03-22T08:29:42.873Z",
    "message": "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 3,
    "timestamp": "2022-03-22T08:29:44.778Z",
    "message": "Requirement already satisfied: numpy\u003e=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  ...
]

[
  (LogRow(
    (line = 1),
    (message =
      "Cloning into '/outputs/repo'...\n"),
    (timestamp = "2022-03-22T08:29:20.836Z")
  )
]
[
  (LogRow(
    (line = 2),
    (message =
      "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)"),
    (timestamp = "2022-03-22T08:29:42.873Z")
  ),
  LogRow(
    (line = 3),
    (message =
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)"),
    (timestamp = "2022-03-22T08:29:44.778Z")
  ),
  ...
];

run

Run workflow spec

Examples

CLI
cURL
Python

gradient workflows run \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --path "workflow-spec.yaml"

curl -X POST 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
	"spec": {
		"jobs": {
			"CloneRepo": {
				"resources": {
					"instance-type": "C5"
				},
				"outputs": {
					"repo": {
						"type": "volume"
					}
				},
				"uses": "git-checkout@v1",
				"with": {
					"url": "https://github.com/NVlabs/stylegan2.git"
				}
			},
			"StyleGan2": {
				"resources": {
					"instance-type": "P4000"
				},
				"needs": ["CloneRepo"],
				"inputs": {
					"repo": "CloneRepo.outputs.repo"
				},
				"outputs": {
					"generatedFaces": {
						"type": "dataset",
						"with": {
							"ref": "dsx8kjbbkxtc5v3"
						}
					}
				},
				"uses": "script@v1",
				"with": {
					"script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces",
					"image": "tensorflow/tensorflow:1.14.0-gpu-py3"
				}
			}
		}
	},
	"cluster": {
        "id": "clg07azjl"
    },
	"run": "True",
	"markDefault": "False"
}'

from gradient import WorkflowsClient
import yaml

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

spec_path = "./workflow-spec.yaml"

yaml_spec = open(spec_path, 'r')
spec = yaml.safe_load(yaml_spec)

print(workflow_client.run_workflow(
    workflow_id='553d666c-2306-4bf3-a3db-9bd1275a9657',
    spec=spec,
    cluster_id='clg07azjl',
    inputs=None
))

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--clusterId`	string	optional	Cluster ID
`--path`	string	required	Path to workflow specification
`--inputPath`	string	optional	Path to inputs
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
cURL
Python

Created workflow run wfr627e7d055e0146ad94677d32f9380991

{
  "id": 2,
  "uuid": "627e7d05-5e01-46ad-9467-7d32f9380991",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\\npip install requests==2.22.0\\npip install Pillow==6.2.1\\ncd /inputs/repo\\npython run_generator.py generate-images \\\\\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\\\\n  --seeds=6600-6605 \\\\\\n  --truncation-psi=0.5 \\\\\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUBMITTING",
    "logId": "wfr627e7d055e0146ad94677d32f9380991",
    "jobs": {
      "CloneRepo": {
        "phase": "SUBMITTING",
        "logId": "wfrj2ac144f8c3174efeac14aea8f522edaf"
      },
      "StyleGan2": {
        "phase": "SUBMITTING",
        "logId": "wfrj5634fae547234e7598741a17126a174e",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:hypjdmy", "isCommitted": false }
          }
        }
      }
    }
  }
}

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      9,
      21,
      29,
      214000,
      (tzinfo = tzutc())
    ))
  ),
];

workflows

create

Examples

Options

Response

get

Examples

Options

Response

list

Examples

Options

Response

logs

Examples

Options

Response

run

Examples

Options

Response

We can't find any results for your search.