workflows

Validated on 14 Dec 2023 • Last edited on 15 May 2024
Note
A new Paperspace API is now available. As of 15 July 2024, Gradient and Core API endpoints are retired and no longer available for Paperspace users.

create

Create workflow

Examples

gradient workflows create \
  --name "NewWorkflow" \
  --projectId "prjpkflqz"

curl -X POST 'https://api.paperspace.io/workflows' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
    "name": "New-Workflow",
    "projectId": "prjpkflqz"
}'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.create(
    name='New-Workflow',
    project_id='prjpkflqz'
))

Options

Name Type Attributes Description
--name string required Workflow name
--projectId string required Project ID
--apiKey string optional API key to use this time only
--optionsFile string optional Path to YAML with predefined options
--createOptionsFile string optional Generate template options file

Response

+--------------+--------------------------------------+
| Name         | ID                                   |
+--------------+--------------------------------------+
| New-Workflow | 075615ac-a036-408c-a95e-36e5b18c73aa |
+--------------+--------------------------------------+

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

get

Get workflow info

Examples

gradient workflows get \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa"

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.get(
    workflow_id='075615ac-a036-408c-a95e-36e5b18c73aa'
))

Options

Name Type Attributes Description
--id string required Workflow ID
--show-runs optional Fetch runs
--run string optional Specify workload run
--apiKey string optional API key to use this time only
--optionsFile string optional Path to YAML with predefined options
--createOptionsFile string optional Generate template options file

Response

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

list

List workflows

Examples

gradient workflows list \
  --projectId "prjpkflqz"

curl -g -X GET 'https://api.paperspace.io/workflows?filter[where][projectId]=prjpkflqz' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.list(
    project_id='prjpkflqz'
))

Options

Name Type Attributes Description
--projectId string required Project ID
--apiKey string optional API key to use this time only
--optionsFile string optional Path to YAML with predefined options
--createOptionsFile string optional Generate template options file

Response

+---------------+--------------------------------------+
| Name          | ID                                   |
+---------------+--------------------------------------+
| New-Workflow  | 075615ac-a036-408c-a95e-36e5b18c73aa |
+---------------+--------------------------------------+

[
  {
    "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
    "name": "New-Workflow",
    "dtCreated": "2022-03-22T08:10:29.677Z",
    "dtModified": "2022-03-22T08:10:29.677Z"
  }
]

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    ))
  ),
];

logs

List logs for specific workflow

Examples

gradient workflows logs \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --run 1

# First query to get logId values which is used for jobId in subsequent steps

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs/1' \
-H 'x-api-key: d44808a2785d6a...'

# Subsequent queries to get logs for each job

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrj67d8451ea1bf49749925755f5e61a46e' \
-H 'x-api-key: d44808a2785d6a...'

# Another query

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrjaedc275373ec4d239f3ea1af029dfd60' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(
    workflow_client.logs(job_id='wfrj67d8451ea1bf49749925755f5e61a46e'),
    workflow_client.logs(job_id='wfrjaedc275373ec4d239f3ea1af029dfd60')
)

Options

Name Type Attributes Description
--id string required Workflow ID
--run string required Specify workload run
--logId string optional Workflow log ID
--line integer optional Specific line number of logs
--limit integer optional Limit of logs to fetch
--apiKey string optional API key to use this time only
--optionsFile string optional Path to YAML with predefined options
--createOptionsFile string optional Generate template options file

Response

+Workflows wfrjaedc275373ec4d239f3ea1af029dfd60 logs------------------------------------------------------------------------------------------------------------------------------------+
| LINE | MESSAGE                                                                                                                                                                        |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1    | Collecting scipy==1.3.3                                                                                                                                                        |
| 2    |   Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB) |
| 3    | Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)                                                            |
| 4    | Installing collected packages: scipy                                                                                                                                           |
| 5    | Successfully installed scipy-1.3.3                                                                                                                                             |
| 6    | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 7    | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 8    | Collecting requests==2.22.0                                                                                                                                                    |
| 9    |   Downloading https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl (57kB)           |
| 10   | Collecting chardet<3.1.0,>=3.0.2 (from requests==2.22.0)                                                                                                                       |
| 11   |   Downloading https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl (133kB)            |
| 12   | Collecting certifi>=2017.4.17 (from requests==2.22.0)                                                                                                                          |
| 13   |   Downloading https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl (149kB)        |
| 14   | Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests==2.22.0)                                                                                                     |
| 15   |   Downloading https://files.pythonhosted.org/packages/56/aa/4ef5aa67a9a62505db124a5cb5262332d1d4153462eb8fd89c9fa41e5d92/urllib3-1.25.11-py2.py3-none-any.whl (127kB)          |
| 16   | Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests==2.22.0) (2.6)                                                                  |
| 17   | Installing collected packages: chardet, certifi, urllib3, requests                                                                                                             |
| 18   | Successfully installed certifi-2021.10.8 chardet-3.0.4 requests-2.22.0 urllib3-1.25.11                                                                                         |
| 19   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 20   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 21   | Collecting Pillow==6.2.1                                                                                                                                                       |
| 22   |   Downloading https://files.pythonhosted.org/packages/10/5c/0e94e689de2476c4c5e644a3bd223a1c1b9e2bdb7c510191750be74fa786/Pillow-6.2.1-cp36-cp36m-manylinux1_x86_64.whl (2.1MB) |
| 23   | Installing collected packages: Pillow                                                                                                                                          |
| 24   | Successfully installed Pillow-6.2.1                                                                                                                                            |
| 25   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 26   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

// Response to first query
{
  "id": 1,
  "uuid": "4dbd74f4-03d1-46fc-97cb-15a76805f3a1",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUCCEEDED",
    "logId": "wfr4dbd74f403d146fc97cb15a76805f3a1",
    "started": "2022-03-22T08:28:53.000Z",
    "finished": "2022-03-22T08:31:37.833Z",
    "jobs": {
      "CloneRepo": {
        "phase": "SUCCEEDED",
        "logId": "wfrj67d8451ea1bf49749925755f5e61a46e",
        "started": "2022-03-22T08:29:07.000Z",
        "finished": "2022-03-22T08:29:22.000Z"
      },
      "StyleGan2": {
        "phase": "SUCCEEDED",
        "logId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
        "started": "2022-03-22T08:29:23.000Z",
        "finished": "2022-03-22T08:31:16.000Z",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:3w8iqvi", "isCommitted": true }
          }
        }
      }
    }
  }
}


// Response to second query

[
  {
    "jobId": "wfrj67d8451ea1bf49749925755f5e61a46e",
    "line": 1,
    "timestamp": "2022-03-22T08:29:20.836Z",
    "message": "Cloning into '/outputs/repo'...\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-3308432544",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  }
]

// Response to third query

[
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 1,
    "timestamp": "2022-03-22T08:29:42.347Z",
    "message": "Collecting scipy==1.3.3\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 2,
    "timestamp": "2022-03-22T08:29:42.873Z",
    "message": "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 3,
    "timestamp": "2022-03-22T08:29:44.778Z",
    "message": "Requirement already satisfied: numpy\u003e=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  ...
]

[
  (LogRow(
    (line = 1),
    (message =
      "Cloning into '/outputs/repo'...\n"),
    (timestamp = "2022-03-22T08:29:20.836Z")
  )
]
[
  (LogRow(
    (line = 2),
    (message =
      "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)"),
    (timestamp = "2022-03-22T08:29:42.873Z")
  ),
  LogRow(
    (line = 3),
    (message =
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)"),
    (timestamp = "2022-03-22T08:29:44.778Z")
  ),
  ...
];

run

Run workflow spec

Examples

gradient workflows run \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --path "workflow-spec.yaml"

curl -X POST 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
	"spec": {
		"jobs": {
			"CloneRepo": {
				"resources": {
					"instance-type": "C5"
				},
				"outputs": {
					"repo": {
						"type": "volume"
					}
				},
				"uses": "git-checkout@v1",
				"with": {
					"url": "https://github.com/NVlabs/stylegan2.git"
				}
			},
			"StyleGan2": {
				"resources": {
					"instance-type": "P4000"
				},
				"needs": ["CloneRepo"],
				"inputs": {
					"repo": "CloneRepo.outputs.repo"
				},
				"outputs": {
					"generatedFaces": {
						"type": "dataset",
						"with": {
							"ref": "dsx8kjbbkxtc5v3"
						}
					}
				},
				"uses": "script@v1",
				"with": {
					"script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces",
					"image": "tensorflow/tensorflow:1.14.0-gpu-py3"
				}
			}
		}
	},
	"cluster": {
        "id": "clg07azjl"
    },
	"run": "True",
	"markDefault": "False"
}'

from gradient import WorkflowsClient
import yaml

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

spec_path = "./workflow-spec.yaml"

yaml_spec = open(spec_path, 'r')
spec = yaml.safe_load(yaml_spec)

print(workflow_client.run_workflow(
    workflow_id='553d666c-2306-4bf3-a3db-9bd1275a9657',
    spec=spec,
    cluster_id='clg07azjl',
    inputs=None
))

Options

Name Type Attributes Description
--id string required Workflow ID
--clusterId string optional Cluster ID
--path string required Path to workflow specification
--inputPath string optional Path to inputs
--apiKey string optional API key to use this time only
--optionsFile string optional Path to YAML with predefined options
--createOptionsFile string optional Generate template options file

Response

Created workflow run wfr627e7d055e0146ad94677d32f9380991

{
  "id": 2,
  "uuid": "627e7d05-5e01-46ad-9467-7d32f9380991",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\\npip install requests==2.22.0\\npip install Pillow==6.2.1\\ncd /inputs/repo\\npython run_generator.py generate-images \\\\\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\\\\n  --seeds=6600-6605 \\\\\\n  --truncation-psi=0.5 \\\\\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUBMITTING",
    "logId": "wfr627e7d055e0146ad94677d32f9380991",
    "jobs": {
      "CloneRepo": {
        "phase": "SUBMITTING",
        "logId": "wfrj2ac144f8c3174efeac14aea8f522edaf"
      },
      "StyleGan2": {
        "phase": "SUBMITTING",
        "logId": "wfrj5634fae547234e7598741a17126a174e",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:hypjdmy", "isCommitted": false }
          }
        }
      }
    }
  }
}

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      9,
      21,
      29,
      214000,
      (tzinfo = tzutc())
    ))
  ),
];
Control Panel Blog Pricing Careers Terms Privacy Status API Docs Tutorials Support