Skip to main content

Data Processing Pipeline Patterns

Custom GCS Storage​

{
  "region": "YOUR_REGION_NAME",
  "bucket": "YOUR_BUCKET_NAME",
  "service_account": {
  "type": "service_account",
  "project_id": "YOUR_PROJECT_ID",
  "private_key_id": "YOUR_PRIVATE_KEY_ID",
  "private_key": "-----BEGIN PRIVATE KEY-----XXXX-----END PRIVATE KEY-----\n",
  "client_email": "SERVICE_ACCOUNT@PROJECT_ID.iam.gserviceaccount.com",
  "client_id": "YOUR_CLIENT_ID",
  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
  "token_uri": "https://oauth2.googleapis.com/token",
  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/YOUR_SERVICE_ACCOUNT.iam.gserviceaccount.com",
  "universe_domain": "googleapis.com"
  }
}

Custom S3 Storage​

{
  "bucket": "XXXXX",
  "region": "XXXXX",
  "provider": "aws",
  "aws_access_key_id": "AKIAXXXXXX",
  "aws_secret_access_key": "rpKn3TXXXXXX+QXXXX"
}