| // Copyright 2015 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package bigquery |
| |
| import ( |
| "io" |
| |
| bq "google.golang.org/api/bigquery/v2" |
| ) |
| |
| // GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute |
| // an input or output to a BigQuery operation. |
| type GCSReference struct { |
| // URIs refer to Google Cloud Storage objects. |
| URIs []string |
| |
| FileConfig |
| |
| // DestinationFormat is the format to use when writing exported files. |
| // Allowed values are: CSV, Avro, JSON. The default is CSV. |
| // CSV is not supported for tables with nested or repeated fields. |
| DestinationFormat DataFormat |
| |
| // Compression specifies the type of compression to apply when writing data |
| // to Google Cloud Storage, or using this GCSReference as an ExternalData |
| // source with CSV or JSON SourceFormat. Default is None. |
| // |
| // Avro files allow additional compression types: DEFLATE and SNAPPY. |
| Compression Compression |
| } |
| |
| // NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination. |
| // In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object. |
| // Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided. |
| // Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name. |
| // For more information about the treatment of wildcards and multiple URIs, |
| // see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple |
| func NewGCSReference(uri ...string) *GCSReference { |
| return &GCSReference{URIs: uri} |
| } |
| |
| // Compression is the type of compression to apply when writing data to Google Cloud Storage. |
| type Compression string |
| |
| const ( |
| // None specifies no compression. |
| None Compression = "NONE" |
| // Gzip specifies gzip compression. |
| Gzip Compression = "GZIP" |
| // Deflate specifies DEFLATE compression for Avro files. |
| Deflate Compression = "DEFLATE" |
| // Snappy specifies SNAPPY compression for Avro files. |
| Snappy Compression = "SNAPPY" |
| ) |
| |
| func (gcs *GCSReference) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader { |
| lc.SourceUris = gcs.URIs |
| gcs.FileConfig.populateLoadConfig(lc) |
| return nil |
| } |
| |
| func (gcs *GCSReference) toBQ() bq.ExternalDataConfiguration { |
| conf := bq.ExternalDataConfiguration{ |
| Compression: string(gcs.Compression), |
| SourceUris: append([]string{}, gcs.URIs...), |
| } |
| gcs.FileConfig.populateExternalDataConfig(&conf) |
| return conf |
| } |