m-lab / autoloader / 72

Committed 23 Mar 2023 09:40PM UTC coverage: 95.041%. First build

Build # 72

Build Type

Pull #5

travis-ci-com

Committed by

web-flow

Commit Message

<a href="https://github.com/m-lab/autoloader/commit/<a class=hub.com/m-lab/autoloader/commit/7446a805f8548b3056e98d68e64223eb8d5e6e29">7446a805f<a href="https://github.com/m-lab/autoloader/commit/7446a805f8548b3056e98d68e64223eb8d5e6e29">">Merge </a><a class="double-link" href="https://github.com/m-lab/autoloader/commit/<a class="double-link" href="https://github.com/m-lab/autoloader/commit/c324b42810f5ce15f9b81058c553e95789ae00fc">c324b4281</a>">c324b4281</a><a href="https://github.com/m-lab/autoloader/commit/7446a805f8548b3056e98d68e64223eb8d5e6e29"> into a8586d276">a8586d276</a>

Pull Request Pull Request #5: Add travis CI

Run Details

12 of 12 new or added lines in 1 file covered. (100.0%)

115 of 121 relevant lines covered (95.04%)

1.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.88

/bq/client.go

package bq

import (
        "context"

        "cloud.google.com/go/bigquery"
        "github.com/googleapis/google-cloud-go-testing/bigquery/bqiface"
        "github.com/m-lab/autoloader/api"
)

// Client is used to perform BigQuery operations.
type Client struct {
        bqiface.Client
}

// NewClient returns a new instance of Client. Operations performed via the Client
// take place within the specified GCP `project` argument.
func NewClient(c *bigquery.Client) *Client {
        return &Client{bqiface.AdaptClient(c)}
}

// GetDataset returns a handle to the input dataset and an error indicating whether the
// dataset exists.
func (c *Client) GetDataset(ctx context.Context, name string) (bqiface.Dataset, error) {
        ds := c.Dataset(name)
        _, err := ds.Metadata(ctx)
        return ds, err
}

// CreateDataset creates a new dataset for the input `api.Datatype`.
// It returns an error if the dataset already exists.
func (c *Client) CreateDataset(ctx context.Context, dt *api.Datatype) (bqiface.Dataset, error) {
        ds := c.Dataset(dt.Experiment)
        err := ds.Create(ctx, &bqiface.DatasetMetadata{
                DatasetMetadata: bigquery.DatasetMetadata{
                        Name:     dt.Experiment,
                        Location: dt.Location,
                },
        })
        return ds, err
}

// GetTableMetadata returns the metadata for the input table and an error indicating whether
// the table exists.
func (c *Client) GetTableMetadata(ctx context.Context, ds bqiface.Dataset, name string) (*bigquery.TableMetadata, error) {
        t := ds.Table(name)
        md, err := t.Metadata(ctx)
        return md, err
}

// CreateTable creates a new date-partitioned table for the input `api.Datatype`.
// It returns an error if the table already exists.
func (c *Client) CreateTable(ctx context.Context, ds bqiface.Dataset, dt *api.Datatype) error {
        bqSchema, err := bigquery.SchemaFromJSON(dt.Schema)
        if err != nil {
                return err
        }

        t := ds.Table(dt.Name)
        err = t.Create(ctx, &bigquery.TableMetadata{
                Name:   dt.Name,
                Schema: bqSchema,
                TimePartitioning: &bigquery.TimePartitioning{
                        Type:                   "DAY",
                        Field:                  "date",
                        RequirePartitionFilter: true,
                },
        })
        return err
}

// UpdateSchema updates the schema for the input `api.Datatype` table.
func (c *Client) UpdateSchema(ctx context.Context, ds bqiface.Dataset, dt *api.Datatype) error {
        bqSchema, err := bigquery.SchemaFromJSON(dt.Schema)
        if err != nil {
                return err
        }

        t := ds.Table(dt.Name)
        _, err = t.Update(ctx, bigquery.TableMetadataToUpdate{
                Schema: bqSchema,
        }, "")
        return err
}

1	package bq
2
3	import (
4	"context"
5
6	"cloud.google.com/go/bigquery"
7	"github.com/googleapis/google-cloud-go-testing/bigquery/bqiface"
8	"github.com/m-lab/autoloader/api"
9	)
10
11	// Client is used to perform BigQuery operations.
12	type Client struct {
13	bqiface.Client
14	}
15
16	// NewClient returns a new instance of Client. Operations performed via the Client
17	// take place within the specified GCP `project` argument.
18	func NewClient(c bigquery.Client) Client {	×
19	return &Client{bqiface.AdaptClient(c)}	×
20	}	×
21
22	// GetDataset returns a handle to the input dataset and an error indicating whether the
23	// dataset exists.
24	func (c *Client) GetDataset(ctx context.Context, name string) (bqiface.Dataset, error) {	1✔
25	ds := c.Dataset(name)	1✔
26	_, err := ds.Metadata(ctx)	1✔
27	return ds, err	1✔
28	}	1✔
29
30	// CreateDataset creates a new dataset for the input `api.Datatype`.
31	// It returns an error if the dataset already exists.
32	func (c Client) CreateDataset(ctx context.Context, dt api.Datatype) (bqiface.Dataset, error) {	1✔
33	ds := c.Dataset(dt.Experiment)	1✔
34	err := ds.Create(ctx, &bqiface.DatasetMetadata{	1✔
35	DatasetMetadata: bigquery.DatasetMetadata{	1✔
36	Name: dt.Experiment,	1✔
37	Location: dt.Location,	1✔
38	},	1✔
39	})	1✔
40	return ds, err	1✔
41	}	1✔
42
43	// GetTableMetadata returns the metadata for the input table and an error indicating whether
44	// the table exists.
45	func (c Client) GetTableMetadata(ctx context.Context, ds bqiface.Dataset, name string) (bigquery.TableMetadata, error) {	1✔
46	t := ds.Table(name)	1✔
47	md, err := t.Metadata(ctx)	1✔
48	return md, err	1✔
49	}	1✔
50
51	// CreateTable creates a new date-partitioned table for the input `api.Datatype`.
52	// It returns an error if the table already exists.
53	func (c Client) CreateTable(ctx context.Context, ds bqiface.Dataset, dt api.Datatype) error {	1✔
54	bqSchema, err := bigquery.SchemaFromJSON(dt.Schema)	1✔
55	if err != nil {	2✔
56	return err	1✔
57	}	1✔
58
59	t := ds.Table(dt.Name)	1✔
60	err = t.Create(ctx, &bigquery.TableMetadata{	1✔
61	Name: dt.Name,	1✔
62	Schema: bqSchema,	1✔
63	TimePartitioning: &bigquery.TimePartitioning{	1✔
64	Type: "DAY",	1✔
65	Field: "date",	1✔
66	RequirePartitionFilter: true,	1✔
67	},	1✔
68	})	1✔
69	return err	1✔
70	}
71
72	// UpdateSchema updates the schema for the input `api.Datatype` table.
73	func (c Client) UpdateSchema(ctx context.Context, ds bqiface.Dataset, dt api.Datatype) error {	1✔
74	bqSchema, err := bigquery.SchemaFromJSON(dt.Schema)	1✔
75	if err != nil {	2✔
76	return err	1✔
77	}	1✔
78
79	t := ds.Table(dt.Name)	1✔
80	_, err = t.Update(ctx, bigquery.TableMetadataToUpdate{	1✔
81	Schema: bqSchema,	1✔
82	}, "")	1✔
83	return err	1✔
84	}

m-lab / autoloader / 72

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous