4801130633

Committed 25 Apr 2023 06:54PM UTC coverage: 90.273% (-0.2%) from 90.484%

Build Type

push

github

Committed by Bartłomiej Święcki

Commit Message

Fix validation of release docker images

Run Details

1717 of 1902 relevant lines covered (90.27%)

1.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.59

/pkg/cmd/static_datastore/compile.go

/*
Copyright © 2022 Bartłomiej Święcki (byo)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package static_datastore

import (
        "context"
        "encoding/json"
        "fmt"
        "log"
        "os"

        "github.com/cinode/go/pkg/blenc"
        "github.com/cinode/go/pkg/datastore"
        "github.com/cinode/go/pkg/protobuf"
        "github.com/cinode/go/pkg/structure"
        "github.com/jbenet/go-base58"
        "github.com/spf13/cobra"
)

func compileCmd() *cobra.Command {

        var srcDir, dstDir string
        var useStaticBlobs bool
        var useRawFilesystem bool
        var rootWriterInfoStr string
        var rootWriterInfoFile string

        cmd := &cobra.Command{
                Use:   "compile --source <src_dir> --destination <dst_dir>",
                Short: "Compile datastore from static files",
                Long: `
The compile command can be used to create an encrypted datastore from
a content with static files that can then be used to serve through a
simple http server.
`,
                Run: func(cmd *cobra.Command, args []string) {
                        if srcDir == "" || dstDir == "" {
                                cmd.Help()
                                return
                        }

                        enc := json.NewEncoder(os.Stdout)
                        enc.SetIndent("", "  ")

                        fatalResult := func(format string, args ...interface{}) {
                                msg := fmt.Sprintf(format, args...)

                                enc.Encode(map[string]string{
                                        "result": "ERROR",
                                        "msg":    msg,
                                })

                                log.Fatalf(msg)
                        }

                        var wi *protobuf.WriterInfo
                        if len(rootWriterInfoFile) > 0 {
                                data, err := os.ReadFile(rootWriterInfoFile)
                                if err != nil {
                                        fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err)
                                }
                                if len(data) == 0 {
                                        fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile)
                                }
                                rootWriterInfoStr = string(data)
                        }
                        if len(rootWriterInfoStr) > 0 {
                                _wi, err := protobuf.WriterInfoFromBytes(base58.Decode(rootWriterInfoStr))
                                if err != nil {
                                        fatalResult("Couldn't parse writer info: %v", err)
                                }
                                wi = _wi
                        }

                        ep, wi, err := compileFS(srcDir, dstDir, useStaticBlobs, wi, useRawFilesystem)
                        if err != nil {
                                fatalResult("%s", err)
                        }

                        epBytes, err := ep.ToBytes()
                        if err != nil {
                                fatalResult("Couldn't serialize entrypoint: %v", err)
                        }

                        result := map[string]string{
                                "result":     "OK",
                                "entrypoint": base58.Encode(epBytes),
                        }
                        if wi != nil {
                                wiBytes, err := wi.ToBytes()
                                if err != nil {
                                        fatalResult("Couldn't serialize writer info: %v", err)
                                }

                                result["writer-info"] = base58.Encode(wiBytes)
                        }
                        enc.Encode(result)

                        log.Println("DONE")

                },
        }

        cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile")
        cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs")
        cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile only the static dataset, do not create or update dynamic link")
        cmd.Flags().BoolVarP(&useRawFilesystem, "raw-filesystem", "r", false, "If set to true, use raw filesystem instead of the optimized one, can be used to create dataset for a standard http server")
        cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")
        cmd.Flags().StringVarP(&rootWriterInfoFile, "writer-info-file", "f", "", "Name of the file containing writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")

        return cmd
}

func compileFS(
        srcDir, dstDir string,
        static bool,
        writerInfo *protobuf.WriterInfo,
        useRawFS bool,
) (
        *protobuf.Entrypoint,
        *protobuf.WriterInfo,
        error,
) {
        var retWi *protobuf.WriterInfo

        ds, err := func() (datastore.DS, error) {
                if useRawFS {
                        return datastore.InRawFileSystem(dstDir)
                }
                return datastore.InFileSystem(dstDir)
        }()
        if err != nil {
                return nil, nil, fmt.Errorf("could not open datastore: %w", err)
        }

        be := blenc.FromDatastore(ds)

        ep, err := structure.UploadStaticDirectory(context.Background(), os.DirFS(srcDir), be)
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err)
        }

        if !static {
                if writerInfo == nil {
                        ep, retWi, err = structure.CreateLink(context.Background(), be, ep)
                        if err != nil {
                                return nil, nil, fmt.Errorf("failed to update root link: %w", err)
                        }
                } else {
                        ep, err = structure.UpdateLink(context.Background(), be, writerInfo, ep)
                        if err != nil {
                                return nil, nil, fmt.Errorf("failed to update root link: %w", err)
                        }
                }
        }

        return ep, retWi, nil
}

1	/*
2	Copyright © 2022 Bartłomiej Święcki (byo)
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package static_datastore
18
19	import (
20	"context"
21	"encoding/json"
22	"fmt"
23	"log"
24	"os"
25
26	"github.com/cinode/go/pkg/blenc"
27	"github.com/cinode/go/pkg/datastore"
28	"github.com/cinode/go/pkg/protobuf"
29	"github.com/cinode/go/pkg/structure"
30	"github.com/jbenet/go-base58"
31	"github.com/spf13/cobra"
32	)
33
34	func compileCmd() *cobra.Command {	×
35		×
36	var srcDir, dstDir string	×
37	var useStaticBlobs bool	×
38	var useRawFilesystem bool	×
39	var rootWriterInfoStr string	×
40	var rootWriterInfoFile string	×
41		×
42	cmd := &cobra.Command{	×
43	Use: "compile --source <src_dir> --destination <dst_dir>",	×
44	Short: "Compile datastore from static files",	×
45	Long: `	×
46	The compile command can be used to create an encrypted datastore from	×
47	a content with static files that can then be used to serve through a	×
48	simple http server.	×
49	`,	×
50	Run: func(cmd *cobra.Command, args []string) {	×
51	if srcDir == "" \|\| dstDir == "" {	×
52	cmd.Help()	×
53	return	×
54	}	×
55
56	enc := json.NewEncoder(os.Stdout)	×
57	enc.SetIndent("", " ")	×
58		×
59	fatalResult := func(format string, args ...interface{}) {	×
60	msg := fmt.Sprintf(format, args...)	×
61		×
62	enc.Encode(map[string]string{	×
63	"result": "ERROR",	×
64	"msg": msg,	×
65	})	×
66		×
67	log.Fatalf(msg)	×
68	}	×
69
70	var wi *protobuf.WriterInfo	×
71	if len(rootWriterInfoFile) > 0 {	×
72	data, err := os.ReadFile(rootWriterInfoFile)	×
73	if err != nil {	×
74	fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err)	×
75	}	×
76	if len(data) == 0 {	×
77	fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile)	×
78	}	×
79	rootWriterInfoStr = string(data)	×
80	}
81	if len(rootWriterInfoStr) > 0 {	×
82	_wi, err := protobuf.WriterInfoFromBytes(base58.Decode(rootWriterInfoStr))	×
83	if err != nil {	×
84	fatalResult("Couldn't parse writer info: %v", err)	×
85	}	×
86	wi = _wi	×
87	}
88
89	ep, wi, err := compileFS(srcDir, dstDir, useStaticBlobs, wi, useRawFilesystem)	×
90	if err != nil {	×
91	fatalResult("%s", err)	×
92	}	×
93
94	epBytes, err := ep.ToBytes()	×
95	if err != nil {	×
96	fatalResult("Couldn't serialize entrypoint: %v", err)	×
97	}	×
98
99	result := map[string]string{	×
100	"result": "OK",	×
101	"entrypoint": base58.Encode(epBytes),	×
102	}	×
103	if wi != nil {	×
104	wiBytes, err := wi.ToBytes()	×
105	if err != nil {	×
106	fatalResult("Couldn't serialize writer info: %v", err)	×
107	}	×
108
109	result["writer-info"] = base58.Encode(wiBytes)	×
110	}
111	enc.Encode(result)	×
112		×
113	log.Println("DONE")	×
114
115	},
116	}
117
118	cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile")	×
119	cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs")	×
120	cmd.Flags().BoolVarP(&useStaticBlobs, "static", "t", false, "If set to true, compile only the static dataset, do not create or update dynamic link")	×
121	cmd.Flags().BoolVarP(&useRawFilesystem, "raw-filesystem", "r", false, "If set to true, use raw filesystem instead of the optimized one, can be used to create dataset for a standard http server")	×
122	cmd.Flags().StringVarP(&rootWriterInfoStr, "writer-info", "w", "", "Writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")	×
123	cmd.Flags().StringVarP(&rootWriterInfoFile, "writer-info-file", "f", "", "Name of the file containing writer info for the root dynamic link, if neither writer info nor writer info file is specified, a random writer info will be generated and printed out")	×
124		×
125	return cmd	×
126	}
127
128	func compileFS(
129	srcDir, dstDir string,
130	static bool,
131	writerInfo *protobuf.WriterInfo,
132	useRawFS bool,
133	) (
134	*protobuf.Entrypoint,
135	*protobuf.WriterInfo,
136	error,
137	) {	1✔
138	var retWi *protobuf.WriterInfo	1✔
139		1✔
140	ds, err := func() (datastore.DS, error) {	2✔
141	if useRawFS {	1✔
142	return datastore.InRawFileSystem(dstDir)	×
143	}	×
144	return datastore.InFileSystem(dstDir)	1✔
145	}()
146	if err != nil {	1✔
147	return nil, nil, fmt.Errorf("could not open datastore: %w", err)	×
148	}	×
149
150	be := blenc.FromDatastore(ds)	1✔
151		1✔
152	ep, err := structure.UploadStaticDirectory(context.Background(), os.DirFS(srcDir), be)	1✔
153	if err != nil {	1✔
154	return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err)	×
155	}	×
156
157	if !static {	2✔
158	if writerInfo == nil {	2✔
159	ep, retWi, err = structure.CreateLink(context.Background(), be, ep)	1✔
160	if err != nil {	1✔
161	return nil, nil, fmt.Errorf("failed to update root link: %w", err)	×
162	}	×
163	} else {	1✔
164	ep, err = structure.UpdateLink(context.Background(), be, writerInfo, ep)	1✔
165	if err != nil {	1✔
166	return nil, nil, fmt.Errorf("failed to update root link: %w", err)	×
167	}	×
168	}
169	}
170
171	return ep, retWi, nil	1✔
172	}

cinode / go / 4801130633

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous