15633973488

Committed 13 Jun 2025 11:57AM UTC coverage: 95.318%. Remained the same

Build # 15633973488

Build Type

Pull #45

github

Committed by

byo

Commit Message

fixup

Pull Request Pull Request #45: Fix github action issues

Run Details

3013 of 3161 relevant lines covered (95.32%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.57

/pkg/cmd/static_datastore/compile.go

/*
Copyright © 2023 Bartłomiej Święcki (byo)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package static_datastore

import (
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "log"
        "os"
        "strings"

        "github.com/cinode/go/pkg/blenc"
        "github.com/cinode/go/pkg/cinodefs"
        "github.com/cinode/go/pkg/cinodefs/uploader"
        "github.com/cinode/go/pkg/datastore"
        "github.com/spf13/cobra"
)

func compileCmd() *cobra.Command {
        var o compileFSOptions
        var rootWriterInfoStr string
        var rootWriterInfoFile string
        var useRawFilesystem bool

        cmd := &cobra.Command{
                Use:   "compile --source <src_dir> --destination <dst_location>",
                Short: "Compile datastore from static files",
                Long: strings.Join([]string{
                        "The compile command can be used to create an encrypted datastore from",
                        "a content with static files that can then be used to serve through a",
                        "simple http server.",
                }, "\n"),
                RunE: func(cmd *cobra.Command, args []string) error {
                        if o.srcDir == "" || o.dstLocation == "" {
                                return cmd.Help()
                        }

                        enc := json.NewEncoder(cmd.OutOrStdout())
                        enc.SetIndent("", "  ")

                        fatalResult := func(format string, args ...interface{}) error {
                                msg := fmt.Sprintf(format, args...)

                                enc.Encode(map[string]string{
                                        "result": "ERROR",
                                        "msg":    msg,
                                })

                                cmd.SilenceUsage = true
                                cmd.SilenceErrors = true
                                return errors.New(msg)
                        }

                        if len(rootWriterInfoFile) > 0 {
                                data, err := os.ReadFile(rootWriterInfoFile)
                                if err != nil {
                                        return fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err)
                                }
                                if len(data) == 0 {
                                        return fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile)
                                }
                                rootWriterInfoStr = string(data)
                        }
                        if len(rootWriterInfoStr) > 0 {
                                wi, err := cinodefs.WriterInfoFromString(rootWriterInfoStr)
                                if err != nil {
                                        return fatalResult("Couldn't parse writer info: %v", err)
                                }
                                o.writerInfo = wi
                        }

                        if useRawFilesystem {
                                // For backwards compatibility
                                o.dstLocation = "file-raw://" + o.dstLocation
                        }

                        ep, wi, err := compileFS(cmd.Context(), o)
                        if err != nil {
                                return fatalResult("%s", err)
                        }

                        result := map[string]string{
                                "result":     "OK",
                                "entrypoint": ep.String(),
                        }
                        if wi != nil {
                                result["writer-info"] = wi.String()
                        }
                        enc.Encode(result)

                        log.Println("DONE")
                        return nil
                },
        }

        cmd.Flags().StringVarP(
                &o.srcDir, "source", "s", "",
                "Source directory with content to compile",
        )
        cmd.Flags().StringVarP(
                &o.dstLocation, "destination", "d", "",
                "location of destination datastore for blobs, can be a directory "+
                        "or an url prefixed with file://, file-raw://, http://, https://",
        )
        cmd.Flags().BoolVarP(
                &o.static, "static", "t", false,
                "if set to true, compile only the static dataset, do not create or update dynamic link",
        )
        cmd.Flags().BoolVarP(
                &useRawFilesystem, "raw-filesystem", "r", false,
                "if set to true, use raw filesystem instead of the optimized one, "+
                        "can be used to create dataset for a standard http server",
        )
        cmd.Flags().MarkDeprecated(
                "raw-filesystem",
                "use file-raw:// destination prefix instead",
        )
        cmd.Flags().StringVarP(
                &rootWriterInfoStr, "writer-info", "w", "",
                "writer info for the root dynamic link, if neither writer info nor writer info file is specified, "+
                        "a random writer info will be generated and printed out",
        )
        cmd.Flags().StringVarP(
                &rootWriterInfoFile, "writer-info-file", "f", "",
                "name of the file containing writer info for the root dynamic link, "+
                        "if neither writer info nor writer info file is specified, "+
                        "a random writer info will be generated and printed out",
        )
        cmd.Flags().StringVar(
                &o.indexFile, "index-file", "index.html",
                "name of the index file",
        )
        cmd.Flags().BoolVar(
                &o.generateIndexFiles, "generate-index-files", false,
                "automatically generate index html files with directory listing if index file is not present",
        )
        cmd.Flags().BoolVar(
                &o.append, "append", false,
                "append file in existing datastore leaving existing unchanged files as is",
        )

        return cmd
}

type compileFSOptions struct {
        srcDir             string
        dstLocation        string
        static             bool
        writerInfo         *cinodefs.WriterInfo
        generateIndexFiles bool
        indexFile          string
        append             bool
}

func compileFS(
        ctx context.Context,
        o compileFSOptions,
) (
        *cinodefs.Entrypoint,
        *cinodefs.WriterInfo,
        error,
) {
        ds, err := datastore.FromLocation(o.dstLocation)
        if err != nil {
                return nil, nil, fmt.Errorf("could not open datastore: %w", err)
        }

        opts := []cinodefs.Option{}
        if o.static {
                opts = append(opts, cinodefs.NewRootStaticDirectory())
        } else if o.writerInfo == nil {
                opts = append(opts, cinodefs.NewRootDynamicLink())
        } else {
                opts = append(opts, cinodefs.RootWriterInfo(o.writerInfo))
        }

        fs, err := cinodefs.New(
                ctx,
                blenc.FromDatastore(ds),
                opts...,
        )
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't create cinode filesystem instance: %w", err)
        }

        if !o.append {
                err = fs.ResetDir(ctx, []string{})
                if err != nil {
                        return nil, nil, fmt.Errorf("failed to reset the root directory: %w", err)
                }
        }

        var genOpts []uploader.Option
        if o.generateIndexFiles {
                genOpts = append(genOpts, uploader.CreateIndexFile(o.indexFile))
        }

        err = uploader.UploadStaticDirectory(
                ctx,
                os.DirFS(o.srcDir),
                fs,
                genOpts...,
        )
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err)
        }

        err = fs.Flush(ctx)
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't flush after directory upload: %w", err)
        }

        ep, err := fs.RootEntrypoint()
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't get root entrypoint from cinodefs instance: %w", err)
        }

        wi, err := fs.RootWriterInfo(ctx)
        if errors.Is(err, cinodefs.ErrNotALink) {
                return ep, nil, nil
        }
        if err != nil {
                return nil, nil, fmt.Errorf("couldn't get root writer info from cinodefs instance: %w", err)
        }

        return ep, wi, nil
}

1	/*
2	Copyright © 2023 Bartłomiej Święcki (byo)
3
4	Licensed under the Apache License, Version 2.0 (the "License");
5	you may not use this file except in compliance with the License.
6	You may obtain a copy of the License at
7
8	http://www.apache.org/licenses/LICENSE-2.0
9
10	Unless required by applicable law or agreed to in writing, software
11	distributed under the License is distributed on an "AS IS" BASIS,
12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	See the License for the specific language governing permissions and
14	limitations under the License.
15	*/
16
17	package static_datastore
18
19	import (
20	"context"
21	"encoding/json"
22	"errors"
23	"fmt"
24	"log"
25	"os"
26	"strings"
27
28	"github.com/cinode/go/pkg/blenc"
29	"github.com/cinode/go/pkg/cinodefs"
30	"github.com/cinode/go/pkg/cinodefs/uploader"
31	"github.com/cinode/go/pkg/datastore"
32	"github.com/spf13/cobra"
33	)
34
35	func compileCmd() *cobra.Command {	1✔
36	var o compileFSOptions	1✔
37	var rootWriterInfoStr string	1✔
38	var rootWriterInfoFile string	1✔
39	var useRawFilesystem bool	1✔
40		1✔
41	cmd := &cobra.Command{	1✔
42	Use: "compile --source <src_dir> --destination <dst_location>",	1✔
43	Short: "Compile datastore from static files",	1✔
44	Long: strings.Join([]string{	1✔
45	"The compile command can be used to create an encrypted datastore from",	1✔
46	"a content with static files that can then be used to serve through a",	1✔
47	"simple http server.",	1✔
48	}, "\n"),	1✔
49	RunE: func(cmd *cobra.Command, args []string) error {	2✔
50	if o.srcDir == "" \|\| o.dstLocation == "" {	2✔
51	return cmd.Help()	1✔
52	}	1✔
53
54	enc := json.NewEncoder(cmd.OutOrStdout())	1✔
55	enc.SetIndent("", " ")	1✔
56		1✔
57	fatalResult := func(format string, args ...interface{}) error {	2✔
58	msg := fmt.Sprintf(format, args...)	1✔
59		1✔
60	enc.Encode(map[string]string{	1✔
61	"result": "ERROR",	1✔
62	"msg": msg,	1✔
63	})	1✔
64		1✔
65	cmd.SilenceUsage = true	1✔
66	cmd.SilenceErrors = true	1✔
67	return errors.New(msg)	1✔
68	}	1✔
69
70	if len(rootWriterInfoFile) > 0 {	2✔
71	data, err := os.ReadFile(rootWriterInfoFile)	1✔
72	if err != nil {	2✔
73	return fatalResult("Couldn't read data from the writer info file at '%s': %v", rootWriterInfoFile, err)	1✔
74	}	1✔
75	if len(data) == 0 {	2✔
76	return fatalResult("Writer info file at '%s' is empty", rootWriterInfoFile)	1✔
77	}	1✔
78	rootWriterInfoStr = string(data)	1✔
79	}
80	if len(rootWriterInfoStr) > 0 {	2✔
81	wi, err := cinodefs.WriterInfoFromString(rootWriterInfoStr)	1✔
82	if err != nil {	2✔
83	return fatalResult("Couldn't parse writer info: %v", err)	1✔
84	}	1✔
85	o.writerInfo = wi	1✔
86	}
87
88	if useRawFilesystem {	1✔
89	// For backwards compatibility	×
90	o.dstLocation = "file-raw://" + o.dstLocation	×
91	}	×
92
93	ep, wi, err := compileFS(cmd.Context(), o)	1✔
94	if err != nil {	1✔
95	return fatalResult("%s", err)	×
96	}	×
97
98	result := map[string]string{	1✔
99	"result": "OK",	1✔
100	"entrypoint": ep.String(),	1✔
101	}	1✔
102	if wi != nil {	2✔
103	result["writer-info"] = wi.String()	1✔
104	}	1✔
105	enc.Encode(result)	1✔
106		1✔
107	log.Println("DONE")	1✔
108	return nil	1✔
109	},
110	}
111
112	cmd.Flags().StringVarP(	1✔
113	&o.srcDir, "source", "s", "",	1✔
114	"Source directory with content to compile",	1✔
115	)	1✔
116	cmd.Flags().StringVarP(	1✔
117	&o.dstLocation, "destination", "d", "",	1✔
118	"location of destination datastore for blobs, can be a directory "+	1✔
119	"or an url prefixed with file://, file-raw://, http://, https://",	1✔
120	)	1✔
121	cmd.Flags().BoolVarP(	1✔
122	&o.static, "static", "t", false,	1✔
123	"if set to true, compile only the static dataset, do not create or update dynamic link",	1✔
124	)	1✔
125	cmd.Flags().BoolVarP(	1✔
126	&useRawFilesystem, "raw-filesystem", "r", false,	1✔
127	"if set to true, use raw filesystem instead of the optimized one, "+	1✔
128	"can be used to create dataset for a standard http server",	1✔
129	)	1✔
130	cmd.Flags().MarkDeprecated(	1✔
131	"raw-filesystem",	1✔
132	"use file-raw:// destination prefix instead",	1✔
133	)	1✔
134	cmd.Flags().StringVarP(	1✔
135	&rootWriterInfoStr, "writer-info", "w", "",	1✔
136	"writer info for the root dynamic link, if neither writer info nor writer info file is specified, "+	1✔
137	"a random writer info will be generated and printed out",	1✔
138	)	1✔
139	cmd.Flags().StringVarP(	1✔
140	&rootWriterInfoFile, "writer-info-file", "f", "",	1✔
141	"name of the file containing writer info for the root dynamic link, "+	1✔
142	"if neither writer info nor writer info file is specified, "+	1✔
143	"a random writer info will be generated and printed out",	1✔
144	)	1✔
145	cmd.Flags().StringVar(	1✔
146	&o.indexFile, "index-file", "index.html",	1✔
147	"name of the index file",	1✔
148	)	1✔
149	cmd.Flags().BoolVar(	1✔
150	&o.generateIndexFiles, "generate-index-files", false,	1✔
151	"automatically generate index html files with directory listing if index file is not present",	1✔
152	)	1✔
153	cmd.Flags().BoolVar(	1✔
154	&o.append, "append", false,	1✔
155	"append file in existing datastore leaving existing unchanged files as is",	1✔
156	)	1✔
157		1✔
158	return cmd	1✔
159	}
160
161	type compileFSOptions struct {
162	srcDir string
163	dstLocation string
164	static bool
165	writerInfo *cinodefs.WriterInfo
166	generateIndexFiles bool
167	indexFile string
168	append bool
169	}
170
171	func compileFS(
172	ctx context.Context,
173	o compileFSOptions,
174	) (
175	*cinodefs.Entrypoint,
176	*cinodefs.WriterInfo,
177	error,
178	) {	1✔
179	ds, err := datastore.FromLocation(o.dstLocation)	1✔
180	if err != nil {	1✔
181	return nil, nil, fmt.Errorf("could not open datastore: %w", err)	×
182	}	×
183
184	opts := []cinodefs.Option{}	1✔
185	if o.static {	2✔
186	opts = append(opts, cinodefs.NewRootStaticDirectory())	1✔
187	} else if o.writerInfo == nil {	3✔
188	opts = append(opts, cinodefs.NewRootDynamicLink())	1✔
189	} else {	2✔
190	opts = append(opts, cinodefs.RootWriterInfo(o.writerInfo))	1✔
191	}	1✔
192
193	fs, err := cinodefs.New(	1✔
194	ctx,	1✔
195	blenc.FromDatastore(ds),	1✔
196	opts...,	1✔
197	)	1✔
198	if err != nil {	1✔
199	return nil, nil, fmt.Errorf("couldn't create cinode filesystem instance: %w", err)	×
200	}	×
201
202	if !o.append {	2✔
203	err = fs.ResetDir(ctx, []string{})	1✔
204	if err != nil {	1✔
205	return nil, nil, fmt.Errorf("failed to reset the root directory: %w", err)	×
206	}	×
207	}
208
209	var genOpts []uploader.Option	1✔
210	if o.generateIndexFiles {	1✔
211	genOpts = append(genOpts, uploader.CreateIndexFile(o.indexFile))	×
212	}	×
213
214	err = uploader.UploadStaticDirectory(	1✔
215	ctx,	1✔
216	os.DirFS(o.srcDir),	1✔
217	fs,	1✔
218	genOpts...,	1✔
219	)	1✔
220	if err != nil {	1✔
221	return nil, nil, fmt.Errorf("couldn't upload directory content: %w", err)	×
222	}	×
223
224	err = fs.Flush(ctx)	1✔
225	if err != nil {	1✔
226	return nil, nil, fmt.Errorf("couldn't flush after directory upload: %w", err)	×
227	}	×
228
229	ep, err := fs.RootEntrypoint()	1✔
230	if err != nil {	1✔
231	return nil, nil, fmt.Errorf("couldn't get root entrypoint from cinodefs instance: %w", err)	×
232	}	×
233
234	wi, err := fs.RootWriterInfo(ctx)	1✔
235	if errors.Is(err, cinodefs.ErrNotALink) {	2✔
236	return ep, nil, nil	1✔
237	}	1✔
238	if err != nil {	1✔
239	return nil, nil, fmt.Errorf("couldn't get root writer info from cinodefs instance: %w", err)	×
240	}	×
241
242	return ep, wi, nil	1✔
243	}

cinode / go / 15633973488

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous