3736023969

Committed 19 Dec 2022 11:26PM UTC coverage: 79.498%. First build

Build Type

push

github

Committed by Bartek

Commit Message

Basic test for the static_datastore binary

Run Details

919 of 1156 relevant lines covered (79.5%)

0.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.38

/cmd/static_datastore/cmd/compile.go

package cmd

import (
        "bytes"
        "context"
        "encoding/hex"
        "fmt"
        "io"
        "log"
        "mime"
        "net/http"
        "os"
        "path"
        "path/filepath"

        "github.com/cinode/go/blenc"
        "github.com/cinode/go/common"
        "github.com/cinode/go/datastore"
        "github.com/cinode/go/internal/blobtypes"
        "github.com/cinode/go/structure"
        "github.com/spf13/cobra"
        "google.golang.org/protobuf/proto"
)

func compileCmd() *cobra.Command {

        var srcDir, dstDir string

        cmd := &cobra.Command{
                Use:   "compile --source <src_dir> --destination <dst_dir>",
                Short: "Compile datastore from static files",
                Long: `
The compile command can be used to create an encrypted datastore from
a content with static files that can then be used to serve through a
simple http server.

Files stored on disk are encrypted through the blenc layer. However
this tool should not be considered secure since the encryption key
for the root node is stored in plaintext in an 'entrypoint.txt' file.
`,
                Run: func(cmd *cobra.Command, args []string) {
                        if srcDir == "" || dstDir == "" {
                                cmd.Help()
                                return
                        }
                        err := compile(srcDir, dstDir)
                        if err != nil {
                                log.Fatal(err)
                        }
                        fmt.Println("DONE")
                },
        }

        cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile")
        cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs")

        return cmd
}

func compile(srcDir, dstDir string) error {

        be := blenc.FromDatastore(datastore.InFileSystem(dstDir))

        name, key, err := compileOneLevel(srcDir, be)
        if err != nil {
                return err
        }

        fl, err := os.Create(path.Join(dstDir, "entrypoint.txt"))
        if err != nil {
                return err
        }
        keyType, keyKey, keyIV, err := key.GetSymmetricKey()
        if err != nil {
                log.Fatal(err)
        }
        _, err = fmt.Fprintf(
                fl,
                "%s\n%s:%s:%s\n",
                name,
                hex.EncodeToString([]byte{keyType}),
                hex.EncodeToString(keyKey),
                hex.EncodeToString(keyIV),
        )
        if err != nil {
                fl.Close()
                return err
        }

        err = fl.Close()
        if err != nil {
                return err
        }

        return nil
}

func compileOneLevel(path string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {
        st, err := os.Stat(path)
        if err != nil {
                return nil, nil, fmt.Errorf("Couldn't check path: %w", err)
        }

        if st.IsDir() {
                return compileDir(path, be)
        }

        if st.Mode().IsRegular() {
                return compileFile(path, be)
        }

        return nil, nil, fmt.Errorf("Neither dir nor a regular file: %v", path)
}

func compileFile(path string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {
        fmt.Println(" *", path)
        fl, err := os.Open(path)
        if err != nil {
                return nil, nil, fmt.Errorf("Couldn't read file %v: %w", path, err)
        }
        defer fl.Close()
        bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, fl)
        return bn, ki, err
}

func compileDir(p string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {
        fileList, err := os.ReadDir(p)
        if err != nil {
                return nil, nil, fmt.Errorf("Couldn't read contents of dir %v: %w", p, err)
        }
        dirStruct := structure.Directory{
                Entries: make(map[string]*structure.Directory_Entry),
        }
        for _, e := range fileList {
                subPath := path.Join(p, e.Name())
                name, ki, err := compileOneLevel(subPath, be)
                if err != nil {
                        return nil, nil, err
                }
                contentType := "application/cinode-dir"
                if !e.IsDir() {
                        contentType = mime.TypeByExtension(filepath.Ext(e.Name()))
                        if contentType == "" {
                                file, err := os.Open(subPath)
                                if err != nil {
                                        return nil, nil, fmt.Errorf("Can not detect content type for %v: %w", subPath, err)
                                }
                                buffer := make([]byte, 512)
                                n, err := io.ReadFull(file, buffer)
                                file.Close()
                                if err != nil && err != io.ErrUnexpectedEOF {
                                        return nil, nil, fmt.Errorf("Can not detect content type for %v: %w", subPath, err)
                                }
                                contentType = http.DetectContentType(buffer[:n])
                        }
                }
                keyType, keyKey, keyIV, err := ki.GetSymmetricKey()
                if err != nil {
                        return nil, nil, fmt.Errorf("Can not fetch key ifo for %v: %w", subPath, err)

                }
                dirStruct.Entries[e.Name()] = &structure.Directory_Entry{
                        Bid: name,
                        KeyInfo: &structure.KeyInfo{
                                Type: uint32(keyType),
                                Key:  keyKey,
                                Iv:   keyIV,
                        },
                        MimeType: contentType,
                }
        }

        data, err := proto.Marshal(&dirStruct)
        if err != nil {
                return nil, nil, fmt.Errorf("Can not serialize directory %v: %w", p, err)
        }

        bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(data))
        return bn, ki, err
}

1	package cmd
2
3	import (
4	"bytes"
5	"context"
6	"encoding/hex"
7	"fmt"
8	"io"
9	"log"
10	"mime"
11	"net/http"
12	"os"
13	"path"
14	"path/filepath"
15
16	"github.com/cinode/go/blenc"
17	"github.com/cinode/go/common"
18	"github.com/cinode/go/datastore"
19	"github.com/cinode/go/internal/blobtypes"
20	"github.com/cinode/go/structure"
21	"github.com/spf13/cobra"
22	"google.golang.org/protobuf/proto"
23	)
24
25	func compileCmd() *cobra.Command {	×
26		×
27	var srcDir, dstDir string	×
28		×
29	cmd := &cobra.Command{	×
30	Use: "compile --source <src_dir> --destination <dst_dir>",	×
31	Short: "Compile datastore from static files",	×
32	Long: `	×
33	The compile command can be used to create an encrypted datastore from	×
34	a content with static files that can then be used to serve through a	×
35	simple http server.	×
36		×
37	Files stored on disk are encrypted through the blenc layer. However	×
38	this tool should not be considered secure since the encryption key	×
39	for the root node is stored in plaintext in an 'entrypoint.txt' file.	×
40	`,	×
41	Run: func(cmd *cobra.Command, args []string) {	×
42	if srcDir == "" \|\| dstDir == "" {	×
43	cmd.Help()	×
44	return	×
45	}	×
46	err := compile(srcDir, dstDir)	×
47	if err != nil {	×
48	log.Fatal(err)	×
49	}	×
50	fmt.Println("DONE")	×
51	},
52	}
53
54	cmd.Flags().StringVarP(&srcDir, "source", "s", "", "Source directory with content to compile")	×
55	cmd.Flags().StringVarP(&dstDir, "destination", "d", "", "Destination directory for blobs")	×
56		×
57	return cmd	×
58	}
59
60	func compile(srcDir, dstDir string) error {	1✔
61		1✔
62	be := blenc.FromDatastore(datastore.InFileSystem(dstDir))	1✔
63		1✔
64	name, key, err := compileOneLevel(srcDir, be)	1✔
65	if err != nil {	1✔
66	return err	×
67	}	×
68
69	fl, err := os.Create(path.Join(dstDir, "entrypoint.txt"))	1✔
70	if err != nil {	1✔
71	return err	×
72	}	×
73	keyType, keyKey, keyIV, err := key.GetSymmetricKey()	1✔
74	if err != nil {	1✔
75	log.Fatal(err)	×
76	}	×
77	_, err = fmt.Fprintf(	1✔
78	fl,	1✔
79	"%s\n%s:%s:%s\n",	1✔
80	name,	1✔
81	hex.EncodeToString([]byte{keyType}),	1✔
82	hex.EncodeToString(keyKey),	1✔
83	hex.EncodeToString(keyIV),	1✔
84	)	1✔
85	if err != nil {	1✔
86	fl.Close()	×
87	return err	×
88	}	×
89
90	err = fl.Close()	1✔
91	if err != nil {	1✔
92	return err	×
93	}	×
94
95	return nil	1✔
96	}
97
98	func compileOneLevel(path string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {	1✔
99	st, err := os.Stat(path)	1✔
100	if err != nil {	1✔
101	return nil, nil, fmt.Errorf("Couldn't check path: %w", err)	×
102	}	×
103
104	if st.IsDir() {	2✔
105	return compileDir(path, be)	1✔
106	}	1✔
107
108	if st.Mode().IsRegular() {	2✔
109	return compileFile(path, be)	1✔
110	}	1✔
111
112	return nil, nil, fmt.Errorf("Neither dir nor a regular file: %v", path)	×
113	}
114
115	func compileFile(path string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {	1✔
116	fmt.Println(" *", path)	1✔
117	fl, err := os.Open(path)	1✔
118	if err != nil {	1✔
119	return nil, nil, fmt.Errorf("Couldn't read file %v: %w", path, err)	×
120	}	×
121	defer fl.Close()	1✔
122	bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, fl)	1✔
123	return bn, ki, err	1✔
124	}
125
126	func compileDir(p string, be blenc.BE) (common.BlobName, blenc.KeyInfo, error) {	1✔
127	fileList, err := os.ReadDir(p)	1✔
128	if err != nil {	1✔
129	return nil, nil, fmt.Errorf("Couldn't read contents of dir %v: %w", p, err)	×
130	}	×
131	dirStruct := structure.Directory{	1✔
132	Entries: make(map[string]*structure.Directory_Entry),	1✔
133	}	1✔
134	for _, e := range fileList {	2✔
135	subPath := path.Join(p, e.Name())	1✔
136	name, ki, err := compileOneLevel(subPath, be)	1✔
137	if err != nil {	1✔
138	return nil, nil, err	×
139	}	×
140	contentType := "application/cinode-dir"	1✔
141	if !e.IsDir() {	2✔
142	contentType = mime.TypeByExtension(filepath.Ext(e.Name()))	1✔
143	if contentType == "" {	1✔
144	file, err := os.Open(subPath)	×
145	if err != nil {	×
146	return nil, nil, fmt.Errorf("Can not detect content type for %v: %w", subPath, err)	×
147	}	×
148	buffer := make([]byte, 512)	×
149	n, err := io.ReadFull(file, buffer)	×
150	file.Close()	×
151	if err != nil && err != io.ErrUnexpectedEOF {	×
152	return nil, nil, fmt.Errorf("Can not detect content type for %v: %w", subPath, err)	×
153	}	×
154	contentType = http.DetectContentType(buffer[:n])	×
155	}
156	}
157	keyType, keyKey, keyIV, err := ki.GetSymmetricKey()	1✔
158	if err != nil {	1✔
159	return nil, nil, fmt.Errorf("Can not fetch key ifo for %v: %w", subPath, err)	×
160		×
161	}	×
162	dirStruct.Entries[e.Name()] = &structure.Directory_Entry{	1✔
163	Bid: name,	1✔
164	KeyInfo: &structure.KeyInfo{	1✔
165	Type: uint32(keyType),	1✔
166	Key: keyKey,	1✔
167	Iv: keyIV,	1✔
168	},	1✔
169	MimeType: contentType,	1✔
170	}	1✔
171	}
172
173	data, err := proto.Marshal(&dirStruct)	1✔
174	if err != nil {	1✔
175	return nil, nil, fmt.Errorf("Can not serialize directory %v: %w", p, err)	×
176	}	×
177
178	bn, ki, _, err := be.Create(context.Background(), blobtypes.Static, bytes.NewReader(data))	1✔
179	return bn, ki, err	1✔
180	}

cinode / go / 3736023969

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous