Coveralls logob
Coveralls logo
  • Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

sergei-maertens / django-systemjs / 131

29 Aug 2016 - 13:38 coverage: 96.739% (+1.3%) from 95.443%
131

Pull #18

travis-ci

9181eb84f9c35729a3bad740fb7f9d93?size=18&default=identiconweb-flow
Catch error output from jspm api & raise error
Pull Request #18: Feature/more efficient bundling

291 of 297 new or added lines in 9 files covered. (97.98%)

534 of 552 relevant lines covered (96.74%)

11.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.01
/systemjs/base.py
1
from __future__ import unicode_literals
12×
2

3
import hashlib
12×
4
import io
12×
5
import json
12×
6
import logging
12×
7
import os
12×
8
import posixpath
12×
9
import subprocess
12×
10

11
from django.conf import settings
12×
12
from django.contrib.staticfiles.storage import staticfiles_storage
12×
13
from django.utils.encoding import force_text
12×
14

15
import semantic_version
12×
16

17
from .jspm import locate_package_json
12×
18

19
logger = logging.getLogger(__name__)
12×
20

21

22
JSPM_LOG_VERSION = semantic_version.Version('0.16.3')
12×
23

24
SOURCEMAPPING_URL_COMMENT = b'//# sourceMappingURL='
12×
25

26
NODE_ENV_VAR = 'NODE_PATH'
12×
27

28

29
class BundleError(OSError):
12×
30
    pass
12×
31

32

33
class System(object):
12×
34

35
    def __init__(self, **opts):
12×
36
        self.opts = opts
12×
37
        self.stdout = self.stdin = self.stderr = subprocess.PIPE
12×
38
        self.cwd = None
12×
39
        self.version = None  # JSPM version
12×
40

41
    def _has_jspm_log(self):
12×
42
        return self.jspm_version and self.jspm_version >= JSPM_LOG_VERSION
12×
43

44
    def get_jspm_version(self, opts):
12×
45
        jspm_bin = opts['jspm']
12×
46
        cmd = '{} --version'.format(jspm_bin)
12×
47
        proc = subprocess.Popen(
12×
48
            cmd, shell=True, cwd=self.cwd, stdout=self.stdout,
49
            stdin=self.stdin, stderr=self.stderr)
50
        result, err = proc.communicate()  # block until it's done
12×
51
        if err:
12×
52
            raise BundleError("Could not determine JSPM version, error: %s", err)
12×
53
        version_string = result.decode().split()[0]
12×
54
        return semantic_version.Version(version_string, partial=True)
12×
55

56
    @property
12×
57
    def jspm_version(self):
58
        if not self.version:
12×
59
            options = self.opts.copy()
12×
60
            options.setdefault('jspm', settings.SYSTEMJS_JSPM_EXECUTABLE)
12×
61
            self.version = self.get_jspm_version(options)
12×
62
        return self.version
12×
63

64
    def bundle(self, app):
12×
65
        bundle = SystemBundle(self, app, **self.opts)
12×
66
        return bundle.bundle()
12×
67

68
    @staticmethod
12×
69
    def get_bundle_path(app):
70
        """
71
        Returns the path relative to STATIC_URL for the bundle for app.
72
        """
73
        bundle = SystemBundle(None, app)
12×
74
        return bundle.get_paths()[1]
12×
75

76

77
class SystemBundle(object):
12×
78
    """
79
    Represents a single app to be bundled.
80
    """
81

82
    def __init__(self, system, app, **options):
12×
83
        """
84
        Initialize a SystemBundle object.
85

86
        :param system: a System instance that holds the non-bundle specific
87
        meta information (such as jspm version, configuration)
88

89
        :param app: string, the name of the JS package to bundle. This may be
90
        missing the '.js' extension.
91

92
        :param options: dict containing the bundle-specific options. Possible
93
        options:
94
            `jspm`: `jspm` executable (if it's not on $PATH, for example)
95
            `log`: logging mode for jspm, can be ok|warn|err. Only available
96
                   for jspm >= 0.16.3
97
            `minify`: boolean, whether go generate minified bundles or not
98
            `sfx`: boolean, generate a self-executing bundle or not
99
        """
100
        self.system = system
12×
101
        self.app = app
12×
102

103
        # set the bundle options
104
        options.setdefault('jspm', settings.SYSTEMJS_JSPM_EXECUTABLE)
12×
105
        self.opts = options
12×
106

107
        bundle_cmd = 'bundle-sfx' if self.opts.get('sfx') else 'bundle'
12×
108
        self.command = '{jspm} ' + bundle_cmd + ' {app} {outfile}'
12×
109

110
        self.stdout = self.stdin = self.stderr = subprocess.PIPE
12×
111

112
    def get_outfile(self):
12×
113
        js_file = '{app}{ext}'.format(app=self.app, ext='.js' if self.needs_ext() else '')
12×
114
        outfile = os.path.join(settings.STATIC_ROOT, settings.SYSTEMJS_OUTPUT_DIR, js_file)
12×
115
        return outfile
12×
116

117
    def get_paths(self):
12×
118
        """
119
        Return a tuple with the absolute path and relative path (relative to STATIC_URL)
120
        """
121
        outfile = self.get_outfile()
12×
122
        rel_path = os.path.relpath(outfile, settings.STATIC_ROOT)
12×
123
        return outfile, rel_path
12×
124

125
    def needs_ext(self):
12×
126
        """
127
        Check whether `self.app` is missing the '.js' extension and if it needs it.
128
        """
129
        if settings.SYSTEMJS_DEFAULT_JS_EXTENSIONS:
12×
130
            name, ext = posixpath.splitext(self.app)
12×
131
            if not ext:
12×
132
                return True
12×
NEW
133
        return False
!
134

135
    def bundle(self):
12×
136
        """
137
        Bundle the app and return the static url to the bundle.
138
        """
139
        outfile, rel_path = self.get_paths()
12×
140

141
        options = self.opts
12×
142
        if self.system._has_jspm_log():
12×
143
            self.command += ' --log {log}'
12×
144
            options.setdefault('log', 'err')
12×
145

146
        if options.get('minify'):
12×
147
            self.command += ' --minify'
12×
148

149
        try:
12×
150
            cmd = self.command.format(app=self.app, outfile=outfile, **options)
12×
151
            proc = subprocess.Popen(
12×
152
                cmd, shell=True, cwd=self.system.cwd, stdout=self.stdout,
153
                stdin=self.stdin, stderr=self.stderr)
154

155
            result, err = proc.communicate()  # block until it's done
12×
156
            if err and self.system._has_jspm_log():
12×
157
                fmt = 'Could not bundle \'%s\': \n%s'
12×
158
                logger.warn(fmt, self.app, err)
12×
159
                raise BundleError(fmt % (self.app, err))
12×
160
            if result.strip():
12×
161
                logger.info(result)
12×
162
        except (IOError, OSError) as e:
12×
163
            if isinstance(e, BundleError):
12×
164
                raise
12×
165
            raise BundleError('Unable to apply %s (%r): %s' % (
12×
166
                              self.__class__.__name__, cmd, e))
167
        else:
168
            if not options.get('sfx'):
12×
169
                # add the import statement, which is missing for non-sfx bundles
170
                sourcemap = find_sourcemap_comment(outfile)
12×
171
                with open(outfile, 'a') as of:
12×
172
                    of.write("\nSystem.import('{app}{ext}');\n{sourcemap}".format(
12×
173
                        app=self.app,
174
                        ext='.js' if self.needs_ext() else '',
175
                        sourcemap=sourcemap if sourcemap else '',
176
                    ))
177
        return rel_path
12×
178

179

180
class TraceError(Exception):
12×
181
    pass
12×
182

183

184
class SystemTracer(object):
12×
185

186
    def __init__(self, node_path=None):
12×
187
        node_env = os.environ.copy()
12×
188
        if node_path and NODE_ENV_VAR not in node_env:
12×
189
            node_env[NODE_ENV_VAR] = node_path
12×
190
        self.env = node_env
12×
191
        self.name = 'deps.json'
12×
192
        self.storage = staticfiles_storage
12×
193
        self._trace_cache = {}
12×
194
        self._package_json_dir = os.path.dirname(locate_package_json())
12×
195

196
    @property
12×
197
    def cache_file_path(self):
198
        if not os.path.exists(settings.SYSTEMJS_CACHE_DIR):
12×
199
            os.makedirs(settings.SYSTEMJS_CACHE_DIR)
12×
200
        return os.path.join(settings.SYSTEMJS_CACHE_DIR, self.name)
12×
201

202
    def trace(self, app):
12×
203
        """
204
        Trace the dependencies for app.
205

206
        A tracer-instance is shortlived, and re-tracing the same app should
207
        yield the same results. Since tracing is an expensive process, cache
208
        the result on the tracer instance.
209
        """
210
        if app not in self._trace_cache:
12×
211
            process = subprocess.Popen(
12×
212
                "trace-deps.js {}".format(app), shell=True,
213
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
214
                env=self.env, universal_newlines=True, cwd=self._package_json_dir
215
            )
216
            out, err = process.communicate()
12×
217
            if err:
12×
NEW
218
                raise TraceError(err)
!
219
            self._trace_cache[app] = json.loads(out)
12×
220
        return self._trace_cache[app]
12×
221

222
    def get_hash(self, path):
12×
223
        md5 = hashlib.md5()
12×
224
        with self.storage.open(path) as infile:
12×
225
            for chunk in infile.chunks():
12×
226
                md5.update(chunk)
12×
227
        return md5.hexdigest()
12×
228

229
    def write_depcache(self, app_deps, bundle_options):  # TODO: use storage
12×
230
        all_deps = {
12×
231
            'version': 1,
232
            'packages': app_deps,
233
            'hashes': {},
234
            'options': bundle_options,
235
        }
236

237
        for pkg_deptree in app_deps.values():
12×
238
            for module, info in pkg_deptree.items():
12×
239
                path = info['path']
12×
240
                if path not in all_deps['hashes']:
12×
241
                    all_deps['hashes'][path] = self.get_hash(path)
12×
242

243
        with open(self.cache_file_path, 'w') as outfile:
12×
244
            json.dump(all_deps, outfile)
12×
245

246
    @property
12×
247
    def cached_deps(self):
248
        if not hasattr(self, '_depcache'):
12×
249
            with open(self.cache_file_path, 'r') as infile:
12×
250
                self._depcache = json.load(infile)
12×
251
        return self._depcache
12×
252

253
    def get_depcache(self, app):
12×
254
        # cache in memory for faster lookup
255
        if self.cached_deps.get('version') == 1:
12×
256
            return self.cached_deps['packages'].get(app)
12×
257
        else:
258
            raise NotImplementedError  # noqa
259

260
    def get_hashes(self):
12×
261
        if self.cached_deps.get('version') == 1:
12×
262
            return self.cached_deps['hashes']
12×
263
        else:
264
            raise NotImplementedError  # noqa
265

266
    def get_bundle_options(self):
12×
267
        if self.cached_deps.get('version') == 1:
12×
268
            return self.cached_deps.get('options')
12×
269
        else:
270
            raise NotImplementedError  # noqa
271

272
    def hashes_match(self, dep_tree):
12×
273
        """
274
        Compares the app deptree file hashes with the hashes stored in the
275
        cache.
276
        """
277
        hashes = self.get_hashes()
12×
278
        for module, info in dep_tree.items():
12×
279
            md5 = self.get_hash(info['path'])
12×
280
            if md5 != hashes[info['path']]:
12×
281
                return False
12×
282
        return True
12×
283

284
    def check_needs_update(self, app):
12×
285
        cached_deps = self.get_depcache(app)
12×
286
        deps = self.trace(app)
12×
287
        # no re-bundle needed if the trees, mtimes and file hashes match
288
        if deps == cached_deps and self.hashes_match(deps):
12×
289
            return False
12×
290
        return True
12×
291

292

293
def find_sourcemap_comment(filepath, block_size=100):
12×
294
    """
295
    Seeks and removes the sourcemap comment. If found, the sourcemap line is
296
    returned.
297

298
    Bundled output files can have massive amounts of lines, and the sourceMap
299
    comment is always at the end. So, to extract it efficiently, we read out the
300
    lines of the file starting from the end. We look back at most 2 lines.
301

302
    :param:filepath: path to output bundle file containing the sourcemap comment
303
    :param:blocksize: integer saying how many bytes to read at once
304
    :return:string with the sourcemap comment or None
305
    """
306

307
    MAX_TRACKBACK = 2  # look back at most 2 lines, catching potential blank line at the end
12×
308

309
    block_number = -1
12×
310
    # blocks of size block_size, in reverse order starting from the end of the file
311
    blocks = []
12×
312
    sourcemap = None
12×
313

314
    try:
12×
315
        # open file in binary read+write mode, so we can seek with negative offsets
316
        of = io.open(filepath, 'br+')
12×
317
        # figure out what's the end byte
318
        of.seek(0, os.SEEK_END)
12×
319
        block_end_byte = of.tell()
12×
320

321
        # track back for maximum MAX_TRACKBACK lines and while we can track back
322
        while block_end_byte > 0 and MAX_TRACKBACK > 0:
12×
323
            if (block_end_byte - block_size > 0):
12×
324
                # read the last block we haven't yet read
325
                of.seek(block_number*block_size, os.SEEK_END)
12×
326
                blocks.append(of.read(block_size))
12×
327
            else:
328
                # file too small, start from begining
329
                of.seek(0, os.SEEK_SET)
12×
330
                # only read what was not read
331
                blocks = [of.read(block_end_byte)]
12×
332

333
            # update variables that control while loop
334
            content = b''.join(reversed(blocks))
12×
335
            lines_found = content.count(b'\n')
12×
336
            MAX_TRACKBACK -= lines_found
12×
337
            block_end_byte -= block_size
12×
338
            block_number -= 1
12×
339

340
            # early check and bail out if we found the sourcemap comment
341
            if SOURCEMAPPING_URL_COMMENT in content:
12×
342
                offset = 0
12×
343
                # splitlines eats the last \n if its followed by a blank line
344
                lines = content.split(b'\n')
12×
345
                for i, line in enumerate(lines):
12×
346
                    if line.startswith(SOURCEMAPPING_URL_COMMENT):
12×
347
                        offset = len(line)
12×
348
                        sourcemap = line
12×
349
                        break
12×
350
                while i+1 < len(lines):
12×
351
                    offset += 1  # for the newline char
12×
352
                    offset += len(lines[i+1])
12×
353
                    i += 1
12×
354
                # track back until the start of the comment, and truncate the comment
355
                if sourcemap:
12×
356
                    offset += 1  # for the newline before the sourcemap comment
12×
357
                    of.seek(-offset, os.SEEK_END)
12×
358
                    of.truncate()
12×
359
                return force_text(sourcemap)
12×
360
    finally:
361
        of.close()
12×
362
    return sourcemap
12×
Troubleshooting · Open an Issue · Sales · Support · ENTERPRISE · CAREERS · STATUS
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2023 Coveralls, Inc