• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

chanzuckerberg / miniwdl / 27707168593

17 Jun 2026 05:23PM UTC coverage: 95.701% (-0.1%) from 95.806%
27707168593

Pull #887

github

web-flow
Merge a648fe80d into 24eb628b0
Pull Request #887: [WDL 1.2] narrow FileCoercion lint criteria

51 of 61 new or added lines in 2 files covered. (83.61%)

2 existing lines in 2 files now uncovered.

8704 of 9095 relevant lines covered (95.7%)

0.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.39
/WDL/Lint.py
1
# pylint: disable=protected-access
2
"""
1✔
3
Annotate WDL document AST with hygiene warnings (underlies ``miniwdl check``)
4

5
Given a ``doc: WDL.Document``, the lint warnings can be retrieved like so::
6

7
    import WDL
8
    import WDL.Lint
9

10
    lint = WDL.Lint.collect(WDL.Lint.lint(doc, descend_imports=False))
11
    for (pos, lint_class, message, suppressed) in lint:
12
        assert isinstance(pos, WDL.SourcePosition)
13
        assert isinstance(lint_class, str) and isinstance(message, str)
14
        if not suppressed:
15
            print(json.dumps({
16
                "uri"        : pos.uri,
17
                "abspath"    : pos.abspath,
18
                "line"       : pos.line,
19
                "end_line"   : pos.end_line,
20
                "column"     : pos.column,
21
                "end_column" : pos.end_column,
22
                "lint"       : lint_class,
23
                "message"    : message,
24
            }))
25

26
The ``descend_imports`` flag controls whether lint warnings are generated for imported documents
27
recursively (true, default), or otherwise only the given document (false).
28
"""
29

30
import subprocess
1✔
31
import tempfile
1✔
32
import json
1✔
33
import os
1✔
34
import random
1✔
35
import shutil
1✔
36
from typing import Any, Optional, Union
1✔
37
import regex
1✔
38
from . import Error, Type, Env, Expr, Tree, StdLib, Value, Walker, _util
1✔
39
from ._util import WDLVersion, wdl_version_geq, wdl_version_ord
1✔
40

41

42
def _find_doc(obj: Error.SourceNode):
1✔
43
    "find the containing document"
44
    doc = obj
1✔
45
    while not isinstance(doc, Tree.Document):
1✔
46
        if hasattr(doc, "_doc4lint"):
1✔
47
            doc = getattr(doc, "_doc4lint")
1✔
48
        else:
49
            doc = getattr(doc, "parent")
1✔
50
        assert doc
1✔
51
    setattr(obj, "_doc4lint", doc)
1✔
52
    return doc
1✔
53

54

55
def _find_expr_parent(obj: Expr.Base):
1✔
56
    "find closest ancestor of obj that isn't an expression"
57
    pt = obj
1✔
58
    while isinstance(pt, Expr.Base):
1✔
59
        pt = getattr(pt, "parent")
1✔
60
    assert pt
1✔
61
    return pt
1✔
62

63

64
class Linter(Walker.Base):
1✔
65
    """
66
    Linters are Walkers which annotate each Tree node with
67
        ``lint : List[Tuple[SourceNode,str,str]]``
68
    providing lint warnings with a node (possibly more-specific than the
69
    node it's attached to), short codename, and message.
70

71
    Linters initialize the base Walker with ``auto_descend=True`` by default,
72
    but this can be overridden if control of recursive descent is needed.
73
    """
74

75
    def __init__(self, auto_descend: bool = True, descend_imports: bool = True):
1✔
76
        super().__init__(auto_descend=auto_descend, descend_imports=descend_imports)
1✔
77

78
    def add(
1✔
79
        self, obj: Error.SourceNode, message: str, pos: Optional[Error.SourcePosition] = None
80
    ) -> bool:
81
        """
82
        Used by subclasses to attach lint to a node.
83

84
        Note, lint attaches to Tree nodes (Decl, Task, Workflow, Scatter,
85
        Conditional, Document). Warnings about individual expressions will
86
        attach to their parent Tree node.
87
        """
88
        if isinstance(obj, Expr.Base):
1✔
89
            obj = _find_expr_parent(obj)
1✔
90
        if pos is None:
1✔
91
            pos = obj.pos
1✔
92

93
        # check for suppressive comments
94
        suppress = False
1✔
95
        doc = _find_doc(obj)
1✔
96
        for L in [pos.line, pos.end_line]:
1✔
97
            # check the current line
98
            comment = doc.source_comments[L - 1]
1✔
99
            if comment and ("!" + self.__class__.__name__) in comment.text:
1✔
100
                suppress = True
1✔
101
            # check the following line if it has nothing but a comment
102
            comment = doc.source_comments[L] if L < len(doc.source_comments) else None
1✔
103
            if (
1✔
104
                comment
105
                and ("!" + self.__class__.__name__) in comment.text
106
                and comment.text.strip() == doc.source_lines[L].strip()
107
            ):
108
                suppress = True
1✔
109

110
        if not hasattr(obj, "lint"):
1✔
111
            setattr(obj, "lint", [])
1✔
112
        getattr(obj, "lint").append((pos, self.__class__.__name__, message, suppress))
1✔
113
        return True
1✔
114

115

116
_all_linters = []
1✔
117

118

119
def a_linter(cls):
1✔
120
    """
121
    Decorator for subclasses of ``Linter`` to register them for use
122
    """
123
    _all_linters.append(cls)
1✔
124

125

126
def lint(doc, descend_imports: bool = True):
1✔
127
    """
128
    Apply all linters to the document
129
    """
130

131
    # Add additional markups to the AST for use by the linters
132
    Walker.SetParents()(doc)
1✔
133
    Walker.MarkCalled()(doc)
1✔
134
    Walker.Multi([Walker.MarkImportsUsed(), Walker.SetReferrers()])(doc)
1✔
135

136
    # instantiate linters
137
    linter_instances = [cons(descend_imports=descend_imports) for cons in _all_linters]
1✔
138

139
    # run auto-descend linters "concurrently"
140
    Walker.Multi(
1✔
141
        [linter for linter in linter_instances if linter.auto_descend],
142
        descend_imports=descend_imports,
143
    )(doc)
144
    # run each non-auto-descend linter
145
    for linter in linter_instances:
1✔
146
        if not linter.auto_descend:
1✔
147
            linter(doc)
1✔
148

149
    return doc
1✔
150

151

152
class _Collector(Walker.Base):
1✔
153
    def __init__(self):
1✔
154
        super().__init__(auto_descend=True)
1✔
155
        self.lint = []
1✔
156

157
    def __call__(self, obj, descend: Optional[bool] = None):
1✔
158
        if hasattr(obj, "lint"):
1✔
159
            self.lint.extend(getattr(obj, "lint"))
1✔
160
        super().__call__(obj, descend=descend)
1✔
161

162

163
def collect(doc):
1✔
164
    """
165
    Recursively traverse the already-linted document and collect a flat list of
166
    (SourcePosition, linter_class, message, suppressed)
167
    """
168
    collector = _Collector()
1✔
169
    collector(doc)
1✔
170
    return collector.lint
1✔
171

172

173
def _find_input_decl(obj: Tree.Call, name: str) -> Tree.Decl:
1✔
174
    assert isinstance(obj.callee, (Tree.Task, Tree.Workflow))
1✔
175
    return obj.callee.available_inputs[name]
1✔
176

177

178
def _compound_coercion(
1✔
179
    to_type, from_type, base_to_type=Type.Any, from_type_predicate=None, predicates=None
180
):
181
    # helper for StringCoercion and FileCoercion to detect coercions implied
182
    # within compound types like arrays
183
    # TODO: scheme to target error messages to specific offending type parameter (especially for
184
    # struct fields and nested structs)
185
    kwargs = {
1✔
186
        "base_to_type": base_to_type,
187
        "from_type_predicate": from_type_predicate,
188
        "predicates": predicates,
189
    }
190
    if isinstance(to_type, Type.Array) and isinstance(from_type, Type.Array):
1✔
191
        return _compound_coercion(to_type.item_type, from_type.item_type, **kwargs)
1✔
192
    if isinstance(to_type, Type.Map) and isinstance(from_type, Type.Map):
1✔
193
        return _compound_coercion(
1✔
194
            to_type.item_type[0], from_type.item_type[0], **kwargs
195
        ) or _compound_coercion(to_type.item_type[1], from_type.item_type[1], **kwargs)
196
    if isinstance(to_type, Type.Pair) and isinstance(from_type, Type.Pair):
1✔
197
        return _compound_coercion(
1✔
198
            to_type.left_type, from_type.left_type, **kwargs
199
        ) or _compound_coercion(to_type.right_type, from_type.right_type, **kwargs)
200
    if (
1✔
201
        isinstance(to_type, Type.StructInstance)
202
        and to_type.members
203
        and isinstance(from_type, Type.Object)
204
    ):
205
        for field, field_type in from_type.members.items():
1✔
206
            if _compound_coercion(to_type.members[field], field_type, **kwargs):
1✔
207
                return True
1✔
208
        return False
1✔
209
    if isinstance(to_type, base_to_type):
1✔
210
        if predicates:
1✔
211
            return predicates(to_type, from_type)
×
212
        if not from_type_predicate:
1✔
213
            from_type_predicate = lambda ty: (
1✔
214
                not isinstance(  # noqa: E731
215
                    ty, (base_to_type, Type.Any)
216
                )
217
            )
218
        return from_type_predicate(from_type)
1✔
219
    return False
1✔
220

221

222
def _parent_executable(obj: Error.SourceNode) -> Optional[Union[Tree.Task, Tree.Workflow]]:
1✔
223
    if isinstance(obj, (Tree.Task, Tree.Workflow)):
1✔
224
        return obj
1✔
225
    if hasattr(obj, "parent_executable"):
1✔
226
        return getattr(obj, "parent_executable")
1✔
227
    if hasattr(obj, "parent"):
1✔
228
        ans = _parent_executable(getattr(obj, "parent"))
1✔
229
        setattr(obj, "parent_executable", ans)
1✔
230
        return ans
1✔
231
    return None
×
232

233

234
def _file_coercion_literal_status(
1✔
235
    to_type: Type.Base, expr: Expr.Base, source_dir: str
236
) -> Optional[_util.SourceRelativePathKind]:
237
    if not isinstance(to_type, (Type.File, Type.Directory)):
1✔
238
        return None
1✔
239
    if isinstance(expr.type, (Type.Any, Type.File, Type.Directory)):
1✔
NEW
240
        return None
×
241
    if not isinstance(expr.type, Type.String):
1✔
NEW
242
        return _util.SourceRelativePathKind.WRONG_KIND
×
243
    literal = expr.literal
1✔
244
    if not isinstance(literal, Value.String):
1✔
245
        return None
1✔
246
    if "://" in literal.value:
1✔
NEW
247
        return _util.SourceRelativePathKind.ABSOLUTE
×
248
    return _util.source_relative_path_kind(
1✔
249
        source_dir, literal.value, directory=isinstance(to_type, Type.Directory)
250
    )
251

252

253
def _file_coercion_decl_suspicious(to_type: Type.Base, expr: Expr.Base, source_dir: str) -> bool:
1✔
254
    status = _file_coercion_literal_status(to_type, expr, source_dir)
1✔
255
    if status is not None:
1✔
256
        return status in (
1✔
257
            _util.SourceRelativePathKind.ABSOLUTE,
258
            _util.SourceRelativePathKind.ESCAPES,
259
            _util.SourceRelativePathKind.WRONG_KIND,
260
        ) or (status == _util.SourceRelativePathKind.MISSING and not to_type.optional)
261

262
    if isinstance(to_type, Type.Array):
1✔
263
        if isinstance(expr, Expr.Array):
1✔
264
            return any(
1✔
265
                _file_coercion_decl_suspicious(to_type.item_type, item, source_dir)
266
                for item in expr.items
267
            )
NEW
268
        return False
×
269

270
    if isinstance(to_type, Type.Map):
1✔
271
        if isinstance(expr, Expr.Map):
1✔
272
            key_type, value_type = to_type.item_type
1✔
273
            return any(
1✔
274
                _file_coercion_decl_suspicious(key_type, key, source_dir)
275
                or _file_coercion_decl_suspicious(value_type, value, source_dir)
276
                for key, value in expr.items
277
            )
NEW
278
        return False
×
279

280
    if isinstance(to_type, Type.Pair):
1✔
NEW
281
        if isinstance(expr, Expr.Pair):
×
NEW
282
            return _file_coercion_decl_suspicious(
×
283
                to_type.left_type, expr.left, source_dir
284
            ) or _file_coercion_decl_suspicious(to_type.right_type, expr.right, source_dir)
NEW
285
        return False
×
286

287
    if (
1✔
288
        isinstance(to_type, Type.StructInstance)
289
        and to_type.members
290
        and isinstance(expr, Expr.Struct)
291
    ):
NEW
292
        return any(
×
293
            name in to_type.members
294
            and _file_coercion_decl_suspicious(to_type.members[name], member, source_dir)
295
            for name, member in expr.members.items()
296
        )
297

298
    return False
1✔
299

300

301
@a_linter
1✔
302
class StringCoercion(Linter):
1✔
303
    # String declaration with non-String rhs expression
304
    # File-to-String coercions are normal in tasks, but flagged at the workflow level.
305

306
    def decl(self, obj: Tree.Decl) -> Any:
1✔
307
        if obj.expr and _compound_coercion(
1✔
308
            obj.type,
309
            obj.expr.type,
310
            (Type.String,),
311
            lambda from_type: (
312
                not isinstance(
313
                    from_type,
314
                    (
315
                        (Type.Any, Type.String, Type.File, Type.Directory)
316
                        if isinstance(_parent_executable(obj), Tree.Task)
317
                        else (Type.Any, Type.String)
318
                    ),
319
                )
320
            ),
321
        ):
322
            self.add(obj, "{} {} = :{}:".format(str(obj.type), obj.name, str(obj.expr.type)))
1✔
323

324
    def expr(self, obj: Expr.Base) -> Any:
1✔
325
        if isinstance(obj, Expr.Apply):
1✔
326
            # String function operands with non-String expression
327
            if obj.function_name in ("_add", "_interpolation_add"):
1✔
328
                # TODO: should this apply to _interpolation_add, where coercion to String is
329
                # "obviously" intended?
330
                any_string = False
1✔
331
                any_string_literal = False
1✔
332
                non_string = None
1✔
333
                for arg in obj.arguments:
1✔
334
                    if isinstance(arg.type, Type.String):
1✔
335
                        any_string = True
1✔
336
                        if isinstance(arg, Expr.String):
1✔
337
                            any_string_literal = True
1✔
338
                    elif not isinstance(arg.type, (Type.File, Type.Directory)):
1✔
339
                        non_string = arg.type
1✔
340
                if any_string and non_string:
1✔
341
                    allowed = not wdl_version_geq(
1✔
342
                        _find_doc(obj).effective_wdl_version, WDLVersion.V1_1
343
                    )
344
                    if not allowed:
1✔
345
                        self.add(
1✔
346
                            obj,
347
                            "use interpolation instead of concatenating :String:"
348
                            f" + :{non_string}: [deprecated in WDL >=1.1]",
349
                            obj.pos,
350
                        )
351
                    elif not any_string_literal and obj.function_name != "_interpolation_add":
1✔
352
                        # Prior to WDL 1.1, + could implicitly coerce a non-String argument to
353
                        # concatenate with a String argument. Warn about this unless one side is a
354
                        # a String literal or we're inside an interpolation (as those cases make
355
                        # the intent clear)
356
                        self.add(
1✔
357
                            obj,
358
                            f"consider interpolation instead of concatenating :String: + :{non_string}:",
359
                            obj.pos,
360
                        )
361
            else:
362
                F = getattr(
1✔
363
                    StdLib.TaskOutputs(_find_doc(obj).effective_wdl_version), obj.function_name
364
                )
365
                if isinstance(F, StdLib.StaticFunction) and obj.function_name not in (
1✔
366
                    "basename",  # ok to take either String or File
367
                    "write_lines",  # clear intent
368
                    "write_tsv",  # clear intent
369
                ):
370
                    for i in range(min(len(F.argument_types), len(obj.arguments))):
1✔
371
                        F_i = F.argument_types[i]
1✔
372
                        arg_i = obj.arguments[i]
1✔
373
                        if _compound_coercion(
1✔
374
                            F_i,
375
                            arg_i.type,
376
                            (Type.String,),
377
                            lambda from_type: (
378
                                not isinstance(
379
                                    from_type,
380
                                    (
381
                                        (
382
                                            Type.Any,
383
                                            Type.String,
384
                                            Type.File,
385
                                            Type.Directory,
386
                                        )
387
                                        if isinstance(_parent_executable(obj), Tree.Task)
388
                                        else (Type.Any, Type.String)
389
                                    ),
390
                                )
391
                            ),
392
                        ):
393
                            msg = "{} argument of {}() = :{}:".format(
1✔
394
                                str(F_i), F.name, str(arg_i.type)
395
                            )
396
                            self.add(obj, msg, arg_i.pos)
1✔
397
        elif isinstance(obj, Expr.Array):
1✔
398
            # Array literal with mixed item types, one of which is String,
399
            # causing coercion of the others
400
            any_string = False
1✔
401
            all_string = True
1✔
402
            item_types = []
1✔
403
            for elt in obj.items:
1✔
404
                if isinstance(elt.type, Type.String):
1✔
405
                    any_string = True
1✔
406
                elif not isinstance(elt.type, (Type.File, Type.Directory, Type.Any)):
1✔
407
                    all_string = False
1✔
408
                item_types.append(str(elt.type))
1✔
409
            if any_string and not all_string:
1✔
410
                msg = "{} literal = [{}]".format(
1✔
411
                    str(obj.type), ", ".join(":{}:".format(ty) for ty in item_types)
412
                )
413
                self.add(obj, msg, obj.pos)
1✔
414

415
    def call(self, obj: Tree.Call) -> Any:
1✔
416
        for name, inp_expr in obj.inputs.items():
1✔
417
            decl = _find_input_decl(obj, name)
1✔
418
            if _compound_coercion(decl.type, inp_expr.type, (Type.String,)):
1✔
419
                msg = "input {} {} = :{}:".format(str(decl.type), decl.name, str(inp_expr.type))
1✔
420
                self.add(obj, msg, inp_expr.pos)
1✔
421

422

423
@a_linter
1✔
424
class FileCoercion(Linter):
1✔
425
    # String-to-File coercions are typical in task outputs, but potentially
426
    # problematic elsewhere.
427

428
    def __init__(self, descend_imports: bool = True):
1✔
429
        super().__init__(auto_descend=False, descend_imports=descend_imports)
1✔
430

431
    def task(self, obj: Tree.Task) -> Any:
1✔
432
        # descend into everything but outputs
433
        for d in obj.inputs or []:
1✔
434
            self(d)
1✔
435
        for d in obj.postinputs:
1✔
436
            self(d)
1✔
437
        self(obj.command)
1✔
438
        for _, ex in obj.runtime.items():
1✔
439
            self(ex)
1✔
440

441
    # File declaration with String rhs expression
442
    # exception: when rhs looks like a URI constant (typically a default reference database)
443
    def decl(self, obj: Tree.Decl) -> Any:
1✔
444
        super().decl(obj)
1✔
445
        if obj.expr and _compound_coercion(obj.type, obj.expr.type, (Type.File, Type.Directory)):
1✔
446
            if (
1✔
447
                isinstance(obj.expr, Expr.String)
448
                and obj.expr.literal
449
                and "://" in obj.expr.literal.value
450
            ):
451
                self.add(
1✔
452
                    obj,
453
                    f'{obj.type} {obj.name} = "URI" may work with miniwdl, but for WDL portability,'
454
                    " provide default URI in inputs JSON file",
455
                )
456
            else:
457
                allowed = wdl_version_geq(
1✔
458
                    _find_doc(obj).effective_wdl_version, WDLVersion.V1_2
459
                ) and not _file_coercion_decl_suspicious(obj.type, obj.expr, obj.source_dir)
460
                if not allowed:
1✔
461
                    self.add(
1✔
462
                        obj, "{} {} = :{}:".format(str(obj.type), obj.name, str(obj.expr.type))
463
                    )
464

465
    def expr(self, obj: Expr.Base) -> Any:
1✔
466
        super().expr(obj)
1✔
467
        if isinstance(obj, Expr.Apply):
1✔
468
            # File function operands with String expression
469
            F = getattr(StdLib.TaskOutputs(_find_doc(obj).effective_wdl_version), obj.function_name)
1✔
470
            if isinstance(F, StdLib.StaticFunction):
1✔
471
                for i in range(min(len(F.argument_types), len(obj.arguments))):
1✔
472
                    F_i = F.argument_types[i]
1✔
473
                    arg_i = obj.arguments[i]
1✔
474
                    if _compound_coercion(F_i, arg_i.type, (Type.File, Type.Directory)):
1✔
475
                        msg = "{} argument of {}() = :{}:".format(str(F_i), F.name, str(arg_i.type))
1✔
476
                        self.add(obj, msg, arg_i.pos)
1✔
477
            elif obj.function_name == "size":
1✔
478
                arg0ty = obj.arguments[0].type
1✔
479
                if not isinstance(arg0ty, Type.File) and not (
1✔
480
                    isinstance(arg0ty, Type.Array) and isinstance(arg0ty.item_type, Type.File)
481
                ):
482
                    self.add(
1✔
483
                        obj,
484
                        "File?/Array[File?] argument of size() = :{}:".format(
485
                            str(obj.arguments[0].type)
486
                        ),
487
                        obj.arguments[0].pos,
488
                    )
489

490
    def call(self, obj: Tree.Call) -> Any:
1✔
491
        super().call(obj)
1✔
492
        for name, inp_expr in obj.inputs.items():
1✔
493
            decl = _find_input_decl(obj, name)
1✔
494
            if _compound_coercion(decl.type, inp_expr.type, (Type.File, Type.Directory)):
1✔
495
                msg = "input {} {} = :{}:".format(str(decl.type), decl.name, str(inp_expr.type))
1✔
496
                self.add(obj, msg, inp_expr.pos)
1✔
497

498

499
def _array_levels(ty: Type.Base, l=0):
1✔
500
    if isinstance(ty, Type.Array):
1✔
501
        return _array_levels(ty.item_type, l + 1)
1✔
502
    return l
1✔
503

504

505
def _is_array_coercion(value_type: Type.Base, expr_type: Type.Base):
1✔
506
    return (
1✔
507
        isinstance(value_type, Type.Array)
508
        and _array_levels(value_type) > _array_levels(expr_type)
509
        and not isinstance(expr_type, Type.Any)
510
        and expr_type != Type.Array(Type.Any())
511
    )
512

513

514
@a_linter
1✔
515
class ArrayCoercion(Linter):
1✔
516
    # implicit promotion of T to Array[T]
517
    def decl(self, obj: Tree.Decl) -> Any:
1✔
518
        if obj.expr and _is_array_coercion(obj.type, obj.expr.type):
1✔
519
            msg = "{} {} = :{}:".format(str(obj.type), obj.name, str(obj.expr.type))
1✔
520
            self.add(obj, msg)
1✔
521

522
    def expr(self, obj: Expr.Base) -> Any:
1✔
523
        if isinstance(obj, Expr.Apply):
1✔
524
            F = getattr(StdLib.TaskOutputs(_find_doc(obj).effective_wdl_version), obj.function_name)
1✔
525
            if isinstance(F, StdLib.StaticFunction):
1✔
526
                for i in range(min(len(F.argument_types), len(obj.arguments))):
1✔
527
                    F_i = F.argument_types[i]
1✔
528
                    arg_i = obj.arguments[i]
1✔
529
                    if _is_array_coercion(F_i, arg_i.type):
1✔
530
                        msg = "{} argument of {}() = :{}:".format(str(F_i), F.name, str(arg_i.type))
1✔
531
                        self.add(obj, msg, arg_i.pos)
1✔
532

533
    def call(self, obj: Tree.Call) -> Any:
1✔
534
        for name, inp_expr in obj.inputs.items():
1✔
535
            decl = _find_input_decl(obj, name)
1✔
536
            if _is_array_coercion(decl.type, inp_expr.type):
1✔
537
                msg = "input {} {} = :{}:".format(str(decl.type), decl.name, str(inp_expr.type))
1✔
538
                self.add(obj, msg, inp_expr.pos)
1✔
539

540

541
@a_linter
1✔
542
class UnverifiedStruct(Linter):
1✔
543
    # non-statically-verified initialization of StructInstance from Map[String,Any]
544

545
    def decl(self, obj: Tree.Decl) -> Any:
1✔
546
        if obj.expr and _compound_coercion(
1✔
547
            obj.type,
548
            obj.expr.type,
549
            (Type.StructInstance,),
550
            lambda from_type: (
551
                isinstance(from_type, Type.Any)
552
                or (isinstance(from_type, Type.Map) and from_type.literal_keys is None)
553
            ),
554
        ):
555
            self.add(
1✔
556
                obj,
557
                "{} {} = :{}: -- struct initializer isn't statically verified".format(
558
                    str(obj.type), obj.name, str(obj.expr.type)
559
                ),
560
            )
561

562
    def call(self, obj: Tree.Call) -> Any:
1✔
563
        for name, inp_expr in obj.inputs.items():
1✔
564
            decl = _find_input_decl(obj, name)
1✔
565
            if _compound_coercion(
1✔
566
                decl.type,
567
                inp_expr.type,
568
                (Type.StructInstance,),
569
                lambda from_type: (
570
                    isinstance(from_type, Type.Any)
571
                    or (isinstance(from_type, Type.Map) and from_type.literal_keys is None)
572
                ),
573
            ):
574
                msg = "input {} {} = :{}: -- struct initializer isn't statically verified".format(
1✔
575
                    str(decl.type), decl.name, str(inp_expr.type)
576
                )
577
                self.add(obj, msg, inp_expr.pos)
1✔
578

579

580
@a_linter
1✔
581
class NonStringKeyMapJSON(Linter):
1✔
582
    # Map values with non-String key types aren't portable through standard WDL JSON
583
    # serialization. miniwdl may support some of these cases; keep this lint-only.
584

585
    _MESSAGE = (
1✔
586
        "Maps with non-String keys are not JSON-serializable per WDL spec (miniwdl may allow)"
587
    )
588

589
    def _map_with_non_string_key(self, ty: Type.Base) -> Optional[Type.Map]:
1✔
590
        if isinstance(ty, Type.Map):
1✔
591
            if ty.item_type[0] != Type.String():
1✔
592
                return ty
1✔
593
        for p in ty.parameters:
1✔
594
            ans = self._map_with_non_string_key(p)
1✔
595
            if ans:
1✔
596
                return ans
1✔
597
        return None
1✔
598

599
    def decl(self, obj: Tree.Decl) -> Any:
1✔
600
        parent = getattr(obj, "parent", None)
1✔
601
        if isinstance(parent, (Tree.Task, Tree.Workflow)):
1✔
602
            section = None
1✔
603
            if getattr(parent, "inputs", None) and obj in getattr(parent, "inputs"):
1✔
604
                section = "input"
1✔
605
            elif getattr(parent, "outputs", None) and obj in getattr(parent, "outputs"):
1✔
606
                section = "output"
1✔
607
            if section:
1✔
608
                bad_map = self._map_with_non_string_key(obj.type)
1✔
609
                if bad_map:
1✔
610
                    self.add(
1✔
611
                        obj,
612
                        f"{section} {obj.type} {obj.name} contains {bad_map}; " + self._MESSAGE,
613
                    )
614

615
    def call(self, obj: Tree.Call) -> Any:
1✔
616
        for name, inp_expr in obj.inputs.items():
1✔
617
            decl = _find_input_decl(obj, name)
1✔
618
            bad_map = self._map_with_non_string_key(decl.type)
1✔
619
            if bad_map:
1✔
620
                self.add(
1✔
621
                    obj,
622
                    f"input {decl.type} {decl.name} contains {bad_map}; " + self._MESSAGE,
623
                    inp_expr.pos,
624
                )
625

626
    def expr(self, obj: Expr.Base) -> Any:
1✔
627
        if isinstance(obj, Expr.Apply) and obj.function_name == "write_json" and obj.arguments:
1✔
628
            bad_map = self._map_with_non_string_key(obj.arguments[0].type)
1✔
629
            if bad_map:
1✔
630
                self.add(
1✔
631
                    obj,
632
                    f"write_json() argument contains {bad_map}; " + self._MESSAGE,
633
                    obj.arguments[0].pos,
634
                )
635

636

637
@a_linter
1✔
638
class OptionalCoercion(Linter):
1✔
639
    # Expression of optional type where a non-optional value is expected
640
    # Normally these fail typechecking, but the enforcement isn't stringent in
641
    # older WDLs.
642
    # TODO: suppress within 'if (defined(x))' consequent
643
    def expr(self, obj: Expr.Base) -> Any:
1✔
644
        if isinstance(obj, Expr.Apply):
1✔
645
            if obj.function_name in ["_add", "_sub", "_mul", "_div", "_land", "_lor"]:
1✔
646
                # excluded _interpolation_add, since interpolations expressly allow this
647
                assert len(obj.arguments) == 2
1✔
648
                arg0ty = obj.arguments[0].type
1✔
649
                arg1ty = obj.arguments[1].type
1✔
650
                if arg0ty.optional or arg1ty.optional:
1✔
651
                    self.add(
1✔
652
                        obj,
653
                        "infix operator has :{}: and :{}: operands".format(
654
                            str(arg0ty), str(arg1ty)
655
                        ),
656
                        obj.pos,
657
                    )
658
            else:
659
                F = getattr(
1✔
660
                    StdLib.TaskOutputs(_find_doc(obj).effective_wdl_version), obj.function_name
661
                )
662
                if isinstance(F, StdLib.StaticFunction):
1✔
663
                    for i in range(min(len(F.argument_types), len(obj.arguments))):
1✔
664
                        F_i = F.argument_types[i]
1✔
665
                        arg_i = obj.arguments[i]
1✔
666
                        if not arg_i.type.coerces(F_i, check_quant=True) and not _is_array_coercion(
1✔
667
                            F_i, arg_i.type
668
                        ):
669
                            msg = "{} argument of {}() = :{}:".format(
1✔
670
                                str(F.argument_types[i]), F.name, str(obj.arguments[i].type)
671
                            )
672
                            self.add(obj, msg, obj.arguments[i].pos)
1✔
673

674
    def decl(self, obj: Tree.Decl) -> Any:
1✔
675
        if (
1✔
676
            obj.expr
677
            and not obj.expr.type.coerces(obj.type, check_quant=True)
678
            and not _is_array_coercion(obj.type, obj.expr.type)
679
        ):
680
            self.add(obj, "{} {} = :{}:".format(str(obj.type), obj.name, str(obj.expr.type)))
1✔
681

682
    def call(self, obj: Tree.Call) -> Any:
1✔
683
        for name, inp_expr in obj.inputs.items():
1✔
684
            decl = _find_input_decl(obj, name)
1✔
685
            # treat input with default as optional, with or without the ? type quantifier
686
            decltype = decl.type.copy(optional=True) if decl.expr else decl.type
1✔
687
            if not inp_expr.type.coerces(decltype, check_quant=True) and not _is_array_coercion(
1✔
688
                decltype, inp_expr.type
689
            ):
690
                msg = "input {} {} = :{}:".format(str(decl.type), decl.name, str(inp_expr.type))
1✔
691
                self.add(obj, msg, inp_expr.pos)
1✔
692

693

694
def _is_nonempty_coercion(value_type: Type.Base, expr_type: Type.Base):
1✔
695
    return (
1✔
696
        isinstance(value_type, Type.Array)
697
        and isinstance(expr_type, Type.Array)
698
        and value_type.nonempty
699
        and not expr_type.nonempty
700
    )
701
    # TODO: descend into compound types
702

703

704
@a_linter
1✔
705
class NonemptyCoercion(Linter):
1✔
706
    # An array of possibly-empty type where a nonempty array is expected
707
    def expr(self, obj: Expr.Base) -> Any:
1✔
708
        if isinstance(obj, Expr.Apply):
1✔
709
            F = getattr(StdLib.TaskOutputs(_find_doc(obj).effective_wdl_version), obj.function_name)
1✔
710
            if isinstance(F, StdLib.StaticFunction):
1✔
711
                for i in range(min(len(F.argument_types), len(obj.arguments))):
1✔
712
                    F_i = F.argument_types[i]
1✔
713
                    arg_i = obj.arguments[i]
1✔
714
                    if _is_nonempty_coercion(F_i, arg_i.type):
1✔
715
                        msg = "{} argument of {}() = :{}:".format(
×
716
                            str(F.argument_types[i]), F.name, str(obj.arguments[i].type)
717
                        )
718
                        self.add(obj, msg, obj.arguments[i].pos)
×
719

720
    def decl(self, obj: Tree.Decl) -> Any:
1✔
721
        # heuristic exception for: Array[File]+ outp = glob(...)
722
        if (
1✔
723
            obj.expr
724
            and _is_nonempty_coercion(obj.type, obj.expr.type)
725
            and not (
726
                isinstance(obj.expr, Expr.Apply)
727
                and obj.expr.function_name in ["glob", "read_lines", "read_tsv", "read_array"]
728
            )
729
        ):
730
            self.add(obj, "{} {} = :{}:".format(str(obj.type), obj.name, str(obj.expr.type)))
1✔
731

732
    def call(self, obj: Tree.Call) -> Any:
1✔
733
        for name, inp_expr in obj.inputs.items():
1✔
734
            decl = _find_input_decl(obj, name)
1✔
735
            if _is_nonempty_coercion(decl.type, inp_expr.type):
1✔
736
                msg = "input {} {} = :{}:".format(str(decl.type), decl.name, str(inp_expr.type))
1✔
737
                self.add(obj, msg, inp_expr.pos)
1✔
738

739

740
@a_linter
1✔
741
class IncompleteCall(Linter):
1✔
742
    # Call without all required inputs (allowed for top-level workflow)
743
    def call(self, obj: Tree.Call) -> Any:
1✔
744
        assert obj.callee is not None
1✔
745
        required_inputs = set(decl.name for decl in obj.callee.required_inputs)
1✔
746
        for name, _ in obj.inputs.items():
1✔
747
            if name in required_inputs:
1✔
748
                required_inputs.remove(name)
1✔
749
        if required_inputs:
1✔
750
            msg = "required input(s) omitted in call to {} ({})".format(
1✔
751
                obj.callee.name, ", ".join(required_inputs)
752
            )
753
            self.add(obj, msg)
1✔
754

755

756
@a_linter
1✔
757
class NameCollision(Linter):
1✔
758
    # Name collisions between
759
    # - call and import
760
    # - call and struct type/alias
761
    # - decl and import
762
    # - decl and workflow
763
    # - decl and task
764
    # - decl and struct type/alias
765
    # - scatter variable and import
766
    # - scatter variable and workflow
767
    # - scatter variable and task
768
    # - scatter variable and struct type/alias
769
    # - workflow and import
770
    # - workflow and struct type/alias
771
    # - task and import
772
    # - task and struct type/alias
773
    # - struct type/alias and import
774
    # These are allowed, but confusing.
775
    def call(self, obj: Tree.Call) -> Any:
1✔
776
        doc = _find_doc(obj)
1✔
777
        for imp in doc.imports:
1✔
778
            if imp.namespace == obj.name:
1✔
779
                msg = "call name '{}' collides with imported document namespace".format(obj.name)
1✔
780
                self.add(obj, msg)
1✔
781
        for stb in doc.struct_typedefs:
1✔
782
            assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
783
            if stb.name == obj.name:
1✔
784
                msg = "call name '{}' collides with {}struct type".format(
1✔
785
                    obj.name, "imported " if stb.value.imported else ""
786
                )
787
                self.add(obj, msg)
1✔
788

789
    def decl(self, obj: Tree.Decl) -> Any:
1✔
790
        doc = _find_doc(obj)
1✔
791
        assert isinstance(doc, Tree.Document)
1✔
792
        for imp in doc.imports:
1✔
793
            if imp.namespace == obj.name:
1✔
794
                msg = "declaration of '{}' collides with imported document namespace".format(
1✔
795
                    obj.name
796
                )
797
                self.add(obj, msg)
1✔
798
        if doc.workflow and doc.workflow.name == obj.name:
1✔
799
            msg = "declaration of '{}' collides with workflow name".format(obj.name)
1✔
800
            self.add(obj, msg)
1✔
801
        for task in doc.tasks:
1✔
802
            if obj.name == task.name:
1✔
803
                msg = "declaration of '{}' collides with a task name".format(obj.name)
1✔
804
                self.add(obj, msg)
1✔
805
        for stb in doc.struct_typedefs:
1✔
806
            assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
807
            if stb.name == obj.name:
1✔
808
                msg = "declaration of '{}' colides with {}struct type".format(
1✔
809
                    obj.name, "imported " if stb.value.imported else ""
810
                )
811
                self.add(obj, msg)
1✔
812

813
    def scatter(self, obj: Tree.Scatter) -> Any:
1✔
814
        doc = _find_doc(obj)
1✔
815
        for imp in doc.imports:
1✔
816
            if imp.namespace == obj.variable:
1✔
817
                msg = "scatter variable '{}' collides with imported document namespace".format(
1✔
818
                    obj.variable
819
                )
820
                self.add(obj, msg)
1✔
821
        if doc.workflow and doc.workflow.name == obj.variable:
1✔
822
            msg = "scatter variable '{}' collides with workflow name".format(obj.variable)
1✔
823
            self.add(obj, msg)
1✔
824
        for task in doc.tasks:
1✔
825
            if obj.variable == task.name:
1✔
826
                msg = "scatter variable '{}' collides with a task name".format(obj.variable)
1✔
827
                self.add(obj, msg)
1✔
828
        for stb in doc.struct_typedefs:
1✔
829
            assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
830
            if stb.name == obj.variable:
1✔
831
                msg = "scatter variable '{}' colides with {}struct type".format(
1✔
832
                    obj.variable, "imported " if stb.value.imported else ""
833
                )
834
                self.add(obj, msg)
1✔
835

836
    def workflow(self, obj: Tree.Workflow) -> Any:
1✔
837
        doc = _find_doc(obj)
1✔
838
        for imp in doc.imports:
1✔
839
            if imp.namespace == obj.name:
1✔
840
                msg = "workflow name '{}' collides with imported document namespace".format(
1✔
841
                    obj.name
842
                )
843
                self.add(obj, msg)
1✔
844
        for stb in doc.struct_typedefs:
1✔
845
            assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
846
            if stb.name == obj.name:
1✔
847
                msg = "workflow name '{}' colides with {}struct type".format(
1✔
848
                    obj.name, "imported " if stb.value.imported else ""
849
                )
850
                self.add(obj, msg)
1✔
851

852
    def task(self, obj: Tree.Task) -> Any:
1✔
853
        doc = _find_doc(obj)
1✔
854
        for imp in doc.imports:
1✔
855
            if imp.namespace == obj.name:
1✔
856
                msg = "task name '{}' collides with imported document namespace".format(obj.name)
1✔
857
                self.add(obj, msg)
1✔
858
        for stb in doc.struct_typedefs:
1✔
859
            assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
860
            if stb.name == obj.name:
1✔
861
                msg = "task name '{}' colides with {}struct type".format(
1✔
862
                    obj.name, "imported " if stb.value.imported else ""
863
                )
864
                self.add(obj, msg)
1✔
865

866
    def document(self, obj: Tree.Document) -> Any:
1✔
867
        for imp in obj.imports:
1✔
868
            for stb in obj.struct_typedefs:
1✔
869
                assert isinstance(stb, Env.Binding) and isinstance(stb.value, Tree.StructTypeDef)
1✔
870
                if stb.name == imp.namespace:
1✔
871
                    msg = "imported document namespace '{}' collides with {}struct type".format(
1✔
872
                        imp.namespace, "imported " if stb.value.imported else ""
873
                    )
874
                    self.add(obj, msg, imp.pos)
1✔
875

876

877
@a_linter
1✔
878
class UnusedImport(Linter):
1✔
879
    # Nothing used from an imported document
880
    # TODO: clarify confusion when none of an imported document D's structs are used because all
881
    #       the same struct definitions were imported from a different document E (probably because
882
    #       E itself imported D)
883
    def document(self, obj: Tree.Document) -> Any:
1✔
884
        for imp in obj.imports:
1✔
885
            if imp.namespace not in getattr(obj, "imports_used"):
1✔
886
                self.add(
1✔
887
                    obj,
888
                    "no use of workflow, tasks, or structs defined in the imported document "
889
                    + imp.namespace,
890
                    pos=imp.pos,
891
                )
892

893

894
@a_linter
1✔
895
class ImportNewerWDL(Linter):
1✔
896
    # Document imports a document with a newer WDL version
897
    def document(self, obj: Tree.Document) -> Any:
1✔
898
        doc_version = wdl_version_ord(obj.effective_wdl_version)
1✔
899
        for imp in obj.imports:
1✔
900
            assert imp.doc
1✔
901
            if wdl_version_ord(imp.doc.effective_wdl_version) > doc_version:
1✔
902
                self.add(
1✔
903
                    obj,
904
                    "imported document has newer WDL version",
905
                    pos=imp.pos,
906
                )
907

908

909
@a_linter
1✔
910
class ForwardReference(Linter):
1✔
911
    # Ident referencing a value or call output lexically precedes Decl/Call
912
    def expr(self, obj: Expr.Base) -> Any:
1✔
913
        if isinstance(obj, Expr.Ident):
1✔
914
            referee = obj.referee
1✔
915
            if isinstance(referee, Tree.Gather):
1✔
916
                referee = referee.final_referee
1✔
917
            if referee.pos.line > obj.pos.line or (  # type: ignore
1✔
918
                referee.pos.line == obj.pos.line  # type: ignore
919
                and referee.pos.column > obj.pos.column  # type: ignore
920
            ):
921
                if isinstance(referee, Tree.Decl):
1✔
922
                    msg = "reference to {} precedes its declaration".format(obj.name)
1✔
923
                elif isinstance(referee, Tree.Call):
1✔
924
                    msg = "reference to output {} precedes the call".format(obj.name)
1✔
925
                else:
926
                    assert False
×
927
                self.add(obj, msg, obj.pos)
1✔
928

929

930
@a_linter
1✔
931
class UnusedDeclaration(Linter):
1✔
932
    # Nothing references a (non-input) Decl
933
    def decl(self, obj: Tree.Decl) -> Any:
1✔
934
        pt = getattr(obj, "parent")
1✔
935
        is_output = (
1✔
936
            isinstance(pt, (Tree.Workflow, Tree.Task))
937
            and getattr(pt, "outputs")
938
            and obj in getattr(pt, "outputs")
939
        )
940
        if not is_output and not getattr(obj, "referrers", []):
1✔
941
            # heuristic exceptions:
942
            # 1. File whose name suggests it's an hts index file; as these
943
            #    commonly need to be localized, but not explicitly used in task
944
            #    command
945
            # 2. dxWDL "native" task stubs, which declare inputs but leave
946
            #    command empty.
947
            # 3. task declaration has "env" decorator and the command uses it
948
            #    as an environment variable
949
            index_suffixes = [
1✔
950
                "index",
951
                "indexes",
952
                "indices",
953
                "idx",
954
                "tbi",
955
                "bai",
956
                "crai",
957
                "csi",
958
                "fai",
959
                "dict",
960
            ]
961
            if not (
1✔
962
                (
963
                    isinstance(obj.type, Type.File)
964
                    and (sum(1 for sfx in index_suffixes if obj.name.lower().endswith(sfx)) > 0)
965
                )
966
                or (
967
                    isinstance(obj.type, Type.Array)
968
                    and isinstance(obj.type.item_type, Type.File)
969
                    and (sum(1 for sfx in index_suffixes if obj.name.lower().endswith(sfx)) > 0)
970
                )
971
                or (
972
                    isinstance(pt, Tree.Task)
973
                    and pt.meta.get("type") == "native"
974
                    and pt.meta.get("id")
975
                )
976
                or self._used_as_command_env_var(obj)
977
            ):
978
                self.add(obj, "nothing references {} {}".format(str(obj.type), obj.name))
1✔
979

980
    def _used_as_command_env_var(self, decl: Tree.Decl) -> bool:
1✔
981
        # Task input declarations with the "env" modifier are intended to be
982
        # used as environment variables in the task command. False-positive
983
        # UnusedDeclaration warnings might result because such references are
984
        # not modeled in our WDL syntax tree.
985
        # Avoid this by searching for apparent usage of the environment
986
        # variable in string literal parts of the task command script. This
987
        # isn't a perfect heuristic (e.g. it could be single-quoted within the
988
        # script), but that's OK for lint warning purposes.
989
        task = getattr(decl, "parent")
1✔
990
        if not (isinstance(task, Tree.Task) and decl.decor.get("env", False)):
1✔
991
            return False
1✔
992
        pat = regex.compile(r"\$\{?" + decl.name + r"([^0-9A-Za-z_]|$)")
1✔
993
        for part in task.command.parts:
1✔
994
            if isinstance(part, str):
1✔
995
                if pat.search(part):
1✔
996
                    return True
1✔
997
        return False
1✔
998

999

1000
@a_linter
1✔
1001
class UnusedCall(Linter):
1✔
1002
    # the outputs of a Call are neither used nor propagated
1003

1004
    def call(self, obj: Tree.Call) -> Any:
1✔
1005
        if obj.effective_outputs and not getattr(obj, "referrers", []):
1✔
1006
            workflow = obj
1✔
1007
            while not isinstance(workflow, Tree.Workflow):
1✔
1008
                workflow = getattr(workflow, "parent")
1✔
1009
            assert isinstance(workflow, Tree.Workflow)
1✔
1010
            if workflow.outputs is not None:
1✔
1011
                self.add(
1✔
1012
                    obj,
1013
                    "nothing references the outputs of the call "
1014
                    + obj.name
1015
                    + " nor are are they output from the workflow "
1016
                    + workflow.name,
1017
                )
1018

1019

1020
@a_linter
1✔
1021
class UnnecessaryQuantifier(Linter):
1✔
1022
    # A declaration like T? x = :T: where the right-hand side can't be null.
1023
    # Caveats:
1024
    # 1. Exception for File? output of tasks, where this is normal.
1025
    # 2. Specific warning when x is an input, and the interpretation is underspecified by WDL
1026
    #    (called with None, does the binding take None or the default?)
1027
    def decl(self, obj: Tree.Decl) -> Any:
1✔
1028
        if obj.type.optional and obj.expr and not obj.expr.type.optional:
1✔
1029
            tw = obj
1✔
1030
            while not isinstance(tw, (Tree.Task, Tree.Workflow)):
1✔
1031
                tw = getattr(tw, "parent")
1✔
1032
            assert isinstance(tw, (Tree.Task, Tree.Workflow))
1✔
1033
            if not (
1✔
1034
                isinstance(tw, Tree.Task)
1035
                and isinstance(obj.type, (Type.File, Type.Directory))
1036
                and obj in tw.outputs
1037
            ):
1038
                if not isinstance(tw.inputs, list) or obj in tw.inputs:
1✔
1039
                    self.add(
1✔
1040
                        obj,
1041
                        f"input {obj.type} {obj.name} is implicitly optional since it has a default;"
1042
                        " consider removing ? quantifier, which may not behave consistently between WDL interpreters",
1043
                    )
1044
                else:
1045
                    self.add(
1✔
1046
                        obj,
1047
                        f"unnecessary optional quantifier (?) for non-input {obj.type} {obj.name}",
1048
                    )
1049

1050

1051
_shellcheck_available = None
1✔
1052

1053

1054
@a_linter
1✔
1055
class CommandShellCheck(Linter):
1✔
1056
    # If ShellCheck is installed, run it on the task command and propagate any
1057
    # additional lint it finds.
1058

1059
    # we suppress
1060
    #   SC1083 This {/} is literal
1061
    #   SC2043 This loop will only ever run once for a constant value
1062
    #   SC2050 This expression is constant
1063
    #   SC2157 Argument to -n is always true due to literal strings
1064
    #   SC2193 The arguments to this comparison can never be equal
1065
    # which can be triggered by dummy values we substitute to write the script
1066
    # also SC1009 and SC1072 are non-informative commentary
1067
    _suppressions = [1009, 1072, 1083, 2043, 2050, 2157, 2193]
1✔
1068

1069
    def __init__(self, *args, **kwargs):
1✔
1070
        super().__init__(*args, **kwargs)
1✔
1071
        self._tmpdir = tempfile.mkdtemp(prefix="miniwdl_shellcheck_")
1✔
1072
        global _shellcheck_available
1073
        if _shellcheck_available is None:
1✔
1074
            _shellcheck_available = shutil.which("shellcheck") is not None
1✔
1075

1076
    def __del__(self):
1✔
1077
        shutil.rmtree(self._tmpdir, ignore_errors=True)
1✔
1078

1079
    def task(self, obj: Tree.Task) -> Any:
1✔
1080
        global _shellcheck_available
1081
        if not _shellcheck_available:
1✔
1082
            return
1✔
1083

1084
        # for each expression placeholder in the command, make up a dummy value
1085
        # of the appropriate type that shouldn't trigger shellcheck
1086
        command = []
1✔
1087
        for part in obj.command.parts:
1✔
1088
            if isinstance(part, Expr.Placeholder):
1✔
1089
                command.append(_shellcheck_dummy_value(part.expr.type, part.pos))
1✔
1090
            else:
1091
                assert isinstance(part, str)
1✔
1092
                command.append(part)
1✔
1093
        col_offset, command_str = _util.strip_leading_whitespace("".join(command))
1✔
1094

1095
        # write out a temp file with this fake script
1096
        tfn = os.path.join(self._tmpdir, obj.name)
1✔
1097
        with open(tfn, "w") as outfile:
1✔
1098
            outfile.write(command_str)
1✔
1099

1100
        # run shellcheck on it & collect JSON results
1101
        shellcheck_items = None
1✔
1102
        try:
1✔
1103
            shellcheck_items = subprocess.check_output(
1✔
1104
                [
1105
                    "shellcheck",
1106
                    "-s",
1107
                    "bash",
1108
                    "-f",
1109
                    "json",
1110
                    "-e",
1111
                    ",".join(str(c) for c in self.__class__._suppressions),
1112
                    tfn,
1113
                ]
1114
            )
1115
        except subprocess.CalledProcessError as cpe:
1✔
1116
            if cpe.returncode in (0, 1):
1✔
1117
                shellcheck_items = cpe.stdout
1✔
1118
            else:
1119
                self.add(
×
1120
                    obj,
1121
                    "shellcheck failed on the task command; update shellcheck version or use --no-shellcheck "
1122
                    "to suppress this warning",
1123
                    obj.command.pos,
1124
                )
1125

1126
        if shellcheck_items:
1✔
1127
            env_decls = set(
1✔
1128
                decl.name
1129
                for decl in ((obj.inputs or []) + obj.postinputs)
1130
                if decl.decor.get("env", False)
1131
            )
1132
            try:
1✔
1133
                shellcheck_items = json.loads(shellcheck_items)
1✔
1134
                assert isinstance(shellcheck_items, list)
1✔
1135

1136
                # annotate on tree, adding appropriate offsets to line/column positions
1137
                for item in shellcheck_items:
1✔
1138
                    if item["code"] == 2154 and item["message"].split(" ")[0] in env_decls:
1✔
1139
                        # Suppress SC2154 "var is referenced but not assigned" specifically when
1140
                        # var corresponds to a declaration with the "env" modifier. ShellCheck
1141
                        # doesn't know that command expects this var to be set in its environment.
1142
                        continue
1✔
1143
                    line = obj.command.pos.line + item["line"] - 1
1✔
1144
                    column = col_offset + item["column"] - 1
1✔
1145
                    self.add(
1✔
1146
                        obj,
1147
                        "SC{} {}".format(item["code"], item["message"]),
1148
                        Error.SourcePosition(
1149
                            uri=obj.command.pos.uri,
1150
                            abspath=obj.command.pos.abspath,
1151
                            line=line,
1152
                            column=column,
1153
                            end_line=line,
1154
                            end_column=column,
1155
                        ),
1156
                    )
1157
            except Exception:
×
1158
                self.add(
×
1159
                    obj,
1160
                    "error parsing shellcheck output JSON; update shellcheck version or use --no-shellcheck "
1161
                    "to suppress this warning",
1162
                    obj.command.pos,
1163
                )
1164

1165

1166
def _shellcheck_dummy_value(ty, pos):
1✔
1167
    if isinstance(ty, Type.Array):
1✔
1168
        return _shellcheck_dummy_value(ty.item_type, pos)
1✔
1169
    if isinstance(ty, Type.Boolean):
1✔
1170
        return "false"
1✔
1171
    # estimate the length of the interpolation in the original source, so that
1172
    # shellcheck will see the same column numbers. + 3 accounts for "~{" and "}"
1173
    desired_length = max(1, pos.end_column - pos.column) + 3
1✔
1174
    if isinstance(ty, (Type.Int, Type.Float)):
1✔
1175
        return "4" * desired_length
1✔
1176
    # assert ty.coerces(Type.String), str(ty)
1177
    # https://github.com/HumanCellAtlas/skylab/blob/a99b8ddffdb3c0ebdea1a8905d28f01a4d365af5/pipelines/10x/count/count.wdl#L325
1178
    # https://github.com/openwdl/wdl/blob/master/versions/draft-2/SPEC.md#map-serialization
1179
    return "".join(
1✔
1180
        chr(ord(random.choice(["A", "a"])) + random.randrange(26)) for _ in range(desired_length)
1181
    )
1182

1183

1184
@a_linter
1✔
1185
class MixedIndentation(Linter):
1✔
1186
    # Line of task command mixes tab and space indentation
1187
    def task(self, obj: Tree.Task) -> Any:
1✔
1188
        command_lines = "".join(
1✔
1189
            (s if isinstance(s, str) else "$") for s in obj.command.parts
1190
        ).split("\n")
1191
        for ofs, line in enumerate(command_lines):
1✔
1192
            indentation = line[: (len(line) - len(line.lstrip()))]
1✔
1193
            if " " in indentation and "\t" in indentation:
1✔
1194
                self.add(
1✔
1195
                    obj,
1196
                    "command indented with both spaces & tabs",
1197
                    Error.SourcePosition(
1198
                        uri=obj.command.pos.uri,
1199
                        abspath=obj.command.pos.abspath,
1200
                        line=obj.command.pos.line + ofs,
1201
                        column=1,
1202
                        end_line=obj.command.pos.line + ofs,
1203
                        end_column=len(line),
1204
                    ),
1205
                )
1206
                break
1✔
1207

1208

1209
@a_linter
1✔
1210
class SelectArray(Linter):
1✔
1211
    # application of select_first or select_all on a non-optional array
1212
    def expr(self, obj: Expr.Base) -> Any:
1✔
1213
        if isinstance(obj, Expr.Apply) and obj.function_name in ["select_first", "select_all"]:
1✔
1214
            arg0 = obj.arguments[0]
1✔
1215
            if isinstance(arg0.type, Type.Array) and not arg0.type.item_type.optional:
1✔
1216
                self.add(
1✔
1217
                    obj,
1218
                    "array of non-optional items passed to " + obj.function_name,
1219
                    obj.arguments[0].pos,
1220
                )
1221

1222

1223
@a_linter
1✔
1224
class UnknownRuntimeKey(Linter):
1✔
1225
    # refs:
1226
    # https://cromwell.readthedocs.io/en/develop/RuntimeAttributes/
1227
    # https://github.com/broadinstitute/cromwell/blob/develop/wom/src/main/scala/wom/RuntimeAttributes.scala
1228
    # https://github.com/broadinstitute/cromwell/blob/develop/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala
1229
    # https://github.com/broadinstitute/cromwell/blob/develop/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala
1230
    # https://github.com/openwdl/wdl/pull/315
1231
    # https://github.com/dnanexus/dxWDL/blob/master/doc/ExpertOptions.md
1232
    # https://cromwell.readthedocs.io/en/develop/backends/TES/
1233
    # https://aws.github.io/amazon-genomics-cli/docs/workflow-engines/cromwell/#aws-batch-retries
1234
    known_keys = set(
1✔
1235
        [
1236
            "awsBatchRetryAttempts",
1237
            "bootDiskSizeGb",
1238
            "container",
1239
            "continueOnReturnCode",
1240
            "cpu",
1241
            "cpuPlatform",
1242
            "disk",
1243
            "disks",
1244
            "docker",
1245
            "dockerWorkingDir",
1246
            "dx_instance_type",
1247
            "gpu",
1248
            "gpuCount",
1249
            "gpuType",
1250
            "inlineDockerfile",
1251
            "maxRetries",
1252
            "memory",
1253
            "noAddress",
1254
            "preemptible",
1255
            "queueArn",
1256
            "returnCodes",
1257
            "time",
1258
            "zones",
1259
        ]
1260
    )
1261

1262
    def task(self, obj: Tree.Task) -> Any:
1✔
1263
        known_keys = self.known_keys
1✔
1264
        if wdl_version_geq(obj.effective_wdl_version, WDLVersion.V1_2):
1✔
1265
            known_keys = known_keys | {"return_codes"}
1✔
1266
        for k in obj.runtime:
1✔
1267
            if k not in known_keys:
1✔
1268
                self.add(obj, "unknown entry in task runtime section: " + k, obj.runtime[k].pos)
1✔
1269

1270

1271
@a_linter
1✔
1272
class UnexpectedRuntimeValue(Linter):
1✔
1273
    expected = {
1✔
1274
        "cpu": (Type.Int, Type.Float, Type.String),
1275
        "memory": (Type.Int, Type.String),
1276
        "docker": (Type.String, Type.Array),
1277
        "gpu": (Type.Boolean,),
1278
    }
1279

1280
    def task(self, obj: Tree.Task) -> Any:
1✔
1281
        for k in obj.runtime:
1✔
1282
            if not isinstance(obj.runtime[k].type, self.expected.get(k, Type.Base)):
1✔
1283
                self.add(
1✔
1284
                    obj,
1285
                    f"expected {'/'.join(ty.__name__ for ty in self.expected[k])} for task runtime.{k}",
1286
                    obj.runtime[k].pos,
1287
                )
1288

1289
        if "cpu" in obj.runtime and isinstance(obj.runtime["cpu"].type, Type.String):
1✔
1290
            # for historical reasons, allow strings that are int literals, or a single placeholder
1291
            # for an int value
1292
            cpu: Optional[Expr.Base] = obj.runtime["cpu"]
1✔
1293
            if isinstance(cpu, Expr.String) and len(cpu.parts) == 3:
1✔
1294
                cpu_part = cpu.parts[1]
1✔
1295
                if (isinstance(cpu_part, str) and cpu_part.isdigit()) or (
1✔
1296
                    isinstance(cpu_part, Expr.Placeholder)
1297
                    and isinstance(cpu_part.expr.type, Type.Int)
1298
                ):
1299
                    cpu = None
1✔
1300
            if cpu:
1✔
1301
                self.add(obj, "expected Int for task runtime.cpu", cpu.pos)
1✔
1302

1303
        if "memory" in obj.runtime:
1✔
1304
            memory = obj.runtime["memory"]
1✔
1305
            if isinstance(memory, Expr.String) and len(memory.parts) == 3:
1✔
1306
                lit = memory.parts[1]
1✔
1307
                if isinstance(lit, str):
1✔
1308
                    try:
1✔
1309
                        _util.parse_byte_size(lit)
1✔
1310
                    except Exception:
1✔
1311
                        self.add(
1✔
1312
                            obj,
1313
                            "runtime.memory doesn't follow expected format like '8G' or '1024 MiB'",
1314
                            memory.pos,
1315
                        )
1316

1317

1318
@a_linter
1✔
1319
class MissingVersion(Linter):
1✔
1320
    def document(self, obj: Tree.Document) -> Any:
1✔
1321
        first_sloc = next(
1✔
1322
            (
1323
                p
1324
                for p in enumerate(line.lstrip() for line in obj.source_lines)
1325
                if p[1] and p[1][0] != "#"
1326
            ),
1327
            None,
1328
        )
1329
        # (don't bother with this warning if the document is effectively empty)
1330
        if first_sloc and obj.wdl_version is None:
1✔
1331
            line = (first_sloc[0] + 1) if first_sloc else obj.pos.line
1✔
1332
            self.add(
1✔
1333
                obj,
1334
                "document should declare WDL version; draft-2 assumed",
1335
                Error.SourcePosition(
1336
                    uri=obj.pos.uri,
1337
                    abspath=obj.pos.abspath,
1338
                    line=line,
1339
                    end_line=line,
1340
                    column=1,
1341
                    end_column=1,
1342
                ),
1343
            )
1344

1345

1346
@a_linter
1✔
1347
class UnboundDeclaration(Linter):
1✔
1348
    # Unbound declaration outside of input{} section in WDL 1.0+
1349
    def decl(self, obj: Tree.Decl) -> Any:
1✔
1350
        if not obj.expr:
1✔
1351
            if _find_doc(obj).effective_wdl_version != "draft-2":
1✔
1352
                exe = obj
1✔
1353
                while not isinstance(exe, (Tree.Task, Tree.Workflow)):
1✔
1354
                    exe = getattr(exe, "parent")
1✔
1355
                assert isinstance(exe, (Tree.Task, Tree.Workflow))
1✔
1356
                if obj not in (exe.inputs or []):
1✔
1357
                    self.add(
1✔
1358
                        obj,
1359
                        f"{obj.type} {obj.name} should either be in the input section or bound to an expression",
1360
                    )
1361

1362

1363
@a_linter
1✔
1364
class Deprecated(Linter):
1✔
1365
    def task(self, obj: Tree.Task) -> Any:
1✔
1366
        if not wdl_version_geq(obj.effective_wdl_version, WDLVersion.V1_2):
1✔
1367
            return
1✔
1368
        if obj.runtime_section_name == "runtime":
1✔
1369
            self.add(
1✔
1370
                obj,
1371
                "use task requirements/hints sections instead of runtime section [WDL >= 1.2]",
1372
                obj.runtime_section_pos,
1373
            )
1374
        if "docker" in obj.runtime:
1✔
1375
            self.add(
1✔
1376
                obj,
1377
                "use container instead of docker runtime/requirements attribute [WDL >= 1.2]",
1378
                obj.runtime["docker"].pos,
1379
            )
1380
        if "returnCodes" in obj.runtime:
1✔
1381
            self.add(
1✔
1382
                obj,
1383
                "use return_codes instead of returnCodes runtime/requirements attribute [WDL >= 1.2]",
1384
                obj.runtime["returnCodes"].pos,
1385
            )
1386

1387
    def expr(self, obj: Expr.Base) -> Any:
1✔
1388
        if (
1✔
1389
            isinstance(obj, Expr.Placeholder)
1390
            and obj.options
1391
            and wdl_version_geq(_find_doc(obj).effective_wdl_version, WDLVersion.V1_1)
1392
        ):
1393
            self.add(
1✔
1394
                obj,
1395
                "use sep()/select_first()/if-then-else expressions instead of"
1396
                " sep/default/true/false placeholder options [WDL >= 1.1]",
1397
                obj.pos,
1398
            )
1399
        elif (
1✔
1400
            isinstance(obj, Expr.Struct)
1401
            and not obj.struct_type_name
1402
            and wdl_version_geq(_find_doc(obj).effective_wdl_version, WDLVersion.V1_1)
1403
        ):
1404
            self.add(obj, "replace 'object' with specific struct type [WDL >= 1.1]", obj.pos)
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc