CityOfZion/neo3-boa | Build 942 | boa3/internal/compiler/codegenerator/codegeneratorvisitor.py | Coveralls

1

import ast

2

import os.path

3

from inspect import isclass

4

from typing import Dict, List, Optional

5

6

from boa3.internal import constants

7

from boa3.internal.analyser.astanalyser import IAstAnalyser

8

from boa3.internal.compiler.codegenerator.codegenerator import CodeGenerator

9

from boa3.internal.compiler.codegenerator.generatordata import GeneratorData

10

from boa3.internal.compiler.codegenerator.variablegenerationdata import VariableGenerationData

11

from boa3.internal.compiler.codegenerator.vmcodemapping import VMCodeMapping

12

from boa3.internal.constants import SYS_VERSION_INFO

13

from boa3.internal.model.builtin.builtin import Builtin

14

from boa3.internal.model.builtin.interop.interop import Interop

15

from boa3.internal.model.builtin.method.builtinmethod import IBuiltinMethod

16

from boa3.internal.model.expression import IExpression

17

from boa3.internal.model.imports.package import Package

18

from boa3.internal.model.method import Method

19

from boa3.internal.model.operation.binary.binaryoperation import BinaryOperation

20

from boa3.internal.model.operation.binaryop import BinaryOp

21

from boa3.internal.model.operation.operation import IOperation

22

from boa3.internal.model.operation.unary.unaryoperation import UnaryOperation

23

from boa3.internal.model.property import Property

24

from boa3.internal.model.symbol import ISymbol

25

from boa3.internal.model.type.classes.classtype import ClassType

26

from boa3.internal.model.type.classes.userclass import UserClass

27

from boa3.internal.model.type.collection.sequence.sequencetype import SequenceType

28

from boa3.internal.model.type.type import IType, Type

29

from boa3.internal.model.variable import Variable

30

31

32

class VisitorCodeGenerator(IAstAnalyser):

33

"""

34

    This class is responsible for walk through the ast.

35

36

    The methods with the name starting with 'visit_' are implementations of methods from the :class:`NodeVisitor` class.

37

    These methods are used to walk through the Python abstract syntax tree.

38

39

    :ivar generator:

40

"""

41

42

    def __init__(self, generator: CodeGenerator, filename: str = None):

43

        super().__init__(ast.parse(""), filename=filename, log=True)

44

45

        self.generator = generator

46

        self.current_method: Optional[Method] = None

47

        self.current_class: Optional[UserClass] = None

48

        self.symbols = generator.symbol_table

49

50

        self.global_stmts: List[ast.AST] = []

51

        self._is_generating_initialize = False

52

        self._root_module: ast.AST = self._tree

53

54

    @property

55

    def _symbols(self) -> Dict[str, ISymbol]:

56

        symbol_table = self.symbols.copy()

57

58

        if isinstance(self.current_class, UserClass):

59

            symbol_table.update(self.current_class.symbols)

60

61

        return symbol_table

62

63

    def include_instruction(self, node: ast.AST, address: int):

64

        if self.current_method is not None and address in VMCodeMapping.instance().code_map:

65

            bytecode = VMCodeMapping.instance().code_map[address]

66

            from boa3.internal.model.debuginstruction import DebugInstruction

67

            self.current_method.include_instruction(DebugInstruction.build(node, bytecode))

68

69

    def build_data(self, origin_node: Optional[ast.AST],

70

                   symbol_id: Optional[str] = None,

71

                   symbol: Optional[ISymbol] = None,

72

                   result_type: Optional[IType] = None,

73

                   index: Optional[int] = None,

74

                   origin_object_type: Optional[ISymbol] = None,

75

                   already_generated: bool = False) -> GeneratorData:

76

77

        if isinstance(symbol, IType) and result_type is None:

78

            result_type = symbol

79

            symbol = None

80

81

        if symbol is None and symbol_id is not None:

82

            # try to find the symbol if the id is known

83

            if symbol_id in self._symbols:

84

                found_symbol = self._symbols[symbol_id]

85

            else:

86

                _, found_symbol = self.generator.get_symbol(symbol_id)

87

                if found_symbol is Type.none:

88

                    # generator get_symbol returns Type.none if the symbol is not found

89

                    found_symbol = None

90

91

            if found_symbol is not None:

92

                symbol = found_symbol

93

94

        return GeneratorData(origin_node, symbol_id, symbol, result_type, index, origin_object_type, already_generated)

95

96

    def visit_and_update_analyser(self, node: ast.AST, target_analyser) -> GeneratorData:

97

        result = self.visit(node)

98

        if hasattr(target_analyser, '_update_logs'):

99

            target_analyser._update_logs(self)

100

        return result

101

102

    def visit(self, node: ast.AST) -> GeneratorData:

103

        result = super().visit(node)

104

        if not isinstance(result, GeneratorData):

105

            result = self.build_data(node)

106

        return result

107

108

    def visit_to_map(self, node: ast.AST, generate: bool = False) -> GeneratorData:

109

        address: int = VMCodeMapping.instance().bytecode_size

110

        if isinstance(node, ast.Expr):

111

            value = self.visit_Expr(node, generate)

112

        elif generate:

113

            value = self.visit_to_generate(node)

114

        else:

115

            value = self.visit(node)

116

117

        if not isinstance(node, (ast.For, ast.While, ast.If)):

118

            # control flow nodes must map each of their instructions

119

            self.include_instruction(node, address)

120

        return value

121

122

    def visit_to_generate(self, node) -> GeneratorData:

123

"""

124

        Visitor to generate the nodes that the primary visitor is used to retrieve value

125

126

        :param node: an ast node

127

"""

128

        if isinstance(node, ast.AST):

129

            result = self.visit(node)

130

131

            if not result.already_generated and result.symbol_id is not None:

132

                if self.is_exception_name(result.symbol_id):

133

                    self.generator.convert_new_exception()

134

                else:

135

                    # TODO: validate function calls

136

                    is_internal = hasattr(node, 'is_internal_call') and node.is_internal_call

137

                    class_type = result.type

138

139

                    if (self.is_implemented_class_type(result.type)

140

                            and len(result.symbol_id.split(constants.ATTRIBUTE_NAME_SEPARATOR)) > 1):

141

                        # if the symbol id has the attribute separator and the top item on the stack is a user class,

142

                        # then this value is an attribute from that class

143

                        # change the id for correct generation

144

                        result.symbol_id = result.symbol_id.split(constants.ATTRIBUTE_NAME_SEPARATOR)[-1]

145

146

                    elif isinstance(result.index, Package):

147

                        class_type = None

148

149

                    self.generator.convert_load_symbol(result.symbol_id, is_internal=is_internal, class_type=class_type if self.current_method is None else None)

150

151

                result.already_generated = True

152

153

            return result

154

        else:

155

            index = self.generator.convert_literal(node)

156

            return self.build_data(node, index=index)

157

158

    def is_exception_name(self, exc_id: str) -> bool:

159

        global_symbols = globals()

160

        if exc_id in global_symbols or exc_id in global_symbols['__builtins__']:

161

            symbol = (global_symbols[exc_id]

162

                      if exc_id in global_symbols

163

                      else global_symbols['__builtins__'][exc_id])

164

            if isclass(symbol) and issubclass(symbol, BaseException):

165

                return True

166

        return False

167

168

    def _remove_inserted_opcodes_since(self, last_address: int, last_stack_size: Optional[int] = None):

169

        self.generator._remove_inserted_opcodes_since(last_address, last_stack_size)

170

171

    def _get_unique_name(self, name_id: str, node: ast.AST) -> str:

172

        return '{0}{2}{1}'.format(node.__hash__(), name_id, constants.VARIABLE_NAME_SEPARATOR)

173

174

    def set_filename(self, filename: str):

175

        if isinstance(filename, str) and os.path.isfile(filename):

176

            if constants.PATH_SEPARATOR != os.path.sep:

177

                self.filename = filename.replace(constants.PATH_SEPARATOR, os.path.sep)

178

            else:

179

                self.filename = filename

180

181

    def visit_Module(self, module: ast.Module) -> GeneratorData:

182

"""

183

        Visitor of the module node

184

185

        Fills module symbol table

186

187

        :param module:

188

"""

189

        global_stmts = [node for node in module.body if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))]

190

        function_stmts = module.body[len(global_stmts):]

191

        mandatory_global_stmts = []

192

        for stmt in global_stmts:

193

            if isinstance(stmt, ast.ClassDef):

194

                class_symbol = self.get_symbol(stmt.name)

195

                if isinstance(class_symbol, UserClass) and len(class_symbol.class_variables) > 0:

196

                    mandatory_global_stmts.append(stmt)

197

            elif not isinstance(stmt, (ast.Import, ast.ImportFrom)):

198

                mandatory_global_stmts.append(stmt)

199

200

        for stmt in function_stmts:

201

            self.visit(stmt)

202

203

        if self.generator.initialize_static_fields():

204

            last_symbols = self.symbols  # save to revert in the end and not compromise consequent visits

205

            class_non_static_stmts = []

206

207

            for node in global_stmts.copy():

208

                if isinstance(node, ast.ClassDef):

209

                    class_origin_module = None

210

                    if hasattr(node, 'origin'):

211

                        class_origin_module = node.origin

212

                        if (node.origin is not self._root_module

213

                                and hasattr(node.origin, 'symbols')):

214

                            # symbols unique to imports are not included in the symbols

215

                            self.symbols = node.origin.symbols

216

217

                    class_variables = []

218

                    class_functions = []

219

                    for stmt in node.body:

220

                        if isinstance(stmt, (ast.Assign, ast.AugAssign, ast.AnnAssign)):

221

                            class_variables.append(stmt)

222

                        else:

223

                            class_functions.append(stmt)

224

225

                    if len(class_functions) > 0:

226

                        if class_origin_module is not None and not hasattr(node, 'origin'):

227

                            node.origin = class_origin_module

228

229

                        cls_fun = node

230

                        if len(class_variables) > 0:

231

                            cls_var = node

232

                            cls_fun = self.clone(node)

233

                            cls_fun.body = class_functions

234

                            cls_var.body = class_variables

235

                        else:

236

                            global_stmts.remove(cls_fun)

237

238

                        class_non_static_stmts.append(cls_fun)

239

240

            # to generate the 'initialize' method for Neo

241

            self._log_info(f"Compiling '{constants.INITIALIZE_METHOD_ID}' function")

242

            self._is_generating_initialize = True

243

            for stmt in global_stmts:

244

                cur_tree = self._tree

245

                cur_filename = self.filename

246

                if hasattr(stmt, 'origin'):

247

                    if hasattr(stmt.origin, 'filename'):

248

                        self.set_filename(stmt.origin.filename)

249

                    self._tree = stmt.origin

250

251

                self.visit(stmt)

252

                self.filename = cur_filename

253

                self._tree = cur_tree

254

255

            self._is_generating_initialize = False

256

            self.generator.end_initialize()

257

258

            # generate any symbol inside classes that's not variables AFTER generating 'initialize' method

259

            for stmt in class_non_static_stmts:

260

                cur_tree = self._tree

261

                if hasattr(stmt, 'origin'):

262

                    self._tree = stmt.origin

263

                self.visit(stmt)

264

                self._tree = cur_tree

265

266

            self.symbols = last_symbols  # revert in the end to not compromise consequent visits

267

            self.generator.additional_symbols = None

268

269

        elif len(function_stmts) > 0 or len(mandatory_global_stmts) > 0:

270

            # to organize syntax tree nodes from other modules

271

            for stmt in global_stmts:

272

                if not hasattr(stmt, 'origin'):

273

                    stmt.origin = module

274

275

            module.symbols = self._symbols

276

            self.global_stmts.extend(global_stmts)

277

        else:

278

            # to generate objects when there are no static variables to generate 'initialize'

279

            for stmt in global_stmts:

280

                self.visit(stmt)

281

282

        return self.build_data(module)

283

284

    def visit_ClassDef(self, node: ast.ClassDef) -> GeneratorData:

285

        if node.name in self.symbols:

286

            class_symbol = self.symbols[node.name]

287

            if isinstance(class_symbol, UserClass):

288

                self.current_class = class_symbol

289

290

            if self._is_generating_initialize:

291

                address = self.generator.bytecode_size

292

                self.generator.convert_new_empty_array(len(class_symbol.class_variables), class_symbol)

293

                self.generator.convert_store_variable(node.name, address)

294

            else:

295

                init_method = class_symbol.constructor_method()

296

                if isinstance(init_method, Method) and init_method.start_address is None:

297

                    self.generator.generate_implicit_init_user_class(init_method)

298

299

        for stmt in node.body:

300

            self.visit(stmt)

301

302

        self.current_class = None

303

        return self.build_data(node)

304

305

    def visit_FunctionDef(self, function: ast.FunctionDef) -> GeneratorData:

306

"""

307

        Visitor of the function definition node

308

309

        Generates the Neo VM code for the function

310

311

        :param function: the python ast function definition node

312

"""

313

        method = self._symbols[function.name]

314

315

        if isinstance(method, Property):

316

            method = method.getter

317

318

        if isinstance(method, Method):

319

            self.current_method = method

320

            if isinstance(self.current_class, ClassType):

321

                function_name = self.current_class.identifier + constants.ATTRIBUTE_NAME_SEPARATOR + function.name

322

            else:

323

                function_name = function.name

324

325

            self._log_info(f"Compiling '{function_name}' function")

326

327

            if method.is_public or method.is_called:

328

                if not isinstance(self.current_class, ClassType) or not self.current_class.is_interface:

329

                    self.generator.convert_begin_method(method)

330

331

                    for stmt in function.body:

332

                        self.visit_to_map(stmt)

333

334

                    self.generator.convert_end_method(function.name)

335

336

            self.current_method = None

337

338

        return self.build_data(function, symbol=method, symbol_id=function.name)

339

340

    def visit_Return(self, ret: ast.Return) -> GeneratorData:

341

"""

342

        Visitor of a function return node

343

344

        :param ret: the python ast return node

345

"""

346

        if self.generator.stack_size > 0:

347

            self.generator.clear_stack(True)

348

        if self.current_method.return_type is not Type.none:

349

            result = self.visit_to_generate(ret.value)

350

            if result.type is Type.none and not self.generator.is_none_inserted():

351

                self.generator.convert_literal(None)

352

        self.generator.insert_return()

353

354

        return self.build_data(ret)

355

356

    def store_variable(self, *var_ids: VariableGenerationData, value: ast.AST):

357

        # if the value is None, it is a variable declaration

358

        if value is not None:

359

            if len(var_ids) == 1:

360

                # it's a simple assignment

361

                var_id, index, address = var_ids[0]

362

                if index is None:

363

                    # if index is None, then it is a variable assignment

364

                    result_data = self.visit_to_generate(value)

365

366

                    if result_data.type is Type.none and not self.generator.is_none_inserted():

367

                        self.generator.convert_literal(None)

368

                    self.generator.convert_store_variable(var_id, address, self.current_class if self.current_method is None else None)

369

                else:

370

                    # if not, it is an array assignment

371

                    self.generator.convert_load_symbol(var_id)

372

                    self.visit_to_generate(index)

373

374

                    aux_index = VMCodeMapping.instance().bytecode_size

375

                    value_data = self.visit_to_generate(value)

376

                    value_address = value_data.index if value_data.index is not None else aux_index

377

378

                    self.generator.convert_set_item(value_address)

379

380

            elif len(var_ids) > 0:

381

                # it's a chained assignment

382

                self.visit_to_generate(value)

383

                for pos, (var_id, index, address) in enumerate(reversed(var_ids)):

384

                    if index is None:

385

                        # if index is None, then it is a variable assignment

386

                        if pos < len(var_ids) - 1:

387

                            self.generator.duplicate_stack_top_item()

388

                        self.generator.convert_store_variable(var_id, address)

389

                    else:

390

                        # if not, it is an array assignment

391

                        if pos < len(var_ids) - 1:

392

                            self.generator.convert_load_symbol(var_id)

393

                            self.visit_to_generate(index)

394

                            fix_index = VMCodeMapping.instance().bytecode_size

395

                            self.generator.duplicate_stack_item(3)

396

                        else:

397

                            self.visit_to_generate(index)

398

                            fix_index = VMCodeMapping.instance().bytecode_size

399

                            self.generator.convert_load_symbol(var_id)

400

                            self.generator.swap_reverse_stack_items(3)

401

                        self.generator.convert_set_item(fix_index)

402

403

    def visit_AnnAssign(self, ann_assign: ast.AnnAssign) -> GeneratorData:

404

"""

405

        Visitor of an annotated assignment node

406

407

        :param ann_assign: the python ast variable assignment node

408

"""

409

        var_value_address = self.generator.bytecode_size

410

        var_data = self.visit(ann_assign.target)

411

        var_id = var_data.symbol_id

412

        # filter to find the imported variables

413

        if (var_id not in self.generator.symbol_table

414

                and hasattr(ann_assign, 'origin')

415

                and isinstance(ann_assign.origin, ast.AST)):

416

            var_id = self._get_unique_name(var_id, ann_assign.origin)

417

        self.store_variable(VariableGenerationData(var_id, None, var_value_address), value=ann_assign.value)

418

        return self.build_data(ann_assign)

419

420

    def visit_Assign(self, assign: ast.Assign) -> GeneratorData:

421

"""

422

        Visitor of an assignment node

423

424

        :param assign: the python ast variable assignment node

425

"""

426

        vars_ids: List[VariableGenerationData] = []

427

        for target in assign.targets:

428

            var_value_address = self.generator.bytecode_size

429

            target_data: GeneratorData = self.visit(target)

430

431

            var_id = target_data.symbol_id

432

            var_index = target_data.index

433

434

            # filter to find the imported variables

435

            if (var_id not in self.generator.symbol_table

436

                    and hasattr(assign, 'origin')

437

                    and isinstance(assign.origin, ast.AST)):

438

                var_id = self._get_unique_name(var_id, assign.origin)

439

440

            vars_ids.append(VariableGenerationData(var_id, var_index, var_value_address))

441

442

        self.store_variable(*vars_ids, value=assign.value)

443

        return self.build_data(assign)

444

445

    def visit_AugAssign(self, aug_assign: ast.AugAssign) -> GeneratorData:

446

"""

447

        Visitor of an augmented assignment node

448

449

        :param aug_assign: the python ast augmented assignment node

450

"""

451

        start_address = self.generator.bytecode_size

452

        var_data = self.visit(aug_assign.target)

453

        var_id = var_data.symbol_id

454

        # filter to find the imported variables

455

        if (var_id not in self.generator.symbol_table

456

                and hasattr(aug_assign, 'origin')

457

                and isinstance(aug_assign.origin, ast.AST)):

458

            var_id = self._get_unique_name(var_id, aug_assign.origin)

459

460

        if isinstance(var_data.type, UserClass):

461

            self.generator.duplicate_stack_top_item()

462

        elif hasattr(var_data.origin_object_type, 'identifier') and var_data.already_generated:

463

            VMCodeMapping.instance().remove_opcodes(start_address)

464

            self.generator.convert_load_symbol(var_data.origin_object_type.identifier)

465

466

        self.generator.convert_load_symbol(var_id, class_type=var_data.origin_object_type)

467

        value_address = self.generator.bytecode_size

468

        self.visit_to_generate(aug_assign.value)

469

        self.generator.convert_operation(aug_assign.op)

470

        self.generator.convert_store_variable(var_id, value_address, user_class=var_data.origin_object_type)

471

        return self.build_data(aug_assign)

472

473

    def visit_Subscript(self, subscript: ast.Subscript) -> GeneratorData:

474

"""

475

        Visitor of a subscript node

476

477

        :param subscript: the python ast subscript node

478

"""

479

        if isinstance(subscript.slice, ast.Slice):

480

            return self.visit_Subscript_Slice(subscript)

481

482

        return self.visit_Subscript_Index(subscript)

483

484

    def visit_Subscript_Index(self, subscript: ast.Subscript) -> GeneratorData:

485

"""

486

        Visitor of a subscript node with index

487

488

        :param subscript: the python ast subscript node

489

"""

490

        index = None

491

        symbol_id = None

492

        if isinstance(subscript.ctx, ast.Load):

493

            # get item

494

            value_data = self.visit_to_generate(subscript.value)

495

            slice = subscript.slice.value if isinstance(subscript.slice, ast.Index) else subscript.slice

496

            self.visit_to_generate(slice)

497

            self.generator.convert_get_item()

498

499

            value_type = value_data.type

500

        else:

501

            # set item

502

            var_data = self.visit(subscript.value)

503

504

            index = subscript.slice.value if isinstance(subscript.slice, ast.Index) else subscript.slice

505

            symbol_id = var_data.symbol_id

506

            value_type = var_data.type

507

508

        result_type = value_type.item_type if isinstance(value_type, SequenceType) else value_type

509

        return self.build_data(subscript, result_type=result_type,

510

                               symbol_id=symbol_id, index=index)

511

512

    def visit_Subscript_Slice(self, subscript: ast.Subscript) -> GeneratorData:

513

"""

514

        Visitor of a subscript node with slice

515

516

        :param subscript: the python ast subscript node

517

"""

518

        lower_omitted = subscript.slice.lower is None

519

        upper_omitted = subscript.slice.upper is None

520

        step_omitted = subscript.slice.step is None

521

522

        self.visit_to_generate(subscript.value)

523

524

        step_negative = True if not step_omitted and subscript.slice.step.n < 0 else False

525

        # if step is negative, then consider the value reversed

526

        if step_negative:

527

            self.generator.convert_array_negative_stride()

528

529

        # if both are explicit

530

        if not lower_omitted and not upper_omitted:

531

            addresses = [VMCodeMapping.instance().bytecode_size]

532

            self.visit_to_generate(subscript.slice.lower)

533

534

            # length of slice

535

            addresses.append(VMCodeMapping.instance().bytecode_size)

536

            self.visit_to_generate(subscript.slice.upper)

537

538

            self.generator.convert_get_sub_array(addresses, step_negative)

539

        # only one of them is omitted

540

        elif lower_omitted != upper_omitted:

541

            # start position is omitted

542

            if lower_omitted:

543

                self.visit_to_generate(subscript.slice.upper)

544

                self.generator.convert_get_array_beginning(step_negative)

545

            # end position is omitted

546

            else:

547

                self.generator.duplicate_stack_top_item()

548

                # length of slice

549

                self.generator.convert_builtin_method_call(Builtin.Len)

550

                self.visit_to_generate(subscript.slice.lower)

551

                self.generator.convert_get_array_ending(step_negative)

552

        else:

553

            self.generator.convert_copy()

554

555

        if not step_omitted:

556

            self.visit_to_generate(subscript.slice.step)

557

            self.generator.convert_get_stride()

558

559

        return self.build_data(subscript)

560

561

    def _convert_unary_operation(self, operand, op):

562

        self.visit_to_generate(operand)

563

        self.generator.convert_operation(op)

564

565

    def _convert_binary_operation(self, left, right, op):

566

        self.visit_to_generate(left)

567

        self.visit_to_generate(right)

568

        self.generator.convert_operation(op)

569

570

    def visit_BinOp(self, bin_op: ast.BinOp) -> GeneratorData:

571

"""

572

        Visitor of a binary operation node

573

574

        :param bin_op: the python ast binary operation node

575

"""

576

        if isinstance(bin_op.op, BinaryOperation):

577

            self._convert_binary_operation(bin_op.left, bin_op.right, bin_op.op)

578

579

        return self.build_data(bin_op)

580

581

    def visit_UnaryOp(self, un_op: ast.UnaryOp) -> GeneratorData:

582

"""

583

        Visitor of a binary operation node

584

585

        :param un_op: the python ast binary operation node

586

"""

587

        if isinstance(un_op.op, UnaryOperation):

588

            self._convert_unary_operation(un_op.operand, un_op.op)

589

590

        return self.build_data(un_op)

591

592

    def visit_Compare(self, compare: ast.Compare) -> GeneratorData:

593

"""

594

        Visitor of a compare operation node

595

596

        :param compare: the python ast compare operation node

597

"""

598

        operation_symbol: IOperation = BinaryOp.And

599

        converted: bool = False

600

        left = compare.left

601

        for index, op in enumerate(compare.ops):

602

            right = compare.comparators[index]

603

            if isinstance(op, IOperation):

604

                if isinstance(op, BinaryOperation):

605

                    self._convert_binary_operation(left, right, op)

606

                else:

607

                    operand = left

608

                    if isinstance(operand, ast.NameConstant) and operand.value is None:

609

                        operand = right

610

                    self._convert_unary_operation(operand, op)

611

                # if it's more than two comparators, must include AND between the operations

612

                if not converted:

613

                    converted = True

614

                    operation_symbol = op

615

                else:

616

                    operation_symbol = BinaryOp.And

617

                    self.generator.convert_operation(BinaryOp.And)

618

            left = right

619

620

        return self.build_data(compare, symbol=operation_symbol, result_type=operation_symbol.result)

621

622

    def visit_BoolOp(self, bool_op: ast.BoolOp) -> GeneratorData:

623

"""

624

        Visitor of a compare operation node

625

626

        :param bool_op: the python ast boolean operation node

627

"""

628

        if isinstance(bool_op.op, BinaryOperation):

629

            left = bool_op.values[0]

630

            self.visit_to_generate(left)

631

            for index, right in enumerate(bool_op.values[1:]):

632

                self.visit_to_generate(right)

633

                self.generator.convert_operation(bool_op.op)

634

635

        return self.build_data(bool_op)

636

637

    def visit_While(self, while_node: ast.While) -> GeneratorData:

638

"""

639

        Visitor of a while statement node

640

641

        :param while_node: the python ast while statement node

642

"""

643

        start_addr: int = self.generator.convert_begin_while()

644

        for stmt in while_node.body:

645

            self.visit_to_map(stmt, generate=True)

646

647

        test_address: int = VMCodeMapping.instance().bytecode_size

648

        test_data = self.visit_to_map(while_node.test, generate=True)

649

        self.generator.convert_end_while(start_addr, test_address)

650

651

        else_begin_address: int = self.generator.last_code_start_address

652

        for stmt in while_node.orelse:

653

            self.visit_to_map(stmt, generate=True)

654

655

        self.generator.convert_end_loop_else(start_addr, else_begin_address, len(while_node.orelse) > 0)

656

        return self.build_data(while_node, index=start_addr)

657

658

    def visit_For(self, for_node: ast.For) -> GeneratorData:

659

"""

660

        Visitor of for statement node

661

662

        :param for_node: the python ast for node

663

"""

664

        self.visit_to_generate(for_node.iter)

665

        start_address = self.generator.convert_begin_for()

666

667

        if isinstance(for_node.target, tuple):

668

            for target in for_node.target:

669

                var_data = self.visit_to_map(target)

670

                self.generator.convert_store_variable(var_data.symbol_id)

671

        else:

672

            var_data = self.visit(for_node.target)

673

            self.generator.convert_store_variable(var_data.symbol_id)

674

675

        for stmt in for_node.body:

676

            self.visit_to_map(stmt, generate=True)

677

678

        # TODO: remove when optimizing for generation

679

        if self.current_method is not None:

680

            self.current_method.remove_instruction(for_node.lineno, for_node.col_offset)

681

682

        condition_address = self.generator.convert_end_for(start_address)

683

        self.include_instruction(for_node, condition_address)

684

        else_begin = self.generator.last_code_start_address

685

686

        for stmt in for_node.orelse:

687

            self.visit_to_map(stmt, generate=True)

688

689

        self.generator.convert_end_loop_else(start_address,

690

                                             else_begin,

691

                                             has_else=len(for_node.orelse) > 0,

692

                                             is_for=True)

693

        return self.build_data(for_node)

694

695

    def visit_If(self, if_node: ast.If) -> GeneratorData:

696

"""

697

        Visitor of if statement node

698

699

        :param if_node: the python ast if statement node

700

"""

701

        test = self.visit_to_map(if_node.test, generate=True)

702

703

        if not Type.bool.is_type_of(test.type) and test.type is not None:

704

            self.generator.convert_builtin_method_call(Builtin.Bool)

705

706

        start_addr: int = self.generator.convert_begin_if()

707

        for stmt in if_node.body:

708

            self.visit_to_map(stmt, generate=True)

709

710

        ends_with_if = len(if_node.body) > 0 and isinstance(if_node.body[-1], ast.If)

711

712

        if len(if_node.orelse) > 0:

713

            start_addr = self.generator.convert_begin_else(start_addr, ends_with_if)

714

            for stmt in if_node.orelse:

715

                self.visit_to_map(stmt, generate=True)

716

717

        self.generator.convert_end_if(start_addr)

718

        return self.build_data(if_node)

719

720

    def visit_Expr(self, expr: ast.Expr, generate: bool = False) -> GeneratorData:

721

"""

722

        Visitor of an expression node

723

724

        :param expr: the python ast expression node

725

        :param generate: if it should convert the value

726

"""

727

        last_stack = self.generator.stack_size

728

        if generate:

729

            value = self.visit_to_generate(expr.value)

730

        else:

731

            value = self.visit(expr.value)

732

733

        new_stack = self.generator.stack_size

734

        for x in range(last_stack, new_stack):

735

            self.generator.remove_stack_top_item()

736

737

        return value

738

739

    def visit_IfExp(self, if_node: ast.IfExp) -> GeneratorData:

740

"""

741

        Visitor of if expression node

742

743

        :param if_node: the python ast if statement node

744

"""

745

        self.visit_to_map(if_node.test, generate=True)

746

747

        start_addr: int = self.generator.convert_begin_if()

748

        body_data = self.visit_to_map(if_node.body, generate=True)

749

750

        start_addr = self.generator.convert_begin_else(start_addr)

751

        else_data = self.visit_to_map(if_node.orelse, generate=True)

752

753

        self.generator.convert_end_if(start_addr)

754

        return self.build_data(if_node, result_type=Type.union.build([body_data.type, else_data.type]))

755

756

    def visit_Assert(self, assert_node: ast.Assert) -> GeneratorData:

757

"""

758

        Visitor of the assert node

759

760

        :param assert_node: the python ast assert node

761

"""

762

        self.visit_to_generate(assert_node.test)

763

764

        if assert_node.msg is not None:

765

            self.generator.duplicate_stack_top_item()

766

            self.generator.insert_not()

767

768

            # if assert is false, log the message

769

            start_addr: int = self.generator.convert_begin_if()

770

771

            self.visit_to_generate(assert_node.msg)

772

            self.generator.convert_builtin_method_call(Interop.Log)

773

774

            self.generator.convert_end_if(start_addr)

775

776

        self.generator.convert_assert()

777

        return self.build_data(assert_node)

778

779

    def visit_Call(self, call: ast.Call) -> GeneratorData:

780

"""

781

        Visitor of a function call node

782

783

        :param call: the python ast function call node

784

        :returns: The called function return type

785

"""

786

        # the parameters are included into the stack in the reversed order

787

        last_address = VMCodeMapping.instance().bytecode_size

788

        last_stack = self.generator.stack_size

789

790

        func_data = self.visit(call.func)

791

        function_id = func_data.symbol_id

792

        # if the symbol is not a method, check if it is a class method

793

        if (isinstance(func_data.type, ClassType) and func_data.symbol_id in func_data.type.symbols

794

                and isinstance(func_data.type.symbols[func_data.symbol_id], IExpression)):

795

            symbol = func_data.type.symbols[func_data.symbol_id]

796

        else:

797

            symbol = func_data.symbol

798

799

        if not isinstance(symbol, Method):

800

            is_internal = hasattr(call, 'is_internal_call') and call.is_internal_call

801

            _, symbol = self.generator.get_symbol(function_id, is_internal=is_internal)

802

803

        if self.is_implemented_class_type(symbol):

804

            self.generator.convert_init_user_class(symbol)

805

            symbol = symbol.constructor_method()

806

        args_addresses: List[int] = []

807

808

        has_cls_or_self_argument = isinstance(symbol, Method) and symbol.has_cls_or_self

809

        if not has_cls_or_self_argument:

810

            self._remove_inserted_opcodes_since(last_address, last_stack)

811

812

        if isinstance(symbol, Method):

813

            # self or cls is already generated

814

            call_args = (call.args[1:]

815

                         if has_cls_or_self_argument and len(call.args) == len(symbol.args)

816

                         else call.args)

817

            args_to_generate = [arg for index, arg in enumerate(call_args) if index in symbol.args_to_be_generated()]

818

            keywords_dict = {keyword.arg: keyword.value for keyword in call.keywords}

819

            keywords_with_index = {index: keywords_dict[arg_name]

820

                                   for index, arg_name in enumerate(symbol.args)

821

                                   if arg_name in keywords_dict}

822

823

            for index in keywords_with_index:

824

                if index < len(args_to_generate):

825

                    # override default values

826

                    args_to_generate[index] = keywords_with_index[index]

827

                else:

828

                    # put keywords

829

                    args_to_generate.append(keywords_with_index[index])

830

831

        else:

832

            args_to_generate = call.args

833

834

        if isinstance(symbol, IBuiltinMethod):

835

            reordered_args = []

836

            for index in symbol.generation_order:

837

                if 0 <= index < len(args_to_generate):

838

                    reordered_args.append(args_to_generate[index])

839

840

            args = reordered_args

841

        else:

842

            args = reversed(args_to_generate)

843

844

        args_begin_address = self.generator.last_code_start_address

845

        for arg in args:

846

            args_addresses.append(

847

                VMCodeMapping.instance().bytecode_size

848

849

            self.visit_to_generate(arg)

850

        if has_cls_or_self_argument:

851

            num_args = len(args_addresses)

852

            if self.generator.stack_size > num_args:

853

                value = self.generator._stack_pop(-num_args - 1)

854

                self.generator._stack_append(value)

855

            end_address = VMCodeMapping.instance().move_to_end(last_address, args_begin_address)

856

            if not symbol.is_init:

857

                args_addresses.append(end_address)

858

859

        if self.is_exception_name(function_id):

860

            self.generator.convert_new_exception(len(call.args))

861

        elif isinstance(symbol, type(Builtin.Super)) and len(args_to_generate) == 0:

862

            self_or_cls_id = list(self.current_method.args)[0]

863

            self.generator.convert_load_symbol(self_or_cls_id)

864

        elif isinstance(symbol, IBuiltinMethod):

865

            self.generator.convert_builtin_method_call(symbol, args_addresses)

866

        else:

867

            self.generator.convert_load_symbol(function_id, args_addresses)

868

869

        return self.build_data(call, symbol=symbol, symbol_id=function_id,

870

                               result_type=symbol.type if isinstance(symbol, IExpression) else symbol,

871

                               already_generated=True)

872

873

    def visit_Raise(self, raise_node: ast.Raise) -> GeneratorData:

874

"""

875

        Visitor of the raise node

876

877

        :param raise_node: the python ast raise node

878

"""

879

        self.visit_to_map(raise_node.exc, generate=True)

880

        self.generator.convert_raise_exception()

881

        return self.build_data(raise_node)

882

883

    def visit_Try(self, try_node: ast.Try) -> GeneratorData:

884

"""

885

        Visitor of the try node

886

887

        :param try_node: the python ast try node

888

"""

889

        try_address: int = self.generator.convert_begin_try()

890

        try_end: Optional[int] = None

891

        for stmt in try_node.body:

892

            self.visit_to_map(stmt, generate=True)

893

894

        if len(try_node.handlers) == 1:

895

            handler = try_node.handlers[0]

896

            try_end = self.generator.convert_try_except(handler.name)

897

            for stmt in handler.body:

898

                self.visit_to_map(stmt, generate=True)

899

900

        else_address = None

901

        if len(try_node.orelse) > 0:

902

            else_start_address = self.generator.convert_begin_else(try_end)

903

            else_address = self.generator.bytecode_size

904

            for stmt in try_node.orelse:

905

                self.visit_to_map(stmt, generate=True)

906

            self.generator.convert_end_if(else_start_address)

907

908

        except_end = self.generator.convert_end_try(try_address, try_end, else_address)

909

        for stmt in try_node.finalbody:

910

            self.visit_to_map(stmt, generate=True)

911

        self.generator.convert_end_try_finally(except_end, try_address, len(try_node.finalbody) > 0)

912

913

        return self.build_data(try_node)

914

915

    def visit_Name(self, name_node: ast.Name) -> GeneratorData:

916

"""

917

        Visitor of a name node

918

919

        :param name_node: the python ast name identifier node

920

        :return: the identifier of the name

921

"""

922

        name = name_node.id

923

        if name not in self._symbols:

924

            hashed_name = self._get_unique_name(name, self._tree)

925

            if hashed_name not in self._symbols and hasattr(name_node, 'origin'):

926

                hashed_name = self._get_unique_name(name, name_node.origin)

927

928

            if hashed_name in self._symbols:

929

                name = hashed_name

930

        return self.build_data(name_node, name)

931

932

    def visit_Starred(self, starred: ast.Starred) -> GeneratorData:

933

        value_data = self.visit_to_generate(starred.value)

934

        self.generator.convert_starred_variable()

935

936

        starred_data = value_data.copy(starred)

937

        starred_data.already_generated = True

938

        return starred_data

939

940

    def visit_Attribute(self, attribute: ast.Attribute) -> GeneratorData:

941

"""

942

        Visitor of a attribute node

943

944

        :param attribute: the python ast attribute node

945

        :return: the identifier of the attribute

946

"""

947

        last_address = VMCodeMapping.instance().bytecode_size

948

        last_stack = self.generator.stack_size

949

950

        _attr, attr = self.generator.get_symbol(attribute.attr)

951

        value = attribute.value

952

        value_symbol = None

953

        value_type = None

954

        value_data = self.visit(value)

955

        need_to_visit_again = True

956

957

        if value_data.symbol_id is not None and not value_data.already_generated:

958

            value_id = value_data.symbol_id

959

            if value_data.symbol is not None:

960

                value_symbol = value_data.symbol

961

            else:

962

                _, value_symbol = self.generator.get_symbol(value_id)

963

            value_type = value_symbol.type if hasattr(value_symbol, 'type') else value_symbol

964

965

            if hasattr(value_type, 'symbols') and attribute.attr in value_type.symbols:

966

                attr = value_type.symbols[attribute.attr]

967

            elif isinstance(value_type, Package) and attribute.attr in value_type.inner_packages:

968

                attr = value_type.inner_packages[attribute.attr]

969

970

            if isinstance(value_symbol, UserClass):

971

                if isinstance(attr, Method) and attr.has_cls_or_self:

972

                    self.generator.convert_load_symbol(value_id)

973

                    need_to_visit_again = False

974

975

                if isinstance(attr, Variable):

976

                    cur_bytesize = self.generator.bytecode_size

977

                    self.visit_to_generate(attribute.value)

978

                    self.generator.convert_load_symbol(attribute.attr, class_type=value_symbol)

979

980

                    symbol_id = _attr if isinstance(_attr, str) else None

981

                    return self.build_data(attribute,

982

                                           already_generated=self.generator.bytecode_size > cur_bytesize,

983

                                           symbol=attr, symbol_id=symbol_id,

984

                                           origin_object_type=value_symbol)

985

        else:

986

            if isinstance(value, ast.Attribute) and value_data.already_generated:

987

                need_to_visit_again = False

988

            else:

989

                need_to_visit_again = value_data.already_generated

990

                self._remove_inserted_opcodes_since(last_address, last_stack)

991

992

        # the verification above only verify variables, this one will should work with literals and constants

993

        if isinstance(value, (ast.Constant if SYS_VERSION_INFO >= (3, 8) else (ast.Num, ast.Str, ast.Bytes))) \

994

                and len(attr.args) > 0 and isinstance(attr, IBuiltinMethod) and attr.has_self_argument:

995

            attr = attr.build(value_data.type)

996

997

        if attr is not Type.none and not hasattr(attribute, 'generate_value'):

998

            value_symbol_id = (value_symbol.identifier

999

                               if self.is_implemented_class_type(value_symbol)

1000

                               else value_data.symbol_id)

1001

            attribute_id = f'{value_symbol_id}{constants.ATTRIBUTE_NAME_SEPARATOR}{attribute.attr}'

1002

            index = value_type if isinstance(value_type, Package) else None

1003

            return self.build_data(attribute, symbol_id=attribute_id, symbol=attr, index=index)

1004

1005

        if isinstance(value, ast.Attribute) and need_to_visit_again:

1006

            value_data = self.visit(value)

1007

        elif hasattr(attribute, 'generate_value') and attribute.generate_value:

1008

            current_bytecode_size = self.generator.bytecode_size

1009

            if need_to_visit_again:

1010

                value_data = self.visit_to_generate(attribute.value)

1011

1012

            result = value_data.type

1013

            generation_result = value_data.symbol

1014

            if result is None and value_data.symbol_id is not None:

1015

                _, result = self.generator.get_symbol(value_data.symbol_id)

1016

                if isinstance(result, IExpression):

1017

                    generation_result = result

1018

                    result = result.type

1019

            elif isinstance(attribute.value, ast.Attribute) and isinstance(value_data.index, int):

1020

                result = self.get_type(generation_result)

1021

1022

            if self.is_implemented_class_type(result):

1023

                class_attr_id = f'{result.identifier}.{attribute.attr}'

1024

                symbol_id = class_attr_id

1025

                symbol = None

1026

                result_type = None

1027

                symbol_index = None

1028

                is_load_context_variable_from_class = (isinstance(attribute.ctx, ast.Load) and

1029

                                                       isinstance(attr, Variable) and

1030

                                                       isinstance(result, UserClass))

1031

1032

                if self.generator.bytecode_size > current_bytecode_size and isinstance(result, UserClass) and not is_load_context_variable_from_class:

1033

                    # it was generated already, don't convert again

1034

                    generated = False

1035

                    symbol_id = attribute.attr if isinstance(generation_result, Variable) else class_attr_id

1036

                    result_type = result

1037

                else:

1038

                    index = self.generator.convert_class_symbol(result,

1039

                                                                attribute.attr,

1040

                                                                isinstance(attribute.ctx, ast.Load))

1041

                    generated = True

1042

                    symbol = result

1043

                    if not isinstance(result, UserClass):

1044

                        if isinstance(index, int):

1045

                            symbol_index = index

1046

                        else:

1047

                            symbol_id = index

1048

1049

                return self.build_data(attribute,

1050

                                       symbol_id=symbol_id, symbol=symbol,

1051

                                       result_type=result_type, index=symbol_index,

1052

                                       already_generated=generated)

1053

1054

        if value_data is not None and value_symbol is None:

1055

            value_symbol = value_data.symbol_id

1056

1057

        if value_data is not None and value_data.symbol_id is not None:

1058

            value_id = f'{value_data.symbol_id}{constants.ATTRIBUTE_NAME_SEPARATOR}{attribute.attr}'

1059

        else:

1060

            value_id = attribute.attr

1061

1062

        return self.build_data(attribute, symbol_id=value_id, symbol=value_symbol)

1063

1064

    def visit_Continue(self, continue_node: ast.Continue) -> GeneratorData:

1065

"""

1066

        :param continue_node: the python ast continue statement node

1067

"""

1068

        self.generator.convert_loop_continue()

1069

        return self.build_data(continue_node)

1070

1071

    def visit_Break(self, break_node: ast.Break) -> GeneratorData:

1072

"""

1073

        :param break_node: the python ast break statement node

1074

"""

1075

        self.generator.convert_loop_break()

1076

        return self.build_data(break_node)

1077

1078

    def visit_Constant(self, constant: ast.Constant) -> GeneratorData:

1079

"""

1080

        Visitor of constant values node

1081

1082

        :param constant: the python ast constant value node

1083

"""

1084

        index = self.generator.convert_literal(constant.value)

1085

        result_type = self.get_type(constant.value)

1086

        return self.build_data(constant, result_type=result_type, index=index, already_generated=True)

1087

1088

    def visit_NameConstant(self, constant: ast.NameConstant) -> GeneratorData:

1089

"""

1090

        Visitor of constant names node

1091

1092

        :param constant: the python ast name constant node

1093

"""

1094

        index = self.generator.convert_literal(constant.value)

1095

        result_type = self.get_type(constant.value)

1096

        return self.build_data(constant, result_type=result_type, index=index, already_generated=True)

1097

1098

    def visit_Num(self, num: ast.Num) -> GeneratorData:

1099

"""

1100

        Visitor of literal number node

1101

1102

        :param num: the python ast number node

1103

"""

1104

        index = self.generator.convert_literal(num.n)

1105

        result_type = self.get_type(num.n)

1106

        return self.build_data(num, result_type=result_type, index=index, already_generated=True)

1107

1108

    def visit_Str(self, string: ast.Str) -> GeneratorData:

1109

"""

1110

        Visitor of literal string node

1111

1112

        :param string: the python ast string node

1113

"""

1114

        index = self.generator.convert_literal(string.s)

1115

        result_type = self.get_type(string.s)

1116

        return self.build_data(string, result_type=result_type, index=index, already_generated=True)

1117

1118

    def visit_Bytes(self, bts: ast.Bytes) -> GeneratorData:

1119

"""

1120

        Visitor of literal bytes node

1121

1122

        :param bts: the python ast bytes node

1123

"""

1124

        index = self.generator.convert_literal(bts.s)

1125

        result_type = self.get_type(bts.s)

1126

        return self.build_data(bts, result_type=result_type, index=index, already_generated=True)

1127

1128

    def visit_Tuple(self, tup_node: ast.Tuple) -> GeneratorData:

1129

"""

1130

        Visitor of literal tuple node

1131

1132

        :param tup_node: the python ast tuple node

1133

"""

1134

        result_type = Type.tuple

1135

        self._create_array(tup_node.elts, result_type)

1136

        return self.build_data(tup_node, result_type=result_type, already_generated=True)

1137

1138

    def visit_List(self, list_node: ast.List) -> GeneratorData:

1139

"""

1140

        Visitor of literal list node

1141

1142

        :param list_node: the python ast list node

1143

"""

1144

        result_type = Type.list

1145

        self._create_array(list_node.elts, result_type)

1146

        return self.build_data(list_node, result_type=result_type, already_generated=True)

1147

1148

    def visit_Dict(self, dict_node: ast.Dict) -> GeneratorData:

1149

"""

1150

        Visitor of literal dict node

1151

1152

        :param dict_node: the python ast dict node

1153

"""

1154

        result_type = Type.dict

1155

        length = min(len(dict_node.keys), len(dict_node.values))

1156

        self.generator.convert_new_map(result_type)

1157

        for key_value in range(length):

1158

            self.generator.duplicate_stack_top_item()

1159

            self.visit_to_generate(dict_node.keys[key_value])

1160

            value_data = self.visit_to_generate(dict_node.values[key_value])

1161

            self.generator.convert_set_item(value_data.index)

1162

1163

        return self.build_data(dict_node, result_type=result_type, already_generated=True)

1164

1165

    def visit_Pass(self, pass_node: ast.Pass) -> GeneratorData:

1166

"""

1167

        Visitor of pass node

1168

1169

        :param pass_node: the python ast dict node

1170

"""

1171

        # only generates if the scope is a function

1172

        result_type = None

1173

        generated = False

1174

1175

        if isinstance(self.current_method, Method):

1176

            self.generator.insert_nop()

1177

            generated = True

1178

1179

        return self.build_data(pass_node, result_type=result_type, already_generated=generated)

1180

1181

    def _create_array(self, values: List[ast.AST], array_type: IType):

1182

"""

1183

        Creates a new array from a literal sequence

1184

1185

        :param values: list of values of the new array items

1186

"""

1187

        length = len(values)

1188

        if length > 0:

1189

            for value in reversed(values):

1190

                self.visit_to_generate(value)

1191

        self.generator.convert_new_array(length, array_type)

CityOfZion / neo3-boa / 942e0a4f-38aa-47a9-b65e-f3dc399b992f

Source File
Press 'n' to go to next uncovered line, 'b' for previous

CityOfZion / neo3-boa / 942e0a4f-38aa-47a9-b65e-f3dc399b992f

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous