30ae3248-fada-4b8d-8735-31794b222593

Committed 17 Jan 2024 01:43PM UTC coverage: 92.103%. Remained the same

Build # 30ae3248-fada-4b8d-8735-31794b222593

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #1170 from CityOfZion/CU-86a1hepnw

CU-86a1hepnw - Fix tests using Python 3.12 (parsing manifest JSON)

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

253 existing lines in 51 files now uncovered.

20773 of 22554 relevant lines covered (92.1%)

2.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.28

/boa3/internal/compiler/codegenerator/vmcodemapping.py

from __future__ import annotations

from typing import Dict, List, Optional, Union

from boa3.internal.compiler.codegenerator.methodtokencollection import MethodTokenCollection
from boa3.internal.compiler.codegenerator.vmcodemap import VMCodeMap
from boa3.internal.compiler.compileroutput import CompilerOutput
from boa3.internal.model.builtin.method import IBuiltinMethod
from boa3.internal.neo.vm.VMCode import VMCode
from boa3.internal.neo.vm.opcode import OpcodeHelper
from boa3.internal.neo.vm.opcode.OpcodeInformation import OpcodeInformation
from boa3.internal.neo3.contracts.contracttypes import CallFlags


class VMCodeMapping:
    """
    This class is responsible for managing the Neo VM instruction during the bytecode generation.
    """
    _instance: VMCodeMapping = None

    @classmethod
    def instance(cls):
        """
        :return: the singleton instance
        """
        if cls._instance is None:
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        self._code_map: VMCodeMap = VMCodeMap()
        self._method_tokens: MethodTokenCollection = MethodTokenCollection()

    @classmethod
    def reset(cls):
        """
        Resets the map to the first state
        """
        if cls._instance is not None:
            cls._instance._code_map.clear()
            cls._instance._method_tokens.clear()

    def add_method_token(self, method: IBuiltinMethod, call_flag: CallFlags) -> Optional[int]:
        """
        Creates a new method token if the method call another contract and return its id.
        Otherwise, returns None
        """
        if hasattr(method, 'contract_script_hash'):
            return self._method_tokens.append(method, call_flag)
        return None

    def get_method_token(self, method_token_id: int):
        """
        Returns the method token with given id if it exists.
        Otherwise, returns None

        :rtype: boa3.internal.neo3.contracts.nef.MethodToken or None
        """
        return self._method_tokens[method_token_id]

    @property
    def codes(self) -> List[VMCode]:
        """
        Gets a list with the included vm codes

        :return: a list of vm codes ordered by its address in the bytecode
        """
        return self._code_map.get_code_list()

    @property
    def code_map(self) -> Dict[int, VMCode]:
        """
        Gets a dictionary that maps each vm code with its address.

        :return: a dictionary that maps each instruction with its address. The keys are ordered by the address.
        """
        return self._code_map.get_code_map()

    def targeted_address(self) -> Dict[int, List[int]]:
        """
        Gets a dictionary that maps each address to the opcodes that targets it

        :return: a dictionary that maps the targeted instructions to its source.
        """
        target_maps = {}
        for code in self._code_map.get_code_with_target_list():
            if code.target is not None and code.target is not code:
                address = self.get_start_address(code)
                target = self.get_start_address(code.target)
                if target not in target_maps:
                    target_maps[target] = [address]
                else:
                    target_maps[target].append(address)
        return target_maps

    def bytecode(self) -> bytes:
        """
        Gets the bytecode of the translated code

        :return: the generated bytecode
        """
        self._remove_empty_targets()
        self._update_larger_codes()

        bytecode = bytearray()
        for code in self.codes:
            bytecode += code.opcode
            if code.data is not None:
                bytecode += code.data
        return bytes(bytecode)

    def result(self) -> CompilerOutput:
        """
        Gets the complete output of the translated code
        """
        bytecode = self.bytecode()
        return CompilerOutput(bytecode, self._method_tokens.to_list())

    @property
    def bytecode_size(self) -> int:
        return self._code_map.get_bytecode_size()

    def insert_code(self, vm_code: VMCode):
        return self._code_map.insert_code(vm_code, has_target=OpcodeHelper.has_target(vm_code.opcode))

    def get_code(self, address: int) -> Optional[VMCode]:
        """
        Gets the VM Opcode at the given position

        :param address: the position of the opcode
        :return: the opcode if it exists. None otherwise
        :rtype: VMCode or None
        """
        return self._code_map.get_code(address)

    def get_addresses(self, start_address: int, end_address: int) -> List[int]:
        return self._code_map.get_addresses(start_address, end_address)

    def get_start_address(self, vm_code: VMCode) -> int:
        """
        Gets the vm code's first byte address

        :param vm_code: the instruction to get the address
        :return: the vm code's address if it's in the map. Otherwise, return's zero.
        """
        return self._code_map.get_start_address(vm_code)

    def get_end_address(self, vm_code: VMCode) -> int:
        """
        Gets the vm code's last byte address

        :param vm_code: the instruction to get the address
        :return: the vm code's last address if it's in the map. Otherwise, return's zero.
        """
        return self._code_map.get_end_address(vm_code)

    def get_opcodes(self, addresses: List[int]) -> List[VMCode]:
        return self._code_map.get_opcodes(addresses)

    def update_vm_code(self, vm_code: VMCode, opcode: OpcodeInformation, data: bytes = bytes()):
        """
        Updates the information from an inserted code

        :param vm_code: code to be updated
        :param opcode: updated opcode information
        :param data: updated opcode data
        """
        code_size = vm_code.size
        vm_code._info = opcode
        vm_code._data = data
        if vm_code.size != code_size:
            self._update_addresses(self.get_start_address(vm_code))

    def _update_addresses(self, start_address: int = 0):
        """
        Updates the instruction map's keys when a opcode is changed

        :param start_address: the address from the changed opcode
        """
        return self._code_map.update_addresses(start_address)

    def _update_targets(self):
        from boa3.internal.neo.vm.type.Integer import Integer
        for code in self._code_map.get_code_with_target_list():
            if code.target is None:
                relative = Integer.from_bytes(code.data)
                absolute = self._code_map.get_start_address(code) + relative
                if absolute in self.code_map:
                    code.set_target(self.code_map[absolute])

    def _update_larger_codes(self):
        """
        Checks if each instruction data fits in its opcode maximum size and updates the opcode from those that don't
        """
        # gets a list with all instructions which its opcode has a larger equivalent, ordered by its address
        instr_with_small_codes = [code for code in self._code_map.get_code_list() if OpcodeHelper.has_larger_opcode(code.opcode)]
        instr_with_small_codes.sort(key=lambda code: self.get_start_address(code), reverse=True)

        from boa3.internal.neo.vm.opcode.OpcodeInfo import OpcodeInfo
        # total_len is initialized with zero because the loop must run at least once
        total_len = 0
        current_size = self.bytecode_size

        # if any instruction is updated, the following instruction addresses and the total size will change as well
        # with the change, previous instruction data may have overflowed the opcode maximum value
        # to make sure, it must check the opcodes that haven't changed again
        while total_len != current_size:
            total_len = current_size

            # verifies each instruction data length
            for code in instr_with_small_codes.copy():  # it's a copy because the list may change during the iteration
                if len(code.raw_data) > code.info.max_data_len:
                    # gets the shortest opcode equivalent that fits the instruction data
                    info = OpcodeInfo.get_info(OpcodeHelper.get_larger_opcode(code.opcode))
                    while len(code.raw_data) > info.max_data_len and OpcodeHelper.has_larger_opcode(info.opcode):
                        info = OpcodeInfo.get_info(OpcodeHelper.get_larger_opcode(code.opcode))

                    self.update_vm_code(code, info)
                    if info.opcode == OpcodeHelper.get_larger_opcode(info.opcode):
                        # if it's the largest equivalent, it won't be updated anymore
                        instr_with_small_codes.remove(code)
            current_size = self.bytecode_size

    def _validate_targets(self, code_or_address: Union[int, VMCode]):
        if isinstance(code_or_address, int):
            address = code_or_address
            code = self.get_code(address)
        else:
            code = code_or_address
            address = self.get_start_address(code)

        targeted_addresses = self.targeted_address()

        if address in targeted_addresses:
            next_address = self.get_end_address(code) + 1
            if next_address < self.bytecode_size:
                next_code = self._code_map.get_code(next_address)
                for source in targeted_addresses[address]:
                    self._code_map.get_code(source).set_target(next_code)

    def move_to_end(self, first_code_address: int, last_code_address: int) -> int:
        """
        Moves a set of instructions to the end of the current bytecode

        :param first_code_address: first instruction start address
        :param last_code_address: last instruction end address
        """
        result = self._code_map.move_to_end(first_code_address, last_code_address)
        if not isinstance(result, int):
            return self.bytecode_size

        self._update_targets()
        return result

    def remove_opcodes(self, first_code_address: int, last_code_address: int = None):
        if not isinstance(last_code_address, int):
            last_code_address = self.bytecode_size
        addresses_to_remove = self._code_map.get_addresses(first_code_address, last_code_address)
        for address in addresses_to_remove:
            self._validate_targets(address)
        return self._code_map.remove_opcodes_by_addresses(addresses_to_remove)

    def remove_opcodes_by_code(self, codes: List[VMCode]):
        addresses_to_remove = self._code_map.get_addresses_from_codes(codes)
        for address in addresses_to_remove:
            self._validate_targets(address)
        return self._code_map.remove_opcodes_by_addresses(addresses_to_remove)

    def _remove_empty_targets(self):
        """
        Checks if each instruction that requires a target has one set and remove those that don't
        """
        addresses_to_remove = []
        for code in self._code_map.get_code_with_target_list():
            if code.target is None or code.target is code:
                address = self.get_start_address(code)
                self._validate_targets(address)
                addresses_to_remove.append(address)

        if len(addresses_to_remove) > 0:
            self._code_map.remove_opcodes_by_addresses(addresses_to_remove)

1	from __future__ import annotations	3✔
2
3	from typing import Dict, List, Optional, Union	3✔
4
5	from boa3.internal.compiler.codegenerator.methodtokencollection import MethodTokenCollection	3✔
6	from boa3.internal.compiler.codegenerator.vmcodemap import VMCodeMap	3✔
7	from boa3.internal.compiler.compileroutput import CompilerOutput	3✔
8	from boa3.internal.model.builtin.method import IBuiltinMethod	3✔
9	from boa3.internal.neo.vm.VMCode import VMCode	3✔
10	from boa3.internal.neo.vm.opcode import OpcodeHelper	3✔
11	from boa3.internal.neo.vm.opcode.OpcodeInformation import OpcodeInformation	3✔
12	from boa3.internal.neo3.contracts.contracttypes import CallFlags	3✔
13
14
15	class VMCodeMapping:	3✔
16	"""
17	This class is responsible for managing the Neo VM instruction during the bytecode generation.
18	"""
19	_instance: VMCodeMapping = None	3✔
20
21	@classmethod	3✔
22	def instance(cls):	3✔
23	"""
24	:return: the singleton instance
25	"""
26	if cls._instance is None:	3✔
27	cls._instance = cls()	3✔
28	return cls._instance	3✔
29
30	def __init__(self):	3✔
31	self._code_map: VMCodeMap = VMCodeMap()	3✔
32	self._method_tokens: MethodTokenCollection = MethodTokenCollection()	3✔
33
34	@classmethod	3✔
35	def reset(cls):	3✔
36	"""
37	Resets the map to the first state
38	"""
39	if cls._instance is not None:	3✔
40	cls._instance._code_map.clear()	3✔
41	cls._instance._method_tokens.clear()	3✔
42
43	def add_method_token(self, method: IBuiltinMethod, call_flag: CallFlags) -> Optional[int]:	3✔
44	"""
45	Creates a new method token if the method call another contract and return its id.
46	Otherwise, returns None
47	"""
48	if hasattr(method, 'contract_script_hash'):	3✔
49	return self._method_tokens.append(method, call_flag)	3✔
50	return None	×
51
52	def get_method_token(self, method_token_id: int):	3✔
53	"""
54	Returns the method token with given id if it exists.
55	Otherwise, returns None
56
57	:rtype: boa3.internal.neo3.contracts.nef.MethodToken or None
58	"""
59	return self._method_tokens[method_token_id]	3✔
60
61	@property	3✔
62	def codes(self) -> List[VMCode]:	3✔
63	"""
64	Gets a list with the included vm codes
65
66	:return: a list of vm codes ordered by its address in the bytecode
67	"""
68	return self._code_map.get_code_list()	3✔
69
70	@property	3✔
71	def code_map(self) -> Dict[int, VMCode]:	3✔
72	"""
73	Gets a dictionary that maps each vm code with its address.
74
75	:return: a dictionary that maps each instruction with its address. The keys are ordered by the address.
76	"""
77	return self._code_map.get_code_map()	3✔
78
79	def targeted_address(self) -> Dict[int, List[int]]:	3✔
80	"""
81	Gets a dictionary that maps each address to the opcodes that targets it
82
83	:return: a dictionary that maps the targeted instructions to its source.
84	"""
85	target_maps = {}	3✔
86	for code in self._code_map.get_code_with_target_list():	3✔
87	if code.target is not None and code.target is not code:	3✔
88	address = self.get_start_address(code)	3✔
89	target = self.get_start_address(code.target)	3✔
90	if target not in target_maps:	3✔
91	target_maps[target] = [address]	3✔
92	else:
93	target_maps[target].append(address)	3✔
94	return target_maps	3✔
95
96	def bytecode(self) -> bytes:	3✔
97	"""
98	Gets the bytecode of the translated code
99
100	:return: the generated bytecode
101	"""
102	self._remove_empty_targets()	3✔
103	self._update_larger_codes()	3✔
104
105	bytecode = bytearray()	3✔
106	for code in self.codes:	3✔
107	bytecode += code.opcode	3✔
108	if code.data is not None:	3✔
109	bytecode += code.data	3✔
110	return bytes(bytecode)	3✔
111
112	def result(self) -> CompilerOutput:	3✔
113	"""
114	Gets the complete output of the translated code
115	"""
116	bytecode = self.bytecode()	3✔
117	return CompilerOutput(bytecode, self._method_tokens.to_list())	3✔
118
119	@property	3✔
120	def bytecode_size(self) -> int:	3✔
121	return self._code_map.get_bytecode_size()	3✔
122
123	def insert_code(self, vm_code: VMCode):	3✔
124	return self._code_map.insert_code(vm_code, has_target=OpcodeHelper.has_target(vm_code.opcode))	3✔
125
126	def get_code(self, address: int) -> Optional[VMCode]:	3✔
127	"""
128	Gets the VM Opcode at the given position
129
130	:param address: the position of the opcode
131	:return: the opcode if it exists. None otherwise
132	:rtype: VMCode or None
133	"""
134	return self._code_map.get_code(address)	3✔
135
136	def get_addresses(self, start_address: int, end_address: int) -> List[int]:	3✔
137	return self._code_map.get_addresses(start_address, end_address)	3✔
138
139	def get_start_address(self, vm_code: VMCode) -> int:	3✔
140	"""
141	Gets the vm code's first byte address
142
143	:param vm_code: the instruction to get the address
144	:return: the vm code's address if it's in the map. Otherwise, return's zero.
145	"""
146	return self._code_map.get_start_address(vm_code)	3✔
147
148	def get_end_address(self, vm_code: VMCode) -> int:	3✔
149	"""
150	Gets the vm code's last byte address
151
152	:param vm_code: the instruction to get the address
153	:return: the vm code's last address if it's in the map. Otherwise, return's zero.
154	"""
155	return self._code_map.get_end_address(vm_code)	3✔
156
157	def get_opcodes(self, addresses: List[int]) -> List[VMCode]:	3✔
158	return self._code_map.get_opcodes(addresses)	3✔
159
160	def update_vm_code(self, vm_code: VMCode, opcode: OpcodeInformation, data: bytes = bytes()):	3✔
161	"""
162	Updates the information from an inserted code
163
164	:param vm_code: code to be updated
165	:param opcode: updated opcode information
166	:param data: updated opcode data
167	"""
168	code_size = vm_code.size	3✔
169	vm_code._info = opcode	3✔
170	vm_code._data = data	3✔
171	if vm_code.size != code_size:	3✔
172	self._update_addresses(self.get_start_address(vm_code))	3✔
173
174	def _update_addresses(self, start_address: int = 0):	3✔
175	"""
176	Updates the instruction map's keys when a opcode is changed
177
178	:param start_address: the address from the changed opcode
179	"""
180	return self._code_map.update_addresses(start_address)	3✔
181
182	def _update_targets(self):	3✔
183	from boa3.internal.neo.vm.type.Integer import Integer	3✔
184	for code in self._code_map.get_code_with_target_list():	3✔
185	if code.target is None:	3✔
186	relative = Integer.from_bytes(code.data)	3✔
187	absolute = self._code_map.get_start_address(code) + relative	3✔
188	if absolute in self.code_map:	3✔
189	code.set_target(self.code_map[absolute])	3✔
190
191	def _update_larger_codes(self):	3✔
192	"""
193	Checks if each instruction data fits in its opcode maximum size and updates the opcode from those that don't
194	"""
195	# gets a list with all instructions which its opcode has a larger equivalent, ordered by its address
196	instr_with_small_codes = [code for code in self._code_map.get_code_list() if OpcodeHelper.has_larger_opcode(code.opcode)]	3✔
197	instr_with_small_codes.sort(key=lambda code: self.get_start_address(code), reverse=True)	3✔
198
199	from boa3.internal.neo.vm.opcode.OpcodeInfo import OpcodeInfo	3✔
200	# total_len is initialized with zero because the loop must run at least once
201	total_len = 0	3✔
202	current_size = self.bytecode_size	3✔
203
204	# if any instruction is updated, the following instruction addresses and the total size will change as well
205	# with the change, previous instruction data may have overflowed the opcode maximum value
206	# to make sure, it must check the opcodes that haven't changed again
207	while total_len != current_size:	3✔
208	total_len = current_size	3✔
209
210	# verifies each instruction data length
211	for code in instr_with_small_codes.copy(): # it's a copy because the list may change during the iteration	3✔
212	if len(code.raw_data) > code.info.max_data_len:	3✔
213	# gets the shortest opcode equivalent that fits the instruction data
214	info = OpcodeInfo.get_info(OpcodeHelper.get_larger_opcode(code.opcode))	3✔
215	while len(code.raw_data) > info.max_data_len and OpcodeHelper.has_larger_opcode(info.opcode):	3✔
216	info = OpcodeInfo.get_info(OpcodeHelper.get_larger_opcode(code.opcode))	×
217
218	self.update_vm_code(code, info)	3✔
219	if info.opcode == OpcodeHelper.get_larger_opcode(info.opcode):	3✔
220	# if it's the largest equivalent, it won't be updated anymore
221	instr_with_small_codes.remove(code)	3✔
222	current_size = self.bytecode_size	3✔
223
224	def _validate_targets(self, code_or_address: Union[int, VMCode]):	3✔
225	if isinstance(code_or_address, int):	3✔
226	address = code_or_address	3✔
227	code = self.get_code(address)	3✔
228	else:
229	code = code_or_address	×
230	address = self.get_start_address(code)	×
231
232	targeted_addresses = self.targeted_address()	3✔
233
234	if address in targeted_addresses:	3✔
235	next_address = self.get_end_address(code) + 1	3✔
236	if next_address < self.bytecode_size:	3✔
237	next_code = self._code_map.get_code(next_address)	3✔
238	for source in targeted_addresses[address]:	3✔
239	self._code_map.get_code(source).set_target(next_code)	3✔
240
241	def move_to_end(self, first_code_address: int, last_code_address: int) -> int:	3✔
242	"""
243	Moves a set of instructions to the end of the current bytecode
244
245	:param first_code_address: first instruction start address
246	:param last_code_address: last instruction end address
247	"""
248	result = self._code_map.move_to_end(first_code_address, last_code_address)	3✔
249	if not isinstance(result, int):	3✔
250	return self.bytecode_size	3✔
251
252	self._update_targets()	3✔
253	return result	3✔
254
255	def remove_opcodes(self, first_code_address: int, last_code_address: int = None):	3✔
256	if not isinstance(last_code_address, int):	3✔
UNCOV 257	last_code_address = self.bytecode_size	2✔
258	addresses_to_remove = self._code_map.get_addresses(first_code_address, last_code_address)	3✔
259	for address in addresses_to_remove:	3✔
260	self._validate_targets(address)	3✔
261	return self._code_map.remove_opcodes_by_addresses(addresses_to_remove)	3✔
262
263	def remove_opcodes_by_code(self, codes: List[VMCode]):	3✔
264	addresses_to_remove = self._code_map.get_addresses_from_codes(codes)	3✔
265	for address in addresses_to_remove:	3✔
266	self._validate_targets(address)	3✔
267	return self._code_map.remove_opcodes_by_addresses(addresses_to_remove)	3✔
268
269	def _remove_empty_targets(self):	3✔
270	"""
271	Checks if each instruction that requires a target has one set and remove those that don't
272	"""
273	addresses_to_remove = []	3✔
274	for code in self._code_map.get_code_with_target_list():	3✔
275	if code.target is None or code.target is code:	3✔
276	address = self.get_start_address(code)	3✔
277	self._validate_targets(address)	3✔
278	addresses_to_remove.append(address)	3✔
279
280	if len(addresses_to_remove) > 0:	3✔
281	self._code_map.remove_opcodes_by_addresses(addresses_to_remove)	3✔

CityOfZion / neo3-boa / 30ae3248-fada-4b8d-8735-31794b222593

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous