12877283541

Committed 20 Jan 2025 11:39PM UTC coverage: 95.796% (+15.2%) from 80.557%

Build # 12877283541

Build Type

Pull #70

github

Specific Base bc1070

Committed by

kevdevg

Commit Message

fix: variable name

Pull Request Pull Request #70: feat(litellm): multimodal

Run Details

101 of 105 new or added lines in 14 files covered. (96.19%)

45 existing lines in 9 files now uncovered.

2142 of 2236 relevant lines covered (95.8%)

3.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.29

/scope3ai/lib.py

import importlib.metadata
import importlib.util
import logging
from contextlib import contextmanager
from contextvars import ContextVar
from functools import partial
from os import getenv
from typing import Optional, List
from uuid import uuid4
import atexit

from .api.client import Client, AsyncClient
from .api.tracer import Tracer
from .api.types import ImpactRow, ImpactResponse, Scope3AIContext
from .api.defaults import DEFAULT_API_URL
from .constants import PROVIDERS

from .worker import BackgroundWorker

logger = logging.getLogger("scope3ai.lib")


def init_anthropic_instrumentor() -> None:
    if importlib.util.find_spec("anthropic") is not None:
        from scope3ai.tracers.anthropic.instrument import AnthropicInstrumentor

        instrumentor = AnthropicInstrumentor()
        instrumentor.instrument()


def init_cohere_instrumentor() -> None:
    if importlib.util.find_spec("cohere") is not None:
        from scope3ai.tracers.cohere.instrument import CohereInstrumentor

        instrumentor = CohereInstrumentor()
        instrumentor.instrument()


def init_openai_instrumentor() -> None:
    if importlib.util.find_spec("openai") is not None:
        from scope3ai.tracers.openai.instrument import OpenAIInstrumentor

        instrumentor = OpenAIInstrumentor()
        instrumentor.instrument()


def init_huggingface_hub_instrumentor() -> None:
    if importlib.util.find_spec("huggingface_hub") is not None:
        from scope3ai.tracers.huggingface.instrument import HuggingfaceInstrumentor

        instrumentor = HuggingfaceInstrumentor()
        instrumentor.instrument()


def init_litellm_instrumentor() -> None:
    if importlib.util.find_spec("litellm") is not None:
        from scope3ai.tracers.litellm.instrument import LiteLLMInstrumentor

        instrumentor = LiteLLMInstrumentor()
        instrumentor.instrument()


def init_mistral_v1_instrumentor() -> None:
    if importlib.util.find_spec("mistralai") is not None:
        from scope3ai.tracers.mistralai.instrument import MistralAIInstrumentor

        instrumentor = MistralAIInstrumentor()
        instrumentor.instrument()


def init_response_instrumentor() -> None:
    from scope3ai.response_interceptor.instrument import ResponseInterceptor

    instrumentor = ResponseInterceptor()
    instrumentor.instrument()


_INSTRUMENTS = {
    PROVIDERS.ANTROPIC.value: init_anthropic_instrumentor,
    PROVIDERS.COHERE.value: init_cohere_instrumentor,
    PROVIDERS.OPENAI.value: init_openai_instrumentor,
    PROVIDERS.HUGGINGFACE_HUB.value: init_huggingface_hub_instrumentor,
    PROVIDERS.LITELLM.value: init_litellm_instrumentor,
    PROVIDERS.MISTRALAI.value: init_mistral_v1_instrumentor,
    PROVIDERS.RESPONSE.value: init_response_instrumentor,
}

_RE_INIT_PROVIDERS = [PROVIDERS.RESPONSE.value]


def generate_id() -> str:
    return uuid4().hex


class Scope3AIError(Exception):
    pass


class Scope3AI:
    """
    Scope3AI tracer class

    This class is a singleton that provides a context manager for tracing
    inference metadata and submitting impact requests to the Scope3 AI API.
    """

    _instance: Optional["Scope3AI"] = None
    _tracer: ContextVar[List[Tracer]] = ContextVar("tracer", default=[])
    _worker: Optional[BackgroundWorker] = None
    _providers: List[str] = []
    _keep_tracers: bool = False

    def __new__(cls, *args, **kwargs):
        if cls._instance is None:
            cls._instance = super(Scope3AI, cls).__new__(cls)
        return cls._instance

    @classmethod
    def init(
        cls,
        api_key: str = None,
        api_url: str = None,
        sync_mode: bool = False,
        enable_debug_logging: bool = False,
        providers: Optional[List[str]] = None,
    ) -> None:
        if cls._instance is not None:
            raise Scope3AIError("Scope3AI is already initialized")
        cls._instance = self = Scope3AI()
        self.api_key = api_key or getenv("SCOPE3AI_API_KEY")
        self.api_url = api_url or getenv("SCOPE3AI_API_URL") or DEFAULT_API_URL
        self.sync_mode = sync_mode or bool(getenv("SCOPE3AI_SYNC_MODE", False))
        if not self.api_key:
            raise Scope3AIError(
                "The scope3 api_key option must be set either by "
                "passing the API key to the Scope3AI.init(api_key='xxx') "
                "or by setting the SCOPE3AI_API_KEY environment variable"
            )
        if not self.api_url:
            raise Scope3AIError(
                "The api_url option must be set either by "
                "passing the API URL to the Scope3AI.init(api_url='xxx') "
                "or by setting the SCOPE3AI_API_URL environment variable"
            )

        if enable_debug_logging:
            self._init_logging()

        if providers is None:
            providers = list(_INSTRUMENTS.keys())

        http_client_options = {"api_key": self.api_key, "api_url": self.api_url}
        self._sync_client = Client(**http_client_options)
        self._async_client = AsyncClient(**http_client_options)
        self._init_providers(providers)
        self._init_atexit()
        return cls._instance

    @classmethod
    def get_instance(cls) -> "Scope3AI":
        """
        Return the instance of the Scope3AI singleton.

        This method provides access to the default global state of the
        Scope3AI library. The returned instance can be used to trace
        inference metadata and submit impact requests to the Scope3 AI
        API from anywhere in the application.

        Returns:
            Scope3AI: The singleton instance of the Scope3AI class.
        """
        return cls._instance

    def submit_impact(
        self,
        impact_row: ImpactRow,
    ) -> Scope3AIContext:
        """
        Submit an impact request to the Scope3 AI API.

        This function sends an impact request represented by the `impact_row`
        to the Scope3 AI API and optionally returns the response.

        Args:
            impact_row (ImpactRow): The impact request data
                that needs to be submitted to the Scope3 AI API.

        Returns:
            Scope3AIContext: A context object containing the request data and
            the response from the API.
        """

        def submit_impact(
            impact_row: ImpactRow,
            ctx: Scope3AIContext,
        ) -> Optional[ImpactResponse]:
            response = self._sync_client.impact(
                rows=[impact_row],
                with_response=True,
            )
            ctx.set_impact(response.rows[0])
            if ctx._tracer:
                ctx._tracer.add_impact(response.rows[0])
                ctx._tracer._unlink_trace(ctx)
            return response

        tracer = self.current_tracer
        ctx = Scope3AIContext(request=impact_row)
        ctx._tracer = tracer
        if tracer:
            tracer._link_trace(ctx)

        if self.sync_mode:
            submit_impact(impact_row, ctx=ctx)
            return ctx

        self._ensure_worker()
        self._worker.submit(partial(submit_impact, impact_row=impact_row, ctx=ctx))
        return ctx

    async def asubmit_impact(
        self,
        impact_row: ImpactRow,
    ) -> Scope3AIContext:
        """
        Async version of Scope3AI::submit_impact.
        """

        if not self.sync_mode:
            # in non sync-mode, it uses the background worker,
            # and the background worker is not async (does not have to be).
            # so we just redirect the call to the sync version.
            return self.submit_impact(impact_row)
        tracer = self.current_tracer
        ctx = Scope3AIContext(request=impact_row)
        ctx._tracer = tracer
        if tracer:
            tracer._link_trace(ctx)

        response = await self._async_client.impact(
            rows=[impact_row],
            with_response=True,
        )
        ctx.set_impact(response.rows[0])
        if tracer:
            tracer.add_impact(response.rows[0])
            tracer._unlink_trace(ctx)

        return ctx

    @property
    def root_tracer(self):
        """
        Return the root tracer.

        The root tracer is the first tracer in the current execution context
        (tracer stack). If no tracers are currently active, it returns None.

        Returns:
            Tracer: The root tracer if available, otherwise None.
        """
        tracers = self._tracer.get()
        return tracers[0] if tracers else None

    @property
    def current_tracer(self):
        """
        Return the current tracer.

        The current tracer is the last tracer in the current execution context
        (tracer stack). If no tracers are currently active, it returns None.

        Returns:
            Tracer: The current tracer if available, otherwise None.
        """
        tracers = self._tracer.get()
        return tracers[-1] if tracers else None

    @contextmanager
    def trace(self, keep_traces=False):
        tracer = Tracer(keep_traces=keep_traces)
        try:
            self._push_tracer(tracer)
            yield tracer
        finally:
            self._pop_tracer(tracer)

    def close(self):
        if self._worker:
            self._worker.kill()
        self.__class__._instance = None

    #
    # Internals
    #

    def _push_tracer(self, tracer: Tracer) -> None:
        tracer._link_parent(self.current_tracer)
        self._tracer.get().append(tracer)

    def _pop_tracer(self, tracer: Tracer) -> None:
        self._tracer.get().remove(tracer)
        tracer._unlink_parent(self.current_tracer)

    def _init_providers(self, providers: List[str]) -> None:
        """Initialize the specified providers."""
        for provider in providers:
            if provider not in _INSTRUMENTS:
                raise Scope3AIError(
                    f"Could not find tracer for the `{provider}` provider."
                )
            if provider in self._providers and provider not in _RE_INIT_PROVIDERS:
                # already initialized
                continue
            init_func = _INSTRUMENTS[provider]
            init_func()
            self._providers.append(provider)

    def _ensure_worker(self) -> None:
        if not self._worker:
            self._worker = BackgroundWorker(-1)

    def _init_logging(self) -> None:
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
            handlers=[logging.StreamHandler()],
        )
        logging.getLogger("scope3ai").setLevel(logging.DEBUG)

    def _init_atexit(self):
        @atexit.register
        def _shutdown():
            # do not reinstanciate the singleton here if somehow it was deleted
            scope3ai = Scope3AI._instance
            if not scope3ai:
                return
            if scope3ai._worker and scope3ai._worker._queue:
                logging.debug("Waiting background informations to be processed")
                scope3ai._worker._queue.join()
                logging.debug("Shutting down Scope3AI")

1	import importlib.metadata	4✔
2	import importlib.util	4✔
3	import logging	4✔
4	from contextlib import contextmanager	4✔
5	from contextvars import ContextVar	4✔
6	from functools import partial	4✔
7	from os import getenv	4✔
8	from typing import Optional, List	4✔
9	from uuid import uuid4	4✔
10	import atexit	4✔
11
12	from .api.client import Client, AsyncClient	4✔
13	from .api.tracer import Tracer	4✔
14	from .api.types import ImpactRow, ImpactResponse, Scope3AIContext	4✔
15	from .api.defaults import DEFAULT_API_URL	4✔
16	from .constants import PROVIDERS	4✔
17
18	from .worker import BackgroundWorker	4✔
19
20	logger = logging.getLogger("scope3ai.lib")	4✔
21
22
23	def init_anthropic_instrumentor() -> None:	4✔
24	if importlib.util.find_spec("anthropic") is not None:	4✔
25	from scope3ai.tracers.anthropic.instrument import AnthropicInstrumentor	4✔
26
27	instrumentor = AnthropicInstrumentor()	4✔
28	instrumentor.instrument()	4✔
29
30
31	def init_cohere_instrumentor() -> None:	4✔
32	if importlib.util.find_spec("cohere") is not None:	4✔
33	from scope3ai.tracers.cohere.instrument import CohereInstrumentor	4✔
34
35	instrumentor = CohereInstrumentor()	4✔
36	instrumentor.instrument()	4✔
37
38
39	def init_openai_instrumentor() -> None:	4✔
40	if importlib.util.find_spec("openai") is not None:	4✔
41	from scope3ai.tracers.openai.instrument import OpenAIInstrumentor	4✔
42
43	instrumentor = OpenAIInstrumentor()	4✔
44	instrumentor.instrument()	4✔
45
46
47	def init_huggingface_hub_instrumentor() -> None:	4✔
48	if importlib.util.find_spec("huggingface_hub") is not None:	4✔
49	from scope3ai.tracers.huggingface.instrument import HuggingfaceInstrumentor	4✔
50
51	instrumentor = HuggingfaceInstrumentor()	4✔
52	instrumentor.instrument()	4✔
53
54
55	def init_litellm_instrumentor() -> None:	4✔
56	if importlib.util.find_spec("litellm") is not None:	4✔
57	from scope3ai.tracers.litellm.instrument import LiteLLMInstrumentor	4✔
58
59	instrumentor = LiteLLMInstrumentor()	4✔
60	instrumentor.instrument()	4✔
61
62
63	def init_mistral_v1_instrumentor() -> None:	4✔
64	if importlib.util.find_spec("mistralai") is not None:	4✔
65	from scope3ai.tracers.mistralai.instrument import MistralAIInstrumentor	4✔
66
67	instrumentor = MistralAIInstrumentor()	4✔
68	instrumentor.instrument()	4✔
69
70
71	def init_response_instrumentor() -> None:	4✔
72	from scope3ai.response_interceptor.instrument import ResponseInterceptor	4✔
73
74	instrumentor = ResponseInterceptor()	4✔
75	instrumentor.instrument()	4✔
76
77
78	_INSTRUMENTS = {	4✔
79	PROVIDERS.ANTROPIC.value: init_anthropic_instrumentor,
80	PROVIDERS.COHERE.value: init_cohere_instrumentor,
81	PROVIDERS.OPENAI.value: init_openai_instrumentor,
82	PROVIDERS.HUGGINGFACE_HUB.value: init_huggingface_hub_instrumentor,
83	PROVIDERS.LITELLM.value: init_litellm_instrumentor,
84	PROVIDERS.MISTRALAI.value: init_mistral_v1_instrumentor,
85	PROVIDERS.RESPONSE.value: init_response_instrumentor,
86	}
87
88	_RE_INIT_PROVIDERS = [PROVIDERS.RESPONSE.value]	4✔
89
90
91	def generate_id() -> str:	4✔
UNCOV 92	return uuid4().hex	×
93
94
95	class Scope3AIError(Exception):	4✔
96	pass	4✔
97
98
99	class Scope3AI:	4✔
100	"""
101	Scope3AI tracer class
102
103	This class is a singleton that provides a context manager for tracing
104	inference metadata and submitting impact requests to the Scope3 AI API.
105	"""
106
107	_instance: Optional["Scope3AI"] = None	4✔
108	_tracer: ContextVar[List[Tracer]] = ContextVar("tracer", default=[])	4✔
109	_worker: Optional[BackgroundWorker] = None	4✔
110	_providers: List[str] = []	4✔
111	_keep_tracers: bool = False	4✔
112
113	def __new__(cls, args, *kwargs):	4✔
114	if cls._instance is None:	4✔
115	cls._instance = super(Scope3AI, cls).__new__(cls)	4✔
116	return cls._instance	4✔
117
118	@classmethod	4✔
119	def init(	4✔
120	cls,
121	api_key: str = None,
122	api_url: str = None,
123	sync_mode: bool = False,
124	enable_debug_logging: bool = False,
125	providers: Optional[List[str]] = None,
126	) -> None:
127	if cls._instance is not None:	4✔
UNCOV 128	raise Scope3AIError("Scope3AI is already initialized")	×
129	cls._instance = self = Scope3AI()	4✔
130	self.api_key = api_key or getenv("SCOPE3AI_API_KEY")	4✔
131	self.api_url = api_url or getenv("SCOPE3AI_API_URL") or DEFAULT_API_URL	4✔
132	self.sync_mode = sync_mode or bool(getenv("SCOPE3AI_SYNC_MODE", False))	4✔
133	if not self.api_key:	4✔
UNCOV 134	raise Scope3AIError(	×
135	"The scope3 api_key option must be set either by "
136	"passing the API key to the Scope3AI.init(api_key='xxx') "
137	"or by setting the SCOPE3AI_API_KEY environment variable"
138	)
139	if not self.api_url:	4✔
UNCOV 140	raise Scope3AIError(	×
141	"The api_url option must be set either by "
142	"passing the API URL to the Scope3AI.init(api_url='xxx') "
143	"or by setting the SCOPE3AI_API_URL environment variable"
144	)
145
146	if enable_debug_logging:	4✔
147	self._init_logging()	4✔
148
149	if providers is None:	4✔
150	providers = list(_INSTRUMENTS.keys())	4✔
151
152	http_client_options = {"api_key": self.api_key, "api_url": self.api_url}	4✔
153	self._sync_client = Client(**http_client_options)	4✔
154	self._async_client = AsyncClient(**http_client_options)	4✔
155	self._init_providers(providers)	4✔
156	self._init_atexit()	4✔
157	return cls._instance	4✔
158
159	@classmethod	4✔
160	def get_instance(cls) -> "Scope3AI":	4✔
161	"""
162	Return the instance of the Scope3AI singleton.
163
164	This method provides access to the default global state of the
165	Scope3AI library. The returned instance can be used to trace
166	inference metadata and submit impact requests to the Scope3 AI
167	API from anywhere in the application.
168
169	Returns:
170	Scope3AI: The singleton instance of the Scope3AI class.
171	"""
172	return cls._instance	4✔
173
174	def submit_impact(	4✔
175	self,
176	impact_row: ImpactRow,
177	) -> Scope3AIContext:
178	"""
179	Submit an impact request to the Scope3 AI API.
180
181	This function sends an impact request represented by the `impact_row`
182	to the Scope3 AI API and optionally returns the response.
183
184	Args:
185	impact_row (ImpactRow): The impact request data
186	that needs to be submitted to the Scope3 AI API.
187
188	Returns:
189	Scope3AIContext: A context object containing the request data and
190	the response from the API.
191	"""
192
193	def submit_impact(	4✔
194	impact_row: ImpactRow,
195	ctx: Scope3AIContext,
196	) -> Optional[ImpactResponse]:
197	response = self._sync_client.impact(	4✔
198	rows=[impact_row],
199	with_response=True,
200	)
201	ctx.set_impact(response.rows[0])	4✔
202	if ctx._tracer:	4✔
203	ctx._tracer.add_impact(response.rows[0])	4✔
204	ctx._tracer._unlink_trace(ctx)	4✔
205	return response	4✔
206
207	tracer = self.current_tracer	4✔
208	ctx = Scope3AIContext(request=impact_row)	4✔
209	ctx._tracer = tracer	4✔
210	if tracer:	4✔
211	tracer._link_trace(ctx)	4✔
212
213	if self.sync_mode:	4✔
214	submit_impact(impact_row, ctx=ctx)	4✔
215	return ctx	4✔
216
217	self._ensure_worker()	4✔
218	self._worker.submit(partial(submit_impact, impact_row=impact_row, ctx=ctx))	4✔
219	return ctx	4✔
220
221	async def asubmit_impact(	4✔
222	self,
223	impact_row: ImpactRow,
224	) -> Scope3AIContext:
225	"""
226	Async version of Scope3AI::submit_impact.
227	"""
228
229	if not self.sync_mode:	4✔
230	# in non sync-mode, it uses the background worker,
231	# and the background worker is not async (does not have to be).
232	# so we just redirect the call to the sync version.
233	return self.submit_impact(impact_row)	4✔
234	tracer = self.current_tracer	4✔
235	ctx = Scope3AIContext(request=impact_row)	4✔
236	ctx._tracer = tracer	4✔
237	if tracer:	4✔
UNCOV 238	tracer._link_trace(ctx)	×
239
240	response = await self._async_client.impact(	4✔
241	rows=[impact_row],
242	with_response=True,
243	)
244	ctx.set_impact(response.rows[0])	4✔
245	if tracer:	4✔
UNCOV 246	tracer.add_impact(response.rows[0])	×
UNCOV 247	tracer._unlink_trace(ctx)	×
248
249	return ctx	4✔
250
251	@property	4✔
252	def root_tracer(self):	4✔
253	"""
254	Return the root tracer.
255
256	The root tracer is the first tracer in the current execution context
257	(tracer stack). If no tracers are currently active, it returns None.
258
259	Returns:
260	Tracer: The root tracer if available, otherwise None.
261	"""
UNCOV 262	tracers = self._tracer.get()	×
UNCOV 263	return tracers[0] if tracers else None	×
264
265	@property	4✔
266	def current_tracer(self):	4✔
267	"""
268	Return the current tracer.
269
270	The current tracer is the last tracer in the current execution context
271	(tracer stack). If no tracers are currently active, it returns None.
272
273	Returns:
274	Tracer: The current tracer if available, otherwise None.
275	"""
276	tracers = self._tracer.get()	4✔
277	return tracers[-1] if tracers else None	4✔
278
279	@contextmanager	4✔
280	def trace(self, keep_traces=False):	4✔
281	tracer = Tracer(keep_traces=keep_traces)	4✔
282	try:	4✔
283	self._push_tracer(tracer)	4✔
284	yield tracer	4✔
285	finally:
286	self._pop_tracer(tracer)	4✔
287
288	def close(self):	4✔
289	if self._worker:	4✔
290	self._worker.kill()	4✔
291	self.__class__._instance = None	4✔
292
293	#
294	# Internals
295	#
296
297	def _push_tracer(self, tracer: Tracer) -> None:	4✔
298	tracer._link_parent(self.current_tracer)	4✔
299	self._tracer.get().append(tracer)	4✔
300
301	def _pop_tracer(self, tracer: Tracer) -> None:	4✔
302	self._tracer.get().remove(tracer)	4✔
303	tracer._unlink_parent(self.current_tracer)	4✔
304
305	def _init_providers(self, providers: List[str]) -> None:	4✔
306	"""Initialize the specified providers."""
307	for provider in providers:	4✔
308	if provider not in _INSTRUMENTS:	4✔
UNCOV 309	raise Scope3AIError(	×
310	f"Could not find tracer for the `{provider}` provider."
311	)
312	if provider in self._providers and provider not in _RE_INIT_PROVIDERS:	4✔
313	# already initialized
314	continue	4✔
315	init_func = _INSTRUMENTS[provider]	4✔
316	init_func()	4✔
317	self._providers.append(provider)	4✔
318
319	def _ensure_worker(self) -> None:	4✔
320	if not self._worker:	4✔
321	self._worker = BackgroundWorker(-1)	4✔
322
323	def _init_logging(self) -> None:	4✔
324	logging.basicConfig(	4✔
325	level=logging.INFO,
326	format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
327	handlers=[logging.StreamHandler()],
328	)
329	logging.getLogger("scope3ai").setLevel(logging.DEBUG)	4✔
330
331	def _init_atexit(self):	4✔
332	@atexit.register	4✔
333	def _shutdown():	4✔
334	# do not reinstanciate the singleton here if somehow it was deleted
UNCOV 335	scope3ai = Scope3AI._instance	×
UNCOV 336	if not scope3ai:	×
UNCOV 337	return	×
UNCOV 338	if scope3ai._worker and scope3ai._worker._queue:	×
UNCOV 339	logging.debug("Waiting background informations to be processed")	×
UNCOV 340	scope3ai._worker._queue.join()	×
UNCOV 341	logging.debug("Shutting down Scope3AI")	×

scope3data / scope3ai-py / 12877283541

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous